Merge discvr-26.7 to develop

.gitignore

@@ -22,4 +22,7 @@ SequenceAnalysis/resources/credits/jars.txt
 SequenceAnalysis/resources/credits/dependencies.txt
 
 OpenLdapSync/resources/credits/dependencies.txt
-OpenLdapSync/resources/credits/jars.txt
+OpenLdapSync/resources/credits/jars.txt
+
+cluster/resources/credits/jars.txt
+cluster/resources/credits/dependencies.txt

OpenLdapSync/src/org/labkey/openldapsync/ldap/LdapSyncRunner.java

@@ -269,16 +269,16 @@ private void setUserActive(User u, boolean active, String reason)
         try
         {
             log("Changing active state of user: " + u.getEmail() + " to " + active + (reason == null ? "" : ", reason: " + reason));
-            _usersInactivated++;
 
             if (!_previewOnly)
             {
                 UserManager.setUserActive(_settings.getLabKeyAdminUser(), u, active);
             }
+            _usersInactivated++;
         }
         catch (SecurityManager.UserManagementException e)
         {
-            _log.error("Unable to deactive user: " + u.getEmail());
+            _log.error("Unable to deactivate user: " + u.getEmail(), e);
         }
     }
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java

@@ -16,6 +16,7 @@
 package org.labkey.api.sequenceanalysis;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
@@ -24,6 +25,11 @@
 import org.labkey.api.exp.api.ExpData;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.pipeline.PipelineJobService;
+import org.labkey.api.security.User;
+import org.labkey.api.security.permissions.Permission;
+import org.labkey.api.security.permissions.ReadPermission;
+import org.labkey.api.util.logging.LogHelper;
+import org.labkey.api.view.UnauthorizedException;
 
 import java.io.File;
 import java.io.Serializable;
@@ -34,6 +40,8 @@
  */
 public class SequenceOutputFile implements Serializable
 {
+    private static final Logger _log = LogHelper.getLogger(SequenceOutputFile.class, "Messages related to SequenceOutputFile");
+
     private Integer _rowid;
     private String _name;
     private String _description;
@@ -211,6 +219,28 @@ public void setModified(Date modified)
         _modified = modified;
     }
 
+    public static SequenceOutputFile getForId(Integer rowId, User u)
+    {
+        return getForId(rowId, u, ReadPermission.class);
+    }
+
+    public static SequenceOutputFile getForId(Integer rowId, User u, Class<? extends Permission> perm)
+    {
+        SequenceOutputFile so = getForId(rowId);
+        if (so.getContainerObj() == null)
+        {
+            _log.error("SequenceOutputFile lacks a valid container: " + rowId);
+            return null;
+        }
+
+        if (!so.getContainerObj().hasPermission(u, perm))
+        {
+            throw new UnauthorizedException("Insufficient permissions: " + rowId);
+        }
+
+        return so;
+    }
+
     public static SequenceOutputFile getForId(Integer rowId)
     {
         if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)

SequenceAnalysis/resources/queries/sequenceanalysis/distinctAnalysisSets.sql

@@ -0,0 +1,3 @@
+SELECT
+    DISTINCT rowid, name
+FROM sequenceanalysis.analysisSets

SequenceAnalysis/resources/web/SequenceAnalysis/window/AddFileSetsWindow.js

@@ -50,7 +50,8 @@ Ext4.define('SequenceAnalysis.window.AddFileSetsWindow', {
                         type: 'labkey-store',
                         containerPath: Laboratory.Utils.getQueryContainerPath(),
                         schemaName: 'laboratory',
-                        sql: 'SELECT DISTINCT rowid, name FROM sequenceanalysis.analysisSets',
+                        queryName: 'distinctAnalysisSets',
+                        columns: 'rowid,name',
                         autoLoad: true
                     },
                     valueField: 'rowid',

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -44,6 +44,7 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -81,25 +82,32 @@ public String getName()
         return "DeleteSequenceAnalysisArtifacts";
     }
 
+    private int _jobId = -1;
+
     // NOTE: if there is a more direct way to locate the JobID this hack should be replaced
     private void checkJobCancelled(Logger log)
     {
-        // Make the assumption there is only one active maintenance job at a time:
-        SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION).
-                addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()).
-                addCondition(FieldKey.fromString("modified"), new Date(), CompareType.DATE_EQUAL);
-        int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).getMapCollection().stream().filter(map -> {
-            String val = String.valueOf(map.get("status"));
-            return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name()));
-        }).map(rs -> Integer.parseInt(String.valueOf(rs.get("rowid")))).max(Integer::compareTo).orElse(-1);
-
-        if (rowId == -1)
+        if (_jobId == -1)
         {
-            log.warn("Unable to find rowId for job", new Exception("Unable to find rowId for job"));
-            return;
+            // Make the assumption there is only one active maintenance job at a time:
+            SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION).
+                    addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()).
+                    addCondition(FieldKey.fromString("modified"), LocalDate.now().minusDays(2), CompareType.DATE_GTE);
+            int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).getMapCollection().stream().filter(map -> {
+                String val = String.valueOf(map.get("status"));
+                return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name()));
+            }).map(rs -> Integer.parseInt(String.valueOf(rs.get("rowid")))).max(Integer::compareTo).orElse(-1);
+
+            if (rowId == -1)
+            {
+                log.warn("Unable to find rowId for job", new Exception("Unable to find rowId for job"));
+                return;
+            }
+
+            _jobId = rowId;
         }
 
-        PipelineStatusFile sf = PipelineService.get().getStatusFile(rowId);
+        PipelineStatusFile sf = PipelineService.get().getStatusFile(_jobId);
         if (PipelineJob.TaskStatus.cancelling.name().equalsIgnoreCase(sf.getStatus()))
         {
             throw new CancelledException();

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PrintReadBackedHaplotypesHandler.java

@@ -90,6 +90,12 @@ public boolean doSplitJobs()
         return true;
     }
 
+    @Override
+    public boolean supportsSraArchivedData()
+    {
+        return true;
+    }
+
     public class Processor implements SequenceOutputProcessor
     {
         @Override
@@ -99,7 +105,7 @@ public void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<Recor
             {
                 if (so.getReadset() != null)
                 {
-                    ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser());
+                    ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser(), true);
                 }
                 else
                 {
@@ -148,7 +154,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                     args.addAll(extraArgs);
                 }
 
-                File output = new File(ctx.getWorkingDirectory(), FileUtil.getBaseName(input) + ".txt");
+                File output = FileUtil.appendName(ctx.getWorkingDirectory(), FileUtil.getBaseName(input) + ".txt");
                 Wrapper wrapper = new Wrapper(ctx.getLogger());
                 wrapper.execute(input, ctx.getSequenceSupport().getCachedGenome(so.getLibrary_id()).getWorkingFastaFile(), output, args);
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportInitTask.java

@@ -132,7 +132,7 @@ private List<AnalysisModel> parseAndCreateReadsets() throws PipelineJobException
                         }
                     }
 
-                    getPipelineJob().getSequenceSupport().cacheGenome(SequenceAnalysisService.get().getReferenceGenome(o.getInt("library_id"), getJob().getUser()));
+                    getPipelineJob().getSequenceSupport().cacheGenome(SequenceAnalysisService.get().getReferenceGenome(o.getInt("library_id"), getJob().getUser()), true);
                     getPipelineJob().getSequenceSupport().cacheReadset(r);
                 }
             }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java

@@ -156,7 +156,7 @@ else if (steps.size() > 1)
                     throw new PipelineJobException("Reference file does not exist: " + refFasta.getPath());
                 }
 
-                getPipelineJob().getSequenceSupport().cacheGenome(output.getReferenceGenome());
+                getPipelineJob().getSequenceSupport().cacheGenome(output.getReferenceGenome(), true);
 
                 getHelper().getFileManager().addStepOutputs(action, output);
                 getHelper().getFileManager().cleanup(Collections.singleton(action));

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java

@@ -29,6 +29,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * User: bimber
@@ -125,7 +127,28 @@ public SequenceReadsetImpl getReadset()
 
     public ReferenceGenome getTargetGenome()
     {
-        return getSequenceSupport().getCachedGenomes().isEmpty() ? null : getSequenceSupport().getCachedGenomes().iterator().next();
+        if (getSequenceSupport().getCachedGenomes().isEmpty())
+        {
+            return null;
+        }
+        else if (getSequenceSupport().getCachedGenomes().size() == 1)
+        {
+            return getSequenceSupport().getCachedGenomes().iterator().next();
+        }
+
+        // Cannot infer the correct genome. We assume it was set upstream:
+        if (getSequenceSupport().getPrimaryGenomeForJob() == null)
+        {
+            throw new IllegalStateException("The primary genome ID has not been set");
+        }
+
+        Set<ReferenceGenome> passing = getSequenceSupport().getCachedGenomes().stream().filter(x -> getSequenceSupport().getPrimaryGenomeForJob().equals(x.getGenomeId())).collect(Collectors.toSet());
+        if (passing.isEmpty())
+        {
+            throw new IllegalStateException("No cached genome with ID: " + getSequenceSupport().getPrimaryGenomeForJob());
+        }
+
+        return passing.iterator().next();
     }
 
     public static final String NAME = "sequenceAnalysisPipeline";

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJobSupportImpl.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import htsjdk.samtools.util.IOUtil;
+import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 import org.junit.Test;
 import org.labkey.api.collections.IntHashMap;
@@ -46,6 +47,7 @@ public class SequenceJobSupportImpl implements SequenceAnalysisJobSupport, Seria
     private final Map<Long, AnalysisModel> _cachedAnalyses = new LongHashMap<>();
     private final Map<Integer, ReferenceGenome> _cachedGenomes = new IntHashMap<>();
     private final Map<String, Serializable> _cachedObjects = new HashMap<>();
+    private Integer _primaryGenomeForJob = null;
 
     private transient boolean _modifiedSinceSerialize = false;
 
@@ -194,6 +196,11 @@ public void cacheAnalysis(AnalysisModelImpl m, PipelineJob job, boolean allowRea
 
     @Override
     public void cacheGenome(ReferenceGenome m)
+    {
+        cacheGenome(m, false);
+    }
+
+    public void cacheGenome(ReferenceGenome m, boolean isPrimaryGenomeForJob)
     {
         markModified();
 
@@ -209,6 +216,10 @@ public void cacheGenome(ReferenceGenome m)
         }
 
         _cachedGenomes.put(key, m);
+        if (isPrimaryGenomeForJob)
+        {
+            _primaryGenomeForJob = key;
+        }
     }
 
     @Override
@@ -235,6 +246,11 @@ public Collection<ReferenceGenome> getCachedGenomes()
         return Collections.unmodifiableCollection(_cachedGenomes.values());
     }
 
+    public @Nullable Integer getPrimaryGenomeForJob()
+    {
+        return _primaryGenomeForJob;
+    }
+
     @Override
     public void cacheExpData(ExpData data)
     {
@@ -314,6 +330,7 @@ public void testSerializeWithMap() throws Exception
 
             js1._cachedReadsets.add(rs1);
             js1._cachedFilePaths.put(4L, new File("/"));
+            js1._primaryGenomeForJob = 1000;
 
             HashMap<Integer, Integer> map = new HashMap<>();
             map.put(1, 1);
@@ -339,6 +356,7 @@ public void testSerializeWithMap() throws Exception
             assertEquals("Readset list not serialized properly", 1, deserialized._cachedReadsets.size());
             assertEquals("Readset not deserialized with correct rowid", 100, deserialized._cachedReadsets.get(0).getRowId());
             assertEquals("Readset not deserialized with correct readsetid", 100, deserialized._cachedReadsets.get(0).getReadsetId().intValue());
+            assertEquals("PrimaryGenomeId not deserialized correctly", 1000, (int)deserialized._primaryGenomeForJob);
 
             assertNotNull("File map not serialized properly", deserialized._cachedFilePaths.get(4L));
             assertNotNull("Cached map not serialized properly", deserialized.getCachedObject("cachedMap",Map.class));

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/reference/CustomReferenceLibraryStep.java

@@ -10,6 +10,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceLibraryStep;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
+import org.labkey.api.util.FileUtil;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.sequenceanalysis.pipeline.ReferenceGenomeImpl;
 
@@ -62,7 +63,7 @@ public CustomReferenceLibraryStep create(PipelineContext ctx)
 
     private File getExpectedFastaFile(File outputDirectory) throws PipelineJobException
     {
-        return new File(outputDirectory, "Custom.fasta");
+        return FileUtil.appendName(outputDirectory, "Custom.fasta");
     }
 
     @Override
@@ -79,7 +80,9 @@ public Output createReferenceFasta(File outputDirectory) throws PipelineJobExcep
         try (PrintWriter writer = PrintWriters.getPrintWriter(refFasta))
         {
             if (!refFasta.exists())
-                refFasta.createNewFile();
+            {
+                FileUtil.createNewFile(refFasta);
+            }
             writer.write(">" + name + "\n");
             seq = seq.replaceAll("\\s+", "");
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SnpEffWrapper.java

@@ -145,8 +145,14 @@ else if ("gbk".equalsIgnoreCase(ext))
 
         try
         {
-            Files.createSymbolicLink(FileUtil.appendName(genomeDir, "sequences.fa").toPath(), genome.getSourceFastaFile().toPath());
-            Files.createSymbolicLink(FileUtil.appendName(genomeDir, "genes." + ext).toPath(), genes.toPath());
+            if (!FileUtil.appendName(genomeDir, "sequences.fa").exists())
+            {
+                Files.createSymbolicLink(FileUtil.appendName(genomeDir, "sequences.fa").toPath(), genome.getSourceFastaFile().toPath());
+            }
+            if (!FileUtil.appendName(genomeDir, "genes." + ext).exists())
+            {
+                Files.createSymbolicLink(FileUtil.appendName(genomeDir, "genes." + ext).toPath(), genes.toPath());
+            }
         }
         catch (IOException e)
         {

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/WhatsHapStep.java

@@ -155,7 +155,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
             throw new PipelineJobException(e);
         }
 
-        output.addSequenceOutput(vcfOut, "Phased VCF: " + inputVCF.getName(), "Phased VCF", null, null, genome.getGenomeId(), null);
+        output.setVcf(vcfOut);
 
         return output;
     }

SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java

@@ -230,10 +230,15 @@ private void importReadsetMetadata()
         waitForElement(Locator.tagContainingText("a", "SRA2"));
 
         dr.checkAllOnPage();
+        Assert.assertEquals("Incorrect checked row count", 3, dr.getCheckedCount());
         dr.clickHeaderButtonAndWait("Delete");
         clickButton("OK");
 
         _readsetCt -= 3;
+
+        log("verifying readset count correct");
+        goToProjectHome();
+        waitForElement(LabModuleHelper.getNavPanelItem("Readsets:", _readsetCt.toString()));
     }
 
     /**
@@ -417,7 +422,7 @@ private void importIlluminaTest() throws Exception
     }
 
     /**
-     * This method has several puposes.  It will verify that the records from illuminaImportTest() were
+     * This method has several purposes.  It will verify that the records from illuminaImportTest() were
      * created properly.  It also exercises various features associated with the readset grid, including
      * the FASTQC report and downloading of results
      *