Merge 25.7 to develop

.github/workflows/build.yml

@@ -31,6 +31,11 @@ jobs:
           echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
         id: default-branch
 
+      # Note: use slight delay in case there are associated commits across repos
+      - name: "Sleep for 30 seconds"
+        run: sleep 30s
+        shell: bash
+
       - name: "Build DISCVR"
         uses: bimberlabinternal/DevOps/githubActions/discvr-build@master
         with:

SequenceAnalysis/pipeline_code/extra_tools_install.sh

@@ -325,11 +325,11 @@ then
     echo "Cleaning up previous installs"
     rm -Rf $LKTOOLS_DIR/sawfish*
 
-    wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.0.0/sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz
-    tar -xzf sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz
+    wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.2.0/sawfish-v2.2.0-x86_64-unknown-linux-gnu.tar.gz
+    tar -xzf sawfish-v2.2.0-x86_64-unknown-linux-gnu.tar.gz
 
-    mv sawfish-v2.0.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/
-    ln -s $LKTOOLS_DIR/sawfish-v2.0.0/bin/sawfish $LKTOOLS_DIR/
+    mv sawfish-v2.2.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/
+    ln -s $LKTOOLS_DIR/sawfish-v2.2.0-x86_64-unknown-linux-gnu/bin/sawfish $LKTOOLS_DIR/
 else
     echo "Already installed"
 fi

SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets.js

@@ -1,11 +1,15 @@
-/*
- * Copyright (c) 2012 LabKey Corporation
- *
- * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
- */
+var LABKEY = require("labkey");
+
+var triggerHelper = new org.labkey.sequenceanalysis.query.SequenceTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id);
 
 function beforeDelete(row, errors){
     if (!this.extraContext.deleteFromServer){
         errors._form = 'You cannot directly delete readsets.  To delete these records, use the delete button above the readset grid.';
     }
+}
+
+function afterInsert(row, errors) {
+    if (row.sraAccessions) {
+        triggerHelper.createReaddataForSra(row.rowid, row.sraAccessions);
+    }
 }

SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/Assigned to Run Lacking Data.qview.xml

@@ -1,6 +1,6 @@
 <customView xmlns="http://labkey.org/data/xml/queryCustomView" canOverride="true">
     <sorts>
-        <sort column="name" descending="true"/>
+        <sort column="instrument_run_id/name" descending="false"/>
     </sorts>
     <filters>
         <filter column="totalFiles" operator="eq" value="0"/>

SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js

@@ -2436,7 +2436,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', {
                             },{
                                 xtype: 'textfield',
                                 fieldLabel: 'Delimiter',
-                                value: '_',
+                                value: '[_-]',
                                 itemId: 'delimiter'
                             }],
                             buttons: [{
@@ -2455,7 +2455,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', {
                                         if (prefix) {
                                             fg = fg.replace(new RegExp('^' + prefix), '');
                                         }
-                                        fg = fg.split(delim);
+                                        fg = fg.split(RegExp(delim));
                                         var id = fg[0];
                                         if (Ext4.isNumeric(id)) {
                                             r.set('readset', id);

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java

@@ -44,6 +44,7 @@ public static class Factory extends AbstractSequenceTaskFactory<Factory>
         public Factory()
         {
             super(AlignmentInitTask.class);
+            setLocation("webserver-high-priority");
             setJoin(true);
         }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java

@@ -165,6 +165,11 @@ public RecordedActionSet run() throws PipelineJobException
             libraryMembers = new TableSelector(libraryMembersTable, new SimpleFilter(FieldKey.fromString("library_id"), getPipelineJob().getLibraryId()), new Sort("ref_nt_id/name")).getArrayList(ReferenceLibraryMember.class);
         }
 
+        if (libraryMembers == null)
+        {
+            throw new PipelineJobException("There are no sequences in the library: " + getPipelineJob().getLibraryId());
+        }
+
         getJob().getLogger().info("there are " + libraryMembers.size() + " sequences to process");
 
         //make sure sequence names are unique

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java

@@ -430,6 +430,10 @@ else if (TaskFileManager.InputFileTreatment.compress == inputFileTreatment)
                         moveInputToAnalysisDir(compressed, job, actions, unalteredInputs, outputFiles);
                     }
                 }
+                else
+                {
+                    job.getLogger().debug("Input file does not exist, may have already been moved: " + input.getPath());
+                }
             }
         }
         else
@@ -450,23 +454,40 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti
             File outputDir = job.getAnalysisDirectory();
             File output = new File(outputDir, input.getName());
             job.getLogger().debug("Destination: " + output.getPath());
+            boolean alreadyMoved = false;
             if (output.exists())
             {
+                job.getLogger().debug("output already exists");
                 if (unalteredInputs != null && unalteredInputs.contains(output))
                 {
                     job.getLogger().debug("\tThis input was unaltered during normalization and a copy already exists in the analysis folder so the original will be discarded");
                     input.delete();
-                    TaskFileManagerImpl.swapFilesInRecordedActions(job.getLogger(), input, output, actions, job, null);
-                    return;
+                    alreadyMoved = true;
                 }
                 else
                 {
-                    output = new File(outputDir, FileUtil.getBaseName(input.getName()) + ".orig.gz");
-                    job.getLogger().debug("\tA file with the expected output name already exists, so the original will be renamed: " + output.getPath());
+                    if (input.length() == output.length() && input.lastModified() == output.lastModified())
+                    {
+                        job.getLogger().info("Output exists, but has the same size/modified timestamp. Deleting original");
+                        input.delete();
+                        alreadyMoved = true;
+                    }
+                    else if (input.exists() && input.length() > output.length() && input.lastModified() == output.lastModified())
+                    {
+                        job.getLogger().info("Output exists with same timestamp, but with smaller file size. This probably indicates a truncated/failed copy. Deleting this file.");
+                        output.delete();
+                    }
+                    else
+                    {
+                        throw new PipelineJobException("A file with the expected output name already exists: " + output.getPath());
+                    }
                 }
             }
 
-            FileUtils.moveFile(input, output);
+            if (!alreadyMoved)
+            {
+                FileUtils.moveFile(input, output);
+            }
             if (!output.exists())
             {
                 throw new PipelineJobException("Unable to move file: " + input.getPath());
@@ -488,7 +509,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti
 
             TaskFileManagerImpl.swapFilesInRecordedActions(job.getLogger(), input, output, actions, job, null);
         }
-        catch (IOException e)
+        catch (Exception e)
         {
             throw new PipelineJobException(e);
         }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java

@@ -35,7 +35,7 @@ public static class Factory extends AbstractTaskFactory<AbstractTaskFactorySetti
         public Factory()
         {
             super(SequenceOutputHandlerInitTask.class);
-            setLocation("webserver");
+            setLocation("webserver-high-priority");
         }
 
         @Override

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerInitTask.java

@@ -35,7 +35,7 @@ public static class Factory extends AbstractTaskFactory<AbstractTaskFactorySetti
         public Factory()
         {
             super(SequenceReadsetHandlerInitTask.class);
-            setLocation("webserver");
+            setLocation("webserver-high-priority");
         }
 
         @Override

SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java

@@ -14,9 +14,11 @@
 import org.junit.Assert;
 import org.junit.Test;
 import org.labkey.api.collections.IntHashMap;
+import org.labkey.api.assay.AssayFileWriter;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
 import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.Table;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.exp.api.DataType;
@@ -29,12 +31,21 @@
 import org.labkey.api.security.User;
 import org.labkey.api.security.UserManager;
 import org.labkey.api.sequenceanalysis.RefNtSequenceModel;
+import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.Path;
+import org.labkey.sequenceanalysis.ReadDataImpl;
 import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
+import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl;
+import org.labkey.sequenceanalysis.SequenceReadsetImpl;
+import org.labkey.sequenceanalysis.pipeline.ReadsetImportJob;
+import org.labkey.vfs.FileLike;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -254,4 +265,65 @@ public long createExpData(String relPath) {
 
         return d.getRowId();
     }
+
+    public void createReaddataForSra(int readsetId, String sraAccessions)
+    {
+        SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(Long.valueOf(readsetId), _user);
+        if (rs == null)
+        {
+            throw new IllegalArgumentException("Unable to find readset: " + readsetId);
+        }
+
+        TableInfo rd = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_READ_DATA);
+
+        String[] tokens = StringUtils.split(sraAccessions, ",");
+        for (String token : tokens)
+        {
+            if (rs.getReadData() != null && !rs.getReadData().isEmpty())
+            {
+                throw new IllegalArgumentException("Did not expect readset to have existing readdata: " + rs.getReadsetId());
+            }
+
+            // Create new:
+            ReadDataImpl rd1 = new ReadDataImpl();
+            rd1.setReadset(Long.valueOf(readsetId));
+            rd1.setContainer(rs.getContainer());
+            rd1.setCreated(new Date());
+            rd1.setModified(new Date());
+            rd1.setCreatedBy(_user.getUserId());
+            rd1.setModifiedBy(_user.getUserId());
+            rd1.setSra_accession(token);
+            rd1.setArchived(true);
+
+            // NOTE: this is a fragile assumption. We might need to eventually query SRA to figure out whether data is paired:
+            Container c = ContainerManager.getForId(rs.getContainer());
+            PipeRoot pr = PipelineService.get().findPipelineRoot(c);
+            if (pr == null)
+            {
+                throw new IllegalStateException("Unable to find pipeline root for: " + c.getPath());
+            }
+
+            String folderName = "SequenceImport_RS" + rs.getRowId() + "_" + FileUtil.getTimestamp();
+            FileLike sequenceImport = FileUtil.appendPath(pr.getRootFileLike(), Path.parse(ReadsetImportJob.NAME));
+            FileLike outDir = FileUtil.findUniqueFileName(folderName, sequenceImport);
+
+            FileLike expectedFile1 = FileUtil.appendPath(outDir, Path.parse(token + "_1.fastq.gz"));
+            ExpData exp1 = ExperimentService.get().createData(c, new DataType("Data"));
+            exp1.setDataFileURI(expectedFile1.toURI());
+            exp1.setContainer(c);
+            exp1.setName(expectedFile1.getName());
+            exp1.save(_user);
+            rd1.setFileId1(exp1.getRowId());
+
+            FileLike expectedFile2 = FileUtil.appendPath(outDir, Path.parse(token + "_2.fastq.gz"));
+            ExpData exp2 = ExperimentService.get().createData(c, new DataType("Data"));
+            exp2.setDataFileURI(expectedFile2.toURI());
+            exp2.setContainer(c);
+            exp2.setName(expectedFile2.getName());
+            exp2.save(_user);
+            rd1.setFileId2(exp2.getRowId());
+
+            Table.insert(_user, rd, rd1);
+        }
+    }
 }