From f0bc6b8556044e3dfc0d16ad3d78e9fc49310351 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 3 Sep 2025 12:21:01 -0700 Subject: [PATCH 01/59] Update view sort --- .../sequence_readsets/Assigned to Run Lacking Data.qview.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/Assigned to Run Lacking Data.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/Assigned to Run Lacking Data.qview.xml index 2a9801077..a9bfce5ec 100644 --- a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/Assigned to Run Lacking Data.qview.xml +++ b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/Assigned to Run Lacking Data.qview.xml @@ -1,6 +1,6 @@ - + From f3ffedba002770019d15d3b91eb4d005c3b3476f Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Sep 2025 09:04:01 -0700 Subject: [PATCH 02/59] Add wrapper for IdentifyAndStoreActiveClonotypes --- .../chunks/IdentifyAndStoreActiveClonotypes.R | 13 +++++++ .../labkey/singlecell/SingleCellModule.java | 2 + .../IdentifyAndStoreActiveClonotypes.java | 37 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R create mode 100644 singlecell/src/org/labkey/singlecell/pipeline/singlecell/IdentifyAndStoreActiveClonotypes.java diff --git a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R new file mode 100644 index 000000000..d82d600d1 --- /dev/null +++ b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R @@ -0,0 +1,13 @@ +for (datasetId in names(seuratObjects)) { + printName(datasetId) + seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) + + Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRA') + Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRB') + + saveData(seuratObj, datasetId) + + # Cleanup + rm(seuratObj) + gc() +} \ No newline at end of file diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java index def91fcd7..bd13979e5 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellModule.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellModule.java @@ -70,6 +70,7 @@ import org.labkey.singlecell.pipeline.singlecell.FilterRawCounts; import org.labkey.singlecell.pipeline.singlecell.FindClustersAndDimRedux; import org.labkey.singlecell.pipeline.singlecell.FindMarkers; +import org.labkey.singlecell.pipeline.singlecell.IdentifyAndStoreActiveClonotypes; import org.labkey.singlecell.pipeline.singlecell.IntegrateData; import org.labkey.singlecell.pipeline.singlecell.MergeSeurat; import org.labkey.singlecell.pipeline.singlecell.NormalizeAndScale; @@ -305,6 +306,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new ApplyKnownClonotypicData.Provider()); SequencePipelineService.get().registerPipelineStep(new CalculateTcrRepertoireStats.Provider()); SequencePipelineService.get().registerPipelineStep(new PredictTcellActivation.Provider()); + SequencePipelineService.get().registerPipelineStep(new IdentifyAndStoreActiveClonotypes.Provider()); SequenceAnalysisService.get().registerReadsetListener(new SingleCellReadsetListener()); } diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/IdentifyAndStoreActiveClonotypes.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/IdentifyAndStoreActiveClonotypes.java new file mode 100644 index 000000000..f5c67047f --- /dev/null +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/IdentifyAndStoreActiveClonotypes.java @@ -0,0 +1,37 @@ +package org.labkey.singlecell.pipeline.singlecell; + +import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.singlecell.pipeline.SingleCellStep; + +import java.util.List; + +public class IdentifyAndStoreActiveClonotypes extends AbstractRDiscvrStep +{ + public IdentifyAndStoreActiveClonotypes(PipelineContext ctx, IdentifyAndStoreActiveClonotypes.Provider provider) + { + super(provider, ctx); + } + + public static class Provider extends AbstractPipelineStepProvider + { + public Provider() + { + super("IdentifyAndStoreActiveClonotypes", "Identify And Store Active Clonotypes", "Rdiscvr", "This uses RDiscvr::IdentifyAndStoreActiveClonotypes to predict TCR-triggered T cells and save the results to the database", List.of(), null, null); + } + + + @Override + public IdentifyAndStoreActiveClonotypes create(PipelineContext ctx) + { + return new IdentifyAndStoreActiveClonotypes(ctx, this); + } + } + + @Override + public String getFileSuffix() + { + return "is"; + } +} + From 35b8849ec0d862ce1e4a6e41130a088f9f6be415 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Sep 2025 09:36:27 -0700 Subject: [PATCH 03/59] Error checking for CommonFilters --- singlecell/resources/chunks/CommonFilters.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/CommonFilters.R b/singlecell/resources/chunks/CommonFilters.R index 7e76aafbc..94aef6815 100644 --- a/singlecell/resources/chunks/CommonFilters.R +++ b/singlecell/resources/chunks/CommonFilters.R @@ -171,7 +171,10 @@ for (datasetId in names(seuratObjects)) { } toDrop <- is.na(seuratObj@meta.data$scGateConsensus) - if (sum(toDrop) > 0) { + if (sum(toDrop) == ncol(seuratObj)) { + print(paste0('There were no cells remaining after dropping cells without a scGateConsensus value')) + seuratObj <- NULL + } else if (sum(toDrop) > 0) { cells <- colnames(seuratObj)[!is.na(seuratObj@meta.data$scGateConsensus)] seuratObj <- subset(seuratObj, cells = cells) print(paste0('After dropping cells without scGateConsensus: ', length(colnames(x = seuratObj)))) From fb102b906a02cdefbc5c6539f1c0928d30d19245 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Sep 2025 13:49:31 -0700 Subject: [PATCH 04/59] Update defaults --- .../labkey/singlecell/pipeline/singlecell/CommonFilters.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CommonFilters.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CommonFilters.java index 88ccb8696..25365bfec 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CommonFilters.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CommonFilters.java @@ -27,12 +27,12 @@ public Provider() put("minValue", 0); put("maxValue", 1); put("decimalPrecision", 3); - }}, 0.5), + }}, 0.1), SeuratToolParameter.create("saturation.RNA.max", "Saturation.RNA Max", "Saturation.RNA max value", "ldk-numberfield", new JSONObject(){{ put("minValue", 0); put("maxValue", 1); put("decimalPrecision", 3); - }}, 0.9), + }}, 0.99), SeuratToolParameter.create("saturation.ADT.min", "Saturation.ADT Min", "Saturation.ADT min value", "ldk-numberfield", new JSONObject(){{ put("minValue", 0); put("maxValue", 1); From 2e6c23350e00d355c26fc72609dc639cae09e0ac Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 5 Sep 2025 04:29:04 -0700 Subject: [PATCH 05/59] Set baseUrl in script --- .../resources/chunks/IdentifyAndStoreActiveClonotypes.R | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R index d82d600d1..6f5d7f459 100644 --- a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R +++ b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R @@ -1,3 +1,12 @@ +netRc <- paste0(Sys.getenv('USER_HOME'), '/.netrc') +if (!file.exists(netRc)) { + print(list.files(Sys.getenv('USER_HOME'))) + stop(paste0('Unable to find file: ', netRc)) +} + +invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc)) +Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder) + for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) From f005c7667e0a196b2602e520a3a43f39a27c0dc7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 7 Sep 2025 18:26:50 -0700 Subject: [PATCH 06/59] Do not storeStimLevelData for TRA --- singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R index 6f5d7f459..0e4d474c2 100644 --- a/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R +++ b/singlecell/resources/chunks/IdentifyAndStoreActiveClonotypes.R @@ -11,7 +11,7 @@ for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRA') + Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRA', storeStimLevelData = FALSE) Rdiscvr::IdentifyAndStoreActiveClonotypes(seuratObj, chain = 'TRB') saveData(seuratObj, datasetId) From 43fe5e2e0e95cb9b984bf9a3857a5942a29a7765 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Sep 2025 09:38:12 -0700 Subject: [PATCH 07/59] Do not force seurat object name to match readset --- .../analysis/AbstractSingleCellHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java index 4f0b32f92..1f365d003 100644 --- a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java +++ b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java @@ -396,7 +396,16 @@ public void processFilesRemote(List inputFiles, JobContext c currentFiles = new ArrayList<>(); for (SequenceOutputFile so : inputFiles) { - String datasetId = FileUtil.makeLegalName(so.getReadset() != null ? ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() : so.getName()); + String datasetId; + if ("Seurat Object Prototype".equals(so.getCategory())) + { + datasetId = FileUtil.makeLegalName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName()); + } + else + { + datasetId = so.getName(); + } + if (distinctIds.contains(datasetId)) { throw new PipelineJobException("Duplicate dataset Ids in input data: " + datasetId); From feba968c7998c694db47f0d5d57aa6cc33f19b25 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Sep 2025 09:39:19 -0700 Subject: [PATCH 08/59] Do not force seurat object name to match readset --- .../labkey/singlecell/analysis/AbstractSingleCellHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java index 1f365d003..fabc68fe4 100644 --- a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java +++ b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java @@ -401,6 +401,10 @@ public void processFilesRemote(List inputFiles, JobContext c { datasetId = FileUtil.makeLegalName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName()); } + else if (_doProcessRawCounts) + { + datasetId = FileUtil.makeLegalName(so.getReadset() == null ? so.getName() : ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName()); + } else { datasetId = so.getName(); From 64699644d475dc364043727980485c775a88ab1c Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Sep 2025 09:57:47 -0700 Subject: [PATCH 09/59] Drop CS-Core and Tricycle --- singlecell/resources/chunks/RunCsCore.R | 14 ---- singlecell/resources/chunks/RunTricycle.R | 12 ---- .../labkey/singlecell/SingleCellModule.java | 4 -- .../pipeline/singlecell/RunCsCore.java | 67 ------------------- .../pipeline/singlecell/RunTricycle.java | 37 ---------- 5 files changed, 134 deletions(-) delete mode 100644 singlecell/resources/chunks/RunCsCore.R delete mode 100644 singlecell/resources/chunks/RunTricycle.R delete mode 100644 singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCsCore.java delete mode 100644 singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunTricycle.java diff --git a/singlecell/resources/chunks/RunCsCore.R b/singlecell/resources/chunks/RunCsCore.R deleted file mode 100644 index b7c7fd7a8..000000000 --- a/singlecell/resources/chunks/RunCsCore.R +++ /dev/null @@ -1,14 +0,0 @@ -for (datasetId in names(seuratObjects)) { - printName(datasetId) - seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - - outFile <- paste0(outputPrefix, '.', makeLegalFileName(datasetId), '.markers.txt') - module_list <- CellMembrane::RunCsCore(seuratObj, saveFile = paste0(outFile, '.cscore.rds')) - saveRDS(module_list, paste0(outFile, '.cscore.wgcna.rds')) - - saveData(seuratObj, datasetId) - - # Cleanup - rm(seuratObj) - gc() -} \ No newline at end of file diff --git a/singlecell/resources/chunks/RunTricycle.R b/singlecell/resources/chunks/RunTricycle.R deleted file mode 100644 index b04162f1d..000000000 --- a/singlecell/resources/chunks/RunTricycle.R +++ /dev/null @@ -1,12 +0,0 @@ - for (datasetId in names(seuratObjects)) { - printName(datasetId) - seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) - - seuratObj <- CellMembrane::RunTricycle(seuratObj) - - saveData(seuratObj, datasetId) - - # Cleanup - rm(seuratObj) - gc() -} \ No newline at end of file diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java index bd13979e5..9c459da1e 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellModule.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellModule.java @@ -87,7 +87,6 @@ import org.labkey.singlecell.pipeline.singlecell.RunCelltypist; import org.labkey.singlecell.pipeline.singlecell.RunCelltypistCustomModel; import org.labkey.singlecell.pipeline.singlecell.RunConga; -import org.labkey.singlecell.pipeline.singlecell.RunCsCore; import org.labkey.singlecell.pipeline.singlecell.RunDecoupler; import org.labkey.singlecell.pipeline.singlecell.RunEscape; import org.labkey.singlecell.pipeline.singlecell.RunLDA; @@ -98,7 +97,6 @@ import org.labkey.singlecell.pipeline.singlecell.RunScGate; import org.labkey.singlecell.pipeline.singlecell.RunScGateBuiltin; import org.labkey.singlecell.pipeline.singlecell.RunSingleR; -import org.labkey.singlecell.pipeline.singlecell.RunTricycle; import org.labkey.singlecell.pipeline.singlecell.RunVision; import org.labkey.singlecell.pipeline.singlecell.ScoreCellCycle; import org.labkey.singlecell.pipeline.singlecell.SeuratPrototype; @@ -295,14 +293,12 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new TrainScTour.Provider()); SequencePipelineService.get().registerPipelineStep(new PredictScTour.Provider()); SequencePipelineService.get().registerPipelineStep(new RunEscape.Provider()); - SequencePipelineService.get().registerPipelineStep(new RunCsCore.Provider()); SequencePipelineService.get().registerPipelineStep(new CustomGSEA.Provider()); SequencePipelineService.get().registerPipelineStep(new StudyMetadata.Provider()); SequencePipelineService.get().registerPipelineStep(new UpdateSeuratPrototype.Provider()); SequencePipelineService.get().registerPipelineStep(new RunDecoupler.Provider()); SequencePipelineService.get().registerPipelineStep(new PerformDefaultNimbleAppend.Provider()); SequencePipelineService.get().registerPipelineStep(new PerformMhcDimRedux.Provider()); - SequencePipelineService.get().registerPipelineStep(new RunTricycle.Provider()); SequencePipelineService.get().registerPipelineStep(new ApplyKnownClonotypicData.Provider()); SequencePipelineService.get().registerPipelineStep(new CalculateTcrRepertoireStats.Provider()); SequencePipelineService.get().registerPipelineStep(new PredictTcellActivation.Provider()); diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCsCore.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCsCore.java deleted file mode 100644 index f3a2dded2..000000000 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCsCore.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.labkey.singlecell.pipeline.singlecell; - -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.singlecell.pipeline.SingleCellStep; - -import java.io.File; -import java.util.Collections; -import java.util.List; - -public class RunCsCore extends AbstractCellMembraneStep -{ - public RunCsCore(PipelineContext ctx, Provider provider) - { - super(provider, ctx); - } - - public static class Provider extends AbstractPipelineStepProvider - { - public Provider() - { - super("RunCsCore", "CS-CORE", "CS-CORE", "Run CS-CORE on the seurat object to identify gene modules.", Collections.emptyList(), null, null); - } - - @Override - public RunCsCore create(PipelineContext ctx) - { - return new RunCsCore(ctx, this); - } - } - - @Override - public boolean createsSeuratObjects() - { - return false; - } - - @Override - public String getFileSuffix() - { - return "cscore"; - } - - @Override - public Output execute(SequenceOutputHandler.JobContext ctx, List inputObjects, String outputPrefix) throws PipelineJobException - { - Output output = super.execute(ctx, inputObjects, outputPrefix); - - // Add the RDS files: - File[] outputs = ctx.getOutputDir().listFiles(f -> f.isDirectory() && f.getName().endsWith(".cscore.wgcna.rds")); - if (outputs == null || outputs.length == 0) - { - return output; - } - - for (File rds : outputs) - { - String sn = rds.getName().replaceAll(".cscore.wgcna.rds", ""); - - output.addSequenceOutput(rds, "CS-CORE: " + sn, "CS-CORE Results", inputObjects.get(0).getReadsetId(), null, ctx.getSequenceSupport().getCachedGenomes().iterator().next().getGenomeId(), null); - } - - return output; - } -} \ No newline at end of file diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunTricycle.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunTricycle.java deleted file mode 100644 index 4e79d7673..000000000 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunTricycle.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.labkey.singlecell.pipeline.singlecell; - -import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; -import org.labkey.api.singlecell.pipeline.SingleCellStep; - -import java.util.Arrays; - -public class RunTricycle extends AbstractCellMembraneStep -{ - public RunTricycle(PipelineContext ctx, RunTricycle.Provider provider) - { - super(provider, ctx); - } - - public static class Provider extends AbstractPipelineStepProvider - { - public Provider() - { - super("RunTricycle", "Run Tricycle", "CellMembrane/Tricycle", "This will run tricycle on the input object(s) to score cell cycle, and save the results in metadata.", Arrays.asList( - - ), null, null); - } - - @Override - public RunTricycle create(PipelineContext ctx) - { - return new RunTricycle(ctx, this); - } - } - - @Override - public String getFileSuffix() - { - return "tricycle"; - } -} From 4427a6a615151148caa251856213b59d6fdcb060 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Sep 2025 17:41:53 -0700 Subject: [PATCH 10/59] Expand data cleanup --- singlecell/resources/queries/singlecell/samples.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/queries/singlecell/samples.js b/singlecell/resources/queries/singlecell/samples.js index f66be9f90..6b3ec2600 100644 --- a/singlecell/resources/queries/singlecell/samples.js +++ b/singlecell/resources/queries/singlecell/samples.js @@ -23,7 +23,7 @@ function beforeUpsert(row, oldRow, errors){ else if (['No stim', 'No Stim'].indexOf(row.stim) !== -1){ row.stim = 'NoStim'; } - else if (['Infected cells: SIV+', 'Infected Cells: SIV+'].indexOf(row.stim) !== -1){ + else if (['SIV+', 'Infected cells: SIV+', 'Infected Cells: SIV+'].indexOf(row.stim) !== -1){ row.stim = 'SIV-Infected CD4s'; } else if (['Infected cells: SIV-', 'Infected Cells: SIV-'].indexOf(row.stim) !== -1){ From d0053f88417971475c1aed299090400b0e316c2c Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 12 Sep 2025 10:07:43 -0700 Subject: [PATCH 11/59] Minor code cleanup --- .../src/org/labkey/singlecell/run/CellRangerGexCountStep.java | 2 +- .../src/org/labkey/singlecell/run/NimbleAlignmentStep.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java index 62ea31c90..0d7abe3ce 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java @@ -61,7 +61,7 @@ public class CellRangerGexCountStep extends AbstractAlignmentPipelineStep provider, PipelineContext ctx, CellRangerWrapper wrapper) { super(provider, ctx, wrapper); } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 0dcb21bf9..1fd40d513 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -37,7 +37,7 @@ public static class Provider extends AbstractAlignmentStepProvider(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null, true, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); + super("Nimble", "This will run Nimble to generate a supplemental scRNA-seq feature count matrix for the provided libraries", getCellRangerGexParams(getToolParameters()), new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null, true, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); } @Override From ddcfe0855046bee0f2063b40164949c8d915509e Mon Sep 17 00:00:00 2001 From: hextraza Date: Tue, 16 Sep 2025 13:21:06 -0700 Subject: [PATCH 12/59] Add subadapter + reflection pattern to ExtendedVariantAdapter in order to delete VCFTabixAdapter (#353) * Add subadapter + reflection pattern to ExtendedVariantAdapter in order to delete VCFTabixAdapter * Add API wrapper for getMetadata and getHeader --------- Co-authored-by: Sebastian Benjamin --- .../ExtendedVariantAdapter.ts | 74 +++++++- .../ExtendedVariantAdapter/VcfTabixAdapter.ts | 159 ------------------ 2 files changed, 71 insertions(+), 162 deletions(-) delete mode 100644 jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/VcfTabixAdapter.ts diff --git a/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/ExtendedVariantAdapter.ts b/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/ExtendedVariantAdapter.ts index d7973773b..035790c92 100644 --- a/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/ExtendedVariantAdapter.ts +++ b/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/ExtendedVariantAdapter.ts @@ -1,14 +1,57 @@ import QuickLRU from '@jbrowse/core/util/QuickLRU'; -import { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter'; +import { BaseOptions, BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter'; import { NoAssemblyRegion } from '@jbrowse/core/util/types'; import { ObservableCreate } from '@jbrowse/core/util/rxjs'; import { Feature } from '@jbrowse/core/util/simpleFeature'; import ExtendedVcfFeature from './ExtendedVcfFeature'; import { VcfFeature } from '@jbrowse/plugin-variants'; -import { default as VcfTabixAdapter } from './VcfTabixAdapter'; -export default class extends VcfTabixAdapter { +export default class extends BaseFeatureDataAdapter { protected featureCache = new QuickLRU({ maxSize: 20 }) + private subAdapterP?: Promise + + constructor(...args: any[]) { + super(...args) + + // Return a Proxy that forwards any unknown member access to the sub-adapter. + // This avoids re-implementing methods like getHeader/getRefNames/getMetadata/etc. + const self = this + return new Proxy(this, { + get(target, prop, receiver) { + // If we have it already (e.g., getFeatures, getFeaturesAsArray, BaseFeatureDataAdapter-derived properties), use it directly + if (prop in target || typeof prop === 'symbol') { + return Reflect.get(target, prop, receiver) + } + + // Otherwise, forward to the VcfTabixAdapter sub-adapter + return async (...callArgs: any[]) => { + const sub = await self.getVcfSubAdapter() + const value = (sub as any)[prop] + + // If it’s a method, call it; otherwise return the property value + if (typeof value === 'function') { + return value.apply(sub, callArgs) + } + return value + } + }, + }) + } + + private async getVcfSubAdapter(): Promise { + if (!this.subAdapterP) { + const vcfGzLocation = this.getConf('vcfGzLocation') + const index = this.getConf(['index']) + const vcfAdapterConf = { type: 'VcfTabixAdapter', vcfGzLocation, index } + this.subAdapterP = this.getSubAdapter!(vcfAdapterConf) + .then(({ dataAdapter }) => dataAdapter) + .catch(e => { + this.subAdapterP = undefined + throw e + }) + } + return this.subAdapterP + } public getFeatures(query: NoAssemblyRegion, opts: BaseOptions = {}) { return ObservableCreate(async observer => { @@ -50,4 +93,29 @@ export default class extends VcfTabixAdapter { return features } + + // Typescript errors at compile time without these stubs + async configure(opts?: BaseOptions) { + const sub = await this.getVcfSubAdapter() + return sub.configure(opts) + } + + async getRefNames(opts: BaseOptions = {}) { + const sub = await this.getVcfSubAdapter() + return sub.getRefNames(opts) + } + + async getHeader(opts?: BaseOptions) { + const sub = await this.getVcfSubAdapter() + return sub.getHeader(opts) + } + + async getMetadata(opts?: BaseOptions) { + const sub = await this.getVcfSubAdapter() + return sub.getMetadata(opts) + } + + freeResources(): void { + void this.getVcfSubAdapter().then(sub => sub.freeResources?.()) + } } \ No newline at end of file diff --git a/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/VcfTabixAdapter.ts b/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/VcfTabixAdapter.ts deleted file mode 100644 index fca6a3e64..000000000 --- a/jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/VcfTabixAdapter.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { TabixIndexedFile } from '@gmod/tabix' -import VcfParser from '@gmod/vcf' -import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter' -import { - fetchAndMaybeUnzipText, - updateStatus, -} from '@jbrowse/core/util' -import { openLocation } from '@jbrowse/core/util/io' -import { ObservableCreate } from '@jbrowse/core/util/rxjs' - -import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter' -import type { Feature } from '@jbrowse/core/util' -import type { NoAssemblyRegion } from '@jbrowse/core/util/types' -import { VcfFeature } from '@jbrowse/plugin-variants'; - -function shorten2(name: string, max = 70) { - return name.length > max ? `${name.slice(0, max)}...` : name -} - -export default class VcfTabixAdapter extends BaseFeatureDataAdapter { - private configured?: Promise<{ - vcf: TabixIndexedFile - parser: VcfParser - }> - - private async configurePre(_opts?: BaseOptions) { - const vcfGzLocation = this.getConf('vcfGzLocation') - const location = this.getConf(['index', 'location']) - const indexType = this.getConf(['index', 'indexType']) - - const filehandle = openLocation(vcfGzLocation, this.pluginManager) - const isCSI = indexType === 'CSI' - const vcf = new TabixIndexedFile({ - filehandle, - csiFilehandle: isCSI - ? openLocation(location, this.pluginManager) - : undefined, - tbiFilehandle: !isCSI - ? openLocation(location, this.pluginManager) - : undefined, - chunkCacheSize: 50 * 2 ** 20, - }) - - return { - vcf, - parser: new VcfParser({ - header: await vcf.getHeader(), - }), - } - } - - protected async configurePre2() { - if (!this.configured) { - this.configured = this.configurePre().catch((e: unknown) => { - this.configured = undefined - throw e - }) - } - return this.configured - } - - async configure(opts?: BaseOptions) { - const { statusCallback = () => {} } = opts || {} - return updateStatus('Downloading index', statusCallback, () => - this.configurePre2(), - ) - } - public async getRefNames(opts: BaseOptions = {}) { - const { vcf } = await this.configure(opts) - return vcf.getReferenceSequenceNames(opts) - } - - async getHeader(opts?: BaseOptions) { - const { vcf } = await this.configure(opts) - return vcf.getHeader() - } - - async getMetadata(opts?: BaseOptions) { - const { parser } = await this.configure(opts) - return parser.getMetadata() - } - - public getFeatures(query: NoAssemblyRegion, opts: BaseOptions = {}) { - return ObservableCreate(async observer => { - const { refName, start, end } = query - const { statusCallback = () => {} } = opts - const { vcf, parser } = await this.configure(opts) - - await updateStatus('Downloading variants', statusCallback, () => - vcf.getLines(refName, start, end, { - lineCallback: (line, fileOffset) => { - observer.next( - new VcfFeature({ - variant: parser.parseLine(line), - parser, - id: `${this.id}-vcf-${fileOffset}`, - }), - ) - }, - ...opts, - }), - ) - observer.complete() - }, opts.stopToken) - } - - async getSources() { - const conf = this.getConf('samplesTsvLocation') - if (conf.uri === '' || conf.uri === '/path/to/samples.tsv') { - const { parser } = await this.configure() - return parser.samples.map(name => ({ - name, - })) - } else { - const txt = await fetchAndMaybeUnzipText( - openLocation(conf, this.pluginManager), - ) - const lines = txt.split(/\n|\r\n|\r/) - const header = lines[0]!.split('\t') - const { parser } = await this.configure() - const metadataLines = lines - .slice(1) - .filter(f => !!f) - .map(line => { - const [name, ...rest] = line.split('\t') - return { - ...Object.fromEntries( - // force col 0 to be called name - rest.map((c, idx) => [header[idx + 1]!, c] as const), - ), - name: name!, - } - }) - const vcfSampleSet = new Set(parser.samples) - const metadataSet = new Set(metadataLines.map(r => r.name)) - const metadataNotInVcfSamples = [...metadataSet].filter( - f => !vcfSampleSet.has(f), - ) - const vcfSamplesNotInMetadata = [...vcfSampleSet].filter( - f => !metadataSet.has(f), - ) - if (metadataNotInVcfSamples.length) { - console.warn( - `There are ${metadataNotInVcfSamples.length} samples in metadata file (${metadataLines.length} lines) not in VCF (${parser.samples.length} samples):`, - shorten2(metadataNotInVcfSamples.join(',')), - ) - } - if (vcfSamplesNotInMetadata.length) { - console.warn( - `There are ${vcfSamplesNotInMetadata.length} samples in VCF file (${parser.samples.length} samples) not in metadata file (${metadataLines.length} lines):`, - shorten2(vcfSamplesNotInMetadata.map(m => m).join(',')), - ) - } - return metadataLines.filter(f => vcfSampleSet.has(f.name)) - } - } - - public freeResources(/* { region } */): void {} -} From f1b7e52732938e1631081ef78fe32f59ff4a8583 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 18 Sep 2025 09:29:17 -0700 Subject: [PATCH 13/59] Correct HTML syntax --- Studies/resources/views/studiesOverview.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Studies/resources/views/studiesOverview.html b/Studies/resources/views/studiesOverview.html index 2d1d6a280..5fd144f1e 100644 --- a/Studies/resources/views/studiesOverview.html +++ b/Studies/resources/views/studiesOverview.html @@ -43,15 +43,15 @@ }); dataByCategory[row['CategoryId/Label']] = dataByCategory[row['CategoryId/Label']] || [] - dataByCategory[row['CategoryId/Label']].push(`
  • ${row.Label}
    • `) + dataByCategory[row['CategoryId/Label']].push($(`
  • ${row.Label}
    • `)) }) Object.keys(dataByCategory).sort().forEach(category => { div.append('

    ' + category + '

    ').append('
      ') + const ul = div.children('ul')[0] dataByCategory[category].forEach(item => { - div.append(item) + ul.append(item[0]) }) - div.append('
    ') }) $('#' + webpart.wrapperDivId).append(div) From 0c0da7dd91cc390eaf3c139f7320ad819806a540 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Sep 2025 10:01:17 -0700 Subject: [PATCH 14/59] Expand study triggers and update cohort fields --- Studies/resources/schemas/studies.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/Studies/resources/schemas/studies.xml b/Studies/resources/schemas/studies.xml index 5dbd8d185..b4ea92255 100644 --- a/Studies/resources/schemas/studies.xml +++ b/Studies/resources/schemas/studies.xml @@ -196,6 +196,7 @@ DETAILED + Cohort ID true false false From 2f7566ab42a3b398547c60c9965affb2ff8f92a9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Sep 2025 10:43:37 -0700 Subject: [PATCH 15/59] Expose getter for StudiesTriggerFactory --- .../api-src/org/labkey/api/studies/StudiesService.java | 3 +++ Studies/src/org/labkey/studies/StudiesServiceImpl.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/Studies/api-src/org/labkey/api/studies/StudiesService.java b/Studies/api-src/org/labkey/api/studies/StudiesService.java index 848fde966..700355741 100644 --- a/Studies/api-src/org/labkey/api/studies/StudiesService.java +++ b/Studies/api-src/org/labkey/api/studies/StudiesService.java @@ -2,6 +2,7 @@ import org.labkey.api.data.Container; import org.labkey.api.data.TableCustomizer; +import org.labkey.api.data.triggers.TriggerFactory; import org.labkey.api.module.Module; import org.labkey.api.resource.Resource; import org.labkey.api.security.User; @@ -37,4 +38,6 @@ static public void setInstance(StudiesService instance) abstract public List getEventProviders(Container c); abstract public TableCustomizer getStudiesTableCustomizer(); + + abstract public TriggerFactory getStudiesTriggerFactory(); } diff --git a/Studies/src/org/labkey/studies/StudiesServiceImpl.java b/Studies/src/org/labkey/studies/StudiesServiceImpl.java index 4c3910b5b..01ffae684 100644 --- a/Studies/src/org/labkey/studies/StudiesServiceImpl.java +++ b/Studies/src/org/labkey/studies/StudiesServiceImpl.java @@ -5,6 +5,7 @@ import org.labkey.api.data.Container; import org.labkey.api.data.TableCustomizer; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.triggers.TriggerFactory; import org.labkey.api.module.Module; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineService; @@ -25,6 +26,7 @@ import org.labkey.api.util.Path; import org.labkey.api.util.logging.LogHelper; import org.labkey.studies.query.StudiesTableCustomizer; +import org.labkey.studies.query.StudiesTriggerFactory; import java.io.FileNotFoundException; import java.io.IOException; @@ -51,6 +53,12 @@ private StudiesServiceImpl() } + @Override + public TriggerFactory getStudiesTriggerFactory() + { + return new StudiesTriggerFactory(); + } + @Override public void importFolderDefinition(Container container, User user, Module m, Path sourceFolderDirPath) throws IOException { From 04e2b402a8c32758cabe4223daa636a088a7c62b Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Sep 2025 14:05:55 -0700 Subject: [PATCH 16/59] Clean up trigger/customizer layer code --- .../studies/query/StudiesTriggerFactory.java | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java b/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java index b8805dd35..e3fbc5c3d 100644 --- a/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java +++ b/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java @@ -38,33 +38,55 @@ public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map< @Override public void beforeInsert(TableInfo table, Container c, User user, @Nullable Map newRow, ValidationException errors, Map extraContext, @Nullable Map existingRecord) throws ValidationException { - possiblyResolveStudy(newRow, c); + possiblyResolveStudy(table, newRow, existingRecord, c); } @Override public void beforeUpdate(TableInfo table, Container c, User user, @Nullable Map newRow, @Nullable Map oldRow, ValidationException errors, Map extraContext) throws ValidationException { - possiblyResolveStudy(newRow, c); + possiblyResolveStudy(table, newRow, oldRow, c); } /** * This allows incoming data to specify the study using the string name, which is resolved into the rowId */ - private void possiblyResolveStudy(@Nullable Map row, Container c) + private void possiblyResolveStudy(TableInfo table, @Nullable Map row, @Nullable Map oldRow, Container c) { if (row == null) { return; } - possiblyResolveStudy(row, c, "studyId"); - if (row.get("studyId") == null & row.get("studyName") != null) + if (table.getColumn("studyId") != null) { - possiblyResolveStudy(row, c, "studyName"); + possiblyResolveStudy(row, c, "studyId"); + if (row.get("studyId") == null & row.get("studyName") != null) + { + possiblyResolveStudy(row, c, "studyName"); + } + } + + if (table.getColumn("cohortId") != null) + { + possiblyResolveCohort(row, c, "cohortId"); + if (row.get("cohortId") == null & row.get("cohortName") != null) + { + possiblyResolveCohort(row, c, "cohortName"); + } } } private void possiblyResolveStudy(@Nullable Map row, Container c, String sourceProperty) + { + possiblyResolveStudyOrCohort(row, c, sourceProperty, "studyId", "studyName"); + } + + private void possiblyResolveCohort(@Nullable Map row, Container c, String sourceProperty) + { + possiblyResolveStudyOrCohort(row, c, sourceProperty, "cohortId", "cohortName"); + } + + private void possiblyResolveStudyOrCohort(@Nullable Map row, Container c, String sourceProperty, String targetFieldName, String filterFieldName) { if (row == null) { @@ -76,11 +98,11 @@ private void possiblyResolveStudy(@Nullable Map row, Container c if (!NumberUtils.isCreatable(row.get(sourceProperty).toString())) { Container target = c.isWorkbookOrTab() ? c.getParent() : c; - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("container"), target.getEntityId()).addCondition(FieldKey.fromString("studyName"), row.get(sourceProperty)); - List rowIds = new TableSelector(StudiesSchema.getInstance().getSchema().getTable(StudiesSchema.TABLE_STUDIES), PageFlowUtil.set("rowId"), filter, null).getArrayList(Integer.class); + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("container"), target.getEntityId()).addCondition(FieldKey.fromString(filterFieldName), row.get(sourceProperty)); + List rowIds = new TableSelector(StudiesSchema.getInstance().getSchema().getTable(StudiesSchema.TABLE_COHORTS), PageFlowUtil.set("rowId"), filter, null).getArrayList(Integer.class); if (rowIds.size() == 1) { - row.put("studyId", rowIds.get(0)); + row.put(targetFieldName, rowIds.get(0)); } } } From 568817012fdec777b413aa06b482505002acb850 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Sep 2025 14:46:45 -0700 Subject: [PATCH 17/59] Create fields to coalesce name/label for studies --- Studies/resources/schemas/studies.xml | 8 ++--- .../studies/query/StudiesUserSchema.java | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/Studies/resources/schemas/studies.xml b/Studies/resources/schemas/studies.xml index b4ea92255..8420ce4d2 100644 --- a/Studies/resources/schemas/studies.xml +++ b/Studies/resources/schemas/studies.xml @@ -208,7 +208,7 @@ studies studies rowId - label + labelOrName @@ -287,7 +287,7 @@ studies studies rowId - label + labelOrName @@ -352,7 +352,7 @@ studies studies rowId - label + labelOrName @@ -361,7 +361,7 @@ studies studyCohorts rowId - label + labelOrName diff --git a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java index 928d882c6..bb7c741db 100644 --- a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java +++ b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java @@ -138,11 +138,11 @@ else if (TABLE_LOOKUPS.equalsIgnoreCase(name)) } else if (TABLE_STUDIES.equalsIgnoreCase(name)) { - return createStudiesTable(name, cf, false); + return createStudiesTable(name, cf); } else if (TABLE_COHORTS.equalsIgnoreCase(name)) { - return createStudyDesignTable(name, cf, true); + return createCohortsTable(name, cf); } else if (TABLE_ANCHOR_EVENTS.equalsIgnoreCase(name)) { @@ -169,18 +169,41 @@ else if (TABLE_EVENT_TYPES.equalsIgnoreCase(name)) return super.createTable(name, cf); } - private TableInfo createStudiesTable(String name, ContainerFilter cf, boolean addTriggers) + private TableInfo createCohortsTable(String name, ContainerFilter cf) + { + CustomPermissionsTable ret = createStudyDesignTable(name, cf, true); + + SQLFragment sql2 = new SQLFragment("coalesce(" + ExprColumn.STR_TABLE_ALIAS + ".label, " + ExprColumn.STR_TABLE_ALIAS + ".cohortName)"); + ExprColumn col2 = new ExprColumn(ret, "labelOrName", sql2, JdbcType.VARCHAR, ret.getColumn("cohortName"), ret.getColumn("label")); + col2.setLabel("Cohort Name"); + col2.setHidden(true); + col2.setDescription("This column lists the cohort label, and the name if label is blank"); + + ret.addColumn(col2); + + return ret; + } + + private TableInfo createStudiesTable(String name, ContainerFilter cf) { - CustomPermissionsTable ret = createStudyDesignTable(name, cf, addTriggers); + CustomPermissionsTable ret = createStudyDesignTable(name, cf, false); final String chr = ret.getSqlDialect().isPostgreSQL() ? "chr" : "char"; - SQLFragment sql1 = new SQLFragment("(SELECT ").append(ret.getSqlDialect().getGroupConcat(new SQLFragment("c.label"), true, true, new SQLFragment(chr + "(10)"))).append(" as expr FROM " + StudiesSchema.NAME + "." + TABLE_COHORTS + " c WHERE c.studyId = " + ExprColumn.STR_TABLE_ALIAS + ".rowId)"); + SQLFragment sql1 = new SQLFragment("(SELECT ").append(ret.getSqlDialect().getGroupConcat(new SQLFragment("coalesce(c.label, c.cohortName)"), true, true, new SQLFragment(chr + "(10)"))).append(" as expr FROM " + StudiesSchema.NAME + "." + TABLE_COHORTS + " c WHERE c.studyId = " + ExprColumn.STR_TABLE_ALIAS + ".rowId)"); ExprColumn col1 = new ExprColumn(ret, "cohorts", sql1, JdbcType.VARCHAR, ret.getColumn("rowid")); col1.setLabel("Cohort(s)"); col1.setDescription("This column lists the cohort labels for this study"); ret.addColumn(col1); + SQLFragment sql2 = new SQLFragment("coalesce(" + ExprColumn.STR_TABLE_ALIAS + ".label, " + ExprColumn.STR_TABLE_ALIAS + ".studyName)"); + ExprColumn col2 = new ExprColumn(ret, "labelOrName", sql2, JdbcType.VARCHAR, ret.getColumn("studyName"), ret.getColumn("label")); + col2.setLabel("Study Name"); + col2.setHidden(true); + col2.setDescription("This column lists the study label, and the name if label is blank"); + + ret.addColumn(col2); + return ret; } From 14ed86a7affc17b94a3740dcf9e928de52e356c1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 22 Sep 2025 06:12:13 -0700 Subject: [PATCH 18/59] Add null check --- .../org/labkey/singlecell/pipeline/singlecell/VireoHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index d4ddc7326..3976c6155 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -72,7 +72,7 @@ public boolean doSplitJobs() @Override public boolean canProcess(SequenceOutputFile o) { - return CellRangerGexCountStep.LOUPE_CATEGORY.equals(o.getCategory()) & o.getFile().getName().endsWith("cloupe.cloupe"); + return CellRangerGexCountStep.LOUPE_CATEGORY.equals(o.getCategory()) & o.getFile() != null & o.getFile().getName().endsWith("cloupe.cloupe"); } @Override From 733f92dcd0c5d30eb7cf2aedefc4b525ff39d631 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 22 Sep 2025 06:12:48 -0700 Subject: [PATCH 19/59] Switch default --- .../org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java index 7a646eb38..a17c151d8 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/SubsetSeurat.java @@ -34,7 +34,7 @@ public Provider() put("width", 600); put("delimiter", DELIM); }}, null), - ToolParameterDescriptor.create("useDplyr", "Use dplyr", "If checked, the subset will be executed using dplyr::filter rather than Seurat::subset. This should allow more complex expressions to be used, including negations", "checkbox", null, false) + ToolParameterDescriptor.create("useDplyr", "Use dplyr", "If checked, the subset will be executed using dplyr::filter rather than Seurat::subset. This should allow more complex expressions to be used, including negations", "checkbox", null, true) ), List.of("/sequenceanalysis/field/TrimmingTextArea.js"), null); } From f4aebaf2f7dabc58bc105750d0503b9bbce1211d Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 24 Sep 2025 09:22:38 -0700 Subject: [PATCH 20/59] Bugfix to StudiesTriggerFactory --- .../labkey/studies/query/StudiesTriggerFactory.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java b/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java index e3fbc5c3d..c2ca102f2 100644 --- a/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java +++ b/Studies/src/org/labkey/studies/query/StudiesTriggerFactory.java @@ -78,28 +78,28 @@ private void possiblyResolveStudy(TableInfo table, @Nullable Map private void possiblyResolveStudy(@Nullable Map row, Container c, String sourceProperty) { - possiblyResolveStudyOrCohort(row, c, sourceProperty, "studyId", "studyName"); + possiblyResolveStudyOrCohort(StudiesSchema.TABLE_STUDIES, row, c, sourceProperty, "studyId", "studyName"); } private void possiblyResolveCohort(@Nullable Map row, Container c, String sourceProperty) { - possiblyResolveStudyOrCohort(row, c, sourceProperty, "cohortId", "cohortName"); + possiblyResolveStudyOrCohort(StudiesSchema.TABLE_COHORTS, row, c, sourceProperty, "cohortId", "cohortName"); } - private void possiblyResolveStudyOrCohort(@Nullable Map row, Container c, String sourceProperty, String targetFieldName, String filterFieldName) + private void possiblyResolveStudyOrCohort(String tableToQuery, @Nullable Map row, Container c, String sourceProperty, String targetFieldName, String filterFieldName) { if (row == null) { return; } - if (row.get(sourceProperty) != null & row.get(sourceProperty) instanceof String) + if (row.get(sourceProperty) != null & row.get(sourceProperty) instanceof String & !String.valueOf(row.get(sourceProperty)).isEmpty()) { if (!NumberUtils.isCreatable(row.get(sourceProperty).toString())) { Container target = c.isWorkbookOrTab() ? c.getParent() : c; SimpleFilter filter = new SimpleFilter(FieldKey.fromString("container"), target.getEntityId()).addCondition(FieldKey.fromString(filterFieldName), row.get(sourceProperty)); - List rowIds = new TableSelector(StudiesSchema.getInstance().getSchema().getTable(StudiesSchema.TABLE_COHORTS), PageFlowUtil.set("rowId"), filter, null).getArrayList(Integer.class); + List rowIds = new TableSelector(StudiesSchema.getInstance().getSchema().getTable(tableToQuery), PageFlowUtil.set("rowId"), filter, null).getArrayList(Integer.class); if (rowIds.size() == 1) { row.put(targetFieldName, rowIds.get(0)); From 26db8356a5e5082f4cdfa5db8846530a2b27cc54 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 29 Sep 2025 20:24:20 -0700 Subject: [PATCH 21/59] Update default --- .../web/SequenceAnalysis/panel/SequenceImportPanel.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js index 711144294..17212476a 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js @@ -2436,7 +2436,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', { },{ xtype: 'textfield', fieldLabel: 'Delimiter', - value: '_', + value: '[_-]', itemId: 'delimiter' }], buttons: [{ @@ -2455,7 +2455,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', { if (prefix) { fg = fg.replace(new RegExp('^' + prefix), ''); } - fg = fg.split(delim); + fg = fg.split(RegExp(delim)); var id = fg[0]; if (Ext4.isNumeric(id)) { r.set('readset', id); From 608cbe469752e5e0a060e710b82fdd87100ae079 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 1 Oct 2025 11:52:23 -0700 Subject: [PATCH 22/59] Improve SnpEff index check --- .../sequenceanalysis/run/variant/SNPEffStep.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java index e55bf1915..16903cb05 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SNPEffStep.java @@ -75,13 +75,17 @@ public static File checkOrCreateIndex(SequenceAnalysisJobSupport support, Logger SnpEffWrapper wrapper = new SnpEffWrapper(log); File snpEffIndexDir = wrapper.getExpectedIndexDir(snpEffBaseDir, genome.getGenomeId(), geneFileId); - if (!snpEffIndexDir.exists()) + if (snpEffIndexDir.exists()) { - wrapper.buildIndex(snpEffBaseDir, genome, geneFile, geneFileId); + log.debug("previously created index found, re-using: " + snpEffIndexDir.getPath()); + return snpEffBaseDir; } - else + + File binFile = new File(snpEffIndexDir, "snpEffectPredictor.bin"); + if (!binFile.exists()) { - log.debug("previously created index found, re-using: " + snpEffIndexDir.getPath()); + log.debug("existing index not found, expected: " + binFile.getPath()); + wrapper.buildIndex(snpEffBaseDir, genome, geneFile, geneFileId); } return snpEffBaseDir; From 3f372f01f2a5521d9c23b7959da08c5962c02177 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 4 Oct 2025 06:44:37 -0700 Subject: [PATCH 23/59] Switch sequence init tasks to use webserver-high-priority --- .../org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java | 1 + .../pipeline/SequenceOutputHandlerInitTask.java | 2 +- .../pipeline/SequenceReadsetHandlerInitTask.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java index 00df2a90c..356a73781 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java @@ -43,6 +43,7 @@ public static class Factory extends AbstractSequenceTaskFactory public Factory() { super(AlignmentInitTask.class); + setLocation("webserver-high-priority"); setJoin(true); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java index ad8a6011a..905b05169 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java @@ -35,7 +35,7 @@ public static class Factory extends AbstractTaskFactory Date: Sat, 4 Oct 2025 07:44:05 -0700 Subject: [PATCH 24/59] Build short delay into github triggers to aid cross-repo commits --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3578fca0b..eb3c0794e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,11 @@ jobs: echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV id: default-branch + # Note: use slight delay in case there are associated commits across repos + - name: "Sleep for 30 seconds" + run: sleep 30s + shell: bash + - name: "Build DISCVR" uses: bimberlabinternal/DevOps/githubActions/discvr-build@master with: From d78b6d236af2b1ea208741ea9d634f261a160912 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 6 Oct 2025 08:03:04 -0700 Subject: [PATCH 25/59] Switch ETLs to log row count discrepancies --- discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java index cd83a749c..f948c987c 100644 --- a/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java +++ b/discvrcore/src/org/labkey/discvrcore/DiscvrCoreController.java @@ -58,7 +58,6 @@ import java.util.Map; import java.util.TreeMap; -import static javax.swing.Spring.width; import static org.labkey.api.util.DOM.Attribute.valign; import static org.labkey.api.util.DOM.at; import static org.labkey.api.util.DOM.cl; From 689af3ea3d7e83bf0781a45a43eb451c67dbc98b Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 7 Oct 2025 15:34:27 -0700 Subject: [PATCH 26/59] Update dependencies --- jbrowse/package-lock.json | 77 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index b7ba7390c..f0e5f599c 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -3176,9 +3176,10 @@ } }, "node_modules/@labkey/api": { - "version": "1.42.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.42.1.tgz", - "integrity": "sha512-rT+Q/ZM6bE6bU8HDj/7f3DIFuq538e+LZAvBw8P3qJjuAnyO+O+ItZz/YukAKCXXiN2GdedOXDJbt1Ms0bgLsg==" + "version": "1.43.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.43.0.tgz", + "integrity": "sha512-4hOQz+pM/QaCey6ooJEmEbElnR9+TDEzWG+8caFfeIX1iAg1335NXW3+/Xzs6a+L9ysRKds8bNgFPu2sxjPzfg==", + "license": "Apache-2.0" }, "node_modules/@labkey/build": { "version": "8.6.0", @@ -3217,12 +3218,13 @@ } }, "node_modules/@labkey/components": { - "version": "6.58.5", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.58.5.tgz", - "integrity": "sha512-N+WHs9QDAkJ5p2NIwCpWSp3O0Q5bNeBXqoizaZRZh9uEM9iSxg0I4GD/dd1jCk1PFGHs05mPLZ13yrPeixDiHQ==", + "version": "6.63.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.63.0.tgz", + "integrity": "sha512-E1tFkF6/erpzefo3b2Ot+jwFSlgRfeGalIuHGKXENPrubHOr5XkLXmu9lgGeyPD95fCx9Ff3dNHGxiokxsupwQ==", + "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.42.1", + "@labkey/api": "1.43.0", "@testing-library/dom": "~10.4.0", "@testing-library/jest-dom": "~6.6.3", "@testing-library/react": "~16.3.0", @@ -4253,6 +4255,12 @@ "@types/node": "*" } }, + "node_modules/@types/pako": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.4.tgz", + "integrity": "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw==", + "license": "MIT" + }, "node_modules/@types/parse-json": { "version": "4.0.2", "license": "MIT" @@ -4761,16 +4769,6 @@ "util": "^0.12.5" } }, - "node_modules/atob": { - "version": "2.1.2", - "license": "(MIT OR Apache-2.0)", - "bin": { - "atob": "bin/atob.js" - }, - "engines": { - "node": ">= 4.5.0" - } - }, "node_modules/available-typed-arrays": { "version": "1.0.7", "license": "MIT", @@ -5219,16 +5217,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/btoa": { - "version": "1.2.1", - "license": "(MIT OR Apache-2.0)", - "bin": { - "btoa": "bin/btoa.js" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/buffer": { "version": "6.0.3", "funding": [ @@ -6736,6 +6724,23 @@ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, + "node_modules/fast-png": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/fast-png/-/fast-png-6.4.0.tgz", + "integrity": "sha512-kAqZq1TlgBjZcLr5mcN6NP5Rv4V2f22z00c3g8vRrwkcqjerx7BEhPbOnWCPqaHUl2XWQBJQvOT/FQhdMT7X/Q==", + "license": "MIT", + "dependencies": { + "@types/pako": "^2.0.3", + "iobuffer": "^5.3.2", + "pako": "^2.1.0" + } + }, + "node_modules/fast-png/node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==", + "license": "(MIT AND Zlib)" + }, "node_modules/fast-uri": { "version": "3.0.1", "license": "MIT" @@ -7723,6 +7728,12 @@ "node": ">=10.13.0" } }, + "node_modules/iobuffer": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/iobuffer/-/iobuffer-5.4.0.tgz", + "integrity": "sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA==", + "license": "MIT" + }, "node_modules/ipaddr.js": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", @@ -8126,13 +8137,13 @@ } }, "node_modules/jspdf": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/jspdf/-/jspdf-3.0.1.tgz", - "integrity": "sha512-qaGIxqxetdoNnFQQXxTKUD9/Z7AloLaw94fFsOiJMxbfYdBbrBuhWmbzI8TVjrw7s3jBY1PFHofBKMV/wZPapg==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/jspdf/-/jspdf-3.0.3.tgz", + "integrity": "sha512-eURjAyz5iX1H8BOYAfzvdPfIKK53V7mCpBTe7Kb16PaM8JSXEcUQNBQaiWMI8wY5RvNOPj4GccMjTlfwRBd+oQ==", + "license": "MIT", "dependencies": { - "@babel/runtime": "^7.26.7", - "atob": "^2.1.2", - "btoa": "^1.2.1", + "@babel/runtime": "^7.26.9", + "fast-png": "^6.2.0", "fflate": "^0.8.1" }, "optionalDependencies": { From 35b3ae01b1c395e859b3806d4e4d9997b375e7c2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 15 Oct 2025 15:21:39 -0700 Subject: [PATCH 27/59] Support sawfish --sample-csv arg --- .../pipeline_code/extra_tools_install.sh | 6 +- .../run/analysis/SawfishAnalysis.java | 65 ++++++++++++++++++- .../analysis/SawfishJointCallingHandler.java | 48 +++++++++++++- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/SequenceAnalysis/pipeline_code/extra_tools_install.sh b/SequenceAnalysis/pipeline_code/extra_tools_install.sh index 8ecd60f37..bb1097ef4 100755 --- a/SequenceAnalysis/pipeline_code/extra_tools_install.sh +++ b/SequenceAnalysis/pipeline_code/extra_tools_install.sh @@ -326,10 +326,10 @@ then rm -Rf $LKTOOLS_DIR/sawfish* wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.0.0/sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz - tar -xzf sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz + tar -xzf sawfish-v2.0.4-x86_64-unknown-linux-gnu.tar.gz - mv sawfish-v2.0.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ - ln -s $LKTOOLS_DIR/sawfish-v2.0.0/bin/sawfish $LKTOOLS_DIR/ + mv sawfish-v2.0.4-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ + ln -s $LKTOOLS_DIR/sawfish-v2.0.4/bin/sawfish $LKTOOLS_DIR/ else echo "Already installed" fi diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java index 8039ff338..2bb9cf5dc 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishAnalysis.java @@ -33,7 +33,7 @@ public static class Provider extends AbstractAnalysisStepProvider args = new ArrayList<>(); args.add(getExe().getPath()); args.add("discover"); args.add("--bam"); - args.add(inputBam.getPath()); + args.add(inputFile.getPath()); // NOTE: sawfish stores the absolute path of the FASTA in the output JSON, so dont rely on working copies: args.add("--ref"); @@ -102,4 +124,41 @@ private File getExe() { return SequencePipelineService.get().getExeForPackage("SAWFISHPATH", "sawfish"); } + + private static class CramToBam extends SamtoolsRunner + { + public CramToBam(Logger log) + { + super(log); + } + + public void convert(File inputCram, File outputBam, File fasta, @Nullable Integer threads) throws PipelineJobException + { + getLogger().info("Converting CRAM to BAM"); + + execute(getParams(inputCram, outputBam, fasta, threads)); + } + + private List getParams(File inputCram, File outputBam, File fasta, @Nullable Integer threads) + { + List params = new ArrayList<>(); + params.add(getSamtoolsPath().getPath()); + params.add("view"); + params.add("-b"); + params.add("-T"); + params.add(fasta.getPath()); + params.add("-o"); + params.add(outputBam.getPath()); + + if (threads != null) + { + params.add("-@"); + params.add(String.valueOf(threads)); + } + + params.add(inputCram.getPath()); + + return params; + } + } } \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java index 9beae27c6..270c1488f 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SawfishJointCallingHandler.java @@ -1,7 +1,12 @@ package org.labkey.sequenceanalysis.run.analysis; +import au.com.bytecode.opencsv.CSVWriter; +import htsjdk.samtools.util.IOUtil; import org.apache.commons.io.FileUtils; import org.json.JSONObject; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; @@ -9,6 +14,7 @@ import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; @@ -66,6 +72,38 @@ public SequenceOutputProcessor getProcessor() public static class Processor implements SequenceOutputProcessor { + @Override + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + try (CSVWriter csv = new CSVWriter(IOUtil.openFileForBufferedUtf8Writing(getSampleCsvFile(ctx)))) + { + for (SequenceOutputFile so : inputFiles) + { + if (so.getRunId() == null) + { + throw new PipelineJobException("Unable to find ExperimentRun for: " + so.getRowid()); + } + + ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); + List inputs = run.getInputDatas("Input BAM File", null); + if (inputs.isEmpty()) + { + throw new PipelineJobException("Unable to find input BAMs for: " + so.getRowid()); + } + else if (inputs.size() > 1) + { + throw new PipelineJobException("More than one input BAM found for ExperimentRun: " + so.getRunId()); + } + + csv.writeNext(new String[]{so.getFile().getParentFile().getPath(), inputs.get(0).getFile().getPath()}); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + @Override public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { @@ -89,8 +127,6 @@ public void processFilesRemote(List inputFiles, JobContext c outputBaseName = outputBaseName.replaceAll(".vcf$", ""); } - File expectedFinalOutput = new File(ctx.getOutputDir(), outputBaseName + ".vcf.gz"); - File ouputVcf = runSawfishCall(ctx, filesToProcess, genome, outputBaseName); SequenceOutputFile so = new SequenceOutputFile(); @@ -102,6 +138,11 @@ public void processFilesRemote(List inputFiles, JobContext c ctx.addSequenceOutput(so); } + private File getSampleCsvFile(PipelineContext ctx) + { + return new File(ctx.getSourceDirectory(), "sawfish.samples.csv"); + } + private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome genome, String outputBaseName) throws PipelineJobException { if (inputs.isEmpty()) @@ -126,6 +167,9 @@ private File runSawfishCall(JobContext ctx, List inputs, ReferenceGenome g args.add(sample.getParentFile().getPath()); } + args.add("--sample-csv"); + args.add(getSampleCsvFile(ctx).getPath()); + File outDir = new File(ctx.getOutputDir(), "sawfish"); args.add("--output-dir"); args.add(outDir.getPath()); From 1fac863fe9c58d82e33ddc636b97fc9cd5423866 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 05:09:42 -0700 Subject: [PATCH 28/59] Option to create readsets from SRA (#355) * Option to create readsets from SRA * Test fix --- .../sequenceanalysis/sequence_readsets.js | 14 ++-- .../query/SequenceTriggerHelper.java | 69 +++++++++++++++++++ .../external/labModules/SequenceTest.java | 50 ++++++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets.js b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets.js index 4556ab827..9c3c14cb4 100644 --- a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets.js +++ b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets.js @@ -1,11 +1,15 @@ -/* - * Copyright (c) 2012 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 - */ +var LABKEY = require("labkey"); + +var triggerHelper = new org.labkey.sequenceanalysis.query.SequenceTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); function beforeDelete(row, errors){ if (!this.extraContext.deleteFromServer){ errors._form = 'You cannot directly delete readsets. To delete these records, use the delete button above the readset grid.'; } +} + +function afterInsert(row, errors) { + if (row.sraAccessions) { + triggerHelper.createReaddataForSra(row.rowid, row.sraAccessions); + } } \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java index f6eb36e24..13803c667 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java @@ -13,9 +13,11 @@ import org.biojava.nbio.core.sequence.transcription.TranscriptionEngine; import org.junit.Assert; import org.junit.Test; +import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.api.DataType; @@ -28,12 +30,19 @@ import org.labkey.api.security.User; import org.labkey.api.security.UserManager; import org.labkey.api.sequenceanalysis.RefNtSequenceModel; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; +import org.labkey.sequenceanalysis.ReadDataImpl; import org.labkey.sequenceanalysis.SequenceAnalysisSchema; +import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl; +import org.labkey.sequenceanalysis.SequenceReadsetImpl; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; import java.io.StringWriter; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -254,4 +263,64 @@ public int createExpData(String relPath) { return d.getRowId(); } + + public void createReaddataForSra(int readsetId, String sraAccessions) + { + SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(readsetId, _user); + if (rs == null) + { + throw new IllegalArgumentException("Unable to find readset: " + readsetId); + } + + TableInfo rd = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_READ_DATA); + + String[] tokens = StringUtils.split(sraAccessions, ","); + for (String token : tokens) + { + if (rs.getReadData() != null && !rs.getReadData().isEmpty()) + { + throw new IllegalArgumentException("Did not expect readset to have existing readdata: " + rs.getReadsetId()); + } + + // Create new: + ReadDataImpl rd1 = new ReadDataImpl(); + rd1.setReadset(readsetId); + rd1.setContainer(rs.getContainer()); + rd1.setCreated(new Date()); + rd1.setModified(new Date()); + rd1.setCreatedBy(_user.getUserId()); + rd1.setModifiedBy(_user.getUserId()); + rd1.setSra_accession(token); + rd1.setArchived(true); + + // NOTE: this is a fragile assumption. We might need to eventually query SRA to figure out whether data is paired: + Container c = ContainerManager.getForId(rs.getContainer()); + PipeRoot pr = PipelineService.get().findPipelineRoot(c); + if (pr == null) + { + throw new IllegalStateException("Unable to find pipeline root for: " + c.getPath()); + } + + String folderName = "SequenceImport_" + FileUtil.getTimestamp(); + FileLike outDir = AssayFileWriter.findUniqueFileName(folderName, pr.getRootFileLike()); + + FileLike expectedFile1 = FileUtil.appendPath(outDir, Path.parse(token + "_1.fastq.gz")); + ExpData exp1 = ExperimentService.get().createData(c, new DataType("Data")); + exp1.setDataFileURI(expectedFile1.toURI()); + exp1.setContainer(c); + exp1.setName(expectedFile1.getName()); + exp1.save(_user); + rd1.setFileId1(exp1.getRowId()); + + FileLike expectedFile2 = FileUtil.appendPath(outDir, Path.parse(token + "_2.fastq.gz")); + ExpData exp2 = ExperimentService.get().createData(c, new DataType("Data")); + exp2.setDataFileURI(expectedFile2.toURI()); + exp2.setContainer(c); + exp2.setName(expectedFile2.getName()); + exp2.save(_user); + rd1.setFileId2(exp2.getRowId()); + + Table.insert(_user, rd, rd1); + } + } } diff --git a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java index e26795289..c264ef281 100644 --- a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java +++ b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java @@ -195,6 +195,44 @@ private void importReadsetMetadata() log("verifying readset count correct"); waitForText("Sequence Readsets"); waitForElement(LabModuleHelper.getNavPanelItem("Sequence Readsets:", _readsetCt.toString())); + + // Repeat, adding SRA accessions: + goToProjectHome(); + waitAndClick(Locator.linkWithText("Plan Sequence Run (Create Readsets)")); + new Window.WindowFinder(getDriver()).withTitle("Create Readsets").waitFor(); + waitAndClickAndWait(Ext4Helper.Locators.ext4Button("Submit")); + + _helper.waitForField("Sample Id", WAIT_FOR_PAGE); + _ext4Helper.clickTabContainingText("Import Spreadsheet"); + waitForText("Copy/Paste Data"); + + setFormElementJS(Locator.name("text"), getIlluminaSRANames()); + + waitAndClick(Ext4Helper.Locators.ext4Button("Upload")); + new Window.WindowFinder(getDriver()).withTitle("Success").waitFor(); + _readsetCt += 3; + assertTextPresent("Success!"); + waitAndClickAndWait(Ext4Helper.Locators.ext4Button("OK")); + + // This is scoped to this workbook: + log("verifying readset count correct"); + waitForText("Sequence Readsets"); + waitAndClickAndWait(LabModuleHelper.getNavPanelItem("Sequence Readsets:", "3")); + + DataRegionTable.DataRegion(getDriver()).withName("query").waitFor(); + + //verify CSV file creation + DataRegionTable.DataRegion(getDriver()).find().goToView("SRA Info"); + DataRegionTable dr = DataRegionTable.DataRegion(getDriver()).withName("query").waitFor(); + waitForElement(Locator.tagContainingText("a", "SRA0")); + waitForElement(Locator.tagContainingText("a", "SRA1")); + waitForElement(Locator.tagContainingText("a", "SRA2")); + + dr.checkAllOnPage(); + dr.clickHeaderButtonAndWait("Delete"); + clickButton("OK"); + + _readsetCt -= 3; } /** @@ -320,6 +358,18 @@ private String getIlluminaNames() return sb.toString(); } + private String getIlluminaSRANames() + { + StringBuilder sb = new StringBuilder("Name\tPlatform\tsraAccessions\n"); + int i = 0; + while (i < 3) + { + sb.append("IlluminaSRA" + (i + 1) + "\tILLUMINA\tSRA" + i + "\n"); + i++; + } + return sb.toString(); + } + /** * This test will kick off a pipeline import using the illumina pipeline. Verification of the result * is performed by readsetFeaturesTest() From b084e0062488caa99987cdbe5453461f7fd218da Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 05:14:15 -0700 Subject: [PATCH 29/59] Error check --- .../pipeline/CreateReferenceLibraryTask.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java index 785e1e628..a2a3fb949 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java @@ -163,6 +163,11 @@ public RecordedActionSet run() throws PipelineJobException libraryMembers = new TableSelector(libraryMembersTable, new SimpleFilter(FieldKey.fromString("library_id"), getPipelineJob().getLibraryId()), new Sort("ref_nt_id/name")).getArrayList(ReferenceLibraryMember.class); } + if (libraryMembers == null) + { + throw new PipelineJobException("There are no sequences in the library: " + getPipelineJob().getLibraryId()); + } + getJob().getLogger().info("there are " + libraryMembers.size() + " sequences to process"); //make sure sequence names are unique From 604ee349310af2e1e6fe4304c5182ed8e3d29136 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 06:17:35 -0700 Subject: [PATCH 30/59] Add nimble/bulk step --- .../labkey/singlecell/SingleCellModule.java | 2 + .../labkey/singlecell/run/NimbleAnalysis.java | 15 +- .../run/NimbleBulkAlignmentStep.java | 163 ++++++++++++++++++ 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java index 9c459da1e..846fd211b 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellModule.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellModule.java @@ -114,6 +114,7 @@ import org.labkey.singlecell.run.CellRangerVDJWrapper; import org.labkey.singlecell.run.NimbleAlignmentStep; import org.labkey.singlecell.run.NimbleAnalysis; +import org.labkey.singlecell.run.NimbleBulkAlignmentStep; import org.labkey.singlecell.run.RepeatNimbleReportHandler; import org.labkey.singlecell.run.VelocytoAlignmentStep; import org.labkey.singlecell.run.VelocytoAnalysisStep; @@ -219,6 +220,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new CellRangerVDJWrapper.VDJProvider()); SequencePipelineService.get().registerPipelineStep(new NimbleAlignmentStep.Provider()); SequencePipelineService.get().registerPipelineStep(new NimbleAnalysis.Provider()); + SequencePipelineService.get().registerPipelineStep(new NimbleBulkAlignmentStep.Provider()); SequencePipelineService.get().registerPipelineStep(new VelocytoAlignmentStep.Provider()); SequencePipelineService.get().registerPipelineStep(new VelocytoAnalysisStep.Provider()); diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java index 8fa4390b7..84c2f3f2a 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java @@ -1,5 +1,8 @@ package org.labkey.singlecell.run; +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMTag; +import htsjdk.samtools.SamReaderFactory; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.model.AnalysisModel; import org.labkey.api.sequenceanalysis.model.Readset; @@ -29,7 +32,7 @@ public static class Provider extends AbstractAnalysisStepProvider(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null); + super("NimbleAnalysis", "Nimble", null, "This will run Nimble to generate a supplemental feature count matrix for the provided libraries. This should work using either CellRanger/scRNA-seq or bulk input data.", NimbleAlignmentStep.getToolParameters(), new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null); } @Override @@ -58,7 +61,15 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); helper.doNimbleAlign(inputBam, output, rs, FileUtil.getBaseName(inputBam)); - NimbleHelper.write10xBarcodes(inputBam, getPipelineCtx().getLogger(), rs, referenceGenome, output); + SAMFileHeader header = SamReaderFactory.makeDefault().getFileHeader(inputBam); + if (header.getAttribute(SAMTag.CB.name()) != null) + { + NimbleHelper.write10xBarcodes(inputBam, getPipelineCtx().getLogger(), rs, referenceGenome, output); + } + else + { + getPipelineCtx().getLogger().info("BAM lacks CB tag, will not output 10x barcodes to file"); + } return output; } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java new file mode 100644 index 000000000..486d9a4fb --- /dev/null +++ b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java @@ -0,0 +1,163 @@ +package org.labkey.singlecell.run; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; +import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; +import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep; +import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; + +public class NimbleBulkAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep +{ + public static class Provider extends AbstractAnalysisStepProvider + { + public Provider() + { + super("NimbleBulkAnalysis", "Nimble (Bulk)", null, "This will run Nimble to generate a supplemental feature count matrix for the provided libraries. This version is intended for bulk input data. Please use the CellRanger/Nimble version for scRNA-seq", NimbleAlignmentStep.getToolParameters(), new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null); + } + + @Override + public NimbleAnalysis create(PipelineContext ctx) + { + return new NimbleAnalysis(this, ctx); + } + } + + public NimbleBulkAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, NimbleBulkAlignmentStep.NimbleBulkWrapper wrapper) + { + super(provider, ctx, wrapper); + } + + @Override + public IndexOutput createIndex(ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException + { + return null; + } + + @Override + public void init(SequenceAnalysisJobSupport support) throws PipelineJobException + { + NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); + + List genomeIds = helper.getGenomeIds(); + for (int id : genomeIds) + { + helper.prepareGenome(id); + } + } + + @Override + public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nullable List inputFastqs2, File outputDirectory, ReferenceGenome referenceGenome, String basename, String readGroupId, @Nullable String platformUnit) throws PipelineJobException + { + AlignmentOutputImpl output = new AlignmentOutputImpl(); + SamtoolsRunner st = new SamtoolsRunner(getPipelineCtx().getLogger()); + + List outputBams = new ArrayList<>(); + int bamIdx = 0; + while (bamIdx < inputFastqs1.size()) + { + File outputBam = new File(getPipelineCtx().getWorkingDirectory(), FileUtil.makeLegalName(rs.getName()) + ".unmapped." + bamIdx + ".bam"); + List args = new ArrayList<>(Arrays.asList(st.getSamtoolsPath().getPath(), "import", "-o", outputBam.getPath(), "-r", "ID:" + readGroupId)); + if (inputFastqs2 == null || inputFastqs2.isEmpty()) + { + args.add("-O"); + args.add(inputFastqs1.get(bamIdx).getPath()); + } + else + { + args.add("-1"); + args.add(inputFastqs1.get(bamIdx).getPath()); + + if (bamIdx > inputFastqs2.size()) + { + throw new PipelineJobException("Unequal lengths for first/second pair FASTQs"); + } + + args.add("-2"); + args.add(inputFastqs2.get(bamIdx).getPath()); + } + bamIdx++; + + st.execute(args); + outputBams.add(outputBam); + } + + File outputBam; + if (outputBams.size() > 1) + { + outputBam = new File(getPipelineCtx().getWorkingDirectory(), FileUtil.makeLegalName(rs.getName()) + ".unmapped.bam"); + outputBams.forEach(output::addIntermediateFile); + + List args = new ArrayList<>(Arrays.asList(st.getSamtoolsPath().getPath(), "merge", "-o", outputBam.getPath(), "-f")); + Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); + if (maxThreads != null) + { + args.add("-@"); + args.add(maxThreads.toString()); + } + + outputBams.forEach(bam -> args.add(bam.getPath())); + st.execute(args); + } + else + { + outputBam = outputBams.get(0); + } + + // Now run nimble itself: + NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); + helper.doNimbleAlign(outputBam, output, rs, basename); + output.setBAM(outputBam); + + return output; + } + + @Override + public boolean doAddReadGroups() + { + return false; + } + + @Override + public boolean doSortIndexBam() + { + return false; + } + + @Override + public boolean alwaysCopyIndexToWorkingDir() + { + return false; + } + + @Override + public boolean supportsGzipFastqs() + { + return true; + } + + public static class NimbleBulkWrapper extends AbstractCommandWrapper + { + public NimbleBulkWrapper(Logger log) + { + super(log); + } + } +} From 6039efb4be1b24379c43096095ba45048bd07859 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 09:21:21 -0700 Subject: [PATCH 31/59] Allow nimble step to use cached barcodes --- .../singlecell/run/NimbleAlignmentStep.java | 133 ++++++++++++++++-- .../labkey/singlecell/run/NimbleHelper.java | 85 ++++++++++- 2 files changed, 206 insertions(+), 12 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 1fd40d513..d1647c1a7 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -3,7 +3,18 @@ import org.apache.commons.io.FileUtils; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; +import org.labkey.api.data.Container; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; import org.labkey.api.sequenceanalysis.model.Readset; import org.labkey.api.sequenceanalysis.pipeline.AbstractAlignmentStepProvider; import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; @@ -14,19 +25,22 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.api.util.PageFlowUtil; +import org.labkey.singlecell.SingleCellSchema; import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; public class NimbleAlignmentStep extends AbstractCellRangerDependentStep { public static final String REF_GENOMES = "refGenomes"; public static final String MAX_HITS_TO_REPORT = "maxHitsToReport"; - public static final String ALIGN_OUTPUT = "alignmentOutput"; public static final String STRANDEDNESS = "strandedness"; + public static final String REQUIRE_CACHED_BARCODES = "requireCachedBarcodes"; public NimbleAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerWrapper wrapper) { @@ -59,7 +73,10 @@ public static List getToolParameters() }}, null), ToolParameterDescriptor.create(MAX_HITS_TO_REPORT, "Max Hits To Report", "If a given hit has more than this number of references, it is discarded", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); - }}, 4) + }}, 4), + ToolParameterDescriptor.create(REQUIRE_CACHED_BARCODES, "Fail Unless Cached Barcodes Present", "If checked, the pipeline will expect a previously computed map of cellbarcodes and UMIs to be computed. Under default conditions, if this is missing, cellranger will be re-run. This flag can be helpful to avoid that computation if you expect the barcode file to exist.", "checkbox", new JSONObject(){{ + + }}, false) ); } @@ -68,6 +85,96 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu { AlignmentOutputImpl output = new AlignmentOutputImpl(); + boolean throwIfNotFound = getProvider().getParameterByName(REQUIRE_CACHED_BARCODES).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + File cachedBarcodes = getCachedBarcodeFile(rs, throwIfNotFound); + + File localBam; + if (cachedBarcodes == null) + { + localBam = performCellRangerAlignment(output, rs, inputFastqs1, inputFastqs2, outputDirectory, referenceGenome, basename, readGroupId, platformUnit); + } + else + { + localBam = createNimbleBam(output, rs, inputFastqs1, inputFastqs2); + } + + + // Now run nimble itself: + NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); + helper.doNimbleAlign(localBam, output, rs, basename); + output.setBAM(localBam); + + return output; + } + + private File createNimbleBam(AlignmentOutputImpl output, Readset rs, List inputFastqs1, List inputFastqs2) throws PipelineJobException + { + File cellbarcodes = getCachedBarcodeFile(rs, true); + File umiMapping = getUmiMapping(cellbarcodes); + + return NimbleHelper.runFastqToBam(output, getPipelineCtx(), rs, inputFastqs1, inputFastqs2, cellbarcodes, umiMapping); + } + + private File getCachedBarcodeFile(Readset rs, boolean throwIfNotFound) throws PipelineJobException + { + Map map = getPipelineCtx().getSequenceSupport().getCachedObject(CACHE_KEY, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); + Integer dataId = map.get(rs.getReadsetId()); + if (dataId == null) + { + if (throwIfNotFound) + { + throw new PipelineJobException("No cached data found for readset: " + rs.getReadsetId()); + } + + return null; + } + + File ret = getPipelineCtx().getSequenceSupport().getCachedData(dataId); + if (ret == null || ! ret.exists()) + { + throw new PipelineJobException("Missing cached cellbarcode file: " + dataId); + } + + return ret; + } + + private File getUmiMapping(File cellbarcodeFile) throws PipelineJobException + { + File ret = new File(cellbarcodeFile.getPath().replaceAll(".cb.txt.gz", ".umi.txt.gz")); + if (ret == null || ! ret.exists()) + { + throw new PipelineJobException("Missing cached UMI file: " + ret.getPath()); + } + + return ret; + } + + private File findCellBarcodeFiles(Readset rs) throws PipelineJobException + { + Container targetContainer = getPipelineCtx().getJob().getContainer().isWorkbookOrTab() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer(); + UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), targetContainer, SingleCellSchema.SEQUENCE_SCHEMA_NAME); + TableInfo ti = us.getTable("outputfiles"); + + SimpleFilter sf = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + sf.addCondition(FieldKey.fromString("category"), NimbleHelper.CATEGORY_CB); + List cbs = new TableSelector(ti, PageFlowUtil.set("dataid"), sf, new Sort("-rowid")).getArrayList(Integer.class); + if (!cbs.isEmpty()) + { + int dataId = cbs.get(0); + ExpData d = ExperimentService.get().getExpData(dataId); + if (d == null || d.getFile() == null) + { + throw new PipelineJobException("Output lacks a file: " + dataId); + } + + return d.getFile(); + } + + return null; + } + + private File performCellRangerAlignment(AlignmentOutputImpl output, Readset rs, List inputFastqs1, @Nullable List inputFastqs2, File outputDirectory, ReferenceGenome referenceGenome, String basename, String readGroupId, @Nullable String platformUnit) throws PipelineJobException + { // We need to ensure we keep the BAM for post-processing: setAlwaysRetainBam(true); @@ -89,12 +196,7 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu NimbleHelper.write10xBarcodes(localBam, getWrapper().getLogger(), rs, referenceGenome, output); - // Now run nimble itself: - NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); - helper.doNimbleAlign(localBam, output, rs, basename); - output.setBAM(localBam); - - return output; + return localBam; } @Override @@ -109,5 +211,20 @@ public void init(SequenceAnalysisJobSupport support) throws PipelineJobException { helper.prepareGenome(id); } + + // Try to find 10x barcodes: + HashMap readsetToBarcodes = new HashMap<>(); + for (Readset rs : support.getCachedReadsets()) + { + File f = findCellBarcodeFiles(rs); + if (f != null) + { + readsetToBarcodes.put(rs.getReadsetId(), f); + } + } + + support.cacheObject(CACHE_KEY, readsetToBarcodes); } + + private static final String CACHE_KEY = "nimble.cb"; } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index 6afdbe366..a74c1a55e 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -27,9 +27,11 @@ import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.run.DISCVRSeqRunner; import org.labkey.api.sequenceanalysis.run.DockerWrapper; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.PrintWriters; @@ -496,6 +498,9 @@ private Map doAlignment(List genomes, List inputFastqs1, List inputFastqs2, File cellBarcodes, File umiMapping) throws PipelineJobException + { + List outputBams = new ArrayList<>(); + int bamIdx = 0; + while (bamIdx < inputFastqs1.size()) + { + File outputBam = new File(ctx.getWorkingDirectory(), FileUtil.makeLegalName(rs.getName()) + ".unmapped." + bamIdx + ".bam"); + + List args = new ArrayList<>(); + args.add("python3"); + args.add("-m"); + args.add("nimble"); + + args.add("fastq-to-bam"); + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); + if (maxThreads != null) + { + args.add("-c"); + args.add(maxThreads.toString()); + } + + args.add("--r1-fastq"); + args.add(inputFastqs1.get(bamIdx).getPath()); + if (bamIdx > inputFastqs2.size()) + { + throw new PipelineJobException("Unequal lengths for first/second pair FASTQs"); + } + + args.add("--r2-fastq"); + args.add(inputFastqs2.get(bamIdx).getPath()); + + args.add("--cell-barcodes"); + args.add(cellBarcodes.getPath()); + + args.add("--umi-mapping"); + args.add(umiMapping.getPath()); + + args.add("--output"); + args.add(outputBam.getPath()); + + runUsingDocker(args, output, "nimble.fastq-to-bam." + bamIdx, ctx); + outputBams.add(outputBam); + bamIdx++; + } + + File outputBam; + if (outputBams.size() > 1) + { + outputBam = new File(ctx.getWorkingDirectory(), FileUtil.makeLegalName(rs.getName()) + ".unmapped.bam"); + outputBams.forEach(output::addIntermediateFile); + + SamtoolsRunner st = new SamtoolsRunner(ctx.getLogger()); + List args = new ArrayList<>(Arrays.asList(st.getSamtoolsPath().getPath(), "merge", "-o", outputBam.getPath(), "-f")); + Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); + if (maxThreads != null) + { + args.add("-@"); + args.add(maxThreads.toString()); + } + + outputBams.forEach(bam -> args.add(bam.getPath())); + st.execute(args); + } + else + { + outputBam = outputBams.get(0); + } + + return outputBam; + } + public static String DOCKER_CONTAINER_NAME = "ghcr.io/bimberlab/nimble:latest"; private boolean runUsingDocker(List nimbleArgs, PipelineStepOutput output, @Nullable String resumeString) throws PipelineJobException From 8921d0a816f0df8b27e30c8e44c7a0e6c78f2881 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 18:04:50 -0700 Subject: [PATCH 32/59] Bugfix to NimbleAlignmentStep --- .../sequenceanalysis/query/SequenceTriggerHelper.java | 2 +- .../labkey/singlecell/run/NimbleAlignmentStep.java | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java index 13803c667..a5e9b02e2 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java @@ -301,7 +301,7 @@ public void createReaddataForSra(int readsetId, String sraAccessions) throw new IllegalStateException("Unable to find pipeline root for: " + c.getPath()); } - String folderName = "SequenceImport_" + FileUtil.getTimestamp(); + String folderName = "SequenceImport_RS" + rs.getRowId() + "_" + FileUtil.getTimestamp(); FileLike outDir = AssayFileWriter.findUniqueFileName(folderName, pr.getRootFileLike()); FileLike expectedFile1 = FileUtil.appendPath(outDir, Path.parse(token + "_1.fastq.gz")); diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index d1647c1a7..bc720124a 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -149,7 +149,7 @@ private File getUmiMapping(File cellbarcodeFile) throws PipelineJobException return ret; } - private File findCellBarcodeFiles(Readset rs) throws PipelineJobException + private ExpData findCellBarcodeFiles(Readset rs) throws PipelineJobException { Container targetContainer = getPipelineCtx().getJob().getContainer().isWorkbookOrTab() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer(); UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), targetContainer, SingleCellSchema.SEQUENCE_SCHEMA_NAME); @@ -167,7 +167,7 @@ private File findCellBarcodeFiles(Readset rs) throws PipelineJobException throw new PipelineJobException("Output lacks a file: " + dataId); } - return d.getFile(); + return d; } return null; @@ -213,13 +213,14 @@ public void init(SequenceAnalysisJobSupport support) throws PipelineJobException } // Try to find 10x barcodes: - HashMap readsetToBarcodes = new HashMap<>(); + HashMap readsetToBarcodes = new HashMap<>(); for (Readset rs : support.getCachedReadsets()) { - File f = findCellBarcodeFiles(rs); + ExpData f = findCellBarcodeFiles(rs); if (f != null) { - readsetToBarcodes.put(rs.getReadsetId(), f); + support.cacheExpData(f); + readsetToBarcodes.put(rs.getReadsetId(), f.getRowId()); } } From 2253e02a2285ea01ed6b8baa7ea6df5951c47a88 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 18:12:38 -0700 Subject: [PATCH 33/59] Bugfix to NimbleAlignmentStep --- .../singlecell/run/NimbleBulkAlignmentStep.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java index 486d9a4fb..c93a0dcbd 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java @@ -4,7 +4,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.AbstractAlignmentStepProvider; import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; @@ -26,17 +26,22 @@ public class NimbleBulkAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep { - public static class Provider extends AbstractAnalysisStepProvider + public static class Provider extends AbstractAlignmentStepProvider { public Provider() { - super("NimbleBulkAnalysis", "Nimble (Bulk)", null, "This will run Nimble to generate a supplemental feature count matrix for the provided libraries. This version is intended for bulk input data. Please use the CellRanger/Nimble version for scRNA-seq", NimbleAlignmentStep.getToolParameters(), new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), null); + super("Nimble-Bulk", + "This will run Nimble to generate a supplemental feature count matrix for the provided libraries. This version is intended for bulk input data. Please use the CellRanger/Nimble version for scRNA-seq", + NimbleAlignmentStep.getToolParameters(), + new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeField.js", "singlecell/panel/NimbleAlignPanel.js")), + null, + true, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); } @Override - public NimbleAnalysis create(PipelineContext ctx) + public NimbleBulkAlignmentStep create(PipelineContext ctx) { - return new NimbleAnalysis(this, ctx); + return new NimbleBulkAlignmentStep(this, ctx, new NimbleBulkWrapper(ctx.getLogger())); } } From 600cda3b07a3cbd0c6db6070d8f8381c15551b0a Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 19:02:17 -0700 Subject: [PATCH 34/59] Bugfix to NimbleAlignmentStep --- .../src/org/labkey/singlecell/run/NimbleAlignmentStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index bc720124a..460a094e8 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -140,7 +140,7 @@ private File getCachedBarcodeFile(Readset rs, boolean throwIfNotFound) throws Pi private File getUmiMapping(File cellbarcodeFile) throws PipelineJobException { - File ret = new File(cellbarcodeFile.getPath().replaceAll(".cb.txt.gz", ".umi.txt.gz")); + File ret = new File(cellbarcodeFile.getPath().replaceAll("cb.txt.gz", "umi.txt.gz")); if (ret == null || ! ret.exists()) { throw new PipelineJobException("Missing cached UMI file: " + ret.getPath()); From 2a5c789ef9b1091bf67e4a1794da5ebeadc00952 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 19:05:39 -0700 Subject: [PATCH 35/59] Bugfix to NimbleAlignmentStep --- .../src/org/labkey/singlecell/run/NimbleAlignmentStep.java | 2 +- .../src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 460a094e8..03b940baa 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -141,7 +141,7 @@ private File getCachedBarcodeFile(Readset rs, boolean throwIfNotFound) throws Pi private File getUmiMapping(File cellbarcodeFile) throws PipelineJobException { File ret = new File(cellbarcodeFile.getPath().replaceAll("cb.txt.gz", "umi.txt.gz")); - if (ret == null || ! ret.exists()) + if (!ret.exists()) { throw new PipelineJobException("Missing cached UMI file: " + ret.getPath()); } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java index c93a0dcbd..2de7aa17a 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java @@ -8,6 +8,7 @@ import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.IndexOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner; @@ -53,7 +54,7 @@ public NimbleBulkAlignmentStep(AlignmentStepProvider provider, PipelineContex @Override public IndexOutput createIndex(ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException { - return null; + return new IndexOutputImpl(referenceGenome); } @Override From f75ffaa39c8b922fce7d036ae081fabd3e1a0d3a Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 19:08:40 -0700 Subject: [PATCH 36/59] Add CD4_Activation_Axis --- .../pipeline/singlecell/CalculateGeneComponentScores.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java index 43be39118..4bc652684 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateGeneComponentScores.java @@ -24,7 +24,7 @@ public Provider() super("CalculateGeneComponentScores", "Calculate Gene Module Scores", "RIRA", "This will generate UCell scores for a set of pre-defined gene modules", Collections.singletonList( SeuratToolParameter.create("savedComponent", "Saved Component(s)", "This is the name of the saved component (from RIRA) to apply", "ldk-simplecombo", new JSONObject() {{ - put("storeValues", "Tcell_EffectorDifferentiation;TCR_EarlyStimulationComponent;TCR_StimulationComponent1;PLS_Score_1;PLS_Score_2;PLS_Score_3;PLS_Score_4;PLS_Score_5;PLS_Score_6"); + put("storeValues", "Tcell_EffectorDifferentiation;TCR_EarlyStimulationComponent;CD4_Activation_Axis"); put("multiSelect", true); put("allowBlank", false); put("joinReturnValue", true); From e50b12ce54e30f017f14b28f55abcfe6c12248e4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 19:46:14 -0700 Subject: [PATCH 37/59] Better support readsets created directly from SRA --- .../run/RestoreSraDataHandler.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 8779afda1..28581210c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -47,6 +47,7 @@ import org.labkey.sequenceanalysis.SequenceAnalysisSchema; import org.labkey.sequenceanalysis.pipeline.ReadsetCreationTask; import org.labkey.sequenceanalysis.pipeline.SequenceNormalizationTask; +import org.labkey.sequenceanalysis.pipeline.SequenceTaskHelper; import org.labkey.sequenceanalysis.util.SequenceUtil; import java.io.File; @@ -299,6 +300,12 @@ public void complete(PipelineJob job, List readsets, List readsets, JobContext ctx) throws Un File expectedFile1 = ctx.getSequenceSupport().getCachedData(rd.getFileId1()); File expectedFile2 = rd.getFileId2() == null ? null : ctx.getSequenceSupport().getCachedData(rd.getFileId2()); + if (!expectedFile1.getParentFile().exists()) + { + ctx.getLogger().info("Creating folder: " + expectedFile1.getParentFile().getPath()); + expectedFile1.getParentFile().mkdirs(); + } + FastqDumpWrapper wrapper = new FastqDumpWrapper(ctx.getLogger()); Pair files = wrapper.downloadSra(accession, ctx.getOutputDir(), rd.isPairedEnd(), false); long lines1 = SequenceUtil.getLineCount(files.first) / 4; ctx.getJob().getLogger().debug("Reads in " + files.first.getName() + ": " + lines1); - if (lines1 != accessionToReads.get(accession)) + if (accessionToReads.containsKey(accession) && lines1 != accessionToReads.get(accession)) { throw new PipelineJobException("Reads found in file, " + lines1 + ", does not match expected: " + accessionToReads.get(accession) + " for file: " + files.first.getPath()); } @@ -397,7 +410,7 @@ public void processFilesRemote(List readsets, JobContext ctx) throws Un { long lines2 = SequenceUtil.getLineCount(files.second) / 4; ctx.getJob().getLogger().debug("Reads in " + files.second.getName() + ": " + lines2); - if (lines2 != accessionToReads.get(accession)) + if (accessionToReads.containsKey(accession) && lines2 != accessionToReads.get(accession)) { throw new PipelineJobException("Reads found in file, " + lines2 + ", does not match expected: " + accessionToReads.get(accession) + " for file: " + files.second.getPath()); } From 4930f5f51f97513af68f827f764daf3e5031c894 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 16 Oct 2025 20:10:09 -0700 Subject: [PATCH 38/59] Expand BAM header --- .../singlecell/run/NimbleBulkAlignmentStep.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java index 2de7aa17a..4311d3e8b 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleBulkAlignmentStep.java @@ -80,7 +80,22 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu while (bamIdx < inputFastqs1.size()) { File outputBam = new File(getPipelineCtx().getWorkingDirectory(), FileUtil.makeLegalName(rs.getName()) + ".unmapped." + bamIdx + ".bam"); - List args = new ArrayList<>(Arrays.asList(st.getSamtoolsPath().getPath(), "import", "-o", outputBam.getPath(), "-r", "ID:" + readGroupId)); + List args = new ArrayList<>(Arrays.asList(st.getSamtoolsPath().getPath(), "import", "-o", outputBam.getPath())); + args.add("-r"); + args.add("ID:" + readGroupId); + + args.add("-r"); + args.add("LB:" + rs.getReadsetId().toString()); + + args.add("-r"); + args.add("PL:" + (rs.getPlatform() == null ? "ILLUMINA" : rs.getPlatform())); + + args.add("-r"); + args.add("PU:" + (platformUnit == null ? rs.getReadsetId().toString() : platformUnit)); + + args.add("-r"); + args.add("SM:" + rs.getName().replaceAll(" ", "_")); + if (inputFastqs2 == null || inputFastqs2.isEmpty()) { args.add("-O"); From 2ef4efb160851275c58a5162455cd4f35aa21a04 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 17 Oct 2025 07:22:40 -0700 Subject: [PATCH 39/59] Bugfix to RestoreSraDataHandler --- .../labkey/sequenceanalysis/run/RestoreSraDataHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 28581210c..7d70cbdc7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -401,7 +401,7 @@ public void processFilesRemote(List readsets, JobContext ctx) throws Un long lines1 = SequenceUtil.getLineCount(files.first) / 4; ctx.getJob().getLogger().debug("Reads in " + files.first.getName() + ": " + lines1); - if (accessionToReads.containsKey(accession) && lines1 != accessionToReads.get(accession)) + if (accessionToReads.containsKey(accession) && accessionToReads.get(accession) > 0 && lines1 != accessionToReads.get(accession)) { throw new PipelineJobException("Reads found in file, " + lines1 + ", does not match expected: " + accessionToReads.get(accession) + " for file: " + files.first.getPath()); } @@ -410,7 +410,7 @@ public void processFilesRemote(List readsets, JobContext ctx) throws Un { long lines2 = SequenceUtil.getLineCount(files.second) / 4; ctx.getJob().getLogger().debug("Reads in " + files.second.getName() + ": " + lines2); - if (accessionToReads.containsKey(accession) && lines2 != accessionToReads.get(accession)) + if (accessionToReads.containsKey(accession) && accessionToReads.get(accession) > 0 && lines2 != accessionToReads.get(accession)) { throw new PipelineJobException("Reads found in file, " + lines2 + ", does not match expected: " + accessionToReads.get(accession) + " for file: " + files.second.getPath()); } From 4b3fd438ecf233f34574a11332fcd6d9b4e8771d Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 17 Oct 2025 08:11:02 -0700 Subject: [PATCH 40/59] Bugfix to RestoreSraDataHandler for new SRA datasets --- .../run/RestoreSraDataHandler.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 7d70cbdc7..d09e0c428 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -181,7 +181,20 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List 1) + if (readdataToSra.get(accession).size() == 1) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("category"), "Readset"); + filter.addCondition(FieldKey.fromString("container"), rs.getContainer()); + filter.addCondition(FieldKey.fromString("dataId"), toMerge.get(0).getFileId1()); + boolean hasMetrics = new TableSelector(SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_QUALITY_METRICS), PageFlowUtil.set("RowId"), filter, null).exists(); + if (!hasMetrics) + { + job.getLogger().debug("No existing metrics found for: " + accession); + updatedAccessions.add(accession); + } + } + else { job.getLogger().debug("Consolidating multiple readdata for: " + accession); From 9a6abc8ab46d9d9c99e99ac93ed519609f458d3c Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 17 Oct 2025 11:13:42 -0700 Subject: [PATCH 41/59] Bugfix to RestoreSraDataHandler for new SRA datasets --- .../run/RestoreSraDataHandler.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index d09e0c428..01e632a48 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -149,6 +149,36 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List Date: Fri, 17 Oct 2025 12:24:08 -0700 Subject: [PATCH 42/59] Bugfix to RestoreSraDataHandler for new SRA datasets --- .../sequenceanalysis/query/SequenceTriggerHelper.java | 4 +++- .../labkey/sequenceanalysis/run/RestoreSraDataHandler.java | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java index a5e9b02e2..33bff92a9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java @@ -36,6 +36,7 @@ import org.labkey.sequenceanalysis.SequenceAnalysisSchema; import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl; import org.labkey.sequenceanalysis.SequenceReadsetImpl; +import org.labkey.sequenceanalysis.pipeline.ReadsetImportJob; import org.labkey.vfs.FileLike; import java.io.File; @@ -302,7 +303,8 @@ public void createReaddataForSra(int readsetId, String sraAccessions) } String folderName = "SequenceImport_RS" + rs.getRowId() + "_" + FileUtil.getTimestamp(); - FileLike outDir = AssayFileWriter.findUniqueFileName(folderName, pr.getRootFileLike()); + FileLike sequenceImport = FileUtil.appendPath(pr.getRootFileLike(), Path.parse(ReadsetImportJob.NAME)); + FileLike outDir = AssayFileWriter.findUniqueFileName(folderName, sequenceImport); FileLike expectedFile1 = FileUtil.appendPath(outDir, Path.parse(token + "_1.fastq.gz")); ExpData exp1 = ExperimentService.get().createData(c, new DataType("Data")); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 01e632a48..e49fc9f01 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -158,8 +158,13 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List Date: Fri, 17 Oct 2025 16:02:46 -0700 Subject: [PATCH 43/59] Bugfix to RestoreSraDataHandler for new SRA datasets --- .../labkey/sequenceanalysis/run/RestoreSraDataHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index e49fc9f01..fddad50ba 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -364,7 +364,9 @@ public void complete(PipelineJob job, List readsets, List toAdd = new ArrayList<>(rd.getFileId1()); + List toAdd = new ArrayList<>(); + toAdd.add(rd.getFileId1()); + if (rd.getFileId2() != null) { toAdd.add(rd.getFileId2()); From 79c1b4b7a9251792b60e52b23ab8a19c98e50f30 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 19 Oct 2025 07:06:07 -0700 Subject: [PATCH 44/59] Reduce logging --- .../run/util/RnaSeQCWrapper.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/RnaSeQCWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/RnaSeQCWrapper.java index aeb02da02..0f21d45d6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/RnaSeQCWrapper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/RnaSeQCWrapper.java @@ -28,6 +28,8 @@ */ public class RnaSeQCWrapper extends AbstractCommandWrapper { + final static int MAX_WARNINGS = 100; + public RnaSeQCWrapper(@Nullable Logger logger) { super(logger); @@ -122,12 +124,32 @@ public File execute(List inputBams, List sampleIds, @Nullable List } else if (!line.contains("transcript_id")) { - getLogger().info("skipping GTF line " + lineNo + " because it lacks transcript_id"); + if (filteredLines <= MAX_WARNINGS) + { + if (filteredLines == MAX_WARNINGS) + { + getLogger().info("skipping GTF line " + lineNo + " because it lacks transcript_id. No additional warnings will be printed"); + } + else + { + getLogger().info("skipping GTF line " + lineNo + " because it lacks transcript_id"); + } + } filteredLines++; } else if (!line.contains("gene_id")) { - getLogger().info("skipping GTF line " + lineNo + " because it lacks gene_id"); + if (filteredLines <= MAX_WARNINGS) + { + if (filteredLines == MAX_WARNINGS) + { + getLogger().info("skipping GTF line " + lineNo + " because it lacks gene_id. No additional warnings will be printed"); + } + else + { + getLogger().info("skipping GTF line " + lineNo + " because it lacks gene_id"); + } + } filteredLines++; } else From f8029db49d8c6d30367e1cf2f751cff493c388ee Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 24 Oct 2025 13:51:59 -0700 Subject: [PATCH 45/59] Update sawfish install --- SequenceAnalysis/pipeline_code/extra_tools_install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/SequenceAnalysis/pipeline_code/extra_tools_install.sh b/SequenceAnalysis/pipeline_code/extra_tools_install.sh index bb1097ef4..8aa3df5a5 100755 --- a/SequenceAnalysis/pipeline_code/extra_tools_install.sh +++ b/SequenceAnalysis/pipeline_code/extra_tools_install.sh @@ -325,11 +325,11 @@ then echo "Cleaning up previous installs" rm -Rf $LKTOOLS_DIR/sawfish* - wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.0.0/sawfish-v2.0.0-x86_64-unknown-linux-gnu.tar.gz - tar -xzf sawfish-v2.0.4-x86_64-unknown-linux-gnu.tar.gz + wget https://github.com/PacificBiosciences/sawfish/releases/download/v2.2.0/sawfish-v2.2.0-x86_64-unknown-linux-gnu.tar.gz + tar -xzf sawfish-v2.2.0-x86_64-unknown-linux-gnu.tar.gz - mv sawfish-v2.0.4-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ - ln -s $LKTOOLS_DIR/sawfish-v2.0.4/bin/sawfish $LKTOOLS_DIR/ + mv sawfish-v2.2.0-x86_64-unknown-linux-gnu $LKTOOLS_DIR/ + ln -s $LKTOOLS_DIR/sawfish-v2.2.0-x86_64-unknown-linux-gnu/bin/sawfish $LKTOOLS_DIR/ else echo "Already installed" fi From 3cc6db16d2388fb69699075f3b29e068ee3d8271 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 29 Oct 2025 12:42:50 -0700 Subject: [PATCH 46/59] Better error handling --- .../labkey/sequenceanalysis/pipeline/ReadsetInitTask.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java index 92ae0ec75..a4a211179 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java @@ -430,6 +430,10 @@ else if (TaskFileManager.InputFileTreatment.compress == inputFileTreatment) moveInputToAnalysisDir(compressed, job, actions, unalteredInputs, outputFiles); } } + else + { + job.getLogger().debug("Input file does not exist, may have already been moved: " + input.getPath()); + } } } else @@ -452,6 +456,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti job.getLogger().debug("Destination: " + output.getPath()); if (output.exists()) { + job.getLogger().debug("output already exists"); if (unalteredInputs != null && unalteredInputs.contains(output)) { job.getLogger().debug("\tThis input was unaltered during normalization and a copy already exists in the analysis folder so the original will be discarded"); @@ -488,7 +493,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti TaskFileManagerImpl.swapFilesInRecordedActions(job.getLogger(), input, output, actions, job, null); } - catch (IOException e) + catch (Exception e) { throw new PipelineJobException(e); } From 8c35a1382e5a95faf01cf8033f86b6406fcbda5d Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 29 Oct 2025 12:57:43 -0700 Subject: [PATCH 47/59] Updates to Save10xBarcodes --- .../run/AbstractCellRangerDependentStep.java | 2 ++ .../singlecell/run/NimbleAlignmentStep.java | 22 +++------------- .../labkey/singlecell/run/NimbleHelper.java | 25 ++++++------------- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java b/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java index a17351fef..5230c0408 100644 --- a/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java @@ -88,6 +88,8 @@ protected File runCellRanger(AlignmentOutputImpl output, Readset rs, List } } + NimbleHelper.write10xBarcodes(localBam, getWrapper().getLogger(), rs, referenceGenome, output); + return localBam; } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 03b940baa..ffeb830ba 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -109,10 +109,9 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu private File createNimbleBam(AlignmentOutputImpl output, Readset rs, List inputFastqs1, List inputFastqs2) throws PipelineJobException { - File cellbarcodes = getCachedBarcodeFile(rs, true); - File umiMapping = getUmiMapping(cellbarcodes); + File cellBarcodeUmiMap = getCachedBarcodeFile(rs, true); - return NimbleHelper.runFastqToBam(output, getPipelineCtx(), rs, inputFastqs1, inputFastqs2, cellbarcodes, umiMapping); + return NimbleHelper.runFastqToBam(output, getPipelineCtx(), rs, inputFastqs1, inputFastqs2, cellBarcodeUmiMap); } private File getCachedBarcodeFile(Readset rs, boolean throwIfNotFound) throws PipelineJobException @@ -132,18 +131,7 @@ private File getCachedBarcodeFile(Readset rs, boolean throwIfNotFound) throws Pi File ret = getPipelineCtx().getSequenceSupport().getCachedData(dataId); if (ret == null || ! ret.exists()) { - throw new PipelineJobException("Missing cached cellbarcode file: " + dataId); - } - - return ret; - } - - private File getUmiMapping(File cellbarcodeFile) throws PipelineJobException - { - File ret = new File(cellbarcodeFile.getPath().replaceAll("cb.txt.gz", "umi.txt.gz")); - if (!ret.exists()) - { - throw new PipelineJobException("Missing cached UMI file: " + ret.getPath()); + throw new PipelineJobException("Missing cached cellbarcode/UMI file: " + dataId); } return ret; @@ -156,7 +144,7 @@ private ExpData findCellBarcodeFiles(Readset rs) throws PipelineJobException TableInfo ti = us.getTable("outputfiles"); SimpleFilter sf = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - sf.addCondition(FieldKey.fromString("category"), NimbleHelper.CATEGORY_CB); + sf.addCondition(FieldKey.fromString("category"), NimbleHelper.CATEGORY_CB_UMI); List cbs = new TableSelector(ti, PageFlowUtil.set("dataid"), sf, new Sort("-rowid")).getArrayList(Integer.class); if (!cbs.isEmpty()) { @@ -194,8 +182,6 @@ private File performCellRangerAlignment(AlignmentOutputImpl output, Readset rs, } } - NimbleHelper.write10xBarcodes(localBam, getWrapper().getLogger(), rs, referenceGenome, output); - return localBam; } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index a74c1a55e..b3d2eee54 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -498,8 +498,7 @@ private Map doAlignment(List genomes, List inputFastqs1, List inputFastqs2, File cellBarcodes, File umiMapping) throws PipelineJobException + public static File runFastqToBam(PipelineStepOutput output, PipelineContext ctx, Readset rs, List inputFastqs1, List inputFastqs2, File cellBarcodeUmiMap) throws PipelineJobException { List outputBams = new ArrayList<>(); int bamIdx = 0; @@ -632,11 +626,8 @@ public static File runFastqToBam(PipelineStepOutput output, PipelineContext ctx, args.add("--r2-fastq"); args.add(inputFastqs2.get(bamIdx).getPath()); - args.add("--cell-barcodes"); - args.add(cellBarcodes.getPath()); - - args.add("--umi-mapping"); - args.add(umiMapping.getPath()); + args.add("--map"); + args.add(cellBarcodeUmiMap.getPath()); args.add("--output"); args.add(outputBam.getPath()); From 0166aa47000696bfadbc4e090e65b910bbae067c Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 29 Oct 2025 14:19:07 -0700 Subject: [PATCH 48/59] Expand StudyMetadata cohorts --- singlecell/resources/chunks/StudyMetadata.R | 2 ++ .../labkey/singlecell/pipeline/singlecell/StudyMetadata.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/StudyMetadata.R b/singlecell/resources/chunks/StudyMetadata.R index f5fb694ff..45bb03cef 100644 --- a/singlecell/resources/chunks/StudyMetadata.R +++ b/singlecell/resources/chunks/StudyMetadata.R @@ -32,6 +32,8 @@ for (datasetId in names(seuratObjects)) { seuratObj <- Rdiscvr::ApplyEC_Metadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) } else if (studyName == 'PPG_Stims') { seuratObj <- Rdiscvr::ApplyPPG_Stim_Metadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) + ] else if (studyName == 'IMPAC_TB_Human') { + seuratObj <- Rdiscvr::ApplyIMPAC_TB_Human_Metadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) } else { stop(paste0('Unknown study: ', studyName)) } diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java index 27050d587..82ffbd953 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java @@ -24,7 +24,7 @@ public Provider() {{ put("multiSelect", false); put("allowBlank", false); - put("storeValues", "PC475;PC531;TB;Malaria;AcuteNx;EC;PPG_Stims"); + put("storeValues", "PC475;PC531;TB;Malaria;AcuteNx;EC;PPG_Stims;IMPAC_TB_Human"); put("delimiter", ";"); }}, null, null, false, false), SeuratToolParameter.create("errorIfUnknownIdsFound", "Error If Unknown Ids Found", "If true, the job will fail if the seurat object contains ID not present in the metadata", "checkbox", null, true) From 14ffbcf614298a6c208d777805f0f3d74206e684 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 29 Oct 2025 14:47:42 -0700 Subject: [PATCH 49/59] Throw exception when existing file present --- .../org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java index a4a211179..6999edbb2 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java @@ -466,8 +466,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti } else { - output = new File(outputDir, FileUtil.getBaseName(input.getName()) + ".orig.gz"); - job.getLogger().debug("\tA file with the expected output name already exists, so the original will be renamed: " + output.getPath()); + throw new PipelineJobException("A file with the expected output name already exists: " + output.getPath()); } } From 504846cbdcee341b8d3d2702a0a929ecb005ec9e Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 30 Oct 2025 05:56:28 -0700 Subject: [PATCH 50/59] Improve resume for ReadsetInitTask --- .../pipeline/ReadsetInitTask.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java index 6999edbb2..5ec6e5a39 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java @@ -454,6 +454,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti File outputDir = job.getAnalysisDirectory(); File output = new File(outputDir, input.getName()); job.getLogger().debug("Destination: " + output.getPath()); + boolean alreadyMoved = false; if (output.exists()) { job.getLogger().debug("output already exists"); @@ -461,16 +462,27 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti { job.getLogger().debug("\tThis input was unaltered during normalization and a copy already exists in the analysis folder so the original will be discarded"); input.delete(); - TaskFileManagerImpl.swapFilesInRecordedActions(job.getLogger(), input, output, actions, job, null); - return; + alreadyMoved = true; } else { - throw new PipelineJobException("A file with the expected output name already exists: " + output.getPath()); + if (input.length() == output.length() && input.lastModified() == output.lastModified()) + { + job.getLogger().info("Output exists, but has the same size/modified timestamp. Deleting original"); + input.delete(); + alreadyMoved = true; + } + else + { + throw new PipelineJobException("A file with the expected output name already exists: " + output.getPath()); + } } } - FileUtils.moveFile(input, output); + if (!alreadyMoved) + { + FileUtils.moveFile(input, output); + } if (!output.exists()) { throw new PipelineJobException("Unable to move file: " + input.getPath()); From ec8135c682d08a9a7aaa8d1522df0327f4aeb375 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 30 Oct 2025 07:11:07 -0700 Subject: [PATCH 51/59] Improve resume for ReadsetInitTask --- .../labkey/sequenceanalysis/pipeline/ReadsetInitTask.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java index 5ec6e5a39..d0e442ef7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java @@ -472,6 +472,11 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti input.delete(); alreadyMoved = true; } + else if (input.exists() && input.length() > output.length() && input.lastModified() == output.lastModified()) + { + job.getLogger().info("Output exists with same timestamp, but with smaller file size. This probably indicates a truncated/failed copy. Deleting this file."); + output.delete(); + } else { throw new PipelineJobException("A file with the expected output name already exists: " + output.getPath()); From fbe9a1373946a23e0dc73f593b92d2e325b6e7d0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 2 Nov 2025 06:29:02 -0800 Subject: [PATCH 52/59] Bugfix to Save10xBarcodes --- .../labkey/singlecell/run/CellRangerGexCountStep.java | 11 ++++++++--- .../src/org/labkey/singlecell/run/NimbleHelper.java | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java index 0d7abe3ce..c4f47edd8 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java @@ -328,7 +328,7 @@ private boolean shouldDiscardBam() return false; } - return !_alwaysRetainBam && getProvider().getParameterByName(AbstractAlignmentStepProvider.DISCARD_BAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + return !_alwaysRetainBam && getProvider().getParameterByName(AbstractAlignmentStepProvider.DISCARD_BAM).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); } private boolean _alwaysRetainBam = false; @@ -349,7 +349,7 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu AbstractAlignmentStepProvider.ALIGNMENT_MODE mode = AbstractAlignmentStepProvider.ALIGNMENT_MODE.valueOf(alignmentMode); List> inputFastqs = new ArrayList<>(); - for (int i = 0; i < inputFastqs1.size();i++) + for (int i = 0; i < inputFastqs1.size(); i++) { File inputFastq1 = inputFastqs1.get(i); File inputFastq2 = inputFastqs2.get(i); @@ -395,9 +395,14 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu File outdir = new File(outputDirectory, id); outdir = new File(outdir, "outs"); + File bam = new File(outdir, "possorted_genome_bam.bam"); + if (bam.exists()) + { + NimbleHelper.write10xBarcodes(bam, getWrapper().getLogger(), rs, referenceGenome, output); + } + if (!shouldDiscardBam()) { - File bam = new File(outdir, "possorted_genome_bam.bam"); if (!bam.exists()) { throw new PipelineJobException("Unable to find file: " + bam.getPath()); diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index b3d2eee54..66b0f5432 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -502,7 +502,7 @@ private Map doAlignment(List genomes, List barcodeArgs = new ArrayList<>(runner.getBaseArgs("Save10xBarcodes")); barcodeArgs.add("-I"); From 5d58764426193f4250edf45fe9bdb97ef4662f14 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 3 Nov 2025 09:07:30 -0800 Subject: [PATCH 53/59] Bugfix to handling of 10x barcodes --- .../labkey/singlecell/run/CellRangerGexCountStep.java | 4 ++++ .../labkey/singlecell/run/NimbleAlignmentStep.java | 6 +++++- .../src/org/labkey/singlecell/run/NimbleAnalysis.java | 11 +++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java index c4f47edd8..2eff03fac 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java @@ -400,6 +400,10 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu { NimbleHelper.write10xBarcodes(bam, getWrapper().getLogger(), rs, referenceGenome, output); } + else + { + getPipelineCtx().getLogger().info("BAM file does not exist, cannot write 10x barcodes: " + bam.getPath()); + } if (!shouldDiscardBam()) { diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index ffeb830ba..2a4513ef0 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -41,6 +41,7 @@ public class NimbleAlignmentStep extends AbstractCellRangerDependentStep public static final String MAX_HITS_TO_REPORT = "maxHitsToReport"; public static final String STRANDEDNESS = "strandedness"; public static final String REQUIRE_CACHED_BARCODES = "requireCachedBarcodes"; + public static final String WRITE_10X_BARCODES = "write10xBarcodes"; public NimbleAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerWrapper wrapper) { @@ -76,7 +77,10 @@ public static List getToolParameters() }}, 4), ToolParameterDescriptor.create(REQUIRE_CACHED_BARCODES, "Fail Unless Cached Barcodes Present", "If checked, the pipeline will expect a previously computed map of cellbarcodes and UMIs to be computed. Under default conditions, if this is missing, cellranger will be re-run. This flag can be helpful to avoid that computation if you expect the barcode file to exist.", "checkbox", new JSONObject(){{ - }}, false) + }}, false), + ToolParameterDescriptor.create(WRITE_10X_BARCODES, "Write 10x Barcodes To File", "If checked, the pipeline will save the 10x barcodes from the BAM to a TSV. This facilitates future analyses.", "checkbox", new JSONObject(){{ + + }}, true) ); } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java index 84c2f3f2a..55da4f3ac 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java @@ -1,8 +1,5 @@ package org.labkey.singlecell.run; -import htsjdk.samtools.SAMFileHeader; -import htsjdk.samtools.SAMTag; -import htsjdk.samtools.SamReaderFactory; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.model.AnalysisModel; import org.labkey.api.sequenceanalysis.model.Readset; @@ -21,6 +18,8 @@ import java.util.LinkedHashSet; import java.util.List; +import static org.labkey.singlecell.run.NimbleAlignmentStep.WRITE_10X_BARCODES; + public class NimbleAnalysis extends AbstractPipelineStep implements AnalysisStep { public NimbleAnalysis(PipelineStepProvider provider, PipelineContext ctx) @@ -61,14 +60,14 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); helper.doNimbleAlign(inputBam, output, rs, FileUtil.getBaseName(inputBam)); - SAMFileHeader header = SamReaderFactory.makeDefault().getFileHeader(inputBam); - if (header.getAttribute(SAMTag.CB.name()) != null) + boolean write10xBarcodes = getProvider().getParameterByName(WRITE_10X_BARCODES).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); + if (write10xBarcodes) { NimbleHelper.write10xBarcodes(inputBam, getPipelineCtx().getLogger(), rs, referenceGenome, output); } else { - getPipelineCtx().getLogger().info("BAM lacks CB tag, will not output 10x barcodes to file"); + getPipelineCtx().getLogger().info("10x barcodes will not be saved to TSV"); } return output; From 544c4eabfffd69116fdb0c192e25733dc5f4f78c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 4 Nov 2025 11:57:44 -0800 Subject: [PATCH 54/59] Switch nimble/CR barcodes to CB alone --- .../src/org/labkey/singlecell/run/NimbleAlignmentStep.java | 2 +- singlecell/src/org/labkey/singlecell/run/NimbleHelper.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 2a4513ef0..78dbe966e 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -148,7 +148,7 @@ private ExpData findCellBarcodeFiles(Readset rs) throws PipelineJobException TableInfo ti = us.getTable("outputfiles"); SimpleFilter sf = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - sf.addCondition(FieldKey.fromString("category"), NimbleHelper.CATEGORY_CB_UMI); + sf.addCondition(FieldKey.fromString("category"), NimbleHelper.CATEGORY_CB); List cbs = new TableSelector(ti, PageFlowUtil.set("dataid"), sf, new Sort("-rowid")).getArrayList(Integer.class); if (!cbs.isEmpty()) { diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java index 66b0f5432..19379d318 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleHelper.java @@ -498,7 +498,7 @@ private Map doAlignment(List genomes, List Date: Tue, 4 Nov 2025 13:43:48 -0800 Subject: [PATCH 55/59] Omit writing to 10x barcodes --- .../run/AbstractCellRangerDependentStep.java | 2 -- .../singlecell/run/CellRangerGexCountStep.java | 9 --------- .../labkey/singlecell/run/NimbleAlignmentStep.java | 6 +----- .../org/labkey/singlecell/run/NimbleAnalysis.java | 12 ------------ 4 files changed, 1 insertion(+), 28 deletions(-) diff --git a/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java b/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java index 5230c0408..a17351fef 100644 --- a/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/AbstractCellRangerDependentStep.java @@ -88,8 +88,6 @@ protected File runCellRanger(AlignmentOutputImpl output, Readset rs, List } } - NimbleHelper.write10xBarcodes(localBam, getWrapper().getLogger(), rs, referenceGenome, output); - return localBam; } diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java index 2eff03fac..7cd85a062 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java @@ -396,15 +396,6 @@ public AlignmentOutput performAlignment(Readset rs, List inputFastqs1, @Nu outdir = new File(outdir, "outs"); File bam = new File(outdir, "possorted_genome_bam.bam"); - if (bam.exists()) - { - NimbleHelper.write10xBarcodes(bam, getWrapper().getLogger(), rs, referenceGenome, output); - } - else - { - getPipelineCtx().getLogger().info("BAM file does not exist, cannot write 10x barcodes: " + bam.getPath()); - } - if (!shouldDiscardBam()) { if (!bam.exists()) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 78dbe966e..0ce323bb6 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -41,7 +41,6 @@ public class NimbleAlignmentStep extends AbstractCellRangerDependentStep public static final String MAX_HITS_TO_REPORT = "maxHitsToReport"; public static final String STRANDEDNESS = "strandedness"; public static final String REQUIRE_CACHED_BARCODES = "requireCachedBarcodes"; - public static final String WRITE_10X_BARCODES = "write10xBarcodes"; public NimbleAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerWrapper wrapper) { @@ -77,10 +76,7 @@ public static List getToolParameters() }}, 4), ToolParameterDescriptor.create(REQUIRE_CACHED_BARCODES, "Fail Unless Cached Barcodes Present", "If checked, the pipeline will expect a previously computed map of cellbarcodes and UMIs to be computed. Under default conditions, if this is missing, cellranger will be re-run. This flag can be helpful to avoid that computation if you expect the barcode file to exist.", "checkbox", new JSONObject(){{ - }}, false), - ToolParameterDescriptor.create(WRITE_10X_BARCODES, "Write 10x Barcodes To File", "If checked, the pipeline will save the 10x barcodes from the BAM to a TSV. This facilitates future analyses.", "checkbox", new JSONObject(){{ - - }}, true) + }}, false) ); } diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java index 55da4f3ac..b24d546b2 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAnalysis.java @@ -18,8 +18,6 @@ import java.util.LinkedHashSet; import java.util.List; -import static org.labkey.singlecell.run.NimbleAlignmentStep.WRITE_10X_BARCODES; - public class NimbleAnalysis extends AbstractPipelineStep implements AnalysisStep { public NimbleAnalysis(PipelineStepProvider provider, PipelineContext ctx) @@ -60,16 +58,6 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc NimbleHelper helper = new NimbleHelper(getPipelineCtx(), getProvider(), getStepIdx()); helper.doNimbleAlign(inputBam, output, rs, FileUtil.getBaseName(inputBam)); - boolean write10xBarcodes = getProvider().getParameterByName(WRITE_10X_BARCODES).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - if (write10xBarcodes) - { - NimbleHelper.write10xBarcodes(inputBam, getPipelineCtx().getLogger(), rs, referenceGenome, output); - } - else - { - getPipelineCtx().getLogger().info("10x barcodes will not be saved to TSV"); - } - return output; } From 2b95f2f8d17b7bdfc7522cee013ec8ca76aa25e2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 5 Nov 2025 08:44:00 -0800 Subject: [PATCH 56/59] Add another coalesce() term in case name and label are blank --- Studies/resources/schemas/studies.xml | 1 + Studies/src/org/labkey/studies/query/StudiesUserSchema.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Studies/resources/schemas/studies.xml b/Studies/resources/schemas/studies.xml index 8420ce4d2..c28038ff9 100644 --- a/Studies/resources/schemas/studies.xml +++ b/Studies/resources/schemas/studies.xml @@ -140,6 +140,7 @@ Study Name + false Label diff --git a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java index bb7c741db..00401b524 100644 --- a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java +++ b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java @@ -173,7 +173,8 @@ private TableInfo createCohortsTable(String name, ContainerFilter cf) { CustomPermissionsTable ret = createStudyDesignTable(name, cf, true); - SQLFragment sql2 = new SQLFragment("coalesce(" + ExprColumn.STR_TABLE_ALIAS + ".label, " + ExprColumn.STR_TABLE_ALIAS + ".cohortName)"); + SQLFragment lastTerm = ret.getSqlDialect().concatenate(new SQLFragment("'Cohort-'"), new SQLFragment("CAST(" + ExprColumn.STR_TABLE_ALIAS + ".rowId AS VARCHAR)")); + SQLFragment sql2 = new SQLFragment("coalesce(" + ExprColumn.STR_TABLE_ALIAS + ".label, " + ExprColumn.STR_TABLE_ALIAS + ".cohortName, ").append(lastTerm).append(new SQLFragment(")")); ExprColumn col2 = new ExprColumn(ret, "labelOrName", sql2, JdbcType.VARCHAR, ret.getColumn("cohortName"), ret.getColumn("label")); col2.setLabel("Cohort Name"); col2.setHidden(true); From 476ac9087593df145ac834dc7d88cfcc1a5de430 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 5 Nov 2025 10:08:23 -0800 Subject: [PATCH 57/59] Bugfix to study import --- Studies/src/org/labkey/studies/StudiesManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Studies/src/org/labkey/studies/StudiesManager.java b/Studies/src/org/labkey/studies/StudiesManager.java index bf7926dcc..f233e16d1 100644 --- a/Studies/src/org/labkey/studies/StudiesManager.java +++ b/Studies/src/org/labkey/studies/StudiesManager.java @@ -225,7 +225,7 @@ private Map studyToMap(StudyDefinition s) Map m = new HashMap<>(); if (s.getRowId() != null) m.put("rowId", s.getRowId()); - m.put("name", s.getStudyName()); + m.put("studyName", s.getStudyName()); m.put("label", s.getLabel()); m.put("category", s.getCategory()); m.put("description", s.getDescription()); From 3ffa2d4ed2211707799d583c676d56a916d13b20 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 5 Nov 2025 19:12:58 -0800 Subject: [PATCH 58/59] Build fixes --- .../sequenceanalysis/query/SequenceTriggerHelper.java | 6 +++--- .../labkey/sequenceanalysis/run/RestoreSraDataHandler.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java index d49fcef99..e2f485a00 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceTriggerHelper.java @@ -268,7 +268,7 @@ public long createExpData(String relPath) { public void createReaddataForSra(int readsetId, String sraAccessions) { - SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(readsetId, _user); + SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(Long.valueOf(readsetId), _user); if (rs == null) { throw new IllegalArgumentException("Unable to find readset: " + readsetId); @@ -286,7 +286,7 @@ public void createReaddataForSra(int readsetId, String sraAccessions) // Create new: ReadDataImpl rd1 = new ReadDataImpl(); - rd1.setReadset(readsetId); + rd1.setReadset(Long.valueOf(readsetId)); rd1.setContainer(rs.getContainer()); rd1.setCreated(new Date()); rd1.setModified(new Date()); @@ -305,7 +305,7 @@ public void createReaddataForSra(int readsetId, String sraAccessions) String folderName = "SequenceImport_RS" + rs.getRowId() + "_" + FileUtil.getTimestamp(); FileLike sequenceImport = FileUtil.appendPath(pr.getRootFileLike(), Path.parse(ReadsetImportJob.NAME)); - FileLike outDir = AssayFileWriter.findUniqueFileName(folderName, sequenceImport); + FileLike outDir = FileUtil.findUniqueFileName(folderName, sequenceImport); FileLike expectedFile1 = FileUtil.appendPath(outDir, Path.parse(token + "_1.fastq.gz")); ExpData exp1 = ExperimentService.get().createData(c, new DataType("Data")); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 825606303..8145a40c9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -350,7 +350,7 @@ public void complete(PipelineJob job, List readsets, List Date: Thu, 6 Nov 2025 05:28:05 -0800 Subject: [PATCH 59/59] Build fixes --- .../src/org/labkey/singlecell/run/NimbleAlignmentStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java index 0ce323bb6..9c0a3e785 100644 --- a/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java +++ b/singlecell/src/org/labkey/singlecell/run/NimbleAlignmentStep.java @@ -199,7 +199,7 @@ public void init(SequenceAnalysisJobSupport support) throws PipelineJobException } // Try to find 10x barcodes: - HashMap readsetToBarcodes = new HashMap<>(); + HashMap readsetToBarcodes = new HashMap<>(); for (Readset rs : support.getCachedReadsets()) { ExpData f = findCellBarcodeFiles(rs);