From 0b3cdec95f024978a784f1f7606dcbba48dc7263 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 17 Nov 2025 11:43:30 -0800 Subject: [PATCH 1/5] Continue adopting FileLike instead of File or Path --- .../pipeline/AbstractResumer.java | 3 +- .../pipeline/AlignerIndexUtil.java | 5 +- .../pipeline/HasJobParams.java | 4 +- .../OutputIntegrationTests.java | 19 ++--- .../SequenceAnalysisController.java | 8 +-- .../SequenceIntegrationTests.java | 16 ++--- .../SequencePipelineServiceImpl.java | 2 +- .../analysis/AlignmentMetricsHandler.java | 2 +- .../analysis/LiftoverHandler.java | 2 +- .../pipeline/AlignmentAnalysisJob.java | 3 +- .../AlignmentAnalysisRemoteWorkTask.java | 4 +- .../pipeline/AlignmentAnalysisWorkTask.java | 7 +- .../pipeline/AlignmentImportJob.java | 4 +- .../pipeline/AlignmentInitTask.java | 5 +- .../pipeline/AlignmentNormalizationTask.java | 8 ++- .../pipeline/CacheAlignerIndexesTask.java | 10 +-- .../pipeline/CreateReferenceLibraryTask.java | 2 +- .../pipeline/IlluminaImportJob.java | 6 +- .../pipeline/IlluminaImportTask.java | 13 ++-- .../pipeline/JobContextImpl.java | 9 ++- .../pipeline/PrepareAlignerIndexesTask.java | 2 +- .../pipeline/ProcessVariantsHandler.java | 9 ++- .../pipeline/ReadsetImportJob.java | 12 ++-- .../pipeline/ReadsetInitTask.java | 14 ++-- .../pipeline/ReferenceLibraryPipelineJob.java | 2 +- .../pipeline/SequenceAlignmentJob.java | 9 +-- .../pipeline/SequenceAlignmentTask.java | 69 ++++++++----------- .../pipeline/SequenceAnalysisTask.java | 2 +- .../pipeline/SequenceConcatTask.java | 2 +- .../pipeline/SequenceJob.java | 37 +++++----- .../pipeline/SequenceNormalizationTask.java | 26 +++---- .../SequenceOutputHandlerFinalTask.java | 2 +- .../SequenceOutputHandlerInitTask.java | 2 +- .../pipeline/SequenceOutputHandlerJob.java | 10 +-- .../SequenceOutputHandlerWebserverTask.java | 2 +- .../pipeline/SequencePipelineSettings.java | 19 ++--- .../SequenceReadsetHandlerInitTask.java | 2 +- .../pipeline/SequenceReadsetHandlerJob.java | 11 +-- .../SequenceReadsetHandlerRemoteTask.java | 2 +- .../SequenceReadsetHandlerWebserverTask.java | 6 +- .../pipeline/SequenceTaskHelper.java | 2 +- .../pipeline/TaskFileManagerImpl.java | 32 +++++---- .../pipeline/VariantProcessingJob.java | 4 +- .../VariantProcessingRemoteMergeTask.java | 5 +- .../VariantProcessingRemoteSplitTask.java | 7 +- .../run/analysis/SnpCountAnalysis.java | 2 +- .../util/AbstractGenomicsDBImportHandler.java | 10 +-- .../run/util/CombineGVCFsHandler.java | 3 +- .../run/util/GenotypeGVCFsWrapper.java | 2 +- .../run/util/SamToFastqWrapper.java | 6 ++ .../run/variant/SplitVcfBySamplesStep.java | 2 +- .../sequenceanalysis/util/SequenceUtil.java | 5 +- 52 files changed, 244 insertions(+), 208 deletions(-) diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractResumer.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractResumer.java index 147bd71e9..0b6706d27 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractResumer.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractResumer.java @@ -7,6 +7,7 @@ import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.util.Pair; +import org.labkey.vfs.FileSystemLike; import java.io.BufferedInputStream; import java.io.File; @@ -240,7 +241,7 @@ public static T create(SequenceOutputHandler.JobCont { for (File orig : ret.getCopiedInputs().keySet()) { - ctx.getWorkDir().inputFile(orig, ret._copiedInputs.get(orig), false); + ctx.getWorkDir().inputFile(FileSystemLike.wrapFile(orig), FileSystemLike.wrapFile(ret._copiedInputs.get(orig)), false); } } } diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java index 5b86a99d7..39b08ed7f 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java @@ -5,6 +5,7 @@ import org.labkey.api.data.ConvertHelper; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.WorkDirectory; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -89,7 +90,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable if (doCopy) { ctx.getLogger().info("copying index files to work location"); - File localSharedDir = new File(wd.getDir(), "Shared"); + File localSharedDir = new File(wd.getDir().toNioPathForRead().toFile(), "Shared"); File destination = new File(localSharedDir, localName); ctx.getLogger().debug(destination.getPath()); File[] files = webserverIndexDir.listFiles(); @@ -98,7 +99,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable return false; } - destination = wd.inputFile(webserverIndexDir, destination, true); + destination = wd.inputFile(FileSystemLike.wrapFile(webserverIndexDir), FileSystemLike.wrapFile(destination), true).toNioPathForRead().toFile(); if (output != null && !destination.equals(webserverIndexDir)) { ctx.getLogger().debug("adding deferred delete file: " + destination.getPath()); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/HasJobParams.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/HasJobParams.java index 210230f17..a94c8a459 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/HasJobParams.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/HasJobParams.java @@ -1,8 +1,8 @@ package org.labkey.api.sequenceanalysis.pipeline; import org.json.JSONObject; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.Map; /** @@ -14,5 +14,5 @@ public interface HasJobParams JSONObject getParameterJson(); - File getParametersFile(); + FileLike getParametersFile(); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java index 4dc9e91a8..7fdb138e1 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java @@ -34,6 +34,7 @@ import org.labkey.api.view.ViewServlet; import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler; import org.labkey.sequenceanalysis.pipeline.SequenceOutputHandlerJob; +import org.labkey.vfs.FileLike; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.web.bind.annotation.RequestMethod; @@ -110,15 +111,15 @@ public void testVariantProcessing() throws Exception SequenceOutputHandlerJob job = (SequenceOutputHandlerJob)j; Set extraFiles = new HashSet<>(); - extraFiles.add(new File(job.getAnalysisDirectory(), jobName + "." + outputFileId + ".log")); - extraFiles.add(new File(job.getAnalysisDirectory(), "sequenceOutput.json")); - extraFiles.add(new File(job.getAnalysisDirectory(), "sequenceSupport.json.gz")); - extraFiles.add(ProcessVariantsHandler.getPedigreeFile(job.getAnalysisDirectory(), "laboratory.subjects")); - extraFiles.add(new File(job.getAnalysisDirectory(), basename + ".gfiltered.selectVariants.annotated.filtered.vcf.gz")); - extraFiles.add(new File(job.getAnalysisDirectory(), basename + ".gfiltered.selectVariants.annotated.filtered.vcf.gz.tbi")); - extraFiles.add(new File(job.getAnalysisDirectory(), job.getBaseName() + ".pipe.xar.xml")); - - verifyFileOutputs(job.getAnalysisDirectory(), extraFiles); + extraFiles.add(job.getAnalysisDirectory().resolveChild(jobName + "." + outputFileId + ".log").toNioPathForRead().toFile()); + extraFiles.add(job.getAnalysisDirectory().resolveChild("sequenceOutput.json").toNioPathForRead().toFile()); + extraFiles.add(job.getAnalysisDirectory().resolveChild("sequenceSupport.json.gz").toNioPathForRead().toFile()); + extraFiles.add(ProcessVariantsHandler.getPedigreeFile(job.getAnalysisDirectory(), "laboratory.subjects").toNioPathForRead().toFile()); + extraFiles.add(job.getAnalysisDirectory().resolveChild(basename + ".gfiltered.selectVariants.annotated.filtered.vcf.gz").toNioPathForRead().toFile()); + extraFiles.add(job.getAnalysisDirectory().resolveChild(basename + ".gfiltered.selectVariants.annotated.filtered.vcf.gz.tbi").toNioPathForRead().toFile()); + extraFiles.add(job.getAnalysisDirectory().resolveChild(job.getBaseName() + ".pipe.xar.xml").toNioPathForRead().toFile()); + + verifyFileOutputs(job.getAnalysisDirectory().toNioPathForRead().toFile(), extraFiles); //verify outputfile created: TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("runId/jobid/job"), job.getJobGUID()), null); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java index 58553d3d6..86802d501 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java @@ -1876,14 +1876,14 @@ public boolean isSubmitJobToReadsetContainer() return getJobParameters() != null && getJobParameters().optBoolean("submitJobToReadsetContainer", false); } - public List getFiles(PipeRoot pr) throws PipelineValidationException + public List getFiles(PipeRoot pr) throws PipelineValidationException { if (getJobParameters() == null || getJobParameters().get("inputFiles") == null) { return null; } - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); JSONArray inputFiles = getJobParameters().getJSONArray("inputFiles"); String path = getJobParameters().optString("path"); for (JSONObject o : JsonUtil.toJSONObjectList(inputFiles)) @@ -1900,7 +1900,7 @@ else if (!d.getFile().exists()) throw new PipelineValidationException("Missing file for data: " + o.get("dataId")); } - ret.add(d.getFile()); + ret.add(d.getFileLike()); } else if (o.has("relPath") || o.has("fileName")) { @@ -1927,7 +1927,7 @@ else if (o.has("relPath") || o.has("fileName")) throw new PipelineValidationException("Unknown file: " + o.getString("relPath") + " / " + o.getString("fileName")); } - ret.add(f.toNioPathForRead().toFile()); + ret.add(f); } else if (o.opt("filePath") != null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java index 561cc0122..02e3e0dce 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java @@ -1144,7 +1144,7 @@ public void barcodeTest() throws Exception private File getBaseDir(PipelineJob job) { - return ((SequenceJob)job).getAnalysisDirectory(); + return ((SequenceJob)job).getAnalysisDirectory().toNioPathForRead().toFile(); } /** @@ -1489,7 +1489,7 @@ protected void validateAlignmentJob(Set jobs, Collection ad protected void validateAlignmentJob(Set jobs, Collection additionalFiles, SequenceReadsetImpl rs, Integer aligned, Integer unaligned, boolean includeRefFiles) throws Exception { SequenceAlignmentJob job = getAlignmentJob(jobs, rs); - File basedir = job.getAnalysisDirectory(); + File basedir = job.getAnalysisDirectory().toNioPathForRead().toFile(); String outDir = SequenceTaskHelper.getUnzippedBaseName(rs.getReadDataImpl().get(0).getFile1()); Set expectedOutputs = new HashSet<>(); @@ -2982,7 +2982,7 @@ public void testBismarkWithSavedLibraryAndAdapters() throws Exception //job1Files.add("paired1/Alignment/TestReadset1.insertsize.metrics"); //job1Files.add("paired1/Alignment/TestReadset1.insertsize.metrics.pdf"); - File basedir1 = getAlignmentJob(jobs, _readsets.get(0)).getAnalysisDirectory(); + File basedir1 = getAlignmentJob(jobs, _readsets.get(0)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job1Files, basedir1, "paired1/Alignment/CpG_OB_TestReadset1.txt.gz"); addOptionalFile(job1Files, basedir1, "paired1/Alignment/CpG_OT_TestReadset1.txt.gz"); addOptionalFile(job1Files, basedir1, "paired1/Alignment/Non_CpG_OB_TestReadset1.bam.txt.gz"); @@ -3008,7 +3008,7 @@ public void testBismarkWithSavedLibraryAndAdapters() throws Exception //job2Files.add("paired3/Alignment/TestReadset2.summary.metrics"); //job2Files.add("paired3/Alignment/TestReadset2.insertsize.metrics"); - File basedir2 = getAlignmentJob(jobs, _readsets.get(1)).getAnalysisDirectory(); + File basedir2 = getAlignmentJob(jobs, _readsets.get(1)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job2Files, basedir2, "paired3/Alignment/CpG_OB_TestReadset2.txt.gz"); addOptionalFile(job2Files, basedir2, "paired3/Alignment/CpG_OT_TestReadset2.txt.gz"); addOptionalFile(job2Files, basedir2, "paired3/Alignment/Non_CpG_OB_TestReadset2.bam.txt.gz"); @@ -3032,7 +3032,7 @@ public void testBismarkWithSavedLibraryAndAdapters() throws Exception //job3Files.add("paired4/Alignment/TestReadset3.summary.metrics"); //job3Files.add("paired4/Alignment/TestReadset3.insertsize.metrics"); - File basedir3 = getAlignmentJob(jobs, _readsets.get(2)).getAnalysisDirectory(); + File basedir3 = getAlignmentJob(jobs, _readsets.get(2)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job3Files, basedir3, "paired4/Alignment/CpG_OB_TestReadset3.txt.gz"); addOptionalFile(job3Files, basedir3, "paired4/Alignment/CpG_OT_TestReadset3.txt.gz"); addOptionalFile(job3Files, basedir3, "paired4/Alignment/Non_CpG_OB_TestReadset3.bam.txt.gz"); @@ -3109,7 +3109,7 @@ public void testBismarkWithSavedLibraryAdaptersAndDelete() throws Exception job1Files.add("paired1/Alignment/TestReadset1.insertsize.metrics"); job1Files.add("paired1/Alignment/TestReadset1.insertsize.metrics.pdf"); - File basedir1 = getAlignmentJob(jobs, _readsets.get(0)).getAnalysisDirectory(); + File basedir1 = getAlignmentJob(jobs, _readsets.get(0)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job1Files, basedir1, "paired1/Alignment/CpG_OB_TestReadset1.txt.gz"); addOptionalFile(job1Files, basedir1, "paired1/Alignment/CpG_OT_TestReadset1.txt.gz"); addOptionalFile(job1Files, basedir1, "paired1/Alignment/Non_CpG_OB_TestReadset1.bam.txt.gz"); @@ -3132,7 +3132,7 @@ public void testBismarkWithSavedLibraryAdaptersAndDelete() throws Exception job2Files.add("paired3/Alignment/TestReadset2.summary.metrics"); //job2Files.add("paired3/Alignment/TestReadset2.insertsize.metrics"); - File basedir2 = getAlignmentJob(jobs, _readsets.get(1)).getAnalysisDirectory(); + File basedir2 = getAlignmentJob(jobs, _readsets.get(1)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job2Files, basedir2, "paired3/Alignment/CpG_OB_TestReadset2.txt.gz"); addOptionalFile(job2Files, basedir2, "paired3/Alignment/CpG_OT_TestReadset2.txt.gz"); addOptionalFile(job2Files, basedir2, "paired3/Alignment/Non_CpG_OB_TestReadset2.bam.txt.gz"); @@ -3154,7 +3154,7 @@ public void testBismarkWithSavedLibraryAdaptersAndDelete() throws Exception job3Files.add("paired4/Alignment/TestReadset3.summary.metrics"); //job3Files.add("paired4/Alignment/TestReadset3.insertsize.metrics"); - File basedir3 = getAlignmentJob(jobs, _readsets.get(2)).getAnalysisDirectory(); + File basedir3 = getAlignmentJob(jobs, _readsets.get(2)).getAnalysisDirectory().toNioPathForRead().toFile(); addOptionalFile(job3Files, basedir3, "paired4/Alignment/CpG_OB_TestReadset3.txt.gz"); addOptionalFile(job3Files, basedir3, "paired4/Alignment/CpG_OT_TestReadset3.txt.gz"); addOptionalFile(job3Files, basedir3, "paired4/Alignment/Non_CpG_OB_TestReadset3.bam.txt.gz"); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java index 871caab49..847f97ebd 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java @@ -514,7 +514,7 @@ public List getSequenceJobInputFiles(PipelineJob job) return null; } - return ((SequenceJob) job).getInputFiles(); + return ((SequenceJob) job).getInputFiles().stream().map(f -> f.toNioPathForRead().toFile()).toList(); } @Override diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AlignmentMetricsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AlignmentMetricsHandler.java index 002710a0c..17c0b6975 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AlignmentMetricsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/AlignmentMetricsHandler.java @@ -90,7 +90,7 @@ public void processFilesRemote(List inputFiles, JobContext c } List bams = new ArrayList<>(); - bams.addAll(job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles()); + bams.addAll(job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles().stream().map(f -> f.toNioPathForRead().toFile()).toList()); if (bams.isEmpty()) { job.error("No BAMS found, aborting"); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java index 2d579cc28..6da4d9c87 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java @@ -226,7 +226,7 @@ public void processFilesRemote(List inputFiles, JobContext c action.addInput(f.getFile(), "Input File"); action.addInput(chainFile, "Chain File"); - File outDir = ((FileAnalysisJobSupport) job).getAnalysisDirectory(); + File outDir = ((FileAnalysisJobSupport) job).getAnalysisDirectory().toNioPathForRead().toFile(); String baseName = SequenceAnalysisService.get().getUnzippedBaseName(f.getFile().getName()); File lifted = new File(outDir, baseName + ".lifted-" + targetGenomeId + getOutputExtension(f.getFile())); File unmappedOutput = retainUnmapped ? getUnmappedOutputFile(lifted) : null; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisJob.java index 46c8e6474..a510e4a60 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisJob.java @@ -20,6 +20,7 @@ import org.labkey.sequenceanalysis.SequenceReadsetImpl; import org.labkey.sequenceanalysis.model.AnalysisModelImpl; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileSystemLike; import java.io.IOException; import java.util.ArrayList; @@ -104,7 +105,7 @@ public static List createForAnalyses(Container c, User u, AlignmentAnalysisJob j = new AlignmentAnalysisJob(targetContainer, u, jobName, pr, params, model); j.setDescription(description); - j.setInputFiles(Collections.singletonList(model.getAlignmentFileObject())); + j.setInputFiles(Collections.singletonList(FileSystemLike.wrapFile(model.getAlignmentFileObject()))); ret.add(j); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java index 1af40f388..3e7e18749 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java @@ -16,6 +16,7 @@ import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.FileType; import org.labkey.sequenceanalysis.run.util.FastaIndexer; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.ArrayList; @@ -111,8 +112,9 @@ public RecordedActionSet run() throws PipelineJobException Map alignmentMap = getAnalysisMap(); List outputs = new ArrayList<>(); - for (File inputBam : getTaskHelper().getSupport().getInputFiles()) + for (FileLike inputFileLike : getTaskHelper().getSupport().getInputFiles()) { + File inputBam = inputFileLike.toNioPathForRead().toFile(); AnalysisModel m = alignmentMap.get(inputBam.getName()); if (m == null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisWorkTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisWorkTask.java index aa7d69b01..671fb6063 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisWorkTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisWorkTask.java @@ -65,10 +65,9 @@ public List getProtocolActionNames() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public AlignmentAnalysisWorkTask createTask(PipelineJob job) { - AlignmentAnalysisWorkTask task = new AlignmentAnalysisWorkTask(this, job); - return task; + return new AlignmentAnalysisWorkTask(this, job); } @Override @@ -124,7 +123,7 @@ public RecordedActionSet run() throws PipelineJobException throw new PipelineJobException("Unable to find reference FASTA for analysis: " + getPipelineJob().getAnalyisId()); } - File outDir = new File(getTaskHelper().getJob().getAnalysisDirectory(), FileUtil.getBaseName(m.getAlignmentFileObject())); + File outDir = getTaskHelper().getJob().getAnalysisDirectory().resolveChild(FileUtil.getBaseName(m.getAlignmentFileObject())).toNioPathForRead().toFile(); if (!outDir.exists()) { outDir.mkdirs(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportJob.java index 2a7e39889..058e327de 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportJob.java @@ -14,8 +14,8 @@ import org.labkey.api.view.NotFoundException; import org.labkey.sequenceanalysis.SequenceAnalysisModule; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.List; @@ -39,7 +39,7 @@ private AlignmentImportJob(Container c, User u, String jobName, PipeRoot root, J super(SequencePipelineProvider.NAME, c, u, jobName, root, params, new TaskId(FileAnalysisTaskPipeline.class, NAME), FOLDER_NAME); } - public static List create(Container c, User u, String jobName, String description, JSONObject params, Collection inputFiles) throws IOException, PipelineValidationException + public static List create(Container c, User u, String jobName, String description, JSONObject params, Collection inputFiles) throws IOException, PipelineValidationException { PipeRoot pr = PipelineService.get().findPipelineRoot(c); if (pr == null || !pr.isValid()) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java index 249832cff..d7a396657 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java @@ -14,6 +14,7 @@ import org.labkey.api.sequenceanalysis.pipeline.ReferenceLibraryStep; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.FileType; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.util.Collections; @@ -136,13 +137,13 @@ else if (steps.size() > 1) } else { - getHelper().getFileManager().addInput(action, "Job Parameters", getHelper().getJob().getParametersFile()); + getHelper().getFileManager().addInput(action, "Job Parameters", FileSystemLike.toFile(getHelper().getJob().getParametersFile())); getJob().getLogger().info("Creating Reference Library FASTA"); ReferenceLibraryStep step = steps.get(0).getProvider().create(getHelper()); //ensure the FASTA exists - File sharedDirectory = new File(getHelper().getJob().getAnalysisDirectory(), SequenceTaskHelper.SHARED_SUBFOLDER_NAME); + File sharedDirectory = FileSystemLike.toFile(getHelper().getJob().getAnalysisDirectory().resolveChild(SequenceTaskHelper.SHARED_SUBFOLDER_NAME)); if (!sharedDirectory.exists()) { sharedDirectory.mkdirs(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentNormalizationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentNormalizationTask.java index aa2caf833..aa479ff95 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentNormalizationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentNormalizationTask.java @@ -27,6 +27,8 @@ import org.labkey.sequenceanalysis.run.util.CollectInsertSizeMetricsWrapper; import org.labkey.sequenceanalysis.run.util.CollectWgsMetricsWrapper; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -111,10 +113,10 @@ public RecordedActionSet run() throws PipelineJobException //move and make sure BAMs are coordinate sorted and indexed FileType bamFile = new FileType("bam"); Map bamMap = new HashMap<>(); - for (File f : getTaskHelper().getJob().getInputFiles()) + for (FileLike f : getTaskHelper().getJob().getInputFiles()) { getJob().getLogger().debug("input file: " + f.getPath()); - bamMap.put(f.getName(), f); + bamMap.put(f.getName(), FileSystemLike.toFile(f)); } @@ -195,7 +197,7 @@ else if (step.expectToCreateNewBam()) getTaskHelper().getFileManager().addInput(moveAction, "Input BAM", bam); actions.add(moveAction); - File finalDestination = new File(getTaskHelper().getJob().getAnalysisDirectory(), originalFile.getName()); + File finalDestination = FileSystemLike.toFile(getTaskHelper().getJob().getAnalysisDirectory().resolveChild(originalFile.getName())); if (TaskFileManager.InputFileTreatment.leaveInPlace == getTaskHelper().getFileManager().getInputFileTreatment()) { if (bam.equals(originalFile)) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java index a6d7d8935..af7939026 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java @@ -18,6 +18,7 @@ import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.FileType; import org.labkey.sequenceanalysis.run.util.FastaIndexer; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -122,19 +123,20 @@ public RecordedActionSet run() throws PipelineJobException //pre-cache aligner indexes for (PipelineStepProvider provider : SequencePipelineService.get().getProviders(AlignmentStep.class)) { - if (provider instanceof AbstractAlignmentStepProvider) + if (provider instanceof AbstractAlignmentStepProvider aasp) { - if (((AbstractAlignmentStepProvider)provider).isAlwaysCacheIndex()) + if (aasp.isAlwaysCacheIndex()) { getJob().getLogger().info("preparing index for: " + provider.getName()); AlignmentStep alignmentStep = provider.create(ctx); boolean hasIndex = AlignerIndexUtil.hasCachedIndex(alignmentStep.getPipelineCtx(), alignmentStep.getIndexCachedDirName(getJob()), referenceGenome); - File outDir = new File(_wd.getDir(), alignmentStep.getIndexCachedDirName(getJob())); + File dir = FileSystemLike.toFile(_wd.getDir()); + File outDir = new File(dir, alignmentStep.getIndexCachedDirName(getJob())); if (!hasIndex) { //create locally first - alignmentStep.createIndex(referenceGenome, _wd.getDir()); + alignmentStep.createIndex(referenceGenome, dir); //NOTE: the AlignerSteps are doing this themselves. not sure if that is the right behavior if (!AlignerIndexUtil.hasCachedIndex(ctx, provider.getName(), referenceGenome)) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java index ee424a3a5..b2784cf17 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java @@ -293,7 +293,7 @@ public RecordedActionSet run() throws PipelineJobException getPipelineJob().setLibraryId(rowId); String basename = FileUtil.makeLegalName(rowId + "_" + getPipelineJob().getLibraryName().replace(" ", "_")); - File outputDir = new File(getPipelineJob().getAnalysisDirectory(), rowId.toString()); + File outputDir = getPipelineJob().getAnalysisDirectory().resolveChild(rowId.toString()).toNioPathForRead().toFile(); if (!outputDir.exists()) { outputDir.mkdirs(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportJob.java index f96fddba1..33925ed18 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportJob.java @@ -15,8 +15,8 @@ import org.labkey.api.util.FileType; import org.labkey.api.view.NotFoundException; import org.labkey.sequenceanalysis.SequenceAnalysisModule; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -42,14 +42,14 @@ private IlluminaImportJob(Container c, User u, String jobName, PipeRoot root, JS super(SequencePipelineProvider.NAME, c, u, jobName, root, params, new TaskId(FileAnalysisTaskPipeline.class, NAME), FOLDER_NAME); } - public static List create(Container c, User u, String jobName, String description, JSONObject params, Collection inputFiles) throws ClassNotFoundException, IOException, PipelineValidationException + public static List create(Container c, User u, String jobName, String description, JSONObject params, Collection inputFiles) throws ClassNotFoundException, IOException, PipelineValidationException { PipeRoot pr = PipelineService.get().findPipelineRoot(c); if (pr == null || !pr.isValid()) throw new NotFoundException(); List ret = new ArrayList<>(); - for (File csv : inputFiles) + for (FileLike csv : inputFiles) { IlluminaImportJob job = new IlluminaImportJob(c, u, jobName, pr, params); job.setDescription(description); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java index 2ad04ec9c..8c320d331 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java @@ -46,6 +46,7 @@ import org.labkey.api.util.Pair; import org.labkey.sequenceanalysis.ReadDataImpl; import org.labkey.sequenceanalysis.SequenceAnalysisSchema; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -131,7 +132,7 @@ public RecordedActionSet run() throws PipelineJobException job.getLogger().info("Starting analysis"); String prefix = job.getParameters().get("fastqPrefix"); - List inputFiles = getSupport().getInputFiles(); + List inputFiles = getSupport().getInputFiles(); if (inputFiles.isEmpty()) throw new PipelineJobException("No input files"); @@ -140,7 +141,7 @@ public RecordedActionSet run() throws PipelineJobException handleInstrumentRun(schema); //iterate over each CSV - for (File input : inputFiles) + for (FileLike input : inputFiles) { RecordedAction action = new RecordedAction(ACTION_NAME); action.addInputIfNotPresent(input, "Illumina Sample CSV"); @@ -150,9 +151,9 @@ public RecordedActionSet run() throws PipelineJobException //NOTE: it might be just as easy to match filename based on expected pattern //this step will be slow - IlluminaFastqSplitter parser = new IlluminaFastqSplitter<>("Illumina", sampleMap, job.getLogger(), input.getParent(), prefix); + IlluminaFastqSplitter parser = new IlluminaFastqSplitter<>("Illumina", sampleMap, job.getLogger(), input.toNioPathForRead().toFile().getParent(), prefix); parser.setOutputGzip(true); - parser.setDestinationDir(getSupport().getAnalysisDirectory()); + parser.setDestinationDir(getSupport().getAnalysisDirectory().toNioPathForRead().toFile()); // the first element of the pair is the sample ID. the second is either 1 or 2, // depending on whether the file represents the forward or reverse reads @@ -328,10 +329,10 @@ private ExpData createExpData(File f) return SequenceTaskHelper.createExpData(f, getJob()); } - private Map parseCsv(File sampleFile, DbSchema schema) throws PipelineJobException + private Map parseCsv(FileLike sampleFile, DbSchema schema) throws PipelineJobException { getJob().getLogger().info("Parsing Sample File: " + sampleFile.getName()); - try (CSVReader reader = new CSVReader(Readers.getReader(sampleFile))) + try (CSVReader reader = new CSVReader(Readers.getReader(sampleFile.openInputStream()))) { //parse the samples file String [] nextLine; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/JobContextImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/JobContextImpl.java index dea56ee14..df58f7621 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/JobContextImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/JobContextImpl.java @@ -11,6 +11,8 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.TaskFileManager; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.util.Arrays; @@ -30,6 +32,11 @@ public class JobContextImpl implements SequenceOutputHandler.MutableJobContext private TaskFileManager _fileManager; private final WorkDirectory _wd; + public JobContextImpl(SequenceJob job, SequenceAnalysisJobSupport support, JSONObject params, FileLike outputDir, TaskFileManager fileManager, @Nullable WorkDirectory workDirectory) + { + this(job, support, params, FileSystemLike.toFile(outputDir), fileManager, workDirectory); + } + public JobContextImpl(SequenceJob job, SequenceAnalysisJobSupport support, JSONObject params, File outputDir, TaskFileManager fileManager, @Nullable WorkDirectory workDirectory) { _job = job; @@ -110,7 +117,7 @@ public WorkDirectory getWorkDir() @Override public File getWorkingDirectory() { - return _wd.getDir(); + return FileSystemLike.toFile(_wd.getDir()); } @Override diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PrepareAlignerIndexesTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PrepareAlignerIndexesTask.java index ef75c5297..bea77ab7b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PrepareAlignerIndexesTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PrepareAlignerIndexesTask.java @@ -192,7 +192,7 @@ private RecordedAction ensureIndexExists() throws PipelineJobException getJob().getLogger().debug("location of source FASTA: " + getPipelineJob().getTargetGenome().getSourceFastaFile().getPath()); //NOTE: always create the index back in the local working dir, since we'll need to move it back there anyway - File localSharedDirectory = new File(getHelper().getJob().getAnalysisDirectory(), SequenceTaskHelper.SHARED_SUBFOLDER_NAME); + File localSharedDirectory = new File(getHelper().getJob().getAnalysisDirectory().toNioPathForRead().toFile(), SequenceTaskHelper.SHARED_SUBFOLDER_NAME); if (!localSharedDirectory.exists()) { localSharedDirectory.mkdirs(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java index 8cfb0d856..7e588096b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java @@ -52,6 +52,7 @@ import org.labkey.sequenceanalysis.run.util.MergeVcfsAndGenotypesWrapper; import org.labkey.sequenceanalysis.run.variant.OutputVariantsStartingInIntervalsStep; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -356,6 +357,12 @@ public static File getPedigreeFile(File outputDir, String providerName) return new File(outputDir, providerName + ".ped"); } + public static FileLike getPedigreeFile(FileLike outputDir, String providerName) + { + providerName = FileUtil.makeLegalName(providerName); + return outputDir.resolveChild(providerName + ".ped"); + } + public static List getIntervals(JobContext ctx) { PipelineJob pj = ctx.getJob(); @@ -730,7 +737,7 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport @Override public void complete(JobContext ctx, List inputs, List outputsCreated) throws PipelineJobException { - SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(ctx.getJob()), getPipelineJob(ctx.getJob()).getDataDirectory()); + SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(ctx.getJob()), getPipelineJob(ctx.getJob()).getDataDirectory().toNioPathForRead().toFile()); List> providers = SequencePipelineService.get().getSteps(ctx.getJob(), VariantProcessingStep.class); for (PipelineStepCtx stepCtx : providers) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetImportJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetImportJob.java index bfd38bce2..55c04b456 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetImportJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetImportJob.java @@ -26,8 +26,8 @@ import org.labkey.sequenceanalysis.SequenceAnalysisModule; import org.labkey.sequenceanalysis.SequenceReadsetImpl; import org.labkey.sequenceanalysis.util.NucleotideSequenceFileType; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -55,7 +55,7 @@ protected ReadsetImportJob() { } - public static List create(Container c, User u, String jobName, String description, JSONObject params, List inputFiles) throws PipelineJobException, IOException, PipelineValidationException + public static List create(Container c, User u, String jobName, String description, JSONObject params, List inputFiles) throws PipelineJobException, IOException, PipelineValidationException { Map containerToPipeRootMap = new HashMap<>(); @@ -137,7 +137,7 @@ else if (x.startsWith("fileGroup_")) job.getLogger().debug("job was submitted to an alternate folder: " + targetContainer.getPath()); } - List inputFilesSubset = new ArrayList<>(); + List inputFilesSubset = new ArrayList<>(); JSONArray files = fileGroup.getJSONArray("files"); for (int i = 0; i < files.length(); i++) { @@ -165,7 +165,7 @@ else if (x.startsWith("fileGroup_")) } } - private static File findFile(JSONObject file, List inputFiles) + private static FileLike findFile(JSONObject file, List inputFiles) { if (!file.isNull("dataId") && StringUtils.trimToNull(file.get("dataId").toString()) != null) { @@ -178,7 +178,7 @@ private static File findFile(JSONObject file, List inputFiles) throw new IllegalArgumentException("Unable to find file with ID: " + file.getInt("dataId")); } - return d.getFile(); + return d.getFileLike(); } catch (ConversionException | NullPointerException e) { @@ -187,7 +187,7 @@ private static File findFile(JSONObject file, List inputFiles) } else if (!file.isNull("fileName") && StringUtils.trimToNull(file.getString("fileName")) != null) { - List hits = new ArrayList<>(); + List hits = new ArrayList<>(); inputFiles.forEach(x -> { if (x.getName().equals(file.getString("fileName"))) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java index d0e442ef7..d253377f7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java @@ -39,6 +39,7 @@ import org.labkey.sequenceanalysis.model.BarcodeModel; import org.labkey.sequenceanalysis.util.FastqUtils; import org.labkey.sequenceanalysis.util.NucleotideSequenceFileType; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -147,7 +148,7 @@ public RecordedActionSet run() throws PipelineJobException getJob().getLogger().info("Preparing to import sequence data"); getJob().getLogger().info("input files:"); - for (File input : getPipelineJob().getInputFiles()) + for (FileLike input : getPipelineJob().getInputFiles()) { getJob().getLogger().info(input.getPath()); } @@ -280,7 +281,7 @@ private File processFile(File f, RecordedAction action) throws IOException if (!gz.isType(f)) { getJob().getLogger().info("\tcompressing input file: " + f.getName()); - output = new File(getHelper().getJob().getAnalysisDirectory(), f.getName() + ".gz"); + output = new File(getHelper().getJob().getAnalysisDirectory().toNioPathForRead().toFile(), f.getName() + ".gz"); //note: the non-compressed file will potentially be deleted during cleanup, depending on the selection for input handling Compress.compressGzip(f, output); } @@ -293,7 +294,7 @@ private File processFile(File f, RecordedAction action) throws IOException } else { - output = new File(getHelper().getJob().getAnalysisDirectory(), f.getName()); + output = new File(getHelper().getJob().getAnalysisDirectory().toNioPathForRead().toFile(), f.getName()); if (!output.exists()) { if (getHelper().getFileManager().getInputFileTreatment() == TaskFileManager.InputFileTreatment.delete || getHelper().getFileManager().getInputFileTreatment() == TaskFileManager.InputFileTreatment.compress) @@ -352,13 +353,12 @@ private BarcodeModel[] getAdditionalBarcodes() public static File getExtraBarcodesFile(SequenceTaskHelper helper) { - return new File(helper.getJob().getAnalysisDirectory(), "extraBarcodes.txt"); + return helper.getJob().getAnalysisDirectory().resolveChild("extraBarcodes.txt").toNioPathForRead().toFile(); } public static Set handleInputs(SequenceJob job, TaskFileManager.InputFileTreatment inputFileTreatment, Collection actions, Set outputFiles, @Nullable Set unalteredInputs) throws PipelineJobException { - Set inputs = new HashSet<>(); - inputs.addAll(job.getInputFiles()); + Set inputs = new HashSet<>(job.getInputFiles().stream().map(f -> f.toNioPathForRead().toFile()).toList()); job.getLogger().info("Cleaning up input files"); @@ -451,7 +451,7 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti try { //NOTE: we assume the input is gzipped already - File outputDir = job.getAnalysisDirectory(); + File outputDir = job.getAnalysisDirectory().toNioPathForRead().toFile(); File output = new File(outputDir, input.getName()); job.getLogger().debug("Destination: " + output.getPath()); boolean alreadyMoved = false; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java index 37f9421dc..2964d1759 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java @@ -109,7 +109,7 @@ protected void writeParameters(JSONObject params) throws IOException public File getSerializedLibraryMembersFile() { - return FileUtil.appendName(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".json"); + return getDataDirectory().resolveChild(FileUtil.getBaseName(getLogFile()) + ".json").toNioPathForRead().toFile(); } //for recreating an existing library diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java index 3a36fc287..fd22059fb 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java @@ -21,8 +21,9 @@ import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl; import org.labkey.sequenceanalysis.SequenceReadsetImpl; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -74,7 +75,7 @@ public static List createForReadsets(Container c, User u, SequenceAlignmentJob j = new SequenceAlignmentJob(targetContainer, u, jobName, pr, params, readset); j.setDescription(description); - List inputFiles = new ArrayList<>(); + List inputFiles = new ArrayList<>(); for (ReadData rd : readset.getReadData()) { if (rd.getFileId1() != null) @@ -86,7 +87,7 @@ public static List createForReadsets(Container c, User u, } else { - inputFiles.add(d1.getFile()); + inputFiles.add(FileSystemLike.wrapFile(d1.getFile())); } } @@ -99,7 +100,7 @@ public static List createForReadsets(Container c, User u, } else { - inputFiles.add(d2.getFile()); + inputFiles.add(FileSystemLike.wrapFile(d2.getFile())); } } } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java index f485d5199..31f6c67aa 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java @@ -87,6 +87,7 @@ import org.labkey.sequenceanalysis.util.FastqMerger; import org.labkey.sequenceanalysis.util.FastqUtils; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -440,39 +441,32 @@ private void copyReferenceResources() throws PipelineJobException } else if (doCopy) { - try - { - RecordedAction action = new RecordedAction(COPY_REFERENCE_LIBRARY_ACTIONNAME); - Date start = new Date(); - action.setStartTime(start); + RecordedAction action = new RecordedAction(COPY_REFERENCE_LIBRARY_ACTIONNAME); + Date start = new Date(); + action.setStartTime(start); - String basename = FileUtil.getBaseName(refFasta); - File targetDir = new File(_wd.getDir(), "Shared"); - for (File f : refFasta.getParentFile().listFiles()) + String basename = FileUtil.getBaseName(refFasta); + File targetDir = new File(_wd.getDir().toNioPathForRead().toFile(), "Shared"); + for (File f : refFasta.getParentFile().listFiles()) + { + if (f.getName().startsWith(basename)) { - if (f.getName().startsWith(basename)) - { - getJob().getLogger().debug("copying reference file: " + f.getPath()); - File movedFile = _wd.inputFile(f, new File(targetDir, f.getName()), true); - copiedInputs.put(f, movedFile); - action.addInputIfNotPresent(f, "Reference File"); - action.addOutputIfNotPresent(movedFile, "Copied Reference File", true); - } + getJob().getLogger().debug("copying reference file: " + f.getPath()); + File movedFile = _wd.inputFile(f, new File(targetDir, f.getName()), true); + copiedInputs.put(f, movedFile); + action.addInputIfNotPresent(f, "Reference File"); + action.addOutputIfNotPresent(movedFile, "Copied Reference File", true); } + } - Date end = new Date(); - action.setEndTime(end); - getJob().getLogger().info("Copy Reference Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); - actions.add(action); + Date end = new Date(); + action.setEndTime(end); + getJob().getLogger().info("Copy Reference Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); + actions.add(action); - referenceGenome.setWorkingFasta(new File(targetDir, refFasta.getName())); + referenceGenome.setWorkingFasta(new File(targetDir, refFasta.getName())); - getTaskFileManagerImpl().addIntermediateFile(targetDir); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } + getTaskFileManagerImpl().addIntermediateFile(targetDir); } else { @@ -1507,16 +1501,16 @@ public Resumer() private Resumer(SequenceAlignmentTask task) { - super(task.getPipelineJob().getAnalysisDirectory(), task.getJob().getLogger(), task.getHelper().getFileManager()); + super(task.getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), task.getJob().getLogger(), task.getHelper().getFileManager()); } public static Resumer create(SequenceAlignmentJob job, SequenceAlignmentTask task) throws PipelineJobException { //NOTE: allow a file in either local working dir or webserver dir. if both exist, use the file most recently modified File file = null; - for (File dir : Arrays.asList(job.getAnalysisDirectory(), task._wd.getDir())) + for (FileLike dir : Arrays.asList(job.getAnalysisDirectory(), task._wd.getDir())) { - File toCheck = getSerializedJson(dir, JSON_NAME); + File toCheck = getSerializedJson(dir.toNioPathForRead().toFile(), JSON_NAME); if (toCheck.exists()) { job.getLogger().debug("inspecting file: " + toCheck.getPath()); @@ -1554,24 +1548,17 @@ private static Resumer createFromJson(SequenceAlignmentJob job, SequenceAlignmen Resumer ret = readFromJson(file, Resumer.class); ret._isResume = true; ret.setLogger(task.getJob().getLogger()); - ret.setWebserverJobDir(task.getPipelineJob().getAnalysisDirectory()); + ret.setWebserverJobDir(task.getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile()); ret.getFileManager().onResume(job, task._wd); task._taskHelper.setFileManager(ret.getFileManager()); - try + if (!ret._copiedInputs.isEmpty()) { - if (!ret._copiedInputs.isEmpty()) + for (File orig : ret._copiedInputs.keySet()) { - for (File orig : ret._copiedInputs.keySet()) - { - task._wd.inputFile(orig, ret._copiedInputs.get(orig), false); - } + task._wd.inputFile(orig, ret._copiedInputs.get(orig), false); } } - catch (IOException e) - { - throw new PipelineJobException(e); - } //debugging: job.getLogger().debug("loaded from file. total recorded actions: " + ret.getRecordedActions().size()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java index 2bd1175b7..a706466a5 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java @@ -149,7 +149,7 @@ public RecordedActionSet run() throws PipelineJobException SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(), _wd); //we dont delete the resume file during the alignment task, since we've had issues w/ JMS dropping messages and needing to repeat that whole task - File xmlFile = SequenceAlignmentTask.Resumer.getSerializedJson(getPipelineJob().getAnalysisDirectory(), SequenceAlignmentTask.Resumer.JSON_NAME); + File xmlFile = SequenceAlignmentTask.Resumer.getSerializedJson(getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), SequenceAlignmentTask.Resumer.JSON_NAME); if (xmlFile.exists()) { xmlFile.delete(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceConcatTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceConcatTask.java index a8316a789..d3b73b47f 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceConcatTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceConcatTask.java @@ -93,7 +93,7 @@ public RecordedActionSet run() throws PipelineJobException try { - createConcatenatedSequence(sequenceIds, getPipelineJob().getParameterJson().getString("sequenceName"), getPipelineJob().getParameterJson().getString("sequenceDescription"), getJob().getContainer(), getJob().getUser(), getPipelineJob().getAnalysisDirectory()); + createConcatenatedSequence(sequenceIds, getPipelineJob().getParameterJson().getString("sequenceName"), getPipelineJob().getParameterJson().getString("sequenceDescription"), getJob().getContainer(), getJob().getUser(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile()); } catch (IOException e) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java index 90a947d45..b4b44ab6a 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java @@ -51,7 +51,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -73,7 +72,7 @@ public class SequenceJob extends PipelineJob implements FileAnalysisJobSupport, private FileLike _webserverJobDir; private FileLike _parentWebserverJobDir; private String _folderPrefix; - private List _inputFiles; + private List _inputFiles; private List _outputsToCreate = new ArrayList<>(); private PipeRoot _folderFileRoot; private Collection _dockerVolumes; @@ -172,13 +171,13 @@ public TaskPipeline getTaskPipeline() return PipelineJobService.get().getTaskPipeline(_taskPipelineId); } - protected void setInputFiles(Collection inputs) + protected void setInputFiles(Collection inputs) { _inputFiles = inputs == null ? Collections.emptyList() : new ArrayList<>(inputs); } @Override - public List getInputFiles() + public List getInputFiles() { return _inputFiles == null ? Collections.emptyList() : Collections.unmodifiableList(_inputFiles); } @@ -237,7 +236,7 @@ protected FileLike createLocalDirectory(PipeRoot pipeRoot) throws IOException protected void writeParameters(JSONObject params) throws IOException { - try (PrintWriter writer = PrintWriters.getPrintWriter(getParametersFile())) + try (PrintWriter writer = PrintWriters.getPrintWriter(getParametersFile().openOutputStream())) { writer.write(params.isEmpty() ? "" : params.toString(1)); } @@ -256,10 +255,10 @@ public JSONObject getParameterJson() { try { - File paramFile = getParametersFile(); + File paramFile = getParametersFile().toNioPathForRead().toFile(); if (paramFile.exists()) { - try (BufferedReader reader = Readers.getReader(getParametersFile())) + try (BufferedReader reader = Readers.getReader(getParametersFile().openInputStream())) { List lines = IOUtils.readLines(reader); @@ -341,7 +340,7 @@ public String getBaseNameForFileType(FileType fileType) } @Override - public FileLike getDataDirectoryFileLike() + public FileLike getDataDirectory() { return _webserverJobDir; } @@ -359,26 +358,26 @@ public File getWebserverDir(boolean forceParent) } @Override - public File getAnalysisDirectory() + public FileLike getAnalysisDirectory() { - return _webserverJobDir.toNioPathForWrite().toFile(); + return _webserverJobDir; } @Override - public File findOutputFile(String name) + public FileLike findOutputFile(String name) { return findFile(name); } @Override - public File findInputFile(String name) + public FileLike findInputFile(String name) { return findFile(name); } @Override - public File findOutputFile(@NotNull String outputDir, @NotNull String fileName) + public FileLike findOutputFile(@NotNull String outputDir, @NotNull String fileName) { return AbstractFileAnalysisJob.getOutputFile(outputDir, fileName, getPipeRoot(), getLogger(), getAnalysisDirectory()); } @@ -390,10 +389,10 @@ public ParamParser createParamParser() } @Override - public File getParametersFile() + public FileLike getParametersFile() { var dir = _parentWebserverJobDir == null ? _webserverJobDir : _parentWebserverJobDir; - return FileUtil.appendName(dir.toNioPathForWrite().toFile(),_folderPrefix + ".json"); + return dir.resolveFile(org.labkey.api.util.Path.parse(_folderPrefix + ".json")); } @Override @@ -559,9 +558,9 @@ public void setExperimentRunRowId(Long experimentRunRowId) _experimentRunRowId = experimentRunRowId; } - public File findFile(String name) + public FileLike findFile(String name) { - return FileUtil.appendName(getAnalysisDirectory(), name); + return getAnalysisDirectory().resolveChild(name); } protected static XarGeneratorFactorySettings getXarGenerator() throws CloneNotSupportedException @@ -613,7 +612,7 @@ public void testSerializeSupport() throws Exception SequenceJob job = new SequenceJob(); job._support = new SequenceJobSupportImpl(); job._support.cacheExpData(d1); - job.setLogFile(FileUtil.appendName(FileUtil.getTempDirectory(), "testJob.log").toPath()); + job.setLogFile(FileUtil.getTempDirectoryFileLike().resolveChild("testJob.log")); File testFile = FileUtil.appendName(FileUtil.getTempDirectory(), "testJob.json.txt"); File support = job.getCachedSupportFile(); @@ -655,7 +654,7 @@ public File getLocationForCachedInputs(WorkDirectory wd, boolean createIfDoesntE if (localDir == null) { - ret = FileUtil.appendName(wd.getDir(), "cachedData"); + ret = FileUtil.appendName(wd.getDir().toNioPathForRead().toFile(), "cachedData"); } else { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java index 1cb082521..7f91a73ca 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java @@ -55,6 +55,8 @@ import org.labkey.sequenceanalysis.util.FastqUtils; import org.labkey.sequenceanalysis.util.NucleotideSequenceFileType; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.FileNotFoundException; @@ -164,16 +166,16 @@ private ReadsetImportJob getPipelineJob() return (ReadsetImportJob)getJob(); } - private static List getFilesToNormalize(PipelineJob job, List files, boolean allowMissingFiles) throws FileNotFoundException + private static List getFilesToNormalize(PipelineJob job, List files, boolean allowMissingFiles) throws FileNotFoundException { SequencePipelineSettings settings = new SequencePipelineSettings(job.getParameters()); if (settings.isDoBarcode()) { - return files; + return files.stream().map(f -> f.toNioPathForRead().toFile()).toList(); } List toNormalize = new ArrayList<>(); - for (File f : files) + for (FileLike f : files) { if (!f.exists()) { @@ -184,7 +186,7 @@ private static List getFilesToNormalize(PipelineJob job, List files, } if (isNormalizationRequired(job, f)) - toNormalize.add(f); + toNormalize.add(f.toNioPathForRead().toFile()); } try @@ -230,7 +232,7 @@ private static List getFilesToNormalize(PipelineJob job, List files, return toNormalize; } - private static boolean isNormalizationRequired(PipelineJob job, File f) + private static boolean isNormalizationRequired(PipelineJob job, FileLike f) { try { @@ -249,7 +251,7 @@ private static boolean isNormalizationRequired(PipelineJob job, File f) try { - if (!SequenceUtil.hasMinLineCount(f, 4)) + if (!SequenceUtil.hasMinLineCount(f.toNioPathForRead().toFile(), 4)) { job.getLogger().warn("file has fewer than 4 lines: " + f.getPath()); return false; @@ -261,7 +263,7 @@ private static boolean isNormalizationRequired(PipelineJob job, File f) } QualityEncodingDetector detector = new QualityEncodingDetector(); - try (FastqReader reader = new FastqReader(f)) + try (FastqReader reader = new FastqReader(f.toNioPathForRead().toFile())) { detector.add(QualityEncodingDetector.DEFAULT_MAX_RECORDS_TO_ITERATE * 10, reader); } @@ -316,7 +318,7 @@ public RecordedActionSet run() throws PipelineJobException List actions = new ArrayList<>(); getJob().getLogger().info("Starting Normalization Task"); - File normalizationDir = new File(_wd.getDir(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); + File normalizationDir = new File(_wd.getDir().toNioPathForRead().toFile(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); normalizationDir.mkdirs(); try @@ -333,7 +335,7 @@ public RecordedActionSet run() throws PipelineJobException { for (FileGroup.FilePair fp : lane) { - File localCopy = _wd.getWorkingCopyForInput(fp.file1); + File localCopy = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file1)).toNioPathForRead().toFile(); if (localCopy != null) { getJob().getLogger().debug("using local working copy for file: " + fp.file1.getPath()); @@ -362,7 +364,7 @@ public RecordedActionSet run() throws PipelineJobException if (fp.file2 != null) { - localCopy = _wd.getWorkingCopyForInput(fp.file2); + localCopy = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file2)).toNioPathForRead().toFile(); if (localCopy != null) { getJob().getLogger().debug("using local working copy for file: " + fp.file2.getPath()); @@ -634,7 +636,7 @@ else if (lane.isEmpty()) } //add final action for later tracking based on role - File baseDirectory = new File(_wd.getDir(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); + File baseDirectory = new File(_wd.getDir().toNioPathForRead().toFile(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); baseDirectory.mkdirs(); Set finalOutputs = ReadsetInitTask.handleInputs(getPipelineJob(), getHelper().getFileManager().getInputFileTreatment(), actions, _finalOutputs, null); @@ -880,7 +882,7 @@ private File normalizeFastqSequence(File input, List actions) th getJob().getLogger().info("\tFile format: " + type.getPrimaryExtension()); - File baseDirectory = new File(_wd.getDir(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); + File baseDirectory = new File(_wd.getDir().toNioPathForWrite().toFile(), SequenceTaskHelper.NORMALIZATION_SUBFOLDER_NAME); baseDirectory.mkdirs(); File workingDir = new File(baseDirectory, basename); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java index 11d4262a0..7fb4166c3 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java @@ -151,7 +151,7 @@ private SequenceOutputHandlerJob getPipelineJob() } //run final handler - TaskFileManagerImpl manager = new TaskFileManagerImpl(getPipelineJob(), getPipelineJob().getAnalysisDirectory(), null); + TaskFileManagerImpl manager = new TaskFileManagerImpl(getPipelineJob(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), null); JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory(), manager, null); getPipelineJob().getHandler().getProcessor().complete(ctx, getPipelineJob().getFiles(), outputsCreated); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java index 905b05169..0cb33d6fd 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerInitTask.java @@ -131,7 +131,7 @@ public RecordedActionSet run() throws PipelineJobException } } - TaskFileManagerImpl manager = new TaskFileManagerImpl(getPipelineJob(), getPipelineJob().getAnalysisDirectory(), null); + TaskFileManagerImpl manager = new TaskFileManagerImpl(getPipelineJob(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), null); JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory(), manager, null); handler.getProcessor().init(ctx, getPipelineJob().getFiles(), actions, outputsToCreate); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerJob.java index d892e6eb8..9614062ec 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerJob.java @@ -18,6 +18,8 @@ import org.labkey.api.util.FileUtil; import org.labkey.sequenceanalysis.SequenceAnalysisManager; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.BufferedReader; import java.io.File; @@ -164,7 +166,7 @@ public File getSerializedOutputFilesFile() return new File(getWebserverDir(true), logName + ".outputs.json.gz"); } - return new File(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".outputs.json.gz"); + return getDataDirectory().resolveChild(FileUtil.getBaseName(getLogFile()) + ".outputs.json.gz").toNioPathForRead().toFile(); } @Override @@ -199,14 +201,14 @@ public List getFiles() throws PipelineJobException } @Override - public List getInputFiles() + public List getInputFiles() { try { - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); for (SequenceOutputFile o : getFiles()) { - ret.add(o.getFile()); + ret.add(FileSystemLike.wrapFile(o.getFile())); } return ret; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerWebserverTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerWebserverTask.java index 3448cc5e3..1ed50b607 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerWebserverTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerWebserverTask.java @@ -106,7 +106,7 @@ public RecordedActionSet run() throws PipelineJobException getJob().getLogger().warn("there are no sequence output files to process, this is probably an error"); } - handler.getProcessor().processFilesOnWebserver(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getFiles(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory(), actions, outputsToCreate); + handler.getProcessor().processFilesOnWebserver(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getFiles(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), actions, outputsToCreate); if (!outputsToCreate.isEmpty()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequencePipelineSettings.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequencePipelineSettings.java index 531901ba4..2baef650f 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequencePipelineSettings.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequencePipelineSettings.java @@ -31,8 +31,9 @@ import org.labkey.sequenceanalysis.FileGroup; import org.labkey.sequenceanalysis.SequenceReadsetImpl; import org.labkey.sequenceanalysis.model.BarcodeModel; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; -import java.io.File; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -130,12 +131,12 @@ private FileGroup createFileGroup(JSONObject o, @Nullable SequenceJob job, boole if (json.has("file1")) { JSONObject fileJson = json.getJSONObject("file1"); - p.file1 = resolveFile(fileJson, job, allowMissingFiles); + p.file1 = FileSystemLike.toFile(resolveFile(fileJson, job, allowMissingFiles)); } if (json.has("file2")) { - p.file2 = resolveFile(json.getJSONObject("file2"), job, allowMissingFiles); + p.file2 = FileSystemLike.toFile(resolveFile(json.getJSONObject("file2"), job, allowMissingFiles)); } fg.filePairs.add(p); @@ -209,7 +210,7 @@ public SequenceReadsetImpl createReadsetModel(JSONObject o) return model; } - private File resolveFile(JSONObject json, @Nullable SequenceJob job, boolean allowMissingFiles) + private FileLike resolveFile(JSONObject json, @Nullable SequenceJob job, boolean allowMissingFiles) { if (json.has("dataId")) { @@ -224,14 +225,14 @@ private File resolveFile(JSONObject json, @Nullable SequenceJob job, boolean all throw new IllegalArgumentException("Unable to find ExpData: " + json.get("dataId")); } - return d.getFile(); + return d.getFileLike(); } else { if (job != null && job.getSequenceSupport().getCachedData(dataId) != null) { job.getLogger().debug("found using cached ExpData"); - return job.getSequenceSupport().getCachedData(dataId); + return FileSystemLike.wrapFile(job.getSequenceSupport().getCachedData(dataId)); } } } @@ -242,7 +243,7 @@ private File resolveFile(JSONObject json, @Nullable SequenceJob job, boolean all //resolve based on inputs if (job != null) { - for (File input : job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles()) + for (FileLike input : job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles()) { if (input.getName().equals(json.getString("fileName"))) { @@ -253,7 +254,7 @@ private File resolveFile(JSONObject json, @Nullable SequenceJob job, boolean all else { //file might have been a copied input, check in analysis directory - File test = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), input.getName()); + FileLike test = job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory().resolveChild(input.getName()); if (test.exists()) { return test; @@ -268,7 +269,7 @@ private File resolveFile(JSONObject json, @Nullable SequenceJob job, boolean all { job.getLogger().error("unable to find file: " + json + ", active task: " + job.getActiveTaskId().getName(), new Exception()); job.getLogger().debug("input files were: "); - for (File f : job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles()) + for (FileLike f : job.getJobSupport(FileAnalysisJobSupport.class).getInputFiles()) { job.getLogger().debug("[" + f.getPath() + "], exists: " + f.exists()); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerInitTask.java index c95bc0aae..6a34d6fba 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerInitTask.java @@ -119,7 +119,7 @@ public RecordedActionSet run() throws PipelineJobException } } - handler.getProcessor().init(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getReadsets(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory(), actions, outputsToCreate); + handler.getProcessor().init(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getReadsets(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), actions, outputsToCreate); if (!outputsToCreate.isEmpty()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java index eb819f963..8c3b15555 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java @@ -14,8 +14,9 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.sequenceanalysis.SequenceAnalysisManager; import org.labkey.sequenceanalysis.SequenceReadsetImpl; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -74,23 +75,23 @@ public List getReadsets() } @Override - public List getInputFiles() + public List getInputFiles() { try { - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); for (Readset rs : getReadsets()) { for (ReadData d : rs.getReadData()) { if (d.getFile1() != null) { - ret.add(d.getFile1()); + ret.add(FileSystemLike.wrapFile(d.getFile1())); } if (d.getFile2() != null) { - ret.add(d.getFile2()); + ret.add(FileSystemLike.wrapFile(d.getFile2())); } } } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerRemoteTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerRemoteTask.java index d90e0766a..c37be6faa 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerRemoteTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerRemoteTask.java @@ -100,7 +100,7 @@ private SequenceReadsetHandlerJob getPipelineJob() public RecordedActionSet run() throws PipelineJobException { SequenceOutputHandler handler = getPipelineJob().getHandler(); - JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), _wd.getDir(), new TaskFileManagerImpl(getPipelineJob(), _wd.getDir(), _wd), _wd); + JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), _wd.getDir().toNioPathForRead().toFile(), new TaskFileManagerImpl(getPipelineJob(), _wd.getDir().toNioPathForRead().toFile(), _wd), _wd); getJob().setStatus(PipelineJob.TaskStatus.running, "Running: " + handler.getName()); handler.getProcessor().processFilesRemote(getPipelineJob().getReadsets(), ctx); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerWebserverTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerWebserverTask.java index cab5211a3..ada36be14 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerWebserverTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerWebserverTask.java @@ -51,7 +51,7 @@ public String getStatusName() public List getProtocolActionNames() { List allowableNames = new ArrayList<>(); - for (SequenceOutputHandler handler : SequenceAnalysisServiceImpl.get().getFileHandlers(SequenceOutputHandler.TYPE.Readset)) + for (SequenceOutputHandler handler : SequenceAnalysisServiceImpl.get().getFileHandlers(SequenceOutputHandler.TYPE.Readset)) { allowableNames.add(handler.getName()); } @@ -75,7 +75,7 @@ public boolean isParticipant(PipelineJob job) throws IOException } @Override - public PipelineJob.Task createTask(PipelineJob job) + public SequenceReadsetHandlerWebserverTask createTask(PipelineJob job) { return new SequenceReadsetHandlerWebserverTask(this, job); } @@ -106,7 +106,7 @@ public RecordedActionSet run() throws PipelineJobException getJob().getLogger().warn("there are no sequence output files to process, this is probably an error"); } - handler.getProcessor().processFilesOnWebserver(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getReadsets(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory(), actions, outputsToCreate); + handler.getProcessor().processFilesOnWebserver(getJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getReadsets(), getPipelineJob().getParameterJson(), getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), actions, outputsToCreate); if (!outputsToCreate.isEmpty()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java index e29e63aa3..eb03ac8d6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java @@ -93,7 +93,7 @@ private SequenceTaskHelper(SequenceJob job, WorkDirectory wd, @Nullable File wor { _job = job; _wd = wd; - _workLocation = workLocation == null ? wd.getDir() : workLocation; //TODO: is this the right behavior?? + _workLocation = workLocation == null ? wd.getDir().toNioPathForRead().toFile() : workLocation; //TODO: is this the right behavior?? _fileManager = new TaskFileManagerImpl(_job, getWorkingDirectory(), wd); _settings = new SequencePipelineSettings(_job.getParameters()); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java index 90a6418e7..dd1c000b4 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java @@ -36,6 +36,8 @@ import org.labkey.sequenceanalysis.SequenceAnalysisManager; import org.labkey.sequenceanalysis.SequenceAnalysisSchema; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -79,6 +81,11 @@ public TaskFileManagerImpl() } + public TaskFileManagerImpl(SequenceJob job, FileLike workDir, WorkDirectory wd) + { + this(job, workDir.toNioPathForRead().toFile(), wd); + } + public TaskFileManagerImpl(SequenceJob job, File workDir, WorkDirectory wd) { _job = job; @@ -95,7 +102,7 @@ public void onResume(PipelineJob job, WorkDirectory wd) } _job = (SequenceJob)job; - _workLocation = wd.getDir(); + _workLocation = wd.getDir().toNioPathForRead().toFile(); _wd = wd; } @@ -243,7 +250,7 @@ private SequenceAnalysisJobSupport getSequenceSupport() private File getDeferredDeleteLog(boolean create) { - File logFile = new File(getSupport().getAnalysisDirectory(), "toDelete.txt"); + File logFile = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "toDelete.txt"); if (create && !logFile.exists()) { try @@ -262,7 +269,7 @@ private File getDeferredDeleteLog(boolean create) private File getMetricsLog(boolean create) { - File logFile = new File(getSupport().getAnalysisDirectory(), "metricsToCreate.txt"); + File logFile = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "metricsToCreate.txt"); if (create && !logFile.exists()) { try (PrintWriter writer = PrintWriters.getPrintWriter(logFile)) @@ -381,7 +388,7 @@ private File convertRelPathToFile(String line) File f = new File(_workLocation, line); if (!f.exists()) { - File test = new File(getSupport().getAnalysisDirectory(), line); + File test = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), line); if (test.exists()) { f = test; @@ -521,7 +528,7 @@ public void addPicardMetricsFiles(List f metricLog = getMetricsLog(true); try (BufferedWriter writer = new BufferedWriter(new FileWriter(metricLog, true))) { - String bamRelPath = mf.getInputFile() == null ? "" : FilenameUtils.normalize(_wd.getRelativePath(mf.getInputFile())); + String bamRelPath = mf.getInputFile() == null ? "" : FilenameUtils.normalize(_wd.getRelativePath(FileSystemLike.wrapFile(mf.getInputFile()))); String type = mf.getType() == null ? "" : mf.getType().name(); List> metricLines = PicardMetricsUtil.processFile(mf.getMetricFile(), _job.getLogger()); @@ -724,9 +731,9 @@ public boolean isCopyInputsLocally() private Set getInputPaths() { Set inputPaths = new HashSet<>(); - for (File f : getSupport().getInputFiles()) + for (FileLike f : getSupport().getInputFiles()) { - inputPaths.add(f.getPath()); + inputPaths.add(f.toNioPathForRead().toFile().getPath()); } return inputPaths; } @@ -910,7 +917,7 @@ public void cleanup(Collection actions, @Nullable AbstractResume //NOTE: preserving relative locations is a pain. therefore we copy all outputs, including directories //then sort out which files were specified as named outputs later - for (File input : _wd.getDir().listFiles()) + for (File input : _wd.getDir().toNioPathForRead().toFile().listFiles()) { if (input.getName().matches("^core.[0-9]+$") || input.getName().endsWith(".hprof")) { @@ -979,8 +986,8 @@ private void doCopyFile(File input, Collection actions, @Nullabl return; } - String path = _wd.getRelativePath(input); - File dest = new File(getSupport().getAnalysisDirectory(), path); + String path = _wd.getRelativePath(FileSystemLike.wrapFile(input)); + File dest = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), path); _job.getLogger().debug("to: " + dest.getPath()); boolean doMove = true; @@ -1018,7 +1025,7 @@ private void doCopyFile(File input, Collection actions, @Nullabl Files.delete(input.toPath()); } - dest = _wd.outputFile(input, dest); + dest = _wd.outputFile(FileSystemLike.wrapFile(input), FileSystemLike.wrapFile(dest)).toNioPathForRead().toFile(); processCopiedFile(input, dest, actions, resumer); } } @@ -1048,7 +1055,6 @@ public void processUnzippedInputs() @Override public void decompressInputFiles(Pair pair, List actions) { - List list = new ArrayList<>(); if (pair.first != null) pair.first = decompressFile(pair.first, actions); if (pair.second != null) @@ -1070,7 +1076,7 @@ private File decompressFile(File i, List actions) //NOTE: we use relative paths in all cases here _job.getLogger().info("Decompressing file: " + i.getPath()); - unzipped = new File(_wd.getDir(), i.getName().replaceAll(".gz$", "")); + unzipped = new File(_wd.getDir().toNioPathForRead().toFile(), i.getName().replaceAll(".gz$", "")); unzipped = Compress.decompressGzip(i, unzipped); _job.getLogger().debug("\tunzipped: " + unzipped.getPath()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java index 34db8c265..c8f147bfd 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java @@ -270,7 +270,7 @@ public List createSplitJobs() private File getJobToIntervalFile() { - return new File(isSplitJob() ? getDataDirectory().getParentFile() : getDataDirectory(), "jobsToInterval.txt"); + return new File(isSplitJob() ? getDataDirectory().getParent().toNioPathForRead().toFile() : getDataDirectory().toNioPathForRead().toFile(), "jobsToInterval.txt"); } private void writeJobToIntervalMap(Map> jobToIntervalMap) throws IOException @@ -410,7 +410,7 @@ public void intervalSerializeTest() throws Exception { VariantProcessingJob job1 = new VariantProcessingJob(){ @Override - public FileLike getDataDirectoryFileLike() + public FileLike getDataDirectory() { return FileSystemLike.wrapFile(new File(System.getProperty("java.io.tmpdir"))); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java index f8e043f29..8664a80f4 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteMergeTask.java @@ -109,7 +109,8 @@ private VariantProcessingJob getPipelineJob() { SequenceTaskHelper.logModuleVersions(getJob().getLogger()); RecordedAction action = new RecordedAction(ACTION_NAME); - JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), _wd.getDir(), new TaskFileManagerImpl(getPipelineJob(), _wd.getDir(), _wd), _wd); + File dir = _wd.getDir().toNioPathForRead().toFile(); + JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), dir, new TaskFileManagerImpl(getPipelineJob(), dir, _wd), _wd); File finalOut; SequenceOutputHandler handler = getPipelineJob().getHandler(); @@ -207,7 +208,7 @@ else if (!vcf.exists()) if (!toConcat.isEmpty()) { String basename = SequenceAnalysisService.get().getUnzippedBaseName(toConcat.get(0).getName()); - combined = new File(getPipelineJob().getAnalysisDirectory(), basename + ".vcf.gz"); + combined = new File(getPipelineJob().getAnalysisDirectory().toNioPathForRead().toFile(), basename + ".vcf.gz"); File combinedIdx = new File(combined.getPath() + ".tbi"); if (combinedIdx.exists()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java index 4d28fcae2..2f93e15e8 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingRemoteSplitTask.java @@ -11,6 +11,7 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.util.FileType; import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -97,7 +98,7 @@ private VariantProcessingJob getPipelineJob() SequenceTaskHelper.logModuleVersions(getJob().getLogger()); SequenceOutputHandler handler = getPipelineJob().getHandler(); - JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), _wd.getDir(), new TaskFileManagerImpl(getPipelineJob(), _wd.getDir(), _wd), _wd); + JobContextImpl ctx = new JobContextImpl(getPipelineJob(), getPipelineJob().getSequenceSupport(), getPipelineJob().getParameterJson(), _wd.getDir().toNioPathForRead().toFile(), new TaskFileManagerImpl(getPipelineJob(), _wd.getDir().toNioPathForRead().toFile(), _wd), _wd); List intervals = getPipelineJob().getIntervalsForTask(); if (intervals != null) @@ -123,10 +124,10 @@ private VariantProcessingJob getPipelineJob() ctx.getLogger().debug("No output produced, adding null to scatter outputs"); getPipelineJob().getScatterJobOutputs().put(getPipelineJob().getIntervalSetName(), null); } - else if (output.getPath().startsWith(_wd.getDir().getPath())) + else if (output.getPath().startsWith(_wd.getDir().toNioPathForRead().toFile().getPath())) { //NOTE: the VCF will be copied back to the source dir, so translate paths - String path = _wd.getRelativePath(output); + String path = _wd.getRelativePath(FileSystemLike.wrapFile(output)); output = new File(ctx.getSourceDirectory(), path); getPipelineJob().getScatterJobOutputs().put(getPipelineJob().getIntervalSetName(), output); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SnpCountAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SnpCountAnalysis.java index e554bba5d..a53852297 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SnpCountAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SnpCountAnalysis.java @@ -103,7 +103,7 @@ public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, fastaIndexFile = null; } - File outputFile = new File(getPipelineCtx().getJob().getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), FileUtil.getBaseName(inputBam) + ".snps.txt"); + File outputFile = getPipelineCtx().getJob().getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory().resolveChild(FileUtil.getBaseName(inputBam) + ".snps.txt").toNioPathForRead().toFile(); SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(model.getReadset(), getPipelineCtx().getJob().getUser()); try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile, true));IndexedFastaSequenceFile indexedFastaSequenceFile = new IndexedFastaSequenceFile(referenceFasta, new FastaSequenceIndex(fastaIndexFile))) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java index 2115ef1dc..d2db3394d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java @@ -30,10 +30,12 @@ import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.sequenceanalysis.ScatterGatherUtils; import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler; import org.labkey.sequenceanalysis.pipeline.VariantProcessingJob; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; import java.io.BufferedReader; import java.io.File; @@ -278,7 +280,7 @@ public File performVariantMerge(TaskFileManager manager, RecordedAction action, job.setStatus(PipelineJob.TaskStatus.running, "Creating merged workspace from " + jobToIntervalMap.size() + " jobs"); VariantProcessingJob variantProcessingJob = getPipelineJob(job); - File destinationWorkspace = getWorkspaceOutput(variantProcessingJob.getDataDirectory(), variantProcessingJob.getParameterJson().getString("fileBaseName")); + File destinationWorkspace = getWorkspaceOutput(variantProcessingJob.getDataDirectory().toNioPathForRead().toFile(), variantProcessingJob.getParameterJson().getString("fileBaseName")); if (!destinationWorkspace.exists()) { destinationWorkspace.mkdirs(); @@ -296,7 +298,7 @@ public File performVariantMerge(TaskFileManager manager, RecordedAction action, Map scatterOutputs = getPipelineJob(job).getScatterJobOutputs(); for (String name : jobToIntervalMap.keySet()) { - File overallCopyDone = new File(variantProcessingJob.getDataDirectory(), name + "/copyToWebserver.done"); + File overallCopyDone = variantProcessingJob.getDataDirectory().resolveFile(Path.parse(name + "/copyToWebserver.done")).toNioPathForRead().toFile(); if (overallCopyDone.exists()) { overallCopyDone.delete(); @@ -342,12 +344,12 @@ public File performVariantMerge(TaskFileManager manager, RecordedAction action, private Set _contigsInInputs = null; - Set getContigsInInputs(List inputVCFs, Logger log) throws PipelineJobException + Set getContigsInInputs(List inputVCFs, Logger log) throws PipelineJobException { if (_contigsInInputs == null) { Set ret = new HashSet<>(); - for (File f : inputVCFs) + for (FileLike f : inputVCFs) { ret.addAll(SequenceUtil.getContigsInVcf(f)); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CombineGVCFsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CombineGVCFsHandler.java index b4940a6c7..5c2f078af 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CombineGVCFsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/CombineGVCFsHandler.java @@ -20,6 +20,7 @@ import org.labkey.sequenceanalysis.SequenceAnalysisModule; import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler; import org.labkey.sequenceanalysis.pipeline.VariantProcessingJob; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -198,7 +199,7 @@ public void processFilesRemote(List inputFiles, JobContext c //NOTE: the VCF was copied back to the source dir, so translate paths try { - String path = ctx.getWorkDir().getRelativePath(outputFile); + String path = ctx.getWorkDir().getRelativePath(FileSystemLike.wrapFile(outputFile)); File movedOutputFile = new File(ctx.getSourceDirectory(), path); job.getScatterJobOutputs().put(job.getIntervalSetName(), movedOutputFile); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenotypeGVCFsWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenotypeGVCFsWrapper.java index feedc8b42..536277445 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenotypeGVCFsWrapper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GenotypeGVCFsWrapper.java @@ -129,7 +129,7 @@ public static List copyVcfsLocally(SequenceOutputHandler.JobContext ctx, C // If the cache directory is under the current working dir, mark to delete when done. // If localWorkDir is null, this indicates we're using /tmp, so also delete. // Otherwise this indicates a shared cache dir probably used by multiple scatter/gather jobs, and allow the merge task to do cleanup - boolean reportFilesForDeletion = localWorkDir == null || localWorkDir.getAbsolutePath().startsWith(ctx.getWorkDir().getDir().getAbsolutePath()); + boolean reportFilesForDeletion = localWorkDir == null || localWorkDir.getAbsolutePath().startsWith(ctx.getWorkDir().getDir().toNioPathForRead().toFile().getAbsolutePath()); if (localWorkDir == null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/SamToFastqWrapper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/SamToFastqWrapper.java index b0fff1434..09587d00f 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/SamToFastqWrapper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/SamToFastqWrapper.java @@ -6,6 +6,7 @@ import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.run.PicardWrapper; import org.labkey.api.util.Pair; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.Arrays; @@ -47,6 +48,11 @@ public List extractByReadGroup(File file, File outDir) throws PipelineJobE return Arrays.asList(outDir.listFiles()); } + public Pair executeCommand(FileLike f, String outputName1, @Nullable String outputName2) throws PipelineJobException + { + return executeCommand(f.toNioPathForRead().toFile(), outputName1, outputName2); + } + public Pair executeCommand(File file, String outputName1, @Nullable String outputName2) throws PipelineJobException { Date start = new Date(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SplitVcfBySamplesStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SplitVcfBySamplesStep.java index de8480264..dc3addb88 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SplitVcfBySamplesStep.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/SplitVcfBySamplesStep.java @@ -92,7 +92,7 @@ private List findProducedVcfs(File inputVCF, File outputDirectory) public void performAdditionalMergeTasks(SequenceOutputHandler.JobContext ctx, PipelineJob job, ReferenceGenome genome, List orderedScatterOutputs, List orderedJobDirs) throws PipelineJobException { job.getLogger().info("Merging additional track VCFs"); - File inputVCF = ((SequenceJob)getPipelineCtx().getJob()).getInputFiles().get(0); + File inputVCF = ((SequenceJob)getPipelineCtx().getJob()).getInputFiles().get(0).toNioPathForRead().toFile(); List firstJobOutputs = findProducedVcfs(inputVCF, new File(ctx.getSourceDirectory(), orderedJobDirs.get(0))); job.getLogger().info("total VCFs found in job dir: " + firstJobOutputs.size()); if (firstJobOutputs.isEmpty()) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java index a5357d176..4a143ebc0 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java @@ -46,6 +46,7 @@ import org.labkey.api.writer.PrintWriters; import org.labkey.sequenceanalysis.run.util.BgzipRunner; import org.labkey.sequenceanalysis.run.variant.GatherVcfsCloudWrapper; +import org.labkey.vfs.FileLike; import java.io.BufferedReader; import java.io.File; @@ -602,7 +603,7 @@ else if (!samples.equals(header.getGenotypeSamples())) return outputGzip; } - public static Set getContigsInVcf(File vcf) throws PipelineJobException + public static Set getContigsInVcf(FileLike vcf) throws PipelineJobException { try { @@ -610,7 +611,7 @@ public static Set getContigsInVcf(File vcf) throws PipelineJobException try (PrintWriter writer = PrintWriters.getPrintWriter(script)) { writer.println("#!/bin/bash"); - String cat = vcf.getPath().toLowerCase().endsWith(".gz") ? "zcat" : "cat"; + String cat = vcf.getName().toLowerCase().endsWith(".gz") ? "zcat" : "cat"; writer.println(cat + " '" + vcf.getPath() + "' | grep -v '#' | awk ' { print $1 } ' | sort | uniq"); } From 4a30259ebed57e47afa2811f2357c25fdc432ca0 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 1 Dec 2025 10:53:12 -0800 Subject: [PATCH 2/5] More cleanup --- .../SequenceAnalysisController.java | 43 +++++++++---------- .../src/org/labkey/blast/BLASTController.java | 16 +++---- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java index 86802d501..dc40b1ee9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java @@ -2145,27 +2145,24 @@ protected void setContentType(HttpServletResponse response) } @Override - protected File getTargetFile(String filename) throws IOException + protected FileLike getTargetFile(String filename) throws IOException { if (!PipelineService.get().hasValidPipelineRoot(getContainer())) throw new UploadException("Pipeline root must be configured before uploading FASTA files", HttpServletResponse.SC_NOT_FOUND); PipeRoot root = PipelineService.get().getPipelineRootSetting(getContainer()); - - File targetDirectory = root.getRootPath(); - - return FileUtil.findUniqueFileName(filename, targetDirectory); + return FileUtil.findUniqueFileName(filename, root.getRootFileLike()); } @Override - public String getResponse(ImportFastaSequencesForm form, Map> files) + public String getResponse(ImportFastaSequencesForm form, Map> files) { JSONObject resp = new JSONObject(); try { - for (Map.Entry> entry : files.entrySet()) + for (Map.Entry> entry : files.entrySet()) { - File file = entry.getValue().getKey(); + File file = entry.getValue().getKey().toNioPathForRead().toFile(); Map params = form.getNtParams(); Map libraryParams = form.getLibraryParams(); @@ -2444,7 +2441,7 @@ protected void setContentType(HttpServletResponse response) } @Override - protected File getTargetFile(String filename) throws IOException + protected FileLike getTargetFile(String filename) throws IOException { if (!PipelineService.get().hasValidPipelineRoot(getContainer())) throw new UploadException("Pipeline root must be configured before uploading files", HttpServletResponse.SC_NOT_FOUND); @@ -2452,7 +2449,7 @@ protected File getTargetFile(String filename) throws IOException try { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs"); - return FileUtil.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile(); + return FileUtil.findUniqueFileName(filename, targetDirectory); } catch (ExperimentException e) { @@ -2461,14 +2458,14 @@ protected File getTargetFile(String filename) throws IOException } @Override - public String getResponse(ImportOutputFileForm form, Map> files) + public String getResponse(ImportOutputFileForm form, Map> files) { JSONObject resp = new JSONObject(); try { - for (Map.Entry> entry : files.entrySet()) + for (Map.Entry> entry : files.entrySet()) { - File file = entry.getValue().getKey(); + FileLike file = entry.getValue().getKey(); Map params = new CaseInsensitiveHashMap<>(); params.put("name", form.getName()); @@ -2577,7 +2574,7 @@ protected void setContentType(HttpServletResponse response) } @Override - protected File getTargetFile(String filename) throws IOException + protected FileLike getTargetFile(String filename) throws IOException { if (!PipelineService.get().hasValidPipelineRoot(getContainer())) throw new UploadException("Pipeline root must be configured before uploading files", HttpServletResponse.SC_NOT_FOUND); @@ -2585,7 +2582,7 @@ protected File getTargetFile(String filename) throws IOException try { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer()); - return FileUtil.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile(); + return FileUtil.findUniqueFileName(filename, targetDirectory); } catch (ExperimentException e) { @@ -2610,7 +2607,7 @@ public void validateCommand(ImportTrackForm form, Errors errors) } @Override - public String getResponse(ImportTrackForm form, Map> files) throws UploadException + public String getResponse(ImportTrackForm form, Map> files) throws UploadException { JSONObject resp = new JSONObject(); if (form.getTrackName() == null || form.getLibraryId() == null) @@ -2620,9 +2617,9 @@ public String getResponse(ImportTrackForm form, Map> try { - for (Map.Entry> entry : files.entrySet()) + for (Map.Entry> entry : files.entrySet()) { - File file = entry.getValue().getKey(); + File file = entry.getValue().getKey().toNioPathForRead().toFile(); try { @@ -2742,7 +2739,7 @@ protected void setContentType(HttpServletResponse response) } @Override - protected File getTargetFile(String filename) throws IOException + protected FileLike getTargetFile(String filename) throws IOException { if (!PipelineService.get().hasValidPipelineRoot(getContainer())) throw new UploadException("Pipeline root must be configured before uploading files", HttpServletResponse.SC_NOT_FOUND); @@ -2750,7 +2747,7 @@ protected File getTargetFile(String filename) throws IOException try { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer()); - return FileUtil.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile(); + return FileUtil.findUniqueFileName(filename, targetDirectory); } catch (ExperimentException e) { @@ -2759,14 +2756,14 @@ protected File getTargetFile(String filename) throws IOException } @Override - public String getResponse(ImportChainFileForm form, Map> files) + public String getResponse(ImportChainFileForm form, Map> files) { JSONObject resp = new JSONObject(); try { - for (Map.Entry> entry : files.entrySet()) + for (Map.Entry> entry : files.entrySet()) { - File file = entry.getValue().getKey(); + File file = entry.getValue().getKey().toNioPathForRead().toFile(); if (form.getGenomeId1() == null || form.getGenomeId2() == null) { diff --git a/blast/src/org/labkey/blast/BLASTController.java b/blast/src/org/labkey/blast/BLASTController.java index d93458b8a..1764fd2d7 100644 --- a/blast/src/org/labkey/blast/BLASTController.java +++ b/blast/src/org/labkey/blast/BLASTController.java @@ -212,13 +212,13 @@ public void validateCommand(RunBlastForm form, Errors errors) } @Override - protected File getTargetFile(String filename) throws IOException + protected FileLike getTargetFile(String filename) throws IOException { AssayFileWriter writer = new AssayFileWriter(); try { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer()); - return FileUtil.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile(); + return FileUtil.findUniqueFileName(filename, targetDirectory); } catch (ExperimentException e) { @@ -227,7 +227,7 @@ protected File getTargetFile(String filename) throws IOException } @Override - public String getResponse(RunBlastForm form, Map> files) + public String getResponse(RunBlastForm form, Map> files) { JSONObject resp = new JSONObject(); Map params = new HashMap<>(); @@ -251,10 +251,10 @@ public String getResponse(RunBlastForm form, Map> fil params.put("perc_identity", form.getPctIdentity()); } - List inputFiles = new ArrayList<>(); + List inputFiles = new ArrayList<>(); try { - for (Map.Entry> entry : files.entrySet()) + for (Map.Entry> entry : files.entrySet()) { inputFiles.add(entry.getValue().getKey()); } @@ -272,7 +272,7 @@ public String getResponse(RunBlastForm form, Map> fil { fw.write(form.getQuery()); } - inputFiles.add(input.toNioPathForRead().toFile()); + inputFiles.add(input); } catch (ExperimentException e) { @@ -282,9 +282,9 @@ public String getResponse(RunBlastForm form, Map> fil if (!inputFiles.isEmpty()) { - for (File input : inputFiles) + for (FileLike input : inputFiles) { - String jobId = BLASTManager.get().runBLASTN(getContainer(), getUser(), form.getDatabase(), input, form.getTask(), form.getTitle(), form.getSaveResults(), params, true); + String jobId = BLASTManager.get().runBLASTN(getContainer(), getUser(), form.getDatabase(), input.toNioPathForRead().toFile(), form.getTask(), form.getTitle(), form.getSaveResults(), params, true); resp.put("jobId", jobId); } } From a632c45c1846bff8e689e249882010d352660d9a Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 2 Dec 2025 14:45:16 -0800 Subject: [PATCH 3/5] Prep for merge --- .../sequenceanalysis/pipeline/CacheAlignerIndexesTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java index af7939026..0974d0bd3 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java @@ -17,6 +17,7 @@ import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenomeManager; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; import org.labkey.sequenceanalysis.run.util.FastaIndexer; import org.labkey.vfs.FileSystemLike; @@ -132,7 +133,7 @@ public RecordedActionSet run() throws PipelineJobException boolean hasIndex = AlignerIndexUtil.hasCachedIndex(alignmentStep.getPipelineCtx(), alignmentStep.getIndexCachedDirName(getJob()), referenceGenome); File dir = FileSystemLike.toFile(_wd.getDir()); - File outDir = new File(dir, alignmentStep.getIndexCachedDirName(getJob())); + File outDir = FileUtil.appendName(dir, alignmentStep.getIndexCachedDirName(getJob())); if (!hasIndex) { //create locally first From 66325249aa4d5e575b771a1330b4fb734e62585c Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 3 Dec 2025 09:22:27 -0800 Subject: [PATCH 4/5] Handle nulls --- .../sequenceanalysis/pipeline/SequenceNormalizationTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java index 7f91a73ca..71a14e67e 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java @@ -335,7 +335,8 @@ public RecordedActionSet run() throws PipelineJobException { for (FileGroup.FilePair fp : lane) { - File localCopy = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file1)).toNioPathForRead().toFile(); + FileLike localCopyFileLike = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file1)); + File localCopy = localCopyFileLike == null ? null : localCopyFileLike.toNioPathForRead().toFile(); if (localCopy != null) { getJob().getLogger().debug("using local working copy for file: " + fp.file1.getPath()); From 6d0aab370fb49c74b0dff67920af8d93c2398f44 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 3 Dec 2025 11:45:25 -0800 Subject: [PATCH 5/5] Handle nulls --- .../sequenceanalysis/pipeline/SequenceNormalizationTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java index 71a14e67e..6bf251c4b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceNormalizationTask.java @@ -365,7 +365,8 @@ public RecordedActionSet run() throws PipelineJobException if (fp.file2 != null) { - localCopy = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file2)).toNioPathForRead().toFile(); + localCopyFileLike = _wd.getWorkingCopyForInput(FileSystemLike.wrapFile(fp.file2)); + localCopy = localCopyFileLike == null ? null : localCopyFileLike.toNioPathForRead().toFile(); if (localCopy != null) { getJob().getLogger().debug("using local working copy for file: " + fp.file2.getPath());