From c0cc8ead481e7306710d098ecc0a9a075b16d8f3 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sat, 18 Oct 2025 08:52:26 -0700 Subject: [PATCH 1/5] More file path improvements, mostly around pipeline jobs --- .../pipeline/SequenceJob.java | 7 ------ .../external/labModules/SequenceTest.java | 22 ++++++++++++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java index 02a2ac560..9b58391c5 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java @@ -397,13 +397,6 @@ public File getParametersFile() return FileUtil.appendName(dir.toNioPathForWrite().toFile(),_folderPrefix + ".json"); } - @Nullable - @Override - public File getJobInfoFile() - { - return FileUtil.appendName(_webserverJobDir.toNioPathForWrite().toFile(), FileUtil.makeLegalName(_jobName) + ".job.json"); - } - @Override public FileType.gzSupportLevel getGZPreference() { diff --git a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java index e26795289..1e066c54f 100644 --- a/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java +++ b/SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java @@ -23,6 +23,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; import org.labkey.remoteapi.CommandException; import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.query.Filter; @@ -295,13 +296,22 @@ private void createIlluminaSampleSheet() assertTextPresent(fileSearcher, prop_name + "," + prop_value); File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName()); - File importTemplate = new File(fileRoot, ILLUMINA_CSV); + File importTemplate = FileUtil.appendName(fileRoot, ILLUMINA_CSV); if (importTemplate.exists()) importTemplate.delete(); //NOTE: use the text generated directly using JS - TestFileUtils.saveFile(importTemplate.getParentFile(), importTemplate.getName(), outputTable); + File tsvFile = FileUtil.appendName(importTemplate.getParentFile(), importTemplate.getName()); + + try + { + TestFileUtils.writeFile(tsvFile, outputTable); + } + catch (IOException e) + { + e.printStackTrace(System.err); + } goToProjectHome(); } @@ -330,7 +340,7 @@ private void importIlluminaTest() throws Exception File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName()); for (String fn : Arrays.asList("Illumina-F.fastq.gz", "Illumina-R.fastq.gz", "SkipMe.fastq.gz")) { - File target = new File(fileRoot, fn); + File target = FileUtil.appendName(fileRoot, fn); if (target.exists()) { target.delete(); @@ -749,7 +759,7 @@ private void alignmentImportPanelTest(boolean sequencePipelineEnabled) throws Ex File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName()); log("file root: " + fileRoot.getPath()); - File inputBam = new File(fileRoot, "test.bam"); + File inputBam = FileUtil.appendName(fileRoot, "test.bam"); File inputBamIdx = new File(inputBam.getPath() + ".bai"); if (inputBam.exists()) { @@ -763,8 +773,8 @@ private void alignmentImportPanelTest(boolean sequencePipelineEnabled) throws Ex inputBamIdx.delete(); } - FileUtils.copyFile(new File(_sampleData, "test.bam"), inputBam); - FileUtils.copyFile(new File(_sampleData, "test.bam.bai"), inputBamIdx); + FileUtils.copyFile(FileUtil.appendName(_sampleData, "test.bam"), inputBam); + FileUtils.copyFile(FileUtil.appendName(_sampleData, "test.bam.bai"), inputBamIdx); _helper.initiatePipelineJob(_alignmentImportPipelineName, List.of(inputBam.getName()), getProjectName()); waitForText("Job Name"); From 5d3128f905d447421964dd04ca0ae0242946b233 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sun, 19 Oct 2025 16:42:32 -0700 Subject: [PATCH 2/5] Test fixes, --- .../org/labkey/sequenceanalysis/pipeline/SequenceJob.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java index 9b58391c5..e4a460813 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java @@ -161,10 +161,9 @@ protected void addCustomParams(JSONObject params) params.put("labkeyFolderPath", getContainer().isWorkbook() ? getContainer().getParent().getPath() : getContainer().getPath()); } - private Path _getLogFile() + private FileLike _getLogFile() { - var file = FileUtil.findUniqueFileName((FileUtil.makeLegalName(_jobName) + ".log"), getDataDirectoryFileObject()); - return file.toNioPathForWrite(); + return FileUtil.findUniqueFileName((FileUtil.makeLegalName(_jobName) + ".log"), getDataDirectoryFileObject()); } @Override From d64a27e59cbc20d226f3ffc4f1fd827681029870 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 14:01:33 -0700 Subject: [PATCH 3/5] Test fixes --- .../org/labkey/sequenceanalysis/pipeline/SequenceJob.java | 4 ++-- .../sequenceanalysis/pipeline/VariantProcessingJob.java | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java index e4a460813..90a947d45 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java @@ -341,9 +341,9 @@ public String getBaseNameForFileType(FileType fileType) } @Override - public File getDataDirectory() + public FileLike getDataDirectoryFileLike() { - return _webserverJobDir.toNioPathForWrite().toFile(); + return _webserverJobDir; } public FileLike getDataDirectoryFileObject() diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java index d9f7e4462..34db8c265 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java @@ -37,6 +37,8 @@ import org.labkey.api.writer.PrintWriters; import org.labkey.sequenceanalysis.util.ScatterGatherUtils; import org.labkey.sequenceanalysis.util.SequenceUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -408,9 +410,9 @@ public void intervalSerializeTest() throws Exception { VariantProcessingJob job1 = new VariantProcessingJob(){ @Override - public File getDataDirectory() + public FileLike getDataDirectoryFileLike() { - return new File(System.getProperty("java.io.tmpdir")); + return FileSystemLike.wrapFile(new File(System.getProperty("java.io.tmpdir"))); } }; From 101e33fb06b2f93b09db99b876be7954cee053cb Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 20 Oct 2025 17:07:38 -0700 Subject: [PATCH 4/5] More FileLike --- .../labkey/studies/StudiesServiceImpl.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Studies/src/org/labkey/studies/StudiesServiceImpl.java b/Studies/src/org/labkey/studies/StudiesServiceImpl.java index 4c3910b5b..e19ed526a 100644 --- a/Studies/src/org/labkey/studies/StudiesServiceImpl.java +++ b/Studies/src/org/labkey/studies/StudiesServiceImpl.java @@ -25,12 +25,12 @@ import org.labkey.api.util.Path; import org.labkey.api.util.logging.LogHelper; import org.labkey.studies.query.StudiesTableCustomizer; +import org.labkey.vfs.FileLike; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; import java.sql.SQLException; import java.util.HashMap; import java.util.List; @@ -56,28 +56,28 @@ public void importFolderDefinition(Container container, User user, Module m, Pat { Resource root = m.getModuleResource(sourceFolderDirPath); PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(container); - java.nio.file.Path pipeRootPath = pipeRoot.getRootNioPath(); + FileLike pipeRootPath = pipeRoot.getRootFileLike(); - java.nio.file.Path folderXmlPath; + FileLike folderXmlPath; - if (root instanceof DirectoryResource && ((DirectoryResource)root).getDir().equals(pipeRootPath.toFile())) + if (root instanceof DirectoryResource dir && dir.getDir().equals(pipeRootPath.toNioPathForRead().toFile())) { // The pipeline root is already pointed at the folder definition, like it might be on a dev machine. // No need to copy, especially since copying can cause infinite recursion when the paths are nested - folderXmlPath = pipeRootPath.resolve("folder.xml"); + folderXmlPath = pipeRootPath.resolveChild("folder.xml"); } else { - java.nio.file.Path folderPath = pipeRootPath.resolve("moduleFolderImport"); - folderXmlPath = folderPath.resolve("folder.xml"); - if (Files.exists(folderPath)) + FileLike folderPath = pipeRootPath.resolveChild("moduleFolderImport"); + folderXmlPath = folderPath.resolveChild("folder.xml"); + if (folderPath.exists()) { FileUtil.deleteDir(folderPath); } copyResourceToPath(root, folderPath); } - if (!Files.exists(folderXmlPath)) + if (!folderXmlPath.exists()) { throw new FileNotFoundException("Couldn't find an extracted " + folderXmlPath); } @@ -87,21 +87,21 @@ public void importFolderDefinition(Container container, User user, Module m, Pat PipelineService.get().runFolderImportJob(container, user, null, folderXmlPath, "folder.xml", pipeRoot, options); } - private void copyResourceToPath(Resource resource, java.nio.file.Path target) throws IOException + private void copyResourceToPath(Resource resource, FileLike target) throws IOException { if (resource.isCollection()) { - Files.createDirectory(target); + FileUtil.createDirectory(target); for (Resource child : resource.list()) { - java.nio.file.Path childTarget = target.resolve(child.getName()); + FileLike childTarget = target.resolveChild(child.getName()); copyResourceToPath(child, childTarget); } } else { try (InputStream in = resource.getInputStream(); - OutputStream out = Files.newOutputStream(target)) + OutputStream out = target.openOutputStream()) { FileUtil.copyData(in, out); } From e9e0acee344b83798a96e022a138fe452ba73cce Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 28 Oct 2025 16:00:24 -0700 Subject: [PATCH 5/5] Minor cleanup --- .../labkey/sequenceanalysis/pipeline/AlignmentInitTask.java | 3 ++- .../sequenceanalysis/pipeline/ImportFastaSequencesTask.java | 3 ++- .../sequenceanalysis/pipeline/ImportGenomeTrackTask.java | 3 ++- .../pipeline/SequenceOutputHandlerFinalTask.java | 3 ++- .../pipeline/SequenceReadsetHandlerFinalTask.java | 3 ++- blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java | 3 ++- blast/src/org/labkey/blast/pipeline/BlastFinalTask.java | 3 ++- blast/src/org/labkey/blast/pipeline/BlastWorkTask.java | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java index 00df2a90c..78f6a3332 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java @@ -1,5 +1,6 @@ package org.labkey.sequenceanalysis.pipeline; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; @@ -98,7 +99,7 @@ private SequenceAlignmentJob getPipelineJob() } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { _taskHelper = new SequenceTaskHelper(getPipelineJob(), _wd); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java index b19a93470..1bf2b9362 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java @@ -16,6 +16,7 @@ package org.labkey.sequenceanalysis.pipeline; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; @@ -84,7 +85,7 @@ public boolean isJobComplete(PipelineJob job) } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { getJob().getLogger().info("Importing sequences from file(s): "); for (File f : getPipelineJob().getFastas()) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java index d2d63e6aa..53578ab71 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java @@ -32,6 +32,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; @@ -136,7 +137,7 @@ public boolean isJobComplete(PipelineJob job) } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { getJob().getLogger().info("Importing tracks from file(s): "); RecordedAction action = new RecordedAction(ACTION_NAME); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java index 2b5322ce2..11d4262a0 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java @@ -1,5 +1,6 @@ package org.labkey.sequenceanalysis.pipeline; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.CompareType; import org.labkey.api.data.SimpleFilter; @@ -85,7 +86,7 @@ private SequenceOutputHandlerJob getPipelineJob() } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); getPipelineJob().setExperimentRunRowId(runId); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java index da411b8bd..01adb2e8e 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java @@ -1,5 +1,6 @@ package org.labkey.sequenceanalysis.pipeline; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; @@ -69,7 +70,7 @@ private SequenceReadsetHandlerJob getPipelineJob() } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); getPipelineJob().setExperimentRunRowId(runId); diff --git a/blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java b/blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java index d65cc3ab8..96bc82501 100644 --- a/blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java +++ b/blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java @@ -15,6 +15,7 @@ */ package org.labkey.blast.pipeline; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; @@ -88,7 +89,7 @@ public boolean isJobComplete(PipelineJob job) } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { getJob().getLogger().info("creating BLAST database for library: " + getPipelineJob().getLibraryId()); diff --git a/blast/src/org/labkey/blast/pipeline/BlastFinalTask.java b/blast/src/org/labkey/blast/pipeline/BlastFinalTask.java index 9e868b666..33a247aff 100644 --- a/blast/src/org/labkey/blast/pipeline/BlastFinalTask.java +++ b/blast/src/org/labkey/blast/pipeline/BlastFinalTask.java @@ -15,6 +15,7 @@ */ package org.labkey.blast.pipeline; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; @@ -80,7 +81,7 @@ public boolean isJobComplete(PipelineJob job) } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { getPipelineJob().getBlastJob().setComplete(getJob().getUser(), getJob()); diff --git a/blast/src/org/labkey/blast/pipeline/BlastWorkTask.java b/blast/src/org/labkey/blast/pipeline/BlastWorkTask.java index 6f8e34919..725388dcf 100644 --- a/blast/src/org/labkey/blast/pipeline/BlastWorkTask.java +++ b/blast/src/org/labkey/blast/pipeline/BlastWorkTask.java @@ -15,6 +15,7 @@ */ package org.labkey.blast.pipeline; +import org.jetbrains.annotations.NotNull; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; import org.labkey.api.pipeline.PipelineJob; @@ -81,7 +82,7 @@ public boolean isJobComplete(PipelineJob job) } @Override - public RecordedActionSet run() throws PipelineJobException + public @NotNull RecordedActionSet run() throws PipelineJobException { try {