Skip to content

Adopt FileLike for more pipeline APIs #394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-jeckels merged 6 commits into from
Nov 5, 2025
Merged

Adopt FileLike for more pipeline APIs #394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c0cc8ea
More file path improvements, mostly around pipeline jobs
labkey-jeckels Oct 18, 2025
5d3128f
Test fixes,
labkey-jeckels Oct 19, 2025
d64a27e
Test fixes
labkey-jeckels Oct 20, 2025
101e33f
More FileLike
labkey-jeckels Oct 21, 2025
23d5e0d
Merge branch 'develop' into fb_pipelineFileLike
labkey-jeckels Oct 27, 2025
e9e0ace
Minor cleanup
labkey-jeckels Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.labkey.sequenceanalysis.pipeline;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.PipelineJob;
import org.labkey.api.pipeline.PipelineJobException;
import org.labkey.api.pipeline.RecordedAction;
Expand Down Expand Up @@ -98,7 +99,7 @@ private SequenceAlignmentJob getPipelineJob()
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
_taskHelper = new SequenceTaskHelper(getPipelineJob(), _wd);

Expand Down
3 changes: 2 additions & 1 deletion SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.labkey.sequenceanalysis.pipeline;

import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
import org.labkey.api.pipeline.PipelineJob;
Expand Down Expand Up @@ -84,7 +85,7 @@ public boolean isJobComplete(PipelineJob job)
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
getJob().getLogger().info("Importing sequences from file(s): ");
for (File f : getPipelineJob().getFastas())
Expand Down
3 changes: 2 additions & 1 deletion SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.labkey.api.collections.CaseInsensitiveHashMap;
import org.labkey.api.data.CompareType;
Expand Down Expand Up @@ -136,7 +137,7 @@ public boolean isJobComplete(PipelineJob job)
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
getJob().getLogger().info("Importing tracks from file(s): ");
RecordedAction action = new RecordedAction(ACTION_NAME);
Expand Down
16 changes: 4 additions & 12 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,9 @@ protected void addCustomParams(JSONObject params)
params.put("labkeyFolderPath", getContainer().isWorkbook() ? getContainer().getParent().getPath() : getContainer().getPath());
}

private Path _getLogFile()
private FileLike _getLogFile()
{
var file = FileUtil.findUniqueFileName((FileUtil.makeLegalName(_jobName) + ".log"), getDataDirectoryFileObject());
return file.toNioPathForWrite();
return FileUtil.findUniqueFileName((FileUtil.makeLegalName(_jobName) + ".log"), getDataDirectoryFileObject());
}

@Override
Expand Down Expand Up @@ -342,9 +341,9 @@ public String getBaseNameForFileType(FileType fileType)
}

@Override
public File getDataDirectory()
public FileLike getDataDirectoryFileLike()
{
return _webserverJobDir.toNioPathForWrite().toFile();
return _webserverJobDir;
}

public FileLike getDataDirectoryFileObject()
Expand Down Expand Up @@ -397,13 +396,6 @@ public File getParametersFile()
return FileUtil.appendName(dir.toNioPathForWrite().toFile(),_folderPrefix + ".json");
}

@Nullable
@Override
public File getJobInfoFile()
{
return FileUtil.appendName(_webserverJobDir.toNioPathForWrite().toFile(), FileUtil.makeLegalName(_jobName) + ".job.json");
}

@Override
public FileType.gzSupportLevel getGZPreference()
{
Expand Down
3 changes: 2 additions & 1 deletion ...enceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.labkey.sequenceanalysis.pipeline;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.labkey.api.data.CompareType;
import org.labkey.api.data.SimpleFilter;
Expand Down Expand Up @@ -85,7 +86,7 @@ private SequenceOutputHandlerJob getPipelineJob()
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob());
getPipelineJob().setExperimentRunRowId(runId);
Expand Down
3 changes: 2 additions & 1 deletion ...nceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.labkey.sequenceanalysis.pipeline;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
import org.labkey.api.pipeline.PipelineJob;
Expand Down Expand Up @@ -69,7 +70,7 @@ private SequenceReadsetHandlerJob getPipelineJob()
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob());
getPipelineJob().setExperimentRunRowId(runId);
Expand Down
6 changes: 4 additions & 2 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import org.labkey.api.writer.PrintWriters;
import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
import org.labkey.sequenceanalysis.util.SequenceUtil;
import org.labkey.vfs.FileLike;
import org.labkey.vfs.FileSystemLike;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -408,9 +410,9 @@ public void intervalSerializeTest() throws Exception
{
VariantProcessingJob job1 = new VariantProcessingJob(){
@Override
public File getDataDirectory()
public FileLike getDataDirectoryFileLike()
{
return new File(System.getProperty("java.io.tmpdir"));
return FileSystemLike.wrapFile(new File(System.getProperty("java.io.tmpdir")));
Copy link

@labkey-matthewb labkey-matthewb Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, all these usages of System.getProperty("java.io.tmpdir") should be replaced with some call to FileUtil. For the TODO list.

Copy link
Collaborator

@bbimber bbimber Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean something like FileUtil.getTmpDirFileLike(), correct?

Copy link

@labkey-matthewb labkey-matthewb Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, unless the code actually depends on using this temp directory vs whatever path LabKey has chosen as its default (might be the same, I didn't dive in).

Copy link
Collaborator

@bbimber bbimber Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Two comments:

  • In general, I dont see any real problem with LabKey enforcing a layer over the filesystem, and it's probably an overdue and good idea. In experimented with migrating code in 25.7, and found the FileUtil helpers associated with FileLike to be kinda cumbersome; however, it seems like you're doing a lot on this front in the current release.
  • I cannot think of any reason our modules specifically need java.io.tmpdir. That's just the standard way, to my knowledge, for java code to find the tempdir. Having LabKey introduce a layer over this is probably a good idea.

}
};

Expand Down
22 changes: 16 additions & 6 deletions SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.labkey.api.util.FileUtil;
import org.labkey.remoteapi.CommandException;
import org.labkey.remoteapi.Connection;
import org.labkey.remoteapi.query.Filter;
Expand Down Expand Up @@ -295,13 +296,22 @@ private void createIlluminaSampleSheet()
assertTextPresent(fileSearcher, prop_name + "," + prop_value);

File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName());
File importTemplate = new File(fileRoot, ILLUMINA_CSV);
File importTemplate = FileUtil.appendName(fileRoot, ILLUMINA_CSV);
if (importTemplate.exists())
importTemplate.delete();


//NOTE: use the text generated directly using JS
TestFileUtils.saveFile(importTemplate.getParentFile(), importTemplate.getName(), outputTable);
File tsvFile = FileUtil.appendName(importTemplate.getParentFile(), importTemplate.getName());

try
{
TestFileUtils.writeFile(tsvFile, outputTable);
}
catch (IOException e)
{
e.printStackTrace(System.err);
}
goToProjectHome();
}

Expand Down Expand Up @@ -330,7 +340,7 @@ private void importIlluminaTest() throws Exception
File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName());
for (String fn : Arrays.asList("Illumina-F.fastq.gz", "Illumina-R.fastq.gz", "SkipMe.fastq.gz"))
{
File target = new File(fileRoot, fn);
File target = FileUtil.appendName(fileRoot, fn);
if (target.exists())
{
target.delete();
Expand Down Expand Up @@ -749,7 +759,7 @@ private void alignmentImportPanelTest(boolean sequencePipelineEnabled) throws Ex
File fileRoot = TestFileUtils.getDefaultFileRoot(getProjectName());
log("file root: " + fileRoot.getPath());

File inputBam = new File(fileRoot, "test.bam");
File inputBam = FileUtil.appendName(fileRoot, "test.bam");
File inputBamIdx = new File(inputBam.getPath() + ".bai");
if (inputBam.exists())
{
Expand All @@ -763,8 +773,8 @@ private void alignmentImportPanelTest(boolean sequencePipelineEnabled) throws Ex
inputBamIdx.delete();
}

FileUtils.copyFile(new File(_sampleData, "test.bam"), inputBam);
FileUtils.copyFile(new File(_sampleData, "test.bam.bai"), inputBamIdx);
FileUtils.copyFile(FileUtil.appendName(_sampleData, "test.bam"), inputBam);
FileUtils.copyFile(FileUtil.appendName(_sampleData, "test.bam.bai"), inputBamIdx);

_helper.initiatePipelineJob(_alignmentImportPipelineName, List.of(inputBam.getName()), getProjectName());
waitForText("Job Name");
Expand Down
26 changes: 13 additions & 13 deletions Studies/src/org/labkey/studies/StudiesServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@
import org.labkey.api.util.Path;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.studies.query.StudiesTableCustomizer;
import org.labkey.vfs.FileLike;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.List;
Expand All @@ -56,28 +56,28 @@ public void importFolderDefinition(Container container, User user, Module m, Pat
{
Resource root = m.getModuleResource(sourceFolderDirPath);
PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(container);
java.nio.file.Path pipeRootPath = pipeRoot.getRootNioPath();
FileLike pipeRootPath = pipeRoot.getRootFileLike();

java.nio.file.Path folderXmlPath;
FileLike folderXmlPath;

if (root instanceof DirectoryResource && ((DirectoryResource)root).getDir().equals(pipeRootPath.toFile()))
if (root instanceof DirectoryResource dir && dir.getDir().equals(pipeRootPath.toNioPathForRead().toFile()))
{
// The pipeline root is already pointed at the folder definition, like it might be on a dev machine.
// No need to copy, especially since copying can cause infinite recursion when the paths are nested
folderXmlPath = pipeRootPath.resolve("folder.xml");
folderXmlPath = pipeRootPath.resolveChild("folder.xml");
}
else
{
java.nio.file.Path folderPath = pipeRootPath.resolve("moduleFolderImport");
folderXmlPath = folderPath.resolve("folder.xml");
if (Files.exists(folderPath))
FileLike folderPath = pipeRootPath.resolveChild("moduleFolderImport");
folderXmlPath = folderPath.resolveChild("folder.xml");
if (folderPath.exists())
{
FileUtil.deleteDir(folderPath);
}
copyResourceToPath(root, folderPath);
}

if (!Files.exists(folderXmlPath))
if (!folderXmlPath.exists())
{
throw new FileNotFoundException("Couldn't find an extracted " + folderXmlPath);
}
Expand All @@ -87,21 +87,21 @@ public void importFolderDefinition(Container container, User user, Module m, Pat
PipelineService.get().runFolderImportJob(container, user, null, folderXmlPath, "folder.xml", pipeRoot, options);
}

private void copyResourceToPath(Resource resource, java.nio.file.Path target) throws IOException
private void copyResourceToPath(Resource resource, FileLike target) throws IOException
{
if (resource.isCollection())
{
Files.createDirectory(target);
FileUtil.createDirectory(target);
for (Resource child : resource.list())
{
java.nio.file.Path childTarget = target.resolve(child.getName());
FileLike childTarget = target.resolveChild(child.getName());
copyResourceToPath(child, childTarget);
}
}
else
{
try (InputStream in = resource.getInputStream();
OutputStream out = Files.newOutputStream(target))
OutputStream out = target.openOutputStream())
{
FileUtil.copyData(in, out);
}
Expand Down
3 changes: 2 additions & 1 deletion blast/src/org/labkey/blast/pipeline/BlastDatabaseTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package org.labkey.blast.pipeline;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
import org.labkey.api.pipeline.PipelineJob;
Expand Down Expand Up @@ -88,7 +89,7 @@ public boolean isJobComplete(PipelineJob job)
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
getJob().getLogger().info("creating BLAST database for library: " + getPipelineJob().getLibraryId());

Expand Down
3 changes: 2 additions & 1 deletion blast/src/org/labkey/blast/pipeline/BlastFinalTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package org.labkey.blast.pipeline;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
import org.labkey.api.pipeline.PipelineJob;
Expand Down Expand Up @@ -80,7 +81,7 @@ public boolean isJobComplete(PipelineJob job)
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
getPipelineJob().getBlastJob().setComplete(getJob().getUser(), getJob());

Expand Down
3 changes: 2 additions & 1 deletion blast/src/org/labkey/blast/pipeline/BlastWorkTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package org.labkey.blast.pipeline;

import org.jetbrains.annotations.NotNull;
import org.labkey.api.pipeline.AbstractTaskFactory;
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
import org.labkey.api.pipeline.PipelineJob;
Expand Down Expand Up @@ -81,7 +82,7 @@ public boolean isJobComplete(PipelineJob job)
}

@Override
public RecordedActionSet run() throws PipelineJobException
public @NotNull RecordedActionSet run() throws PipelineJobException
{
try
{
Expand Down