diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java index 415643b70..4d7afe5c6 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java @@ -61,8 +61,8 @@ public class RefNtSequenceModel implements Serializable private String _name; //deprecated private String _sequence; - private Integer _sequenceFile; - private Integer _jobId; + private Long _sequenceFile; + private Long _jobId; private String _category; private String _subset; private String _locus; @@ -171,12 +171,12 @@ public void setSequence(String sequence) _sequence = sequence; } - public Integer getSequenceFile() + public Long getSequenceFile() { return _sequenceFile; } - public void setSequenceFile(Integer sequenceFile) + public void setSequenceFile(Long sequenceFile) { _sequenceFile = sequenceFile; } @@ -361,12 +361,12 @@ public void setModified(Date modified) _modified = modified; } - public Integer getJobId() + public Long getJobId() { return _jobId; } - public void setJobId(Integer jobId) + public void setJobId(Long jobId) { _jobId = jobId; } diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java index 924b5b666..80bb0e133 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java @@ -69,7 +69,7 @@ static public void setInstance(SequenceAnalysisService instance) abstract public ReadData getReadData(int rowId, User u); - abstract public Readset getReadset(int readsetId, User u); + abstract public Readset getReadset(long readsetId, User u); abstract public ReferenceGenome getReferenceGenome(int rowId, User u) throws PipelineJobException; @@ -87,7 +87,7 @@ static public void setInstance(SequenceAnalysisService instance) abstract public String getUnzippedBaseName(String filename); - abstract public Integer getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException; + abstract public Long getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException; abstract public List generatePedigree(Collection sampleNames, Container c, User u, DemographicsProvider d); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java index edf0edb0c..f366fb6b7 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java @@ -37,11 +37,11 @@ public class SequenceOutputFile implements Serializable private Integer _rowid; private String _name; private String _description; - private Integer _dataId; + private Long _dataId; private Integer _library_id; - private Integer _readset; - private Integer _analysis_id; - private Integer _runid; + private Long _readset; + private Long _analysis_id; + private Long _runid; private String _category; private Boolean _intermediate; private String _container; @@ -86,12 +86,12 @@ public void setDescription(String description) _description = description; } - public Integer getDataId() + public Long getDataId() { return _dataId; } - public void setDataId(Integer dataId) + public void setDataId(Long dataId) { _dataId = dataId; } @@ -106,32 +106,32 @@ public void setLibrary_id(Integer library_id) _library_id = library_id; } - public Integer getReadset() + public Long getReadset() { return _readset; } - public void setReadset(Integer readset) + public void setReadset(Long readset) { _readset = readset; } - public Integer getAnalysis_id() + public Long getAnalysis_id() { return _analysis_id; } - public void setAnalysis_id(Integer analysis_id) + public void setAnalysis_id(Long analysis_id) { _analysis_id = analysis_id; } - public Integer getRunId() + public Long getRunId() { return _runid; } - public void setRunId(Integer runid) + public void setRunId(Long runid) { _runid = runid; } diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/AnalysisModel.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/AnalysisModel.java index 89cf684c4..7e18e3f3e 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/AnalysisModel.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/AnalysisModel.java @@ -31,24 +31,24 @@ */ public interface AnalysisModel extends Serializable { - Integer getAnalysisId(); + Long getAnalysisId(); - Integer getRunId(); + Long getRunId(); String getContainer(); - Integer getReadset(); + Long getReadset(); - Integer getAlignmentFile(); + Long getAlignmentFile(); File getAlignmentFileObject(); ExpData getAlignmentData(); @Deprecated - Integer getReferenceLibrary(); + Long getReferenceLibrary(); - void setReferenceLibrary(Integer libraryId); + void setReferenceLibrary(Long libraryId); ExpData getReferenceLibraryData(User u) throws PipelineJobException; @@ -64,7 +64,7 @@ public interface AnalysisModel extends Serializable Integer getCreatedby(); - Integer getRowId(); + Long getRowId(); String getDescription(); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java index 0a008fc50..fa5348ccd 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java @@ -24,9 +24,9 @@ */ public interface ReadData extends Serializable { - Integer getRowid(); + Long getRowid(); - Integer getReadset(); + Long getReadset(); String getPlatformUnit(); @@ -34,9 +34,9 @@ public interface ReadData extends Serializable Date getDate(); - Integer getFileId1(); + Long getFileId1(); - Integer getFileId2(); + Long getFileId2(); File getFile1(); @@ -44,7 +44,7 @@ public interface ReadData extends Serializable String getDescription(); - Integer getRunId(); + Long getRunId(); String getContainer(); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/Readset.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/Readset.java index 763ba1345..ee24c422b 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/Readset.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/Readset.java @@ -46,7 +46,7 @@ public interface Readset extends Serializable Integer getInstrumentRunId(); - Integer getReadsetId(); + Long getReadsetId(); String getBarcode5(); @@ -56,7 +56,7 @@ public interface Readset extends Serializable Double getConcentration(); - int getRowId(); + long getRowId(); String getContainer(); @@ -72,7 +72,7 @@ public interface Readset extends Serializable String getStatus(); - Integer getRunId(); + Long getRunId(); boolean hasPairedData(); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java index 2f50e901a..126741539 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/DefaultPipelineStepOutput.java @@ -103,7 +103,7 @@ public List getSequenceOutputs() } @Override - public void addSequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description) + public void addSequenceOutput(File file, String label, String category, @Nullable Long readsetId, @Nullable Long analysisId, @Nullable Integer genomeId, @Nullable String description) { _intermediateFiles.remove(file); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineOutputTracker.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineOutputTracker.java index b535eb9c0..283c0d9c1 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineOutputTracker.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineOutputTracker.java @@ -17,7 +17,7 @@ public interface PipelineOutputTracker /** * Add a SequenceOutputFile for this job. These files are tracked and displayed through the browser UI. */ - void addSequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description); + void addSequenceOutput(File file, String label, String category, @Nullable Long readsetId, @Nullable Long analysisId, @Nullable Integer genomeId, @Nullable String description); /** * Remove a previously added intermediate file diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineStepOutput.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineStepOutput.java index 92aaad56b..ebcedde87 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineStepOutput.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineStepOutput.java @@ -92,12 +92,12 @@ class SequenceOutput private final File _file; private final String _label; private final String _category; - private final Integer _readsetId; - private final Integer _analysisId; + private final Long _readsetId; + private final Long _analysisId; private final Integer _genomeId; private final String _description; - public SequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description) + public SequenceOutput(File file, String label, String category, @Nullable Long readsetId, @Nullable Long analysisId, @Nullable Integer genomeId, @Nullable String description) { _file = file; _label = label; @@ -123,12 +123,12 @@ public String getCategory() return _category; } - public Integer getReadsetId() + public Long getReadsetId() { return _readsetId; } - public Integer getAnalysisId() + public Long getAnalysisId() { return _analysisId; } @@ -148,7 +148,7 @@ class PicardMetricsOutput { File _metricFile; File _inputFile; - Integer _readsetId; + Long _readsetId; TYPE _type; public enum TYPE @@ -157,7 +157,7 @@ public enum TYPE reads() } - public PicardMetricsOutput(File metricFile, File inputFile, Integer readsetId) + public PicardMetricsOutput(File metricFile, File inputFile, Long readsetId) { _metricFile = metricFile; _inputFile = inputFile; @@ -170,7 +170,7 @@ public PicardMetricsOutput(File metricFile, File inputFile, Integer readsetId) * want to denote that these metrics will apply to the final file, but dont know its name yet. This can be used instead of tying the * metrics to a specific file. */ - public PicardMetricsOutput(File metricFile, TYPE type, Integer readsetId) + public PicardMetricsOutput(File metricFile, TYPE type, Long readsetId) { _metricFile = metricFile; _type = type; @@ -187,7 +187,7 @@ public File getInputFile() return _inputFile; } - public Integer getReadsetId() + public Long getReadsetId() { return _readsetId; } diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java index f0e943867..542a43a38 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java @@ -75,7 +75,7 @@ public interface ReferenceGenome extends Serializable * this will correspond to the permanent FASTA, as opposed to the copy used in this job. If this FASTA was created specifically for * this job then the FASTA will be in the analysis directory. */ - Integer getFastaExpDataId(); + Long getFastaExpDataId(); /** * @param name The name used by the aligner to identify its cached directory diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceAnalysisJobSupport.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceAnalysisJobSupport.java index 2e8cd2fc8..dbf795860 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceAnalysisJobSupport.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceAnalysisJobSupport.java @@ -35,19 +35,19 @@ public interface SequenceAnalysisJobSupport extends Serializable { void cacheExpData(ExpData data); - File getCachedData(int dataId); + File getCachedData(long dataId); - Map getAllCachedData(); + Map getAllCachedData(); - Readset getCachedReadset(Integer rowId); + Readset getCachedReadset(Long rowId); - AnalysisModel getCachedAnalysis(int rowId); + AnalysisModel getCachedAnalysis(long rowId); List getCachedReadsets(); - void cacheReadset(int readsetId, User u); + void cacheReadset(long readsetId, User u); - void cacheReadset(int readsetId, User u, boolean allowReadsetsWithArchivedData); + void cacheReadset(long readsetId, User u, boolean allowReadsetsWithArchivedData); List getCachedAnalyses(); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java index e0f8df975..8d037188e 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java @@ -115,7 +115,7 @@ static public void setInstance(SequencePipelineService instance) /** * Throws exception if no run is found */ - abstract public Integer getExpRunIdForJob(PipelineJob job) throws PipelineJobException; + abstract public Long getExpRunIdForJob(PipelineJob job) throws PipelineJobException; abstract public long getLineCount(File f) throws PipelineJobException; @@ -140,7 +140,7 @@ static public void setInstance(SequencePipelineService instance) abstract public boolean hasMinLineCount(File f, long minLines) throws PipelineJobException; - abstract public void updateOutputFile(SequenceOutputFile o, PipelineJob job, Integer runId, Integer analysisId); + abstract public void updateOutputFile(SequenceOutputFile o, PipelineJob job, Long runId, Long analysisId); abstract public PreprocessingStep.Output simpleTrimFastqPair(File fq1, File fq2, List params, File outDir, Logger log) throws PipelineJobException; diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java index 1c211e399..f14bfb3bf 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java @@ -62,11 +62,11 @@ public interface TaskFileManager extends PipelineOutputTracker void addPicardMetricsFiles(List files) throws PipelineJobException; - void writeMetricsToDb(Map readsetMap, Map> typeMap) throws PipelineJobException; + void writeMetricsToDb(Map readsetMap, Map> typeMap) throws PipelineJobException; void deleteIntermediateFiles() throws PipelineJobException; - Set createSequenceOutputRecords(@Nullable Integer analysisId)throws PipelineJobException; + Set createSequenceOutputRecords(@Nullable Long analysisId)throws PipelineJobException; //should be used for remote jobs or local jobs running in a separate working directory void cleanup(Collection actions) throws PipelineJobException; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java index 3b2ba8350..4dc9e91a8 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java @@ -49,6 +49,8 @@ import java.util.Map; import java.util.Set; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * Created by bimber on 9/18/2016. */ @@ -235,7 +237,7 @@ private int createTestVcf(int genomeId, File vcf) params = Table.insert(TestContext.get().getUser(), SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES), params); - return (Integer)params.get("rowid"); + return asInteger(params.get("rowid")); } } } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java index b3e8d19d2..1791c1e5d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java @@ -1,5 +1,6 @@ package org.labkey.sequenceanalysis; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; @@ -14,7 +15,6 @@ import java.io.File; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,24 +23,24 @@ */ public class ReadDataImpl implements ReadData { - private Integer _rowid; - private Integer _readset; + private Long _rowid; + private Long _readset; private String _platformUnit; private String _centerName; private Date _date; - private Integer _fileId1; - private Integer _fileId2; + private Long _fileId1; + private Long _fileId2; private String _description; private String _container; private Date _created; private Integer _createdBy; private Date _modified; private Integer _modifiedBy; - private Integer _runId; + private Long _runId; private boolean _archived = false; private String sra_accession; - private final Map _cachedFiles = new HashMap<>(); + private final Map _cachedFiles = new IntHashMap<>(); public ReadDataImpl() { @@ -48,23 +48,23 @@ public ReadDataImpl() } @Override - public Integer getRowid() + public Long getRowid() { return _rowid; } - public void setRowid(Integer rowid) + public void setRowid(Long rowid) { _rowid = rowid; } @Override - public Integer getReadset() + public Long getReadset() { return _readset; } - public void setReadset(Integer readset) + public void setReadset(Long readset) { _readset = readset; } @@ -103,23 +103,23 @@ public void setDate(Date date) } @Override - public Integer getFileId1() + public Long getFileId1() { return _fileId1; } - public void setFileId1(Integer fileId1) + public void setFileId1(Long fileId1) { _fileId1 = fileId1; } @Override - public Integer getFileId2() + public Long getFileId2() { return _fileId2; } - public void setFileId2(Integer fileId2) + public void setFileId2(Long fileId2) { _fileId2 = fileId2; } @@ -131,12 +131,12 @@ public String getDescription() } @Override - public Integer getRunId() + public Long getRunId() { return _runId; } - public void setRunId(Integer runId) + public void setRunId(Long runId) { _runId = runId; } @@ -258,7 +258,7 @@ public Integer getTotalReads() } @Transient - private File getFile(int fileIdx, Integer fileId, boolean allowArchived) + private File getFile(int fileIdx, Long fileId, boolean allowArchived) { if (isArchived() && !allowArchived) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java index d0b60e255..58553d3d6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java @@ -53,6 +53,8 @@ import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.IntHashSet; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; @@ -207,6 +209,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.labkey.api.exp.api.ExperimentService.asInteger; import static org.labkey.sequenceanalysis.SequenceIntegrationTests.PIPELINE_PROP_NAME; public class SequenceAnalysisController extends SpringActionController @@ -503,15 +506,15 @@ public URLHelper getSuccessURL(Object form) public static class DeleteForm extends QueryForm { - private Integer[] _jobIds; + private Long[] _jobIds; private boolean _doDelete = false; - public Integer[] getJobIds() + public Long[] getJobIds() { return _jobIds; } - public void setJobIds(Integer[] jobIds) + public void setJobIds(Long[] jobIds) { _jobIds = jobIds; } @@ -596,11 +599,11 @@ public ModelAndView getConfirmView(DeleteForm form, BindException errors) throws keys.add(ConvertHelper.convert(key, Integer.class)); } - Set expRunsToDelete = new HashSet<>(); - Set readsetIds = new HashSet<>(); - Set readDataIds = new HashSet<>(); - Set analysisIds = new HashSet<>(); - Set outputFileIds = new HashSet<>(); + Set expRunsToDelete = new IntHashSet(); + Set readsetIds = new IntHashSet(); + Set readDataIds = new IntHashSet(); + Set analysisIds = new IntHashSet(); + Set outputFileIds = new IntHashSet(); StringBuilder msg = new StringBuilder("Are you sure you want to delete the following " + keys.size() + " "); if (SequenceAnalysisSchema.TABLE_ANALYSES.equals(_table.getName())) @@ -753,7 +756,7 @@ public ModelAndView handleRequest(HttpServletRequest request, HttpServletRespons private void getAdditionalRuns(Set readsetIds, Set readDataIds, Set analysisIds, Set outputFileIds, Set expRunsToDelete) { - Set runIdsStillInUse = new HashSet<>(); + Set runIdsStillInUse = new IntHashSet(); //work backwards, adding additional pipeline jobs that will become orphans: runIdsStillInUse.addAll(getRunIdsInUse(SequenceAnalysisSchema.TABLE_READSETS, expRunsToDelete, readsetIds)); @@ -786,7 +789,7 @@ private Set appendTotal(StringBuilder sb, String tableName, String noun { SimpleFilter filter = new SimpleFilter(FieldKey.fromString(filterCol), keys, CompareType.IN); TableSelector ts = new TableSelector(SequenceAnalysisSchema.getInstance().getSchema().getTable(tableName), PageFlowUtil.set(pkCol), filter, null); - Set total = new HashSet<>(ts.getArrayList(Integer.class)); + Set total = new IntHashSet(ts.getArrayList(Integer.class)); sb.append("
" + total.size() + " " + noun); return total; @@ -1164,7 +1167,7 @@ public ApiResponse execute(ValidateReadsetImportForm form, BindException errors) if (readsets1.length > 0 || readsets2.length > 0) { - Set ids = new HashSet<>(); + Set ids = new IntHashSet(); ids.addAll(Arrays.asList(readsets1)); ids.addAll(Arrays.asList(readsets2)); @@ -2916,7 +2919,7 @@ public void exec(ResultSet object) throws SQLException String header = se.eval(rs.getFieldKeyRowMap()); RefNtSequenceModel model = new RefNtSequenceModel(); if (rs.getObject(FieldKey.fromString("sequenceFile")) != null) - model.setSequenceFile(rs.getInt(FieldKey.fromString("sequenceFile"))); + model.setSequenceFile(rs.getLong(FieldKey.fromString("sequenceFile"))); model.setContainer(rs.getString(FieldKey.fromString("container"))); @@ -3237,8 +3240,8 @@ public ApiResponse execute(CheckFileStatusForm form, BindException errors) continue; } - Integer dataId = (Integer) rowMap.get("dataid"); - Integer libraryId = (Integer) rowMap.get("library_id"); + Integer dataId = asInteger(rowMap.get("dataid")); + Integer libraryId = asInteger(rowMap.get("library_id")); ExpData d = ExperimentService.get().getExpData(dataId); if (d == null) { @@ -3888,7 +3891,7 @@ protected PipelineJob createOutputJob(RunSequenceHandlerForm form, Container tar protected void validateGenomes(List inputs, SequenceOutputHandler handler) throws IllegalArgumentException { - Set genomes = new HashSet<>(); + Set genomes = new IntHashSet(); inputs.forEach(x -> { if (x.getLibrary_id() == null && (handler.requiresGenome() || handler.requiresSingleGenome())) { @@ -4897,7 +4900,7 @@ public ApiResponse execute(OutputFilesForm form, BindException errors) throws Ex { Map resp = new HashMap<>(); - Map fileMap = new HashMap<>(); + Map fileMap = new IntHashMap<>(); for (Integer rowId : form.getOutputFileIds()) { SequenceOutputFile f = SequenceOutputFile.getForId(rowId); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java index 128d3448a..6ea1c01a9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.SystemUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -42,7 +43,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -101,7 +101,7 @@ public void run() { try { - Map genomeMap = new HashMap<>(); + Map genomeMap = new IntHashMap<>(); new TableSelector(SequenceAnalysisSchema.getInstance().getSchema().getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARIES), PageFlowUtil.set("rowid", "fasta_file"), new SimpleFilter(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK), null).forEachResults(rs -> { int dataId = rs.getInt(FieldKey.fromString("fasta_file")); if (dataId > -1) @@ -239,7 +239,7 @@ else if (!d.getFile().exists()) } } - private void inspectForCoreFiles(Integer runId, Logger log) + private void inspectForCoreFiles(Long runId, Logger log) { if (runId == null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java index 006ac7031..59fe53d6d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java @@ -91,6 +91,8 @@ import java.util.Map; import java.util.Set; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + public class SequenceAnalysisManager { private static final SequenceAnalysisManager _instance = new SequenceAnalysisManager(); @@ -280,9 +282,9 @@ public Collection deleteOutputFiles(List outputFileIds, User u throw new IllegalArgumentException("Unable to find sequenceanalysis user schema"); } - Set bamsDeleted = new HashSet<>(); + Set bamsDeleted = new HashSet<>(); Set outputFilesWithDataNotDeleted = new HashSet<>(); - Set expDataDeleted = new HashSet<>(); + Set expDataDeleted = new HashSet<>(); List files = new TableSelector(SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES), new SimpleFilter(FieldKey.fromString("rowid"), outputFileIds, CompareType.IN), null).getArrayList(SequenceOutputFile.class); for (SequenceOutputFile so : files) { @@ -728,7 +730,7 @@ public ReferenceLibraryPipelineJob createReferenceLibrary(Container c, User u, S } } - public List importRefSequencesFromFasta(Container c, User u, File file, boolean splitWhitespace, Map params, Logger log, @Nullable File outDir, @Nullable Integer jobId) throws IOException + public List importRefSequencesFromFasta(Container c, User u, File file, boolean splitWhitespace, Map params, Logger log, @Nullable File outDir, @Nullable Long jobId) throws IOException { PipeRoot root = PipelineService.get().getPipelineRootSetting(c); if (root == null) @@ -783,7 +785,7 @@ public List importRefSequencesFromFasta(Container c, User u, File file, map.put("jobId", jobId); map = Table.insert(u, dnaTable, map); - sequenceIds.add((Integer) map.get("rowid")); + sequenceIds.add(asInteger(map.get("rowid"))); RefNtSequenceModel m = new TableSelector(dnaTable, new SimpleFilter(FieldKey.fromString("rowid"), map.get("rowid")), null).getObject(RefNtSequenceModel.class); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java index 9a3305cbd..976c9009e 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java @@ -69,6 +69,8 @@ import java.util.Set; import java.util.function.Function; +import static org.labkey.api.exp.api.ExperimentService.asLong; + /** * User: bimber * Date: 10/27/13 @@ -193,7 +195,7 @@ public ReadDataImpl getReadData(int rowId, User u) } @Override - public SequenceReadsetImpl getReadset(int readsetId, User u) + public SequenceReadsetImpl getReadset(long readsetId, User u) { TableInfo ti = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_READSETS); SimpleFilter filter = new SimpleFilter(FieldKey.fromString("rowid"), readsetId); @@ -231,7 +233,7 @@ public ReferenceGenomeImpl getReferenceGenome(int genomeId, User u) throws Pipel throw new UnauthorizedException("Cannot read data in container: " + c.getPath()); } - ExpData d = ExperimentService.get().getExpData((Integer)map.get("fasta_file")); + ExpData d = ExperimentService.get().getExpData(asLong(map.get("fasta_file"))); if (d.getFile() == null) { throw new PipelineJobException("No FASTA file found for genome: " + genomeId); @@ -363,7 +365,7 @@ public String getUnzippedBaseName(String filename) } @Override - public Integer getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException + public Long getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException { return SequenceTaskHelper.getExpRunIdForJob(job, throwUnlessFound); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java index 304a88709..561cc0122 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java @@ -90,6 +90,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * User: bimber * Date: 11/25/12 @@ -1452,7 +1454,7 @@ private void validateQualityMetrics(SequenceReadsetImpl[] models, JSONObject con } } - private SequenceReadsetImpl[] getReadsetsForJob(int runId) + private SequenceReadsetImpl[] getReadsetsForJob(long runId) { TableInfo ti = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_READSETS); TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("runid"), runId), null); @@ -1753,11 +1755,11 @@ protected Integer createSavedLibrary() throws Exception if (ts.exists()) { Map rowMap = ts.getMap(); - Integer fasta_file = (Integer)rowMap.get("fasta_file"); + Integer fasta_file = asInteger(rowMap.get("fasta_file")); ExpData d = fasta_file == null ? null : ExperimentService.get().getExpData(fasta_file); if (d != null && d.getFile() != null && d.getFile().exists()) { - return (Integer) rowMap.get("rowid"); + return asInteger(rowMap.get("rowid")); } else { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java index 60f02bd15..871caab49 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java @@ -518,7 +518,7 @@ public List getSequenceJobInputFiles(PipelineJob job) } @Override - public Integer getExpRunIdForJob(PipelineJob job) throws PipelineJobException + public Long getExpRunIdForJob(PipelineJob job) throws PipelineJobException { return SequenceTaskHelper.getExpRunIdForJob(job); } @@ -610,7 +610,7 @@ public boolean hasMinLineCount(File f, long minLines) throws PipelineJobExceptio } @Override - public void updateOutputFile(SequenceOutputFile o, PipelineJob job, Integer runId, Integer analysisId) + public void updateOutputFile(SequenceOutputFile o, PipelineJob job, Long runId, Long analysisId) { SequenceOutputHandlerFinalTask.updateOutputFile(o, job, runId, analysisId); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java index 0f93eea55..ba5e99500 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java @@ -34,7 +34,7 @@ */ public class SequenceReadsetImpl implements Readset { - private Integer _rowId; + private Long _rowId; private String _name; private String _comments; private String _platform; @@ -50,7 +50,7 @@ public class SequenceReadsetImpl implements Readset private Double _fragmentSize; private Double _concentration; private Integer _instrument_run_id; - private Integer _runId; + private Long _runId; private String _status; private String _container; private Date _created; @@ -67,7 +67,7 @@ public SequenceReadsetImpl() } @Override - public int getRowId() + public long getRowId() { return _rowId == null ? 0 : _rowId; } @@ -77,7 +77,7 @@ public void unsetRowId() _rowId = null; } - public void setRowId(int rowId) + public void setRowId(long rowId) { _rowId = rowId; } @@ -89,7 +89,7 @@ public boolean existsInDatabase() @Override @JsonIgnore - public Integer getReadsetId() + public Long getReadsetId() { return _rowId; } @@ -259,12 +259,12 @@ public void setInstrument_run_id(Integer instrument_run_id) } @Override - public Integer getRunId() + public Long getRunId() { return _runId; } - public void setRunId(Integer runId) + public void setRunId(Long runId) { _runId = runId; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java index 7aaea197c..bedd9808b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineStarGeneCountsHandler.java @@ -1,6 +1,7 @@ package org.labkey.sequenceanalysis.analysis; import org.json.JSONObject; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; @@ -41,9 +42,9 @@ protected void processOutputFiles(CountResults results, List long totalStrand2 = 0L; results.distinctGenes.addAll(translator.getGeneMap().keySet()); - Map> unstrandedCounts = new HashMap<>(inputFiles.size()); - Map> strand1Counts = new HashMap<>(inputFiles.size()); - Map> strand2Counts = new HashMap<>(inputFiles.size()); + Map> unstrandedCounts = new IntHashMap<>(inputFiles.size()); + Map> strand1Counts = new IntHashMap<>(inputFiles.size()); + Map> strand2Counts = new IntHashMap<>(inputFiles.size()); for (SequenceOutputFile so : inputFiles) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/MultiQCBamHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/MultiQCBamHandler.java index f89ac0fcf..773e07cf6 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/MultiQCBamHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/MultiQCBamHandler.java @@ -77,7 +77,7 @@ public void processFilesRemote(List inputFiles, JobContext c action.setStartTime(new Date()); Set dirs = new HashSet<>(); - Set readsets = new HashSet<>(); + Set readsets = new HashSet<>(); Set genomes = new HashSet<>(); for (SequenceOutputFile so : inputFiles) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java index 0bed04f5f..e157be4fe 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java @@ -141,7 +141,7 @@ public void complete(JobContext ctx, List inputs, List inputs, List createForAnalyses(Container c, User u, Container targetContainer = c; if (submitJobToReadsetContainer) { - Integer readsetId = model.getReadset(); + Long readsetId = model.getReadset(); if (readsetId != null) { SequenceReadsetImpl readset = SequenceAnalysisServiceImpl.get().getReadset(readsetId, u); @@ -141,22 +141,22 @@ public static void register() throws CloneNotSupportedException PipelineJobService.get().addTaskPipeline(settings); } - public int getAnalyisId() + public long getAnalyisId() { return _analyisId; } - public void setAnalyisId(int analyisId) + public void setAnalyisId(long analyisId) { _analyisId = analyisId; } - public int getReadsetId() + public long getReadsetId() { return _readsetId; } - public void setReadsetId(int readsetId) + public void setReadsetId(long readsetId) { _readsetId = readsetId; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java index 43ebcb290..1af40f388 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentAnalysisRemoteWorkTask.java @@ -92,9 +92,9 @@ public RecordedActionSet run() throws PipelineJobException { _taskHelper = new SequenceTaskHelper(getPipelineJob(), _wd); - Map cachedFiles = getTaskHelper().getSequenceSupport().getAllCachedData(); + Map cachedFiles = getTaskHelper().getSequenceSupport().getAllCachedData(); getJob().getLogger().debug("total ExpDatas cached: " + cachedFiles.size()); - for (Integer dataId : cachedFiles.keySet()) + for (Long dataId : cachedFiles.keySet()) { getJob().getLogger().debug("file was cached: " + dataId + " / " + cachedFiles.get(dataId).getPath()); } @@ -142,7 +142,7 @@ public RecordedActionSet run() throws PipelineJobException private File resolveRefFasta(AnalysisModel m, File inputBam) throws PipelineJobException { //first try to find FASTA based on reference_library - Integer refFastaId = m.getReferenceLibrary(); + Long refFastaId = m.getReferenceLibrary(); if (refFastaId != null) { File refFasta = getTaskHelper().getSequenceSupport().getCachedData(refFastaId); @@ -184,7 +184,7 @@ private Map getAnalysisMap() for (AnalysisModel m : getTaskHelper().getSequenceSupport().getCachedAnalyses()) { - int bamId = m.getAlignmentFile(); + long bamId = m.getAlignmentFile(); File bam = getTaskHelper().getSequenceSupport().getCachedData(bamId); if (bam != null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportTask.java index 56c484aca..bf5078a1a 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportTask.java @@ -2,6 +2,7 @@ import org.jetbrains.annotations.NotNull; import org.json.JSONObject; +import org.labkey.api.collections.LongHashMap; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.Table; @@ -103,7 +104,7 @@ private List parseAndCreateAnalyses() throws PipelineJobException //find moved BAM files and build map Map bamMap = new HashMap<>(); - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); ExpRun run = ExperimentService.get().getExpRun(runId); List datas = run.getInputDatas(SequenceAlignmentTask.FINAL_BAM_ROLE, ExpProtocol.ApplicationType.ExperimentRunOutput); if (!datas.isEmpty()) @@ -151,7 +152,7 @@ private List parseAndCreateAnalyses() throws PipelineJobException a.setLibrary_id(o.getInt("library_id")); TableInfo ti = SequenceAnalysisSchema.getInstance().getSchema().getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARIES); - Integer refFastaId = new TableSelector(ti, PageFlowUtil.set("fasta_file")).getObject(o.getInt("library_id"), Integer.class); + Long refFastaId = new TableSelector(ti, PageFlowUtil.set("fasta_file")).getObject(o.getInt("library_id"), Long.class); a.setReferenceLibrary(refFastaId); a.setContainer(getJob().getContainer().getId()); a.setCreated(new Date()); @@ -185,8 +186,8 @@ private List parseAndCreateAnalyses() throws PipelineJobException transaction.commit(); //process metrics - Map readsetToAnalysisMap = new HashMap<>(); - Map> typeMap = new HashMap<>(); + Map readsetToAnalysisMap = new LongHashMap<>(); + Map> typeMap = new LongHashMap<>(); for (AnalysisModel model : ret) { readsetToAnalysisMap.put(model.getReadset(), model.getRowId()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java index 0cca64834..0e3da3e90 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java @@ -1,6 +1,7 @@ package org.labkey.sequenceanalysis.pipeline; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.Container; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; @@ -15,7 +16,6 @@ import org.labkey.sequenceanalysis.SequenceAnalysisModule; import java.io.File; -import java.util.HashMap; import java.util.Map; public class CacheGenomeTrigger implements GenomeTrigger @@ -48,7 +48,7 @@ private void possiblyCache(Container c, User u, Logger log, int genomeId) { try { - Map genomeMap = new HashMap<>(); + Map genomeMap = new IntHashMap<>(); ReferenceGenome rg = SequenceAnalysisService.get().getReferenceGenome(genomeId, u); genomeMap.put(rg.getGenomeId(), rg.getSourceFastaFile()); cacheGenomes(c, u, genomeMap, log, false); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java index 785e1e628..6e7fcc526 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java @@ -69,6 +69,8 @@ import java.util.Map; import java.util.Set; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * User: bbimber * Date: 8/6/12 @@ -204,7 +206,7 @@ public RecordedActionSet run() throws PipelineJobException throw errors; } libraryRow = new CaseInsensitiveHashMap<>(inserted.get(0)); - rowId = (Integer) libraryRow.get("rowid"); + rowId = asInteger(libraryRow.get("rowid")); } else { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaFastqSplitter.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaFastqSplitter.java index 41a633f76..a88fd7f5d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaFastqSplitter.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaFastqSplitter.java @@ -42,6 +42,8 @@ import java.util.Set; import java.util.zip.GZIPOutputStream; +import static org.labkey.api.exp.api.ExperimentService.asLong; + /** * This is designed to parse the FASTQ files produced by a single run on an illumina instructment and produce one gzipped FASTQ * for each sample in that run. Parsing that CSV file to obtain the sample list is upstream of this class. @@ -232,11 +234,11 @@ private FastqWriter getWriter(String illuminaSampleId, File targetDir, int pairN else { String suffix; - if (Integer.valueOf(0).equals(sampleId)) + if (asLong(0L).equals(asLong(sampleId))) { suffix = "Control"; } - else if (sampleId == null || Integer.valueOf(-1).equals(sampleId)) + else if (sampleId == null || asLong(-1L).equals(asLong(sampleId))) { suffix = "Undetermined"; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java index 3928cc015..2ad04ec9c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaImportTask.java @@ -16,6 +16,7 @@ package org.labkey.sequenceanalysis.pipeline; import au.com.bytecode.opencsv.CSVReader; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.biojava.nbio.core.exceptions.CompoundNotFoundException; import org.biojava.nbio.core.sequence.DNASequence; @@ -56,6 +57,8 @@ import java.util.List; import java.util.Map; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * User: bbimber * Date: 4/22/12 @@ -142,18 +145,18 @@ public RecordedActionSet run() throws PipelineJobException RecordedAction action = new RecordedAction(ACTION_NAME); action.addInputIfNotPresent(input, "Illumina Sample CSV"); - Map sampleMap = parseCsv(input, schema); + Map sampleMap = parseCsv(input, schema); //NOTE: it might be just as easy to match filename based on expected pattern //this step will be slow - IlluminaFastqSplitter parser = new IlluminaFastqSplitter<>("Illumina", sampleMap, job.getLogger(), input.getParent(), prefix); + IlluminaFastqSplitter parser = new IlluminaFastqSplitter<>("Illumina", sampleMap, job.getLogger(), input.getParent(), prefix); parser.setOutputGzip(true); parser.setDestinationDir(getSupport().getAnalysisDirectory()); // the first element of the pair is the sample ID. the second is either 1 or 2, // depending on whether the file represents the forward or reverse reads - Map, File> fileMap = parser.parseFastqFiles(); + Map, File> fileMap = parser.parseFastqFiles(); for (File f : parser.getFiles()) { @@ -164,7 +167,7 @@ public RecordedActionSet run() throws PipelineJobException getJob().getLogger().info("Compressing FASTQ files"); FileType gz = new FileType("gz"); - for (Pair sampleKey : fileMap.keySet()) + for (Pair sampleKey : fileMap.keySet()) { File inputFile = fileMap.get(sampleKey); if (!gz.isType(inputFile)) @@ -182,18 +185,18 @@ public RecordedActionSet run() throws PipelineJobException TableInfo rs = schema.getTable(SequenceAnalysisSchema.TABLE_READSETS); TableInfo readDataTable = schema.getTable(SequenceAnalysisSchema.TABLE_READ_DATA); - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob(), false); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob(), false); //update the readsets Map row; for (Object key : new HashSet<>(sampleMap.values())) { - Integer readsetId = (Integer) key; + Long readsetId = (Long) key; Readset readset = SequenceAnalysisService.get().getReadset(readsetId, getJob().getUser()); row = new HashMap<>(); - Pair pair = Pair.of(readsetId, 1); + Pair pair = Pair.of(readsetId, 1); ReadDataImpl rd = new ReadDataImpl(); rd.setReadset(readsetId); rd.setCreated(new Date()); @@ -291,11 +294,11 @@ private void handleInstrumentRun(DbSchema schema) runRow = Table.insert(getJob().getUser(), runTable, runRow); - _instrumentRunId = (Integer)runRow.get("rowid"); + _instrumentRunId = asInteger(runRow.get("rowid")); getJob().getLogger().info("Created run: " + _instrumentRunId); } - private void addQualityMetrics(DbSchema schema, int readsetId, Pair key, IlluminaFastqSplitter parser, ExpData d) + private void addQualityMetrics(DbSchema schema, long readsetId, Pair key, IlluminaFastqSplitter parser, ExpData d) { getJob().getLogger().info("Adding quality metrics for file: " + d.getFile().getName()); Map, Integer> readCounts = parser.getReadCounts(); @@ -325,15 +328,15 @@ private ExpData createExpData(File f) return SequenceTaskHelper.createExpData(f, getJob()); } - private Map parseCsv(File sampleFile, DbSchema schema) throws PipelineJobException + private Map parseCsv(File sampleFile, DbSchema schema) throws PipelineJobException { getJob().getLogger().info("Parsing Sample File: " + sampleFile.getName()); try (CSVReader reader = new CSVReader(Readers.getReader(sampleFile))) { //parse the samples file String [] nextLine; - Map sampleMap = new HashMap<>(); - sampleMap.put("S0", 0); //placeholder for control and unmapped reads + Map sampleMap = new HashMap<>(); + sampleMap.put("S0", 0L); //placeholder for control and unmapped reads boolean inSamples = false; int sampleIdx = 0; @@ -363,10 +366,10 @@ private Map parseCsv(File sampleFile, DbSchema schema) throws P String indexesRC = new DNASequence(nextLine[6]).getReverseComplement().getSequenceAsString() + "+" + nextLine[8]; sampleIdx++; - Integer readsetId; + Long readsetId; try { - readsetId = Integer.parseInt(nextLine[0]); + readsetId = Long.parseLong(nextLine[0]); } catch (NumberFormatException e) { @@ -400,7 +403,7 @@ private Map parseCsv(File sampleFile, DbSchema schema) throws P } //not very efficient, but we only expect a handful of samples per run - private boolean validateReadsetId(Integer id) throws PipelineJobException, PipelineValidationException + private boolean validateReadsetId(Long id) throws PipelineJobException, PipelineValidationException { getJob().getLogger().debug("attempting to resolve readset by Id: " + id); if (id == null) @@ -425,7 +428,7 @@ private boolean validateReadsetId(Integer id) throws PipelineJobException, Pipel return true; } - private Integer tryCreateReadset(DbSchema schema, String[] line) throws PipelineJobException + private Long tryCreateReadset(DbSchema schema, String[] line) throws PipelineJobException { if (!_settings.doAutoCreateReadsets()) throw new PipelineJobException("Unable to find existing readset matching ID: " + line[0]); @@ -453,7 +456,7 @@ private Integer tryCreateReadset(DbSchema schema, String[] line) throws Pipeline if (_instrumentRunId != null) row.put("instrument_run_id", _instrumentRunId); - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob(), false); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob(), false); if (runId != null) row.put("runid", runId); @@ -462,7 +465,7 @@ private Integer tryCreateReadset(DbSchema schema, String[] line) throws Pipeline row.put("created", new Date()); row = Table.insert(getJob().getUser(), rsTable, row); - return (Integer)row.get("rowid"); + return MapUtils.getLong(row,"rowid"); } private String resolveBarcode(String barcode) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaReadsetCreationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaReadsetCreationTask.java index 5dd3ccd0b..87c5abcc8 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaReadsetCreationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/IlluminaReadsetCreationTask.java @@ -87,7 +87,7 @@ public RecordedActionSet run() throws PipelineJobException PipelineJob job = getJob(); job.getLogger().info("Updating readsets"); - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); DbSchema schema = SequenceAnalysisSchema.getInstance().getSchema(); @@ -97,7 +97,7 @@ public RecordedActionSet run() throws PipelineJobException TableInfo readData = schema.getTable(SequenceAnalysisSchema.TABLE_READ_DATA); ExpRun run = ExperimentService.get().getExpRun(runId); - List dataIds = new ArrayList<>(); + List dataIds = new ArrayList<>(); List outputs = run.getDataOutputs(); for (ExpData d : outputs) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java index 464d8a0b7..b19a93470 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java @@ -97,7 +97,7 @@ public RecordedActionSet run() throws PipelineJobException List sequenceIds = new ArrayList<>(); for (File f : getPipelineJob().getFastas()) { - Integer jobId = PipelineService.get().getJobId(getJob().getUser(), getJob().getContainer(), getJob().getJobGUID()); + Long jobId = PipelineService.get().getJobId(getJob().getUser(), getJob().getContainer(), getJob().getJobGUID()); sequenceIds.addAll(SequenceAnalysisManager.get().importRefSequencesFromFasta(getJob().getContainer(), getJob().getUser(), f, getPipelineJob().isSplitWhitespace(), getPipelineJob().getParams(), getJob().getLogger(), getPipelineJob().getOutDir(), jobId)); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java index adfeeee4a..d2d63e6aa 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java @@ -81,6 +81,8 @@ import java.util.Map; import java.util.Set; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * User: bbimber * Date: 8/6/12 @@ -328,7 +330,7 @@ else if (SequenceUtil.FILETYPE.bw.getFileType().isType(file)) map.put("modified", new Date()); map.put("modifiedby", getJob().getUser().getUserId()); map.put("fileid", trackData.getRowId()); - Integer jobId = PipelineService.get().getJobId(getJob().getUser(), genomeContainer, getJob().getJobGUID()); + Long jobId = PipelineService.get().getJobId(getJob().getUser(), genomeContainer, getJob().getJobGUID()); map.put("jobId", jobId); TableInfo trackTable = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_LIBRARY_TRACKS); @@ -338,7 +340,7 @@ else if (SequenceUtil.FILETYPE.bw.getFileType().isType(file)) throw new PipelineJobException("Unable to find rowid for new track"); } - return (Integer)map.get("rowid"); + return asInteger(map.get("rowid")); } private void sortGxf(File gxf) throws PipelineJobException diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/OrphanFilePipelineJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/OrphanFilePipelineJob.java index a73f34d78..7c95e2b0b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/OrphanFilePipelineJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/OrphanFilePipelineJob.java @@ -498,7 +498,7 @@ else if (jobIds.size() > 1) // NOTE: if this file is within a known job path, it still could be an orphan. first check whether the directory has registered files. // If so, remove that path from the set of known job paths List dataUnderPath = ExperimentService.get().getExpDatasUnderPath(dir, c); - Set dataIdsUnderPath = new HashSet<>(); + Set dataIdsUnderPath = new HashSet<>(); for (ExpData d : dataUnderPath) { dataIdsUnderPath.add(d.getRowId()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java index fab9620ad..8cfb0d856 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java @@ -16,6 +16,9 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.IntHashSet; +import org.labkey.api.collections.LongHashSet; import org.labkey.api.data.Container; import org.labkey.api.laboratory.DemographicsProvider; import org.labkey.api.module.Module; @@ -160,7 +163,7 @@ public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File proces public static SequenceOutputFile createSequenceOutput(PipelineJob job, File processed, List inputFiles, String category) { - Set libraryIds = new HashSet<>(); + Set libraryIds = new IntHashSet(); inputFiles.forEach(x -> { if (x.getLibrary_id() != null) libraryIds.add(x.getLibrary_id()); @@ -171,7 +174,7 @@ public static SequenceOutputFile createSequenceOutput(PipelineJob job, File proc throw new IllegalArgumentException("No library ID defined for VCFs"); } - Set readsetIds = new HashSet<>(); + Set readsetIds = new LongHashSet(); inputFiles.forEach(x -> readsetIds.add(x.getReadset())); int sampleCount; @@ -598,7 +601,7 @@ public void processFilesRemote(List inputFiles, JobContext c throw new PipelineJobException("Priority order not supplied for VCFs"); } - Set genomes = new HashSet<>(); + Set genomes = new IntHashSet(); inputFiles.forEach(x -> genomes.add(x.getLibrary_id())); if (genomes.size() != 1) @@ -609,7 +612,7 @@ public void processFilesRemote(List inputFiles, JobContext c ReferenceGenome rg = ctx.getSequenceSupport().getCachedGenome(genomes.iterator().next()); MergeVcfsAndGenotypesWrapper cv = new MergeVcfsAndGenotypesWrapper(ctx.getLogger()); - Map fileMap = new HashMap<>(); + Map fileMap = new IntHashMap<>(); inputFiles.forEach(x -> fileMap.put(x.getRowid(), x.getDataId())); String[] ids = priorityOrder.split(","); @@ -622,7 +625,7 @@ public void processFilesRemote(List inputFiles, JobContext c throw new PipelineJobException("Unable to find file matching priority: " + i); } - int dataId = fileMap.get(i); + long dataId = fileMap.get(i); vcfsInPriority.add(ctx.getSequenceSupport().getCachedData(dataId)); } @@ -676,7 +679,7 @@ public void processFilesRemote(List inputFiles, JobContext c _resumer.markComplete(ctx); } - private void processFile(File input, Integer libraryId, Integer readsetId, JobContext ctx) throws PipelineJobException + private void processFile(File input, Integer libraryId, Long readsetId, JobContext ctx) throws PipelineJobException { File processed = processVCF(input, libraryId, ctx, _resumer, true); if (processed != null && processed.exists()) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetCreationTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetCreationTask.java index 4900f8c99..e62471f8e 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetCreationTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetCreationTask.java @@ -18,6 +18,8 @@ import au.com.bytecode.opencsv.CSVReader; import org.jetbrains.annotations.NotNull; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.DbSchema; @@ -62,7 +64,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -156,7 +157,7 @@ private void importReadsets() throws PipelineJobException SequencePipelineSettings settings = getSettings(); DbSchema schema = SequenceAnalysisSchema.getInstance().getSchema(); - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); ExpRun run = ExperimentService.get().getExpRun(runId); List datas = new ArrayList<>(); datas.addAll(run.getInputDatas(SequenceTaskHelper.NORMALIZED_FASTQ_OUTPUTNAME, ExpProtocol.ApplicationType.ExperimentRunOutput)); @@ -166,10 +167,10 @@ private void importReadsets() throws PipelineJobException List newReadsets = new ArrayList<>(); - Set fileIdsWithExistingMetrics = new HashSet<>(); + Set fileIdsWithExistingMetrics = new LongHashSet(); try (DbScope.Transaction transaction = schema.getScope().ensureTransaction()) { - Map readsetsToDeactivate = new HashMap<>(); + Map readsetsToDeactivate = new LongHashMap<>(); TableInfo readsetTable = schema.getTable(SequenceAnalysisSchema.TABLE_READSETS); TableInfo readDataTable = schema.getTable(SequenceAnalysisSchema.TABLE_READ_DATA); @@ -514,7 +515,7 @@ private void importReadsets() throws PipelineJobException } } - private void runFastqcForFile(Integer fileId) throws PipelineJobException + private void runFastqcForFile(Long fileId) throws PipelineJobException { ExpData d1 = ExperimentService.get().getExpData(fileId); if (d1 != null && d1.getFile().exists()) @@ -535,7 +536,7 @@ private void runFastqcForFile(Integer fileId) throws PipelineJobException } } - private Long getTotalReadsForFile(int fileId, int readsetId) + private Long getTotalReadsForFile(long fileId, long readsetId) { TableInfo metricsTable = SequenceAnalysisManager.get().getTable(SequenceAnalysisSchema.TABLE_QUALITY_METRICS); @@ -554,12 +555,12 @@ private Long getTotalReadsForFile(int fileId, int readsetId) return 0L; } - public static long addQualityMetricsForReadset(Readset rs, int fileId, PipelineJob job) throws PipelineJobException + public static long addQualityMetricsForReadset(Readset rs, long fileId, PipelineJob job) throws PipelineJobException { return addQualityMetricsForReadset(rs, fileId, job, false); } - public static long addQualityMetricsForReadset(Readset rs, int fileId, PipelineJob job, boolean deleteExisting) throws PipelineJobException + public static long addQualityMetricsForReadset(Readset rs, long fileId, PipelineJob job, boolean deleteExisting) throws PipelineJobException { if (deleteExisting) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceGenomeImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceGenomeImpl.java index d4faf3341..5a2b348a0 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceGenomeImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceGenomeImpl.java @@ -34,7 +34,7 @@ public class ReferenceGenomeImpl implements ReferenceGenome private File _sourceFasta; private File _workingFasta; private Integer _genomeId; - private Integer _expDataId; + private Long _expDataId; // Default constructor for serialization protected ReferenceGenomeImpl() @@ -146,13 +146,13 @@ public void setGenomeId(Integer genomeId) _genomeId = genomeId; } - public void setExpDataId(Integer expDataId) + public void setExpDataId(Long expDataId) { _expDataId = expDataId; } @Override - public Integer getFastaExpDataId() + public Long getFastaExpDataId() { return _expDataId; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java index 23195cb0c..3a36fc287 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentJob.java @@ -38,7 +38,7 @@ public class SequenceAlignmentJob extends SequenceJob { public static final String FOLDER_NAME = "sequenceAnalysis"; - private int _readsetId; + private long _readsetId; // Default constructor for serialization protected SequenceAlignmentJob() @@ -112,7 +112,7 @@ public static List createForReadsets(Container c, User u, return ret; } - public int getReadsetId() + public long getReadsetId() { return _readsetId; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java index 5cecca0da..f485d5199 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java @@ -31,6 +31,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongHashMap; import org.labkey.api.data.ConvertHelper; import org.labkey.api.pipeline.ObjectKeySerialization; import org.labkey.api.pipeline.PairSerializer; @@ -1282,7 +1283,7 @@ private File doMergeThenAlign(ReferenceGenome referenceGenome, Readset rs, Map> inputs = new ArrayList<>(files.values()); - Optional minReadData = files.keySet().stream().map(ReadData::getRowid).min(Integer::compareTo); + Optional minReadData = files.keySet().stream().map(ReadData::getRowid).min(Long::compareTo); return doAlignmentForSet(inputs, referenceGenome, rs, minReadData.get(), "", inputs.size() == 1 ? files.keySet().stream().iterator().next().getPlatformUnit() : null); } } @@ -1340,7 +1341,7 @@ private File doAlignThenMerge(ReferenceGenome referenceGenome, Readset rs, Map> inputFiles, ReferenceGenome referenceGenome, Readset rs, int lowestReadDataId, @NotNull String msgSuffix, @Nullable String platformUnit) throws PipelineJobException, IOException + public File doAlignmentForSet(List> inputFiles, ReferenceGenome referenceGenome, Readset rs, long lowestReadDataId, @NotNull String msgSuffix, @Nullable String platformUnit) throws PipelineJobException, IOException { AlignmentStep alignmentStep = getHelper().getSingleStep(AlignmentStep.class).create(getHelper()); @@ -1496,7 +1497,7 @@ public static class Resumer extends AbstractResumer private boolean _alignmentMetricsDone = false; private File _renamedBamFile = null; private boolean _indexBamDone = false; - private Map _readDataBamMap = new HashMap<>(); + private Map _readDataBamMap = new LongHashMap<>(); //for serialization public Resumer() @@ -1795,12 +1796,12 @@ public void setAlignmentMetricsDone(boolean alignmentMetricsDone, RecordedAction saveState(); } - public boolean isReadDataAlignmentDone(int readDataId) + public boolean isReadDataAlignmentDone(long readDataId) { return _readDataBamMap.containsKey(readDataId); } - public void setReadDataAlignmentDone(int readDataId, List actions, File bam) throws PipelineJobException + public void setReadDataAlignmentDone(long readDataId, List actions, File bam) throws PipelineJobException { getLogger().debug("setting read data alignment done: " + readDataId + ", " + (actions == null ? "0" : actions.size()) + " actions"); _readDataBamMap.put(readDataId, bam); @@ -1811,17 +1812,17 @@ public void setReadDataAlignmentDone(int readDataId, List action saveState(); } - public File getBamForReadData(int readDataId) + public File getBamForReadData(long readDataId) { return _readDataBamMap.get(readDataId); } - public Map getReadDataBamMap() + public Map getReadDataBamMap() { return _readDataBamMap; } - public void setReadDataBamMap(Map readDataBamMap) + public void setReadDataBamMap(Map readDataBamMap) { _readDataBamMap = readDataBamMap; } @@ -1874,8 +1875,8 @@ public void serializeTest() throws Exception ReadDataImpl rd = new ReadDataImpl(); rd.setFile(file1, 1); rd.setFile(file2, 2); - rd.setReadset(-1); - rd.setRowid(-1); + rd.setReadset(-1L); + rd.setRowid(-1L); r._filesToAlign = new LinkedHashMap<>(); r._filesToAlign.put(rd, pair); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java index 0c024abcf..2bd1175b7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAnalysisTask.java @@ -17,6 +17,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.LongHashMap; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; @@ -159,7 +160,7 @@ public RecordedActionSet run() throws PipelineJobException job.getLogger().info("Processing Alignments"); //first validate all analysis records before actually creating any DB records - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); ExpRun run = ExperimentService.get().getExpRun(runId); AnalysisModelImpl analysisModel = null; @@ -313,8 +314,8 @@ else if (d.getFile().exists()) if (SequenceTaskHelper.isAlignmentUsed(getJob())) { //build map used next to import metrics - Map readsetToAnalysisMap = new HashMap<>(); - Map> typeMap = new HashMap<>(); + Map readsetToAnalysisMap = new LongHashMap<>(); + Map> typeMap = new LongHashMap<>(); readsetToAnalysisMap.put(analysisModel.getReadset(), analysisModel.getRowId()); typeMap.put(analysisModel.getReadset(), new HashMap<>()); @@ -356,7 +357,7 @@ else if (d.getFile().exists()) return new RecordedActionSet(); } - private void processAnalyses(AnalysisModelImpl analysisModel, int runId, List actions, SequenceTaskHelper taskHelper, boolean discardBam) throws PipelineJobException + private void processAnalyses(AnalysisModelImpl analysisModel, long runId, List actions, SequenceTaskHelper taskHelper, boolean discardBam) throws PipelineJobException { List outputs = new ArrayList<>(); List> steps = SequencePipelineService.get().getSteps(getJob(), AnalysisStep.class); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java index 8bab610b5..02a2ac560 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java @@ -67,7 +67,7 @@ public class SequenceJob extends PipelineJob implements FileAnalysisJobSupport, HasJobParams, SequenceOutputTracker { private TaskId _taskPipelineId; - private Integer _experimentRunRowId; + private Long _experimentRunRowId; private String _jobName; private String _description; private FileLike _webserverJobDir; @@ -557,12 +557,12 @@ public PipeRoot getFolderPipeRoot() return _folderFileRoot; } - public Integer getExperimentRunRowId() + public Long getExperimentRunRowId() { return _experimentRunRowId; } - public void setExperimentRunRowId(Integer experimentRunRowId) + public void setExperimentRunRowId(Long experimentRunRowId) { _experimentRunRowId = experimentRunRowId; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJobSupportImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJobSupportImpl.java index f84922a68..23e3e68a4 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJobSupportImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJobSupportImpl.java @@ -5,6 +5,8 @@ import htsjdk.samtools.util.IOUtil; import org.junit.Assert; import org.junit.Test; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.LongHashMap; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.pipeline.PipelineJob; @@ -37,10 +39,10 @@ */ public class SequenceJobSupportImpl implements SequenceAnalysisJobSupport, Serializable { - private final Map _cachedFilePaths = new HashMap<>(); + private final Map _cachedFilePaths = new LongHashMap<>(); private final List _cachedReadsets = new ArrayList<>(); - private final Map _cachedAnalyses = new HashMap<>(); - private final Map _cachedGenomes = new HashMap<>(); + private final Map _cachedAnalyses = new LongHashMap<>(); + private final Map _cachedGenomes = new IntHashMap<>(); private final Map _cachedObjects = new HashMap<>(); private transient boolean _modifiedSinceSerialize = false; @@ -53,7 +55,7 @@ public SequenceJobSupportImpl() } @Override - public SequenceReadsetImpl getCachedReadset(Integer rowId) + public SequenceReadsetImpl getCachedReadset(Long rowId) { if (rowId == null || rowId < 1) { @@ -72,19 +74,19 @@ public SequenceReadsetImpl getCachedReadset(Integer rowId) } @Override - public AnalysisModel getCachedAnalysis(int rowId) + public AnalysisModel getCachedAnalysis(long rowId) { return _cachedAnalyses.get(rowId); } @Override - public void cacheReadset(int readsetId, User u) + public void cacheReadset(long readsetId, User u) { cacheReadset(readsetId,u, false); } @Override - public void cacheReadset(int readsetId, User u, boolean allowReadsetsWithArchivedData) + public void cacheReadset(long readsetId, User u, boolean allowReadsetsWithArchivedData) { SequenceReadsetImpl rs = SequenceAnalysisServiceImpl.get().getReadset(readsetId, u); if (rs != null) @@ -243,13 +245,13 @@ public void cacheExpData(ExpData data) } @Override - public File getCachedData(int dataId) + public File getCachedData(long dataId) { return _cachedFilePaths.get(dataId); } @Override - public Map getAllCachedData() + public Map getAllCachedData() { return Collections.unmodifiableMap(_cachedFilePaths); } @@ -303,13 +305,13 @@ public static class TestCase extends Assert public void testSerializeWithMap() throws Exception { SequenceJobSupportImpl js1 = new SequenceJobSupportImpl(); - js1._cachedAnalyses.put(1, new AnalysisModelImpl()); + js1._cachedAnalyses.put(1L, new AnalysisModelImpl()); js1._cachedGenomes.put(2, new ReferenceGenomeImpl()); SequenceReadsetImpl rs1 = new SequenceReadsetImpl(); rs1.setRowId(100); js1._cachedReadsets.add(rs1); - js1._cachedFilePaths.put(4, new File("/")); + js1._cachedFilePaths.put(4L, new File("/")); HashMap map = new HashMap<>(); map.put(1, 1); @@ -324,13 +326,13 @@ public void testSerializeWithMap() throws Exception mapper.writeValue(writer, js1); SequenceJobSupportImpl deserialized = mapper.readValue(new StringReader(writer.toString()), SequenceJobSupportImpl.class); - assertNotNull("Analysis map not serialized properly", deserialized._cachedAnalyses.get(1)); + assertNotNull("Analysis map not serialized properly", deserialized._cachedAnalyses.get(1L)); assertNotNull("Genome map not serialized properly", deserialized._cachedGenomes.get(2)); assertEquals("Readset list not serialized properly", 1, deserialized._cachedReadsets.size()); assertEquals("Readset not deserialized with correct rowid", 100, deserialized._cachedReadsets.get(0).getRowId()); assertEquals("Readset not deserialized with correct readsetid", 100, deserialized._cachedReadsets.get(0).getReadsetId().intValue()); - assertNotNull("File map not serialized properly", deserialized._cachedFilePaths.get(4)); + assertNotNull("File map not serialized properly", deserialized._cachedFilePaths.get(4L)); assertNotNull("Cached map not serialized properly", deserialized.getCachedObject("cachedMap",Map.class)); assertEquals(" Cached int not serialized properly", Integer.valueOf(1), deserialized.getCachedObject("cachedInt", Integer.class)); assertEquals(" Cached string not serialized properly", "foo", deserialized.getCachedObject("cachedString", String.class)); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java index 7fa141ce5..2b5322ce2 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerFinalTask.java @@ -87,7 +87,7 @@ private SequenceOutputHandlerJob getPipelineJob() @Override public RecordedActionSet run() throws PipelineJobException { - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); getPipelineJob().setExperimentRunRowId(runId); //create analysisRecord @@ -119,7 +119,7 @@ public RecordedActionSet run() throws PipelineJobException am.setType(getPipelineJob().getHandler().getAnalysisType(getJob())); TableInfo analysisTable = SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_ANALYSES); - Set readsetIds = new HashSet<>(); + Set readsetIds = new HashSet<>(); for (SequenceOutputFile so : getPipelineJob().getFiles()) { if (so.getReadset() == null) @@ -137,7 +137,7 @@ public RecordedActionSet run() throws PipelineJobException } Table.insert(getJob().getUser(), analysisTable, am); - int analysisId = am.getRowId(); + long analysisId = am.getRowId(); List outputsCreated = new ArrayList<>(); if (!getPipelineJob().getOutputsToCreate().isEmpty()) @@ -164,7 +164,7 @@ public RecordedActionSet run() throws PipelineJobException return new RecordedActionSet(); } - public static Set createOutputFiles(SequenceJob job, int runId, @Nullable Integer analysisId) + public static Set createOutputFiles(SequenceJob job, long runId, @Nullable Long analysisId) { job.getLogger().info("creating " + job.getOutputsToCreate().size() + " new output files for run: " + runId); @@ -205,7 +205,7 @@ public static Set createOutputFiles(SequenceJob job, int run return created; } - public static void updateOutputFile(SequenceOutputFile o, PipelineJob job, Integer runId, Integer analysisId) + public static void updateOutputFile(SequenceOutputFile o, PipelineJob job, Long runId, Long analysisId) { o.setRunId(runId); o.setAnalysis_id(analysisId); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java index b2cf6098d..da411b8bd 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerFinalTask.java @@ -71,7 +71,7 @@ private SequenceReadsetHandlerJob getPipelineJob() @Override public RecordedActionSet run() throws PipelineJobException { - Integer runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); + Long runId = SequenceTaskHelper.getExpRunIdForJob(getJob()); getPipelineJob().setExperimentRunRowId(runId); List outputsCreated = new ArrayList<>(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java index 39612ac3b..eb819f963 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceReadsetHandlerJob.java @@ -27,7 +27,7 @@ public class SequenceReadsetHandlerJob extends SequenceJob implements HasJobParams { private String _handlerClassName; - private List _readsetIds; + private List _readsetIds; // Default constructor for serialization protected SequenceReadsetHandlerJob() @@ -105,12 +105,12 @@ public List getInputFiles() return null; } - public List getReadsetIds() + public List getReadsetIds() { return _readsetIds; } - public void setReadsetIds(List readsetIds) + public void setReadsetIds(List readsetIds) { _readsetIds = readsetIds; } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java index 5805c92d7..e29e63aa3 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java @@ -15,6 +15,7 @@ */ package org.labkey.sequenceanalysis.pipeline; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -182,15 +183,15 @@ public SequencePipelineSettings getSettings() return _settings; } - public static Integer getExpRunIdForJob(PipelineJob job) throws PipelineJobException + public static Long getExpRunIdForJob(PipelineJob job) throws PipelineJobException { return getExpRunIdForJob(job, true); } - public static Integer getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException + public static Long getExpRunIdForJob(PipelineJob job, boolean throwUnlessFound) throws PipelineJobException { - Integer jobId = PipelineService.get().getJobId(job.getUser(), job.getContainer(), job.getJobGUID()); - Integer parentJobId = PipelineService.get().getJobId(job.getUser(), job.getContainer(), job.getParentGUID()); + Long jobId = PipelineService.get().getJobId(job.getUser(), job.getContainer(), job.getJobGUID()); + Long parentJobId = PipelineService.get().getJobId(job.getUser(), job.getContainer(), job.getParentGUID()); TableInfo runs = ExperimentService.get().getSchema().getTable("ExperimentRun"); TableSelector ts = new TableSelector(runs, Collections.singleton("RowId"), new SimpleFilter(FieldKey.fromString("JobId"), jobId), null); @@ -210,7 +211,7 @@ public static Integer getExpRunIdForJob(PipelineJob job, boolean throwUnlessFoun return null; } Map row = rows[0]; - return (Integer)row.get("rowid"); + return MapUtils.getLong(row,"rowid"); } public static String getUnzippedBaseName(File file) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java index 91d06a881..90a6418e7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java @@ -116,7 +116,7 @@ public void addSequenceOutput(SequenceOutputFile o) } @Override - public void addSequenceOutput(File file, String label, String category, @Nullable Integer readsetId, @Nullable Integer analysisId, @Nullable Integer genomeId, @Nullable String description) + public void addSequenceOutput(File file, String label, String category, @Nullable Long readsetId, @Nullable Long analysisId, @Nullable Integer genomeId, @Nullable String description) { SequenceOutputFile so = new SequenceOutputFile(); so.setFile(file); @@ -357,11 +357,11 @@ public void deleteDeferredIntermediateFiles() } @Override - public Set createSequenceOutputRecords(@Nullable Integer analysisId) throws PipelineJobException + public Set createSequenceOutputRecords(@Nullable Long analysisId) throws PipelineJobException { if (!_job.getOutputsToCreate().isEmpty()) { - Integer runId = SequenceTaskHelper.getExpRunIdForJob(_job); + Long runId = SequenceTaskHelper.getExpRunIdForJob(_job); return SequenceOutputHandlerFinalTask.createOutputFiles(_job, runId, analysisId); } else @@ -539,7 +539,7 @@ public void addPicardMetricsFiles(List f } @Override - public void writeMetricsToDb(Map readsetMap, Map> typeMap) throws PipelineJobException + public void writeMetricsToDb(Map readsetMap, Map> typeMap) throws PipelineJobException { if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer) { @@ -574,7 +574,7 @@ public void writeMetricsToDb(Map readsetMap, Map readsetMap, Map _sequenceMap = new HashMap<>(); + private final Map _sequenceMap = new IntHashMap<>(); public SequenceTriggerHelper(int userId, String containerId) { @@ -228,7 +228,7 @@ public void testTranslation() { } } - public int createExpData(String relPath) { + public long createExpData(String relPath) { PipeRoot pr = PipelineService.get().getPipelineRootSetting(getContainer()); if (pr == null) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/BamHaplotyper.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/BamHaplotyper.java index b92066220..8ff1837bb 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/BamHaplotyper.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/BamHaplotyper.java @@ -13,6 +13,8 @@ import org.apache.logging.log4j.LogManager; import org.json.JSONArray; import org.json.JSONObject; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.StringHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -37,7 +39,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -119,7 +120,7 @@ public JSONObject calculateNTHaplotypes(int[] inputFileIds, String[] regions, in if (!found) { - Map map = new HashMap<>(); + Map map = new IntHashMap<>(); map.put(so.getRowid(), newPair.second); combinedResults.add(Pair.of(newPair.first, map)); } @@ -162,7 +163,7 @@ private String getReferenceSequence(ReferenceSequence ref, Map> getInsertionMap(List>> combinedResults) { //build list of all insertions that are present - Map> indels = new HashMap<>(); + Map> indels = new IntHashMap<>(); for (Pair> pair : combinedResults) { for (int idx = 0;idx < pair.first.length;idx++) @@ -189,7 +190,7 @@ private Map> getInsertionMap(List> convertResults(List>> combinedResults, byte[] refBases, Map> indels) { //now iterate each array, convert to string and return results - Map> ret = new HashMap<>(); + Map> ret = new StringHashMap<>(); for (Pair> pair : combinedResults) { StringBuilder sb = new StringBuilder(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java index 8779afda1..b4a00e484 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/RestoreSraDataHandler.java @@ -309,14 +309,15 @@ public void complete(PipelineJob job, List readsets, List toAdd = new ArrayList<>(rd.getFileId1()); + List toAdd = new ArrayList<>(); + toAdd.add(rd.getFileId1()); if (rd.getFileId2() != null) { toAdd.add(rd.getFileId2()); } job.getLogger().debug("adding metrics for " + toAdd.size() + " total fastq files"); - for (int dataId : toAdd) + for (long dataId : toAdd) { //then delete/add: job.getLogger().debug("adding metrics for: " + dataId); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByCodonAggregator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByCodonAggregator.java index 6a55a32f9..4f931c8ca 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByCodonAggregator.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByCodonAggregator.java @@ -18,6 +18,7 @@ import htsjdk.samtools.SAMRecord; import htsjdk.samtools.reference.ReferenceSequence; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.DbScope; import org.labkey.api.data.SimpleFilter; @@ -76,7 +77,7 @@ public void inspectAlignment(SAMRecord record, ReferenceSequence ref, Map> highQualitySnpsByPos = new HashMap<>(); + Map> highQualitySnpsByPos = new IntHashMap<>(); for (Integer pos : snps.keySet()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByReadAggregator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByReadAggregator.java index df8bd0f29..c57e2a442 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByReadAggregator.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AASnpByReadAggregator.java @@ -3,6 +3,7 @@ import htsjdk.samtools.SAMRecord; import htsjdk.samtools.reference.ReferenceSequence; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.Container; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.security.User; @@ -51,7 +52,7 @@ public void inspectAlignment(SAMRecord record, ReferenceSequence ref, Map> highQualitySnpsByPos = new HashMap<>(); + Map> highQualitySnpsByPos = new IntHashMap<>(); for (Integer pos : snps.keySet()) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AbstractAlignmentAggregator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AbstractAlignmentAggregator.java index 6309f89db..a775ba902 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AbstractAlignmentAggregator.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AbstractAlignmentAggregator.java @@ -19,6 +19,7 @@ import htsjdk.samtools.reference.ReferenceSequence; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.ReferenceLibraryHelper; import org.labkey.sequenceanalysis.run.util.AASnp; @@ -230,7 +231,7 @@ private TranslatingReferenceSequence getReferenceFromDb(String refName) return ref; } - private final Map>> _qualMap = new HashMap<>(); + private final Map>> _qualMap = new IntHashMap<>(); private Map> getQualsForReference(Integer refId) throws PipelineJobException { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AvgBaseQualityAggregator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AvgBaseQualityAggregator.java index 62e973304..804b3c26b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AvgBaseQualityAggregator.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/AvgBaseQualityAggregator.java @@ -25,6 +25,8 @@ import htsjdk.samtools.util.IntervalList; import htsjdk.samtools.util.SamLocusIterator; import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.StringHashMap; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.sequenceanalysis.util.SequenceUtil; @@ -93,7 +95,7 @@ public void calculateAvgQuals(String refName, int start, int stop) throws Pipeli if (sr == null) throw new IllegalArgumentException("Unknown reference: " + refName); - Map> quals = new HashMap<>(); + Map> quals = new IntHashMap<>(); IntervalList il = new IntervalList(header); il.add(new Interval(refName, start, stop)); quals.put(sr.getSequenceIndex(), calculateAvgQualsForInterval(reader, il)); @@ -115,7 +117,7 @@ public void calculateAvgQuals() throws PipelineJobException SAMFileHeader header = reader.getFileHeader(); List sequences = header.getSequenceDictionary().getSequences(); - Map> quals = new HashMap<>(); + Map> quals = new IntHashMap<>(); for(SAMSequenceRecord sr : sequences) { IntervalList il = new IntervalList(header); il.add(new Interval(sr.getSequenceName(), 1, sr.getSequenceLength())); @@ -136,7 +138,7 @@ public void calculateAvgQuals() throws PipelineJobException */ private Map calculateAvgQualsForInterval(SamReader sam, IntervalList il) { - Map quals = new HashMap<>(); + Map quals = new StringHashMap<>(); try (SamLocusIterator sli = new SamLocusIterator(sam, il, true)) { @@ -210,7 +212,7 @@ public Map> getQualsForReference(Integer refId) thr if (map == null) return null; - Map> ret = new HashMap<>(); + Map> ret = new IntHashMap<>(); for (String key : map.keySet()) { String[] tokens = key.split("\\|\\|"); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/MergeLoFreqVcfHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/MergeLoFreqVcfHandler.java index c33383ff0..a0dc852f1 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/MergeLoFreqVcfHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/MergeLoFreqVcfHandler.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.Logger; import org.json.JSONObject; +import org.labkey.api.collections.LongHashSet; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; @@ -183,7 +184,7 @@ public void processFilesRemote(List inputFiles, JobContext c { writer.writeNext(new String[]{"ReadsetName", "OutputFileId", "ReadsetId", "Source", "Contig", "Start", "End", "Length", "Ref", "AltAllele", "GatkDepth", "LoFreqDepth", "AltCount", "AltAF"}); - Set analysesWithoutPindel = new HashSet<>(); + Set analysesWithoutPindel = new LongHashSet(); for (SequenceOutputFile so : inputFiles) { //This will error if the coverage file is not found. Perform check now to fail fast diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/NextCladeHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/NextCladeHandler.java index 0b02e947c..667b168b9 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/NextCladeHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/NextCladeHandler.java @@ -94,7 +94,7 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport @Override public void complete(JobContext ctx, List inputFiles, List outputsCreated) throws PipelineJobException { - Map readsetToInput = inputFiles.stream().collect(Collectors.toMap(SequenceOutputFile::getReadset, x -> x)); + Map readsetToInput = inputFiles.stream().collect(Collectors.toMap(SequenceOutputFile::getReadset, x -> x)); ctx.getJob().getLogger().info("Parsing NextClade JSON:"); for (SequenceOutputFile so : outputsCreated) @@ -176,7 +176,7 @@ else if (samples.length() != 1) } } - public static void processAndImportNextCladeAa(PipelineJob job, File jsonFile, int analysisId, int libraryId, int alignmentId, int readsetId, File consensusVCF, boolean dbImport) throws PipelineJobException + public static void processAndImportNextCladeAa(PipelineJob job, File jsonFile, long analysisId, int libraryId, long alignmentId, long readsetId, File consensusVCF, boolean dbImport) throws PipelineJobException { JSONObject sample = parseNextClade(jsonFile, job.getLogger()); if (sample == null) @@ -360,7 +360,7 @@ public static void processAndImportNextCladeAa(PipelineJob job, File jsonFile, i } } - private static void saveClade(String clade, int analysisId, int alignmentId, int readsetId, PipelineJob job) throws PipelineJobException + private static void saveClade(String clade, long analysisId, long alignmentId, long readsetId, PipelineJob job) throws PipelineJobException { List> toInsert = new ArrayList<>(); Map row1 = new CaseInsensitiveHashMap<>(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PangolinHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PangolinHandler.java index 2cccb8b9e..37b34285a 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PangolinHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PangolinHandler.java @@ -193,7 +193,7 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport Container targetContainer = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); TableInfo ti = QueryService.get().getUserSchema(job.getUser(), targetContainer, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_QUALITY_METRICS); - Set uniqueAnalyses = inputFiles.stream().map(SequenceOutputFile::getAnalysis_id).collect(Collectors.toSet()); + Set uniqueAnalyses = inputFiles.stream().map(SequenceOutputFile::getAnalysis_id).collect(Collectors.toSet()); SimpleFilter filter = new SimpleFilter(FieldKey.fromString("category"), "Pangolin"); filter.addCondition(FieldKey.fromString("analysis_id"), uniqueAnalyses, CompareType.IN); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SequenceBasedTypingAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SequenceBasedTypingAnalysis.java index 58a4e41af..c3d9343d7 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SequenceBasedTypingAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SequenceBasedTypingAnalysis.java @@ -376,10 +376,10 @@ protected File getSBTSummaryFile(File outputDir, File bam) public static class AlignmentGroupCompare { - private final int analysisId; + private final long analysisId; private final List groups = new ArrayList<>(); - public AlignmentGroupCompare(final int analysisId, Container c, User u) + public AlignmentGroupCompare(final long analysisId, Container c, User u) { this.analysisId = analysisId; @@ -544,7 +544,7 @@ private boolean compareGroupToOthers(AlignmentGroup g1) public static class AlignmentGroup { - int analysisId; + long analysisId; Set alleles = new TreeSet<>(); String lineages; int totalLineages; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/ViralSnpUtil.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/ViralSnpUtil.java index c3d6a2286..443379426 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/ViralSnpUtil.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/ViralSnpUtil.java @@ -3,6 +3,7 @@ import htsjdk.samtools.util.CloseableIterator; import htsjdk.variant.variantcontext.VariantContext; import htsjdk.variant.vcf.VCFFileReader; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; @@ -18,7 +19,6 @@ import java.io.File; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +40,7 @@ public static int resolveGene(int refNtId, String aaName) throws PipelineJobExce public static Map> readVcfToMap(File vcf) { - Map> consensusMap = new HashMap<>(); + Map> consensusMap = new IntHashMap<>(); try (VCFFileReader reader = new VCFFileReader(vcf); CloseableIterator it = reader.iterator()) { while (it.hasNext()) @@ -69,13 +69,13 @@ public static Map> readVcfToMap(File vcf) return consensusMap; } - public static void deleteExistingMetrics(PipelineJob job, int analysisId, String category) throws PipelineJobException + public static void deleteExistingMetrics(PipelineJob job, long analysisId, String category) throws PipelineJobException { SimpleFilter filter = new SimpleFilter(FieldKey.fromString("category"), category); deleteExistingValues(job, analysisId, SequenceAnalysisSchema.TABLE_QUALITY_METRICS, filter); } - public static void deleteExistingValues(PipelineJob job, int analysisId, String queryName, SimpleFilter filter) throws PipelineJobException + public static void deleteExistingValues(PipelineJob job, long analysisId, String queryName, SimpleFilter filter) throws PipelineJobException { Container targetContainer = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); TableInfo ti = QueryService.get().getUserSchema(job.getUser(), targetContainer, SequenceAnalysisSchema.SCHEMA_NAME).getTable(queryName); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java index 59f9b945a..2115ef1dc 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/AbstractGenomicsDBImportHandler.java @@ -162,7 +162,7 @@ public SequenceOutputFile createFinalSequenceOutput(PipelineJob job, File merged throw new IllegalArgumentException("No library ID defined for VCFs"); } - Set readsetIds = new HashSet<>(); + Set readsetIds = new HashSet<>(); inputFiles.forEach(x -> readsetIds.add(x.getReadset())); int sampleCount = getSamplesForWorkspace(mergedWorkspace.getParentFile()).size(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java index a3715b491..f66d26e94 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ChainFileValidator.java @@ -7,6 +7,8 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.IntHashSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; @@ -59,8 +61,8 @@ public File processChainFile(File chainFile, int sourceGenome, int targetGenome, // reference names must match those in the DB for that genome Pattern SPLITTER = Pattern.compile("\\s"); - Set uniqueIds = new HashSet<>(); - Set encounteredIds = new HashSet<>(); + Set uniqueIds = new IntHashSet(); + Set encounteredIds = new IntHashSet(); //step 1: iterate lines, gather unique chain IDs boolean hasChanges = false; @@ -284,7 +286,7 @@ private int getNextChainId(Integer chainId, Set uniqueIds) return newId; } - private final Map> _cachedReferencesByGenome = new HashMap<>(); + private final Map> _cachedReferencesByGenome = new IntHashMap<>(); private String resolveSequenceId(String refName, int genomeId) { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/TranslatingReferenceSequence.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/TranslatingReferenceSequence.java index 6dce152df..67a74630c 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/TranslatingReferenceSequence.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/TranslatingReferenceSequence.java @@ -17,6 +17,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -29,7 +30,6 @@ import org.labkey.sequenceanalysis.run.util.NTSnp; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -231,7 +231,7 @@ private List translateSnp(NTSnp pi, Map> readSnps) StringBuilder codon = new StringBuilder(); //to track the SNPs that comprise this codon, 0-based - Map snpMap = new HashMap<>(); + Map snpMap = new IntHashMap<>(); Integer positionInCodon = null; //0-based for (Integer p : positions) //1-based { @@ -240,7 +240,7 @@ private List translateSnp(NTSnp pi, Map> readSnps) { StringBuilder thisSegment = new StringBuilder(); int positionInSegment = 0; - Map snpMapForSegment = new HashMap<>(); //0-based + Map snpMapForSegment = new IntHashMap<>(); //0-based //force sorting by idx List otherSnps = readSnps.get(position); diff --git a/Studies/src/org/labkey/studies/StudiesManager.java b/Studies/src/org/labkey/studies/StudiesManager.java index bf7926dcc..4c0d087bb 100644 --- a/Studies/src/org/labkey/studies/StudiesManager.java +++ b/Studies/src/org/labkey/studies/StudiesManager.java @@ -50,6 +50,7 @@ import java.util.stream.Collectors; import static com.fasterxml.jackson.databind.type.LogicalType.Collection; +import static org.labkey.api.exp.api.ExperimentService.asInteger; @RunWith(Enclosed.class) public class StudiesManager @@ -149,7 +150,7 @@ private StudyDefinition upsertStudy(StudyDefinition sd, if (bve.hasErrors()) throw bve; - sd.setRowId((Integer) ret.get(0).get("rowId")); + sd.setRowId(asInteger(ret.get(0).get("rowId"))); return sd; } @@ -208,7 +209,7 @@ private void upsertChildRecords(int studyRowId, { List> ret = qus.insertRows(u, c, inserts, bve, null, null); for (int i = 0; i < ret.size(); i++) - setRowId.set(insertBeans.get(i), (Integer) ret.get(i).get("rowId")); + setRowId.set(insertBeans.get(i), asInteger(ret.get(i).get("rowId"))); } if (!updates.isEmpty()) diff --git a/blast/src/org/labkey/blast/model/BlastJob.java b/blast/src/org/labkey/blast/model/BlastJob.java index fd1e6923d..f6bbaf392 100644 --- a/blast/src/org/labkey/blast/model/BlastJob.java +++ b/blast/src/org/labkey/blast/model/BlastJob.java @@ -61,7 +61,7 @@ public class BlastJob implements Serializable private int _modifiedBy; private Date _modified; private String _jobId; - private Integer _htmlFile; + private Long _htmlFile; private File _outputDir = null; public BlastJob() @@ -231,12 +231,12 @@ public void setJobId(String jobId) _jobId = jobId; } - public Integer getHtmlFile() + public Long getHtmlFile() { return _htmlFile; } - public void setHtmlFile(Integer htmlFile) + public void setHtmlFile(Long htmlFile) { _htmlFile = htmlFile; } @@ -341,7 +341,7 @@ public boolean hasError(User u) return false; } - Integer jobId = PipelineService.get().getJobId(u, ContainerManager.getForId(getContainer()), getJobId()); + Long jobId = PipelineService.get().getJobId(u, ContainerManager.getForId(getContainer()), getJobId()); PipelineStatusFile statusFile = PipelineService.get().getStatusFile(jobId); return "ERROR".equalsIgnoreCase(statusFile.getStatus()); } diff --git a/cluster/src/org/labkey/cluster/pipeline/ClusterJob.java b/cluster/src/org/labkey/cluster/pipeline/ClusterJob.java index afe3c5024..bfc23618d 100644 --- a/cluster/src/org/labkey/cluster/pipeline/ClusterJob.java +++ b/cluster/src/org/labkey/cluster/pipeline/ClusterJob.java @@ -9,7 +9,7 @@ public class ClusterJob { private int _rowId; private String _jobId; - private int _statusFileId; + private long _statusFileId; private String _status; private Boolean _hasStarted; private String _clusterId; @@ -169,12 +169,12 @@ public void setClusterUser(String clusterUser) _clusterUser = clusterUser; } - public int getStatusFileId() + public long getStatusFileId() { return _statusFileId; } - public void setStatusFileId(int statusFileId) + public void setStatusFileId(long statusFileId) { _statusFileId = statusFileId; } diff --git a/jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseSessionTask.java b/jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseSessionTask.java index a4a7b7134..dd8b9b0f6 100644 --- a/jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseSessionTask.java +++ b/jbrowse/src/org/labkey/jbrowse/pipeline/JBrowseSessionTask.java @@ -44,6 +44,8 @@ import java.util.List; import java.util.Map; +import static org.labkey.api.exp.api.ExperimentService.asInteger; + /** * User: bbimber * Date: 8/6/12 @@ -254,7 +256,7 @@ private void createOrAddToSession() throws PipelineJobException trackRecord.put("modified", new Date()); trackRecord.put("modifiedby", getJob().getUser().getUserId()); trackRecord = Table.insert(getJob().getUser(), databaseMembers, trackRecord); - databaseMemberRecordsCreated.add((Integer)trackRecord.get("rowid")); + databaseMemberRecordsCreated.add(asInteger(trackRecord.get("rowid"))); json.prepareResource(getJob().getUser(), getJob().getLogger(), false, false); } @@ -305,7 +307,7 @@ private void createOrAddToSession() throws PipelineJobException trackRecord.put("modified", new Date()); trackRecord.put("modifiedby", getJob().getUser().getUserId()); trackRecord = Table.insert(getJob().getUser(), databaseMembers, trackRecord); - databaseMemberRecordsCreated.add((Integer) trackRecord.get("rowid")); + databaseMemberRecordsCreated.add(asInteger(trackRecord.get("rowid"))); json.prepareResource(getJob().getUser(), getJob().getLogger(), false, false); } diff --git a/singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java b/singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java index 95a1fcf2d..680ff3613 100644 --- a/singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java +++ b/singlecell/api-src/org/labkey/api/singlecell/CellHashingService.java @@ -52,7 +52,7 @@ static public void setInstance(CellHashingService instance) abstract public File generateHashingCallsForRawMatrix(Readset parentReadset, PipelineOutputTracker output, SequenceOutputHandler.JobContext ctx, CellHashingParameters parameters, File rawCountMatrixDir) throws PipelineJobException; - abstract public File getH5FileForGexReadset(SequenceAnalysisJobSupport support, int readsetId, int genomeId) throws PipelineJobException; + abstract public File getH5FileForGexReadset(SequenceAnalysisJobSupport support, long readsetId, int genomeId) throws PipelineJobException; abstract public File getCDNAInfoFile(File sourceDir); @@ -76,7 +76,7 @@ static public void setInstance(CellHashingService instance) abstract public List getHashingCallingParams(boolean allowMethodsNeedingGex); - abstract public Set getHtosForParentReadset(Integer parentReadsetId, File webserverJobDir, SequenceAnalysisJobSupport support, boolean throwIfNotFound) throws PipelineJobException; + abstract public Set getHtosForParentReadset(Long parentReadsetId, File webserverJobDir, SequenceAnalysisJobSupport support, boolean throwIfNotFound) throws PipelineJobException; abstract public File getExistingFeatureBarcodeCountDir(Readset parentReadset, BARCODE_TYPE type, SequenceAnalysisJobSupport support) throws PipelineJobException; @@ -232,7 +232,7 @@ public static CellHashingParameters createFromJson(BARCODE_TYPE type, File webse return Collections.emptyList(); } - public int getEffectiveReadsetId() + public long getEffectiveReadsetId() { return parentReadset != null ? parentReadset.getReadsetId() : htoReadset.getReadsetId(); } diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java index 6bfa58366..b2adf25a0 100644 --- a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java +++ b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java @@ -128,7 +128,7 @@ public Output execute(SequenceOutputHandler.JobContext ctx, List gexReadsetToH5Map = new HashMap<>(); - HashMap readsetToHashingMap = new HashMap<>(); - HashMap readsetToCiteSeqMap = new HashMap<>(); - HashMap readsetToGexMap = new HashMap<>(); - HashMap> gexToPanels = new HashMap<>(); + HashMap readsetToHashingMap = new LongHashMap<>(); + HashMap readsetToCiteSeqMap = new LongHashMap<>(); + HashMap readsetToGexMap = new LongHashMap<>(); + HashMap> gexToPanels = new LongHashMap<>(); List cachedReadsets = support.getCachedReadsets(); job.getLogger().debug("Total cached readsets: " + cachedReadsets.size() + ", using filter on: " + filterField); @@ -279,7 +281,7 @@ public void prepareHashingAndCiteSeqFilesIfNeeded(File sourceDir, PipelineJob jo HashMap readsetToCountMap = new HashMap<>(); if (distinctHTOs.size() > 1) { - Set hashingToRemove = new HashSet<>(); + Set hashingToRemove = new HashSet<>(); readsetToHashingMap.forEach((readsetId, hashingReadsetId) -> { if (cacheCountMatrixFiles) { @@ -342,7 +344,7 @@ else if (distinctHTOs.size() == 1) job.getLogger().info("distinct HTOs: " + distinctHTOs.size()); } - Set citeToRemove = new HashSet<>(); + Set citeToRemove = new HashSet<>(); readsetToCiteSeqMap.forEach((readsetId, citeseqReadsetId) -> { if (cacheCountMatrixFiles) { @@ -433,7 +435,7 @@ else if (distinctHTOs.size() == 1) } } - private Map cacheH5Files(PipelineJob job, SequenceAnalysisJobSupport support, Collection uniqueGex, Map readsetToGexMap) throws PipelineJobException + private Map cacheH5Files(PipelineJob job, SequenceAnalysisJobSupport support, Collection uniqueGex, Map readsetToGexMap) throws PipelineJobException { job.getLogger().debug("Caching H5 Files"); Map gexReadsetToH5Map = new HashMap<>(); @@ -441,7 +443,7 @@ private Map cacheH5Files(PipelineJob job, SequenceAnalysisJobSu TableInfo ti = QueryService.get().getUserSchema(job.getUser(), job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(), SingleCellSchema.SEQUENCE_SCHEMA_NAME).getTable("outputfiles"); Set cachedGenomes = support.getCachedGenomes().stream().map(ReferenceGenome::getGenomeId).collect(Collectors.toSet()); - for (int readsetId : readsetToGexMap.keySet()) + for (long readsetId : readsetToGexMap.keySet()) { boolean isGEX = uniqueGex.contains(readsetId); int gexReadset = readsetToGexMap.get(readsetId); @@ -514,17 +516,17 @@ else if (genomeIds.size() > 1) return gexReadsetToH5Map; } - public File getValidCiteSeqBarcodeFile(File sourceDir, int gexReadsetId) + public File getValidCiteSeqBarcodeFile(File sourceDir, long gexReadsetId) { return new File(sourceDir, "validADTS." + gexReadsetId + ".csv"); } - public File getValidCiteSeqBarcodeMetadataFile(File sourceDir, int gexReadsetId) + public File getValidCiteSeqBarcodeMetadataFile(File sourceDir, long gexReadsetId) { return new File(sourceDir, "validADTS." + gexReadsetId + ".metadata.txt"); } - private void writeCiteSeqBarcodes(PipelineJob job, Map> gexToPanels, File outputDir) throws PipelineJobException + private void writeCiteSeqBarcodes(PipelineJob job, Map> gexToPanels, File outputDir) throws PipelineJobException { Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); UserSchema schema = QueryService.get().getUserSchema(job.getUser(), target, SingleCellSchema.NAME); @@ -539,7 +541,7 @@ private void writeCiteSeqBarcodes(PipelineJob job, Map> gex FieldKey.fromString("antibody/barcodePattern") )); - for (int gexReadsetId : gexToPanels.keySet()) + for (long gexReadsetId : gexToPanels.keySet()) { job.getLogger().info("Writing all unique ADTs for readset: " + gexReadsetId); File barcodeOutput = getValidCiteSeqBarcodeFile(outputDir, gexReadsetId); @@ -595,7 +597,7 @@ public File generateHashingCallsForRawMatrix(Readset parentReadset, PipelineOutp } parameters.validate(true); - Map readsetToHashing = getCachedHashingReadsetMap(ctx.getSequenceSupport()); + Map readsetToHashing = getCachedHashingReadsetMap(ctx.getSequenceSupport()); if (readsetToHashing.isEmpty()) { ctx.getLogger().info("No cached " + parameters.type.name() + " readsets, skipping"); @@ -721,7 +723,7 @@ public File generateHashingCallsForRawMatrix(Readset parentReadset, PipelineOutp return callsFile; } - private Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException + private Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException { return support.getCachedObject(READSET_TO_CITESEQ_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); } @@ -729,7 +731,7 @@ private Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupp @Override public boolean usesCellHashing(SequenceAnalysisJobSupport support, File sourceDir) throws PipelineJobException { - Map gexToHashingMap = getCachedHashingReadsetMap(support); + Map gexToHashingMap = getCachedHashingReadsetMap(support); if (gexToHashingMap == null || gexToHashingMap.isEmpty()) return false; @@ -745,7 +747,7 @@ public boolean usesCellHashing(SequenceAnalysisJobSupport support, File sourceDi @Override public boolean usesCiteSeq(SequenceAnalysisJobSupport support, List inputFiles) throws PipelineJobException { - Map gexToCiteMap = getCachedCiteSeqReadsetMap(support); + Map gexToCiteMap = getCachedCiteSeqReadsetMap(support); if (gexToCiteMap == null || gexToCiteMap.isEmpty()) return false; @@ -761,7 +763,7 @@ public boolean usesCiteSeq(SequenceAnalysisJobSupport support, List map = support.getCachedObject(READSET_AND_GENOME_TO_H5_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, String.class, Integer.class)); String key = readsetId + "-" + genomeId; @@ -797,12 +799,12 @@ public File getCDNAInfoFile(File sourceDir) return new File(sourceDir, "cDNAInfo.txt"); } - public Map getCachedHashingReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException + public Map getCachedHashingReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException { return support.getCachedObject(READSET_TO_HASHING_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); } - public File getCachedReadsetToCountMatrix(SequenceAnalysisJobSupport support, int readsetId, CellHashingService.BARCODE_TYPE type) throws PipelineJobException + public File getCachedReadsetToCountMatrix(SequenceAnalysisJobSupport support, long readsetId, CellHashingService.BARCODE_TYPE type) throws PipelineJobException { Map map = support.getCachedObject(READSET_TO_COUNTS_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, String.class, File.class)); String key = type.name() + "-" + readsetId; @@ -1367,7 +1369,7 @@ public File generateCellHashingCalls(File citeSeqCountOutDir, File outputDir, St @Override public File getExistingFeatureBarcodeCountDir(Readset parentReadset, BARCODE_TYPE type, SequenceAnalysisJobSupport support) throws PipelineJobException { - Integer childId = type == BARCODE_TYPE.hashing ? getCachedHashingReadsetMap(support).get(parentReadset.getReadsetId()) : getCachedCiteSeqReadsetMap(support).get(parentReadset.getReadsetId()); + Long childId = type == BARCODE_TYPE.hashing ? getCachedHashingReadsetMap(support).get(parentReadset.getReadsetId()) : getCachedCiteSeqReadsetMap(support).get(parentReadset.getReadsetId()); if (childId == null) { throw new PipelineJobException("Unable to find cached readset of type " + type.name() + " for parent: " + parentReadset.getReadsetId()); @@ -1416,9 +1418,9 @@ public void copyHtmlLocally(SequenceOutputHandler.JobContext ctx) throws Pipelin } @Override - public Set getHtosForParentReadset(Integer parentReadsetId, File webserverJobDir, SequenceAnalysisJobSupport support, boolean throwIfNotFound) throws PipelineJobException + public Set getHtosForParentReadset(Long parentReadsetId, File webserverJobDir, SequenceAnalysisJobSupport support, boolean throwIfNotFound) throws PipelineJobException { - Integer htoReadset = getCachedHashingReadsetMap(support).get(parentReadsetId); + Long htoReadset = getCachedHashingReadsetMap(support).get(parentReadsetId); if (htoReadset == null) { if (throwIfNotFound) @@ -1434,7 +1436,7 @@ public Set getHtosForParentReadset(Integer parentReadsetId, File webserv return getHtosForReadset(htoReadset, webserverJobDir); } - public Set getHtosForReadset(Integer hashingReadsetId, File webserverJobDir) throws PipelineJobException + public Set getHtosForReadset(Long hashingReadsetId, File webserverJobDir) throws PipelineJobException { Set htosPerReadset = new HashSet<>(); try (CSVReader reader = new CSVReader(Readers.getReader(CellHashingServiceImpl.get().getCDNAInfoFile(webserverJobDir)), '\t')) diff --git a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java index 4f0b32f92..45ce396c2 100644 --- a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java +++ b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java @@ -373,7 +373,7 @@ public void processFilesRemote(List inputFiles, JobContext c String basename; if (inputFiles.size() == 1 && inputFiles.get(0).getReadset() != null) { - Integer readsetId = inputFiles.get(0).getReadset(); + Long readsetId = inputFiles.get(0).getReadset(); Readset rs = ctx.getSequenceSupport().getCachedReadset(readsetId); basename = FileUtil.makeLegalName(rs.getName()).replaceAll(" ", "_"); } diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendCiteSeq.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendCiteSeq.java index 41c92ed63..2e04afa78 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendCiteSeq.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendCiteSeq.java @@ -2,6 +2,7 @@ import org.apache.commons.io.FileUtils; import org.json.JSONObject; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.Readset; @@ -21,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -88,7 +88,7 @@ public String getDockerContainerName() @Override protected Map prepareCountData(SingleCellOutput output, SequenceOutputHandler.JobContext ctx, List inputObjects, String outputPrefix) throws PipelineJobException { - Map dataIdToCalls = new HashMap<>(); + Map dataIdToCalls = new IntHashMap<>(); boolean dropAggregateBarcodes = getProvider().getParameterByName("dropAggregateBarcodes").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); for (SeuratObjectWrapper wrapper : inputObjects) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendSaturation.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendSaturation.java index 3b1a1ebec..389c37fbc 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendSaturation.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendSaturation.java @@ -4,6 +4,8 @@ import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.IntHashSet; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; @@ -25,8 +27,6 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -59,7 +59,7 @@ public void init(SequenceOutputHandler.JobContext ctx, List { try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(getMolInfoTable(ctx)))) { - Map loupeOutputs = new HashMap<>(); + Map loupeOutputs = new IntHashMap<>(); for (SequenceOutputFile so : inputFiles) { if (!LOUPE_TYPE.isType(so.getFile())) @@ -75,7 +75,7 @@ public void init(SequenceOutputHandler.JobContext ctx, List throw new PipelineJobException("Cannot find expected metadata file: " + meta.getPath()); } - Set uniqueIds = new HashSet<>(); + Set uniqueIds = new IntHashSet(); try (CSVReader reader = new CSVReader(Readers.getReader(meta), '_')) { String[] line; @@ -200,7 +200,7 @@ public String getFileSuffix() private void findAdditionalData(SequenceOutputFile loupeFile, CSVWriter writer, PipelineJob job) throws IOException, PipelineJobException { - Set citeReadsets = new HashSet<>(); + Set citeReadsets = new IntHashSet(); Container targetContainer = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); TableSelector ts = new TableSelector(QueryService.get().getUserSchema(job.getUser(), targetContainer, SingleCellSchema.NAME).getTable(SingleCellSchema.TABLE_CDNAS), PageFlowUtil.set("hashingReadsetId", "citeseqReadsetId"), new SimpleFilter(FieldKey.fromString("readsetId"), loupeFile.getReadset()), null); ts.forEachResults(rs -> { diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCellHashing.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCellHashing.java index 1adf350d0..ca91af408 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCellHashing.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/RunCellHashing.java @@ -2,6 +2,7 @@ import au.com.bytecode.opencsv.CSVReader; import org.apache.commons.io.FileUtils; +import org.labkey.api.collections.IntHashMap; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceOutputFile; @@ -25,7 +26,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -80,7 +80,7 @@ protected void onFailure(SequenceOutputHandler.JobContext ctx, String outputPref @Override protected Map prepareCountData(SingleCellOutput output, SequenceOutputHandler.JobContext ctx, List inputObjects, String outputPrefix) throws PipelineJobException { - Map dataIdToCalls = new HashMap<>(); + Map dataIdToCalls = new IntHashMap<>(); for (SeuratObjectWrapper wrapper : inputObjects) { diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerFeatureBarcodeHandler.java b/singlecell/src/org/labkey/singlecell/run/CellRangerFeatureBarcodeHandler.java index b03b7faa0..5f875c4f2 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerFeatureBarcodeHandler.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerFeatureBarcodeHandler.java @@ -194,7 +194,7 @@ public void processFilesRemote(List readsets, JobContext ctx) throws Un action.addInputIfNotPresent(rd.getFile2(), "Input FASTQ"); }); - Integer gexReadsetId = ctx.getSequenceSupport().getCachedObject(FEATURE_TO_GEX, Integer.class); + Long gexReadsetId = ctx.getSequenceSupport().getCachedObject(FEATURE_TO_GEX, Long.class); if (gexReadsetId != null) { ctx.getLogger().info("Adding GEX FASTQs"); diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java index 62ea31c90..2bfb601e4 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerGexCountStep.java @@ -489,7 +489,7 @@ public void complete(SequenceAnalysisJobSupport support, AnalysisModel model, Co } File outsDir = outputForData.getFile().getParentFile(); - Integer dataId = outputForData.getDataId(); + Long dataId = outputForData.getDataId(); if (dataId == null) { throw new PipelineJobException("Unable to find dataId for output file"); diff --git a/singlecell/src/org/labkey/singlecell/run/CellRangerVDJWrapper.java b/singlecell/src/org/labkey/singlecell/run/CellRangerVDJWrapper.java index c78b87d8c..e3ebe6dbe 100644 --- a/singlecell/src/org/labkey/singlecell/run/CellRangerVDJWrapper.java +++ b/singlecell/src/org/labkey/singlecell/run/CellRangerVDJWrapper.java @@ -696,7 +696,7 @@ public void deleteSymlinks(File localFqDir) throws PipelineJobException } } - public void addMetrics(File outDir, AnalysisModel model, int dataId) throws PipelineJobException + public void addMetrics(File outDir, AnalysisModel model, long dataId) throws PipelineJobException { getPipelineCtx().getLogger().debug("adding 10x metrics"); @@ -810,7 +810,7 @@ public void complete(SequenceAnalysisJobSupport support, AnalysisModel model, Co folderWithMetricFile = outputForData.getFile().getParentFile().getParentFile(); } - Integer dataId = outputForData.getDataId(); + Long dataId = outputForData.getDataId(); if (dataId == null) { throw new PipelineJobException("Unable to find dataId for output file");