diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index e5a788736..3dd604ab7 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -33,6 +33,7 @@ usage_future date_of_observations colony + source dam sire objectid diff --git a/mcc/resources/queries/study/demographicsLittermates.sql b/mcc/resources/queries/study/demographicsLittermates.sql index d2edb072b..340fe7d3f 100644 --- a/mcc/resources/queries/study/demographicsLittermates.sql +++ b/mcc/resources/queries/study/demographicsLittermates.sql @@ -3,8 +3,8 @@ SELECT d1.Id, d1.litterId, - (SELECT GROUP_CONCAT(distinct d2.Id, ',') as litterMates FROM study.Demographics d2 WHERE d2.qcstate.publicdata = true AND d1.litterId = d2.litterId AND d1.id != d2.id) as litterMates + (SELECT GROUP_CONCAT(distinct d2.Id, ',') as litterMates FROM study.Demographics d2 WHERE d2.qcstate.publicdata = true AND d2.litterId IS NOT NULL AND d1.litterId = d2.litterId AND d1.id != d2.id) as litterMates FROM study.Demographics d1 -WHERE d1.qcstate.publicdata = true +WHERE d1.qcstate.publicdata = true AND d1.litterId IS NOT NULL diff --git a/mcc/resources/queries/study/demographicsMccTransfer.query.xml b/mcc/resources/queries/study/demographicsMccTransfer.query.xml new file mode 100644 index 000000000..12904afb7 --- /dev/null +++ b/mcc/resources/queries/study/demographicsMccTransfer.query.xml @@ -0,0 +1,29 @@ + + + + + MCC Transfers + Summary of MCC Transfers + + + true + true + + + MCC Request ID + + + PI Last Name + + + PI First Name + + + PI Institution + + + mccRequestId +
+ + + diff --git a/mcc/resources/queries/study/demographicsMccTransfer.sql b/mcc/resources/queries/study/demographicsMccTransfer.sql new file mode 100644 index 000000000..99e1776b6 --- /dev/null +++ b/mcc/resources/queries/study/demographicsMccTransfer.sql @@ -0,0 +1,9 @@ +SELECT + d.Id, + GROUP_CONCAT(DISTINCT d.mccRequestId.rowId, ', ') as mccRequestId, + GROUP_CONCAT(DISTINCT d.mccRequestId.lastName, char(10)) as piLastName, + GROUP_CONCAT(DISTINCT d.mccRequestId.firstName, char(10)) as piFirstName, + GROUP_CONCAT(DISTINCT d.mccRequestId.institutionname, char(10)) as piInstitution +FROM study.departure d +WHERE d.qcstate.publicdata = true AND d.mccRequestId IS NOT NULL +GROUP BY d.Id \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/etl/ProjectAssignmentTransform.java b/mcc/src/org/labkey/mcc/etl/ProjectAssignmentTransform.java index d9cd9a044..a0c73faec 100644 --- a/mcc/src/org/labkey/mcc/etl/ProjectAssignmentTransform.java +++ b/mcc/src/org/labkey/mcc/etl/ProjectAssignmentTransform.java @@ -28,6 +28,7 @@ public class ProjectAssignmentTransform extends ColumnTransform private static final String U24_BREEDER = "U24 breeder"; private static final String U24_OFFSPRING = "U24 offspring"; private static final String U24_OTHER = "Other"; + private static final String U24_INACTIVE = "U24 marmoset breeding- INACTIVE -"; @Override protected Object doTransform(Object inputValue) @@ -42,6 +43,10 @@ protected Object doTransform(Object inputValue) { return getOrCreateFlag(U24_TITLE); } + else if (U24_INACTIVE.equalsIgnoreCase(String.valueOf(inputValue))) + { + return getOrCreateFlag(U24_INACTIVE); + } // SNPRC: else if (U24_BREEDER.equalsIgnoreCase(String.valueOf(inputValue))) { @@ -51,8 +56,11 @@ else if (U24_OFFSPRING.equalsIgnoreCase(String.valueOf(inputValue))) { return getOrCreateFlag(U24_OFFSPRING); } - - return inputValue; + else + { + _log.error("Unknown flag value: " + inputValue + ", in folder: " + getContainerUser().getContainer().getPath()); + return null; + } } private String getOrCreateFlag(String type) diff --git a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java index e917e4ac7..f042eb904 100644 --- a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java @@ -16,10 +16,11 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.LookupForeignKey; import org.labkey.api.query.QueryForeignKey; -import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; import org.labkey.mcc.MccManager; import org.labkey.mcc.MccSchema; +import org.labkey.mcc.security.MccRequestAdminPermission; public class MccEhrCustomizer extends AbstractTableCustomizer { @@ -141,6 +142,39 @@ private void customizeAnimalTable(AbstractTableInfo ti) col.setDescription("Summary of genomic data"); ti.addColumn(col); } + + // Only allow this for authorized users: + possiblyAddRequestSummary(ti); + } + + private void possiblyAddRequestSummary(AbstractTableInfo ti) + { + if (ti.getColumn("mccTransfers") != null) + { + return; + } + + Container animalDataContainer = MccManager.get().getMCCContainer(ti.getUserSchema().getContainer()); + if (animalDataContainer == null) + { + return; + } + + if (!animalDataContainer.equals(ti.getUserSchema().getContainer())) + { + return; + } + + User u = ti.getUserSchema().getUser(); + if (!animalDataContainer.hasPermission(u, MccRequestAdminPermission.class)) + { + return; + } + + var col = getWrappedIdCol(ti.getUserSchema(), ti, "mccTransfers", "demographicsMccTransfer"); + col.setLabel("MCC Transfers"); + col.setDescription("Summarizes MCC Transfer Information"); + ti.addColumn(col); } private BaseColumnInfo getWrappedIdCol(UserSchema us, AbstractTableInfo ds, String name, String queryName) diff --git a/tcrdb/build.gradle b/tcrdb/build.gradle index f22702e73..643cccb99 100644 --- a/tcrdb/build.gradle +++ b/tcrdb/build.gradle @@ -5,38 +5,12 @@ repositories { mavenCentral() } -configurations.all { - resolutionStrategy { - // Related to: https://nvd.nist.gov/vuln/detail/CVE-2025-12183 - dependencySubstitution { - substitute module('org.lz4:lz4-java') using module("at.yawk.lz4:lz4-java:${lz4Version}") - } - } -} - dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "assay"), depProjectConfig: "apiJarFile") - BuildUtils.addExternalDependency( - project, - new ExternalDependency( - "io.repseq:repseqio:${repseqVersion}", - "repseqio", - "repseqio", - "https://github.com/repseqio/repseqio", - ExternalDependency.APACHE_2_LICENSE_NAME, - ExternalDependency.APACHE_2_LICENSE_URL, - "TCR Analysis" - ), - { - // exclude logback to prevent excessive logging - exclude group: "ch.qos.logback", module :"logback-classic" - exclude group: "ch.qos.logback", module :"logback-core" - } - ) implementation "com.github.samtools:htsjdk:${htsjdkVersion}" implementation "net.sf.opencsv:opencsv:${opencsvVersion}" diff --git a/tcrdb/gradle.properties b/tcrdb/gradle.properties index 0e8bb0888..e69de29bb 100644 --- a/tcrdb/gradle.properties +++ b/tcrdb/gradle.properties @@ -1,2 +0,0 @@ -lz4Version=1.10.1 -repseqVersion=1.7.0 \ No newline at end of file diff --git a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml index 20a2fb77e..df07a7e19 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml @@ -142,9 +142,6 @@ TCRdb.window.ExportDataWindow.viewAlignmentHandler(dataRegionName, arguments[0] ? arguments[0].ownerCt : null); - - TCRdb.window.DownloadCloneWindow.buttonHandler(dataRegionName); - TCRdb.window.DownloadCloneWindow.downloadSequenceHandler(dataRegionName); diff --git a/tcrdb/resources/web/tcrdb/window/DownloadCloneWindow.js b/tcrdb/resources/web/tcrdb/window/DownloadCloneWindow.js index f4a1fa3ba..cb5c356e6 100644 --- a/tcrdb/resources/web/tcrdb/window/DownloadCloneWindow.js +++ b/tcrdb/resources/web/tcrdb/window/DownloadCloneWindow.js @@ -2,10 +2,6 @@ Ext4.define('TCRdb.window.DownloadCloneWindow', { extend: 'Ext.window.Window', statics: { - buttonHandler: function(dataRegionName) { - TCRdb.window.DownloadCloneWindow.showWindow(dataRegionName, 'downloadCloneMaterials', 'The goal of this is to download a ZIP with any extracted clone/read data for the selected sample(s), along with the reference sequence for the segments used. These files can be imported into Geneious or a similar program to de novo assemble to construct the FL clone. Note: per sample, it will export all reads overlapping any TCR segments. This at minimum will tend to include both chains (i.e. 2 different CDR3s), and might include reads that either match a defunct TCR or other noise.'); - }, - downloadSequenceHandler: function(dataRegionName){ TCRdb.window.DownloadCloneWindow.showWindow(dataRegionName, 'downloadSequence', 'This will download the full sequence (if available) for the selected rows, along with the reference segments.'); }, diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java index f9c46e2ae..d73939f01 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java @@ -16,78 +16,43 @@ package org.labkey.tcrdb; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.ExportAction; -import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.IntHashMap; -import org.labkey.api.collections.IntHashSet; -import org.labkey.api.collections.StringHashMap; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryAction; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; -import org.labkey.api.reader.Readers; import org.labkey.api.security.IgnoresTermsOfUse; import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.sequenceanalysis.RefNtSequenceModel; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.URLHelper; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.SpringErrorView; -import org.labkey.tcrdb.pipeline.MiXCRWrapper; import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; -import jakarta.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.StringWriter; -import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; public class TCRdbController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(TCRdbController.class); public static final String NAME = "tcrdb"; - private static final FileType FASTA = new FileType("fasta"); - private static final Logger _log = LogManager.getLogger(TCRdbController.class); public TCRdbController() @@ -95,195 +60,6 @@ public TCRdbController() setActionResolver(_actionResolver); } - //TODO: based on set of IDs, run exportAlignmentsPretty - - @RequiresPermission(ReadPermission.class) - public class ExportAlignmentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ExportAlignmentsForm form, BindException errors) throws Exception - { - if (form.getAssayRowIds() == null || form.getAssayRowIds().length == 0) - { - errors.reject(ERROR_MSG, "Must provide IDs to display"); - return new SpringErrorView(errors); - } - - if (StringUtils.isEmpty(form.getSchemaName())) - { - errors.reject(ERROR_MSG, "Must provide the assay schema name"); - return new SpringErrorView(errors); - } - - //find rows - UserSchema us = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); - if (us == null) - { - errors.reject(ERROR_MSG, "Unknown schema: " + form.getSchemaName()); - return new SpringErrorView(errors); - } - - TableInfo ti = us.getTable("data"); - final Map> VDJMap = new HashMap<>(); - List rowIds = new ArrayList<>(Arrays.asList(form.getAssayRowIds())); - - TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowid"), rowIds, CompareType.IN), null); - final StringWriter writer = new StringWriter(); - - ts.forEach(AssayRecord.class, r -> { - if (r.getVdjFile() == null) - { - writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n"); - return; - } - - ExpData d = ExperimentService.get().getExpData(r.getVdjFile()); - if (d == null) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + PageFlowUtil.filter(r.getVdjFile()) + "\n"); - return; - } - - if (!d.getFile().exists()) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + PageFlowUtil.filter(d.getFile().getPath()) + "\n"); - return; - } - - if (!VDJMap.containsKey(d.getFile())) - { - VDJMap.put(d.getFile(), new ArrayList<>()); - } - - VDJMap.get(d.getFile()).add(r); - }); - - if (VDJMap.isEmpty()) - { - errors.reject(ERROR_MSG, "No matching rows found for IDs: " + StringUtils.join(rowIds, ",")); - return new SpringErrorView(errors); - } - - SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd"); - for (File f : VDJMap.keySet()) - { - File tmp = FileUtil.createTempFile("mixcr", ".txt"); - MiXCRWrapper wrapper = new MiXCRWrapper(_log); - //TODO: - //wrapper.setLibraryPath(); - List args = new ArrayList<>(); - if (StringUtils.trimToNull(form.getCdr3Equals()) != null) - { - args.add("--cdr3-equals"); - args.add(form.getCdr3Equals()); - } - - if (StringUtils.trimToNull(form.getReadContains()) != null) - { - args.add("--read-contains"); - args.add(form.getReadContains()); - } - - try - { - wrapper.doExportAlignmentsPretty(f, tmp, args); - - writer.write("File: " + PageFlowUtil.filter(f.getName()) + '\n'); - writer.write("Result Rows From This File: " + '\n'); - for (AssayRecord r : VDJMap.get(f)) - { - writer.write("Sample: " + PageFlowUtil.filter(r.getSampleName()) + '\n'); - writer.write("Sample Date: " + PageFlowUtil.filter(r.getDate() == null ? "" : fmt.format(r.getDate())) + '\n'); - writer.write("CDR3: " + PageFlowUtil.filter(coalesce(r.getCDR3())) + '\n'); - writer.write("vHit: " + PageFlowUtil.filter(coalesce(r.getvHit())) + '\n'); - writer.write("dHit: " + PageFlowUtil.filter(coalesce(r.getdHit())) + '\n'); - writer.write("jHit: " + PageFlowUtil.filter(coalesce(r.getjHit())) + '\n'); - writer.write("cHit: " + PageFlowUtil.filter(coalesce(r.getcHit())) + '\n'); - writer.write("Read Count: " + PageFlowUtil.filter(coalesce(r.getCount())) + '\n'); - writer.write("Fraction: " + PageFlowUtil.filter(coalesce(r.getFraction())) + '\n'); - writer.write("Comments: " + PageFlowUtil.filter(coalesce(r.getComment())) + '\n'); - writer.write('\n'); - } - writer.write('\n'); - try (BufferedReader reader = Readers.getReader(new FileInputStream(tmp))) - { - String line; - boolean inAlignmentBlock = false; - while ((line = reader.readLine()) != null) - { - line = PageFlowUtil.filter(line); - - String trimmed = StringUtils.trimToEmpty(line); - if (StringUtils.isEmpty(trimmed)) - { - inAlignmentBlock = false; - } - - //Highlight mismatches - if (inAlignmentBlock) - { - String[] tokens = trimmed.split("( )+"); - String alignmentRaw = tokens[2]; - StringBuilder sb = new StringBuilder(); - for (int i=0; i"); - sb.append(c); - sb.append(""); - } - else - { - sb.append(PageFlowUtil.filter(c)); - } - } - - line = line.replaceAll(alignmentRaw, sb.toString()); - } - - writer.write(line + '\n'); - - if (trimmed.startsWith("Target")) - { - inAlignmentBlock = true; - } - } - } - - writer.write('\n'); - writer.write("
"); - writer.write('\n'); - writer.write('\n'); - - tmp.delete(); - } - catch (PipelineJobException e){ - writer.write("Unable to run export alignments\n"); - - _log.error("Unable to run exportAlignments:\n" + StringUtils.join(wrapper.getCommandsExecuted(), "\n"), e); - } - } - - //mixcr exportReadsForClones index_file alignments.vdjca.gz 0 1 2 33 54 reads.fastq.gz - //mixcr exportAlignmentsPretty input.vdjca test.txt - - return new HtmlView("MiXCR Alignments", HtmlString.unsafe("
" + writer + "
")); - } - - private String coalesce(Object s) - { - return s == null ? "None" : s.toString(); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("TCR Data Export"); //necessary to set page title, it seems - } - } - public static class AssayRecord { private Integer _rowId; @@ -453,54 +229,6 @@ public void setFraction(Double fraction) } } - public static class ExportAlignmentsForm - { - private Integer[] _assayRowIds; - private String schemaName; - private String _cdr3Equals; - private String _readContains; - - public Integer[] getAssayRowIds() - { - return _assayRowIds; - } - - public void setAssayRowIds(Integer[] assayRowIds) - { - _assayRowIds = assayRowIds; - } - - public String getSchemaName() - { - return schemaName; - } - - public void setSchemaName(String schemaName) - { - this.schemaName = schemaName; - } - - public String getCdr3Equals() - { - return _cdr3Equals; - } - - public void setCdr3Equals(String cdr3Equals) - { - _cdr3Equals = cdr3Equals; - } - - public String getReadContains() - { - return _readContains; - } - - public void setReadContains(String readContains) - { - _readContains = readContains; - } - } - @RequiresPermission(ReadPermission.class) @IgnoresTermsOfUse public static class DownloadSequenceAction extends ExportAction @@ -634,7 +362,7 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response } PageFlowUtil.prepareResponseForFile(response, Collections.emptyMap(), "TCR_Data.fasta", true); - if (fasta.length() == 0) + if (fasta.isEmpty()) { response.getOutputStream().write("No data found".getBytes(StringUtilsLabKey.DEFAULT_CHARSET)); } @@ -645,300 +373,6 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response } } - @RequiresPermission(ReadPermission.class) - @IgnoresTermsOfUse - public static class DownloadCloneMaterials extends ExportAction - { - @Override - public void export(DownloadCloneMaterialsForm form, HttpServletResponse response, BindException errors) throws Exception - { - Container target = getContainer().isWorkbook() ? getContainer().getParent() : getContainer(); - UserSchema us = QueryService.get().getUserSchema(getUser(), target, form.getSchemaName()); - if (us == null) - { - errors.reject(ERROR_MSG, "Unable to find schema: " + form.getSchemaName()); - return; - } - - TableInfo assayData = us.getTable(form.getQueryName()); - if (assayData == null) - { - errors.reject(ERROR_MSG, "Unable to find table: " + form.getQueryName()); - return; - } - - List rowIds = Arrays.asList(form.getRowId()); - if (rowIds.isEmpty()) - { - errors.reject(ERROR_MSG, "No rows provided"); - return; - } - - // find distinct analyses for assay rows and primary segments - SimpleFilter assayFilter = new SimpleFilter(FieldKey.fromString("rowId"), rowIds, CompareType.IN); - Map cols = QueryService.get().getColumns(assayData, PageFlowUtil.set( - FieldKey.fromString("analysisId"), - FieldKey.fromString("samplename"), - FieldKey.fromString("sequence"), - FieldKey.fromString("cdr3"), - FieldKey.fromString("vHit"), - FieldKey.fromString("jHit"), - FieldKey.fromString("dHit"), - FieldKey.fromString("cHit"), - FieldKey.fromString("cloneId"), - FieldKey.fromString("sequence"), - FieldKey.fromString("clonesFile"), - FieldKey.fromString("libraryId/libraryId"))); - - TableSelector ts = new TableSelector(assayData, cols.values(), assayFilter, null); - Set segmentsByName = new HashSet<>(); - Map> segmentsByLibrary = new IntHashMap<>(); - - Map> clnaToCloneMap = new IntHashMap<>(); - Map clnaToCDR3Map = new StringHashMap<>(); - StringBuilder imputedSequences = new StringBuilder(); - Set analyses = new IntHashSet(); - final String[] segmentFields = new String[]{"vHit", "jHit", "cHit"}; - ts.forEachResults(rs -> { - Integer libraryId = rs.getObject(FieldKey.fromString("libraryId/libraryId")) == null ? null : rs.getInt(FieldKey.fromString("libraryId/libraryId")); - for (String fn : segmentFields) - { - if (rs.getString(FieldKey.fromString(fn)) != null) - { - if (libraryId != null) - { - Set map = segmentsByLibrary.getOrDefault(libraryId, new HashSet<>()); - map.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); - segmentsByLibrary.put(libraryId, map); - } - else - { - segmentsByName.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); - } - } - } - - if (rs.getObject(FieldKey.fromString("analysisId")) != null) - { - analyses.add(rs.getInt(FieldKey.fromString("analysisId"))); - } - - if (rs.getObject(FieldKey.fromString("sequence")) != null) - { - imputedSequences.append(">").append(rs.getString(FieldKey.fromString("sampleName"))).append("_").append(rs.getString(FieldKey.fromString("cdr3"))).append("\n"); - imputedSequences.append(rs.getString(FieldKey.fromString("sequence"))).append("\n"); - } - - // This applies to MiXCR - if (rs.getObject(FieldKey.fromString("cloneId")) != null && rs.getObject(FieldKey.fromString("clonesFile")) != null) - { - Integer key = rs.getInt(FieldKey.fromString("clonesFile")); - Set set = clnaToCloneMap.containsKey(key) ? clnaToCloneMap.get(key) : new HashSet<>(); - set.add(rs.getString(FieldKey.fromString("cloneId"))); - - clnaToCloneMap.put(key, set); - - clnaToCDR3Map.put(key + "_" + rs.getString(FieldKey.fromString("cloneId")), rs.getString(FieldKey.fromString("cdr3"))); - } - }); - - if (analyses.isEmpty()) - { - errors.reject(ERROR_MSG, "Unable to find analyses for rows"); - return; - } - - // then find all segments from these analyses - SimpleFilter assayFilter2 = new SimpleFilter(FieldKey.fromString("analysisId"), analyses, CompareType.IN); - new TableSelector(assayData, cols.values(), assayFilter2, null).forEachResults(rs -> { - Integer libraryId = rs.getObject(FieldKey.fromString("libraryId/libraryId")) == null ? null : rs.getInt(FieldKey.fromString("libraryId/libraryId")); - for (String fn : segmentFields) - { - if (rs.getString(FieldKey.fromString(fn)) != null) - { - if (libraryId != null) - { - Set map = segmentsByLibrary.getOrDefault(libraryId, new HashSet<>()); - map.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); - segmentsByLibrary.put(libraryId, map); - } - else - { - segmentsByName.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); - } - } - } - }); - - // look up segments in NT table - Set missingSegments = new HashSet<>(segmentsByName); - for (int libraryId : segmentsByLibrary.keySet()) - { - missingSegments.addAll(segmentsByLibrary.get(libraryId)); - } - - StringBuilder fasta = new StringBuilder(); - if (!segmentsByLibrary.isEmpty()) - { - for (int libraryId : segmentsByLibrary.keySet()) - { - SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("ref_nt_id/name"), segmentsByLibrary.get(libraryId), CompareType.IN); - ntFilter.addCondition(FieldKey.fromString("ref_nt_id/datedisabled"), null, CompareType.ISBLANK); - ntFilter.addCondition(FieldKey.fromString("library_id"), libraryId, CompareType.EQUAL); - new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("reference_library_members"), PageFlowUtil.set("ref_nt_id"), ntFilter, null).forEachResults(rs -> { - RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("ref_nt_id"))); - fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); - missingSegments.remove(nt.getName()); - }); - } - } - - if (!segmentsByName.isEmpty()) - { - SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("name"), segmentsByName, CompareType.IN); - ntFilter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); - new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("ref_nt_sequences"), PageFlowUtil.set("rowid"), ntFilter, null).forEachResults(rs -> { - RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("rowid"))); - fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); - missingSegments.remove(nt.getName()); - }); - } - - if (!missingSegments.isEmpty()) - { - logger.error("Unable to find the following NT sequences: [" + StringUtils.join(missingSegments, "],[") + "]"); - } - - // then grab actual Overlapping Contigs record(s). only bother grabbing if FASTA - SimpleFilter outputFilter = new SimpleFilter(FieldKey.fromString("analysis_id"), analyses, CompareType.IN); - outputFilter.addCondition(FieldKey.fromString("category"), "Overlapping Contigs"); - - Set files = new HashSet<>(); - new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("outputfiles"), PageFlowUtil.set("dataid"), outputFilter, null).forEachResults(rs -> { - ExpData d = ExperimentService.get().getExpData(rs.getInt(FieldKey.fromString("dataid"))); - if (d != null && d.getFile().exists()) - { - if (FASTA.isType(d.getFile())) - { - files.add(d.getFile()); - } - } - }); - - //then exportReadsForClones: - for (Integer expData : clnaToCloneMap.keySet()) - { - MiXCRWrapper wrapper = new MiXCRWrapper(_log); - ExpData d = ExperimentService.get().getExpData(expData); - if (d == null) - { - _log.error("Unable to find exp data with ID: " + expData); - continue; - } - - if (d.getFile() == null || !d.getFile().exists()) - { - _log.error("File not found for ExpData: " + (d.getFile() == null ? expData : d.getFile().getPath())); - continue; - } - - List args = new ArrayList<>(); - args.add("-s"); - args.add("--id"); - args.addAll(clnaToCloneMap.get(expData)); - - String basename = SequenceAnalysisService.get().getUnzippedBaseName(d.getFile().getName()) + ".readsForClones"; - File fqBase = new File(FileUtils.getTempDirectory(), basename + ".fastq.gz"); - - wrapper.doExportReadsForClones(d.getFile(), fqBase, args); - - for (String cloneId : clnaToCloneMap.get(expData)) - { - String cdr3 = clnaToCDR3Map.get(expData + "_" + cloneId); - - File fq1 = new File(fqBase.getParentFile(), basename + "_cln" + cloneId + "_R1.fastq.gz"); - if (!fq1.exists()) - { - _log.error("unable to find file: " + fq1.getPath()); - } - else - { - File fq1m = new File(fqBase.getParentFile(), basename + "_cln" + cloneId + "." + cdr3 + ".R1.fastq.gz"); - FileUtils.moveFile(fq1, fq1m); - - files.add(fq1m); - } - - File fq2 = new File(fqBase.getParentFile(), basename + "_cln" + cloneId + "_R2.fastq.gz"); - if (!fq2.exists()) - { - _log.error("unable to find file: " + fq2.getPath()); - } - else - { - File fq2m = new File(fqBase.getParentFile(), basename + "_cln" + cloneId + "." + cdr3 + ".R2.fastq.gz"); - FileUtils.moveFile(fq2, fq2m); - - files.add(fq2m); - } - } - } - - PageFlowUtil.prepareResponseForFile(response, Collections.emptyMap(), "Clones.zip", true); - Set distinctNames = new HashSet<>(); - try (ZipOutputStream zOut = new ZipOutputStream(response.getOutputStream())) - { - //the segments: - ZipEntry fileEntryFasta = new ZipEntry("segments.fasta"); - distinctNames.add("segments.fasta"); - - zOut.putNextEntry(fileEntryFasta); - zOut.write(fasta.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET)); - zOut.closeEntry(); - - //the FL imputed clones: - ZipEntry fileEntryFasta2 = new ZipEntry("imputedClones.fasta"); - distinctNames.add("imputedClones.fasta"); - - zOut.putNextEntry(fileEntryFasta2); - zOut.write(imputedSequences.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET)); - zOut.closeEntry(); - - for (File f : files) - { - String name = getUnique(f.getName(), distinctNames); - ZipEntry fileEntry = new ZipEntry(name); - zOut.putNextEntry(fileEntry); - - try (FileInputStream in = new FileInputStream(f)) - { - IOUtils.copy(in, zOut); - zOut.closeEntry(); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - } - } - } - } - - private String getUnique(String name, Set distinctNames) - { - int i = 1; - String newName = name; - while (distinctNames.contains(newName)) - { - newName = FileUtil.getBaseName(name) + "." + i + "." + FileUtil.getExtension(name); - i++; - } - - distinctNames.add(newName); - - return newName; - } - } - public static class DownloadCloneMaterialsForm { private String[] _rowId; @@ -975,64 +409,4 @@ public void setQueryName(String queryName) _queryName = queryName; } } - - @RequiresPermission(AdminPermission.class) - public class CreateGenomeFromMixcrAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(CreateGenomeFromMixcrForm form, BindException errors) throws Exception - { - setTitle("Create Genome from MiXCR Library"); - - HtmlView view = new HtmlView("This will create a reference genome from the selected MiXCR library JSON. Note: if there is an existing sequence in this folder with the same name and sequence, it will be re-used rather than creating a new sequence. If there is an existing record of the same name, but with a different shorter sequence, that sequence will be marked as disabled. Do you want to continue?"); - return view; - } - - @Override - public boolean handlePost(CreateGenomeFromMixcrForm form, BindException errors) throws Exception - { - try - { - TCRdbManager.get().createGenomeFromMixcrDb(form.getRowId(), getUser(), getContainer()); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw e; - } - - return true; - } - - @Override - public void validateCommand(CreateGenomeFromMixcrForm form, Errors errors) - { - if (form.getRowId() == null || form.getRowId() == 0) - { - errors.reject(ERROR_MSG, "Must provide the library row Id"); - } - } - - @NotNull - @Override - public URLHelper getSuccessURL(CreateGenomeFromMixcrForm form) - { - return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES); - } - } - - public static class CreateGenomeFromMixcrForm - { - private Integer _rowId; - - public Integer getRowId() - { - return _rowId; - } - - public void setRowId(Integer rowId) - { - _rowId = rowId; - } - } } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java deleted file mode 100644 index e49dc1c74..000000000 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright (c) 2015 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.tcrdb; - -import com.milaboratory.core.sequence.NucleotideSequence; -import io.repseq.core.GeneFeature; -import io.repseq.core.VDJCGene; -import io.repseq.core.VDJCLibrary; -import io.repseq.core.VDJCLibraryRegistry; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.sequenceanalysis.GenomeTrigger; -import org.labkey.api.sequenceanalysis.RefNtSequenceModel; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.tcrdb.query.MixcrLibrary; - -import java.io.File; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public class TCRdbManager -{ - private static final TCRdbManager _instance = new TCRdbManager(); - private static final Logger _log = LogManager.getLogger(TCRdbManager.class); - - private TCRdbManager() - { - - } - - public static TCRdbManager get() - { - return _instance; - } - - public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws Exception - { - MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class); - if (lib == null) - { - throw new IllegalArgumentException("Unable to find MiXCR library: " + mixcrRowId); - } - - File jsonFile = lib.getJsonFile(); - if (jsonFile == null) - { - throw new IllegalArgumentException("Unable to find JSON for MiXCR library: " + mixcrRowId); - } - - if (lib.getLibraryId() != null) - { - throw new IllegalArgumentException("MiXCR library already has a genome associated with it: " + mixcrRowId); - } - - Container target = c.isWorkbookOrTab() ? c.getParent() : c; - UserSchema us = QueryService.get().getUserSchema(u, target, TCRdbSchema.SEQUENCE_ANALYSIS); - TableInfo refNt = us.getTable("ref_nt_sequences"); - - try - { - List sequences = new ArrayList<>(); - - VDJCLibraryRegistry reg = VDJCLibraryRegistry.getDefault(); - reg.addPathResolver(jsonFile.getParentFile().getPath()); - - VDJCLibrary library = reg.getLibrary(lib.getLibraryName(), lib.getSpecies()); - String species = StringUtils.capitalize(lib.getSpecies()); - - OUTER: for (VDJCGene gene : library.getGenes()) - { - String name = gene.getName(); - Map regions = new LinkedHashMap<>(); - switch (gene.getGeneType()) - { - case Variable: - regions.put("L1+VExon2", new GeneFeature(GeneFeature.L1, GeneFeature.VExon2)); - regions.put("VRegion", GeneFeature.VRegion); - break; - case Joining: - regions.put("JRegion", GeneFeature.JRegion); - break; - case Diversity: - regions.put("DRegion", GeneFeature.DRegion); - break; - case Constant: - regions.put("CExon1", GeneFeature.CExon1); - break; - } - - String seq = null; - String regionUsed = null; - for (String regionName : regions.keySet()) - { - GeneFeature region = regions.get(regionName); - try - { - NucleotideSequence nt = gene.getFeature(region); - if (nt != null) - { - seq = nt.toString(); - regionUsed = regionName; - break; - } - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - } - } - - if (seq == null) - { - _log.error("Unable to find segment for sequence: " + gene.getName()); - continue; - } - - List comments = new ArrayList<>(); - comments.add("Region: " + regionUsed); - if (!gene.isFunctional()) - { - comments.add("Functional: " + gene.isFunctional()); - } - - if (gene.getData().getMeta() != null && gene.getData().getMeta().get("In_IMGT") != null) - { - comments.add("In IMGT: " + StringUtils.join(gene.getData().getMeta().get("In_IMGT"), ",")); - - if (gene.getData().getMeta().get("ExtendedFromIMGT") != null) - { - comments.add("ExtendedFromIMGT: " + StringUtils.join(gene.getData().getMeta().get("ExtendedFromIMGT"), ",")); - } - } - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); - filter.addCondition(FieldKey.fromString("species"), species); - TableSelector ts = new TableSelector(refNt, filter, null); - if (ts.exists()) - { - List refs = ts.getArrayList(RefNtSequenceModel.class); - for (RefNtSequenceModel ref : refs) - { - if (seq.equals(ref.getSequence())) - { - _log.info("Using existing: " + ref.getName()); - sequences.add(ref.getRowid()); - - //update fields, as needed: - Map toUpdate = new CaseInsensitiveHashMap<>(); - if (!"TCR".equals(ref.getCategory())) - { - toUpdate.put("category", "TCR"); - - } - - if (ref.getDatedisabled() != null) - { - toUpdate.put("datedisabled", null); - } - - if (ref.getDisabledby() != null) - { - toUpdate.put("disabledby", null); - } - - if (!gene.getGeneName().equals(ref.getSubset())) - { - toUpdate.put("subset", gene.getGeneName()); - } - - String locus = StringUtils.join(gene.getChains(), ","); - if (!locus.equals(ref.getLocus())) - { - toUpdate.put("locus", locus); - } - - if (!gene.getFamilyName().equals(ref.getLineage())) - { - toUpdate.put("lineage", gene.getFamilyName()); - } - - if (!comments.equals(ref.getComments())) - { - toUpdate.put("comments", comments); - } - - if (!toUpdate.isEmpty()) - { - _log.info("Updating existing record: " + ref.getName()); - toUpdate.put("rowid", ref.getRowid()); - Map oldKeys = new CaseInsensitiveHashMap<>(); - oldKeys.put("rowid", ref.getRowid()); - refNt.getUpdateService().updateRows(u, ContainerManager.getForId(ref.getContainer()), List.of(toUpdate), List.of(oldKeys), null, null); - } - - continue OUTER; - } - } - } - - //otherwise create new: - Map row = new CaseInsensitiveHashMap<>(); - row.put("name", name); - row.put("category", "TCR"); - row.put("species", species); - row.put("subset", gene.getGeneName()); - row.put("locus", StringUtils.join(gene.getChains(), ",")); - row.put("lineage", gene.getFamilyName()); - row.put("container", lib.getContainer()); - row.put("sequence", seq); - row.put("comments", StringUtils.join(comments, "\n")); - - BatchValidationException errors = new BatchValidationException(); - List> inserted = refNt.getUpdateService().insertRows(u, target, List.of(row), errors, null, null); - if (errors.hasErrors()) - { - throw errors; - } - - RefNtSequenceModel ref = new TableSelector(refNt).getObject(inserted.get(0).get("rowid"), RefNtSequenceModel.class); - sequences.add(ref.getRowid()); - } - - if (!sequences.isEmpty()) - { - _log.info("Creating mixcr genome with " + sequences.size() + " sequences"); - SequenceAnalysisService.get().createReferenceLibrary(sequences, ContainerManager.getForId(lib.getContainer()), u, lib.getLabel(), null, "Created from MiXCR library: " + lib.getLibraryName(), true, true, PageFlowUtil.set(new MiXCRGenomeTrigger(mixcrRowId))); - } - else - { - _log.error("No sequences found: " + lib.getLibraryName()); - } - } - catch (Exception e) - { - //TODO: improve - _log.error(e.getMessage(), e); - throw e; - - } - } - - public static class MiXCRGenomeTrigger implements GenomeTrigger - { - private Integer _mixcrId = null; - - public MiXCRGenomeTrigger() - { - - } - - public MiXCRGenomeTrigger(int mixcrId) - { - _mixcrId = mixcrId; - } - - public Integer getMixcrId() - { - return _mixcrId; - } - - public void setMixcrId(Integer mixcrId) - { - _mixcrId = mixcrId; - } - - @Override - public String getName() - { - return "MiXCR Library Update"; - } - - @Override - public void onCreate(Container c, User u, Logger log, int genomeId) - { - if (_mixcrId != null) - { - TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES); - List> rows = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - Map row = new CaseInsensitiveHashMap<>(); - row.put("rowid", _mixcrId); - row.put("libraryId", genomeId); - rows.add(row); - - Map keyRow = new CaseInsensitiveHashMap<>(); - keyRow.put("rowid", _mixcrId); - oldKeys.add(keyRow); - - try - { - ti.getUpdateService().updateRows(u, c, rows, oldKeys, null, null); - } - catch (QueryUpdateServiceException | InvalidKeyException | BatchValidationException | SQLException e) - { - _log.error(e.getMessage(), e); - } - } - } - - @Override - public void onRecreate(Container c, User u, Logger log, int genomeId) - { - - } - - @Override - public void onDelete(Container c, User u, Logger log, int genomeId) - { - - } - - @Override - public boolean isAvailable(Container c) - { - return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.class)); - } - } -} \ No newline at end of file