diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index 28f4fec28..4e656554a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -1,9 +1,9 @@ package com.conveyal.datatools.manager.controllers; -import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; +import com.conveyal.datatools.manager.jobs.ValidateGtfsPlusFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; @@ -365,6 +365,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee } else { JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false)); JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser, false)); + JobUtils.heavyExecutor.execute(new ValidateGtfsPlusFeedJob(version, systemUser, false)); } } // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java index 6507f8435..e07cd55e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java @@ -249,17 +249,17 @@ private static String publishGtfsPlusFile(Request req, Response res) { } /** - * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). + * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). If the feed + * version already has GTFS+ validation results, those will be returned instead of re-validating. */ private static GtfsPlusValidation getGtfsPlusValidation(Request req, Response res) { - String feedVersionId = req.params("versionid"); - GtfsPlusValidation gtfsPlusValidation = null; try { - gtfsPlusValidation = GtfsPlusValidation.validate(feedVersionId); + String feedVersionId = req.params("versionid"); + return GtfsPlusValidation.validate(feedVersionId); } catch(Exception e) { logMessageAndHalt(req, 500, "Could not read GTFS+ zip file", e); } - return gtfsPlusValidation; + return null; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 0de656b91..da7c81dfc 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -49,22 +49,47 @@ private GtfsPlusValidation (String feedVersionId) { this.feedVersionId = feedVersionId; } + public GtfsPlusValidation(String feedVersionId, boolean published, List issues) { + this(feedVersionId); + this.published = published; + this.issues = issues; + } + + public GtfsPlusValidation() { + this(null); + // Empty constructor for serialization + } + /** - * Validate a GTFS+ feed and return a list of issues encountered. - * FIXME: For now this uses the MapDB-backed GTFSFeed class. Which actually suggests that this might - * should be contained within a MonitorableJob. + * Overload method to retrieve the feed version. */ public static GtfsPlusValidation validate(String feedVersionId) throws Exception { - GtfsPlusValidation validation = new GtfsPlusValidation(feedVersionId); + FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); + if (feedVersion != null) { + if (feedVersion.gtfsPlusValidation != null) { + return feedVersion.gtfsPlusValidation; + } + GtfsPlusValidation gtfsPlusValidation = validate(feedVersion); + feedVersion.gtfsPlusValidation = gtfsPlusValidation; + Persistence.feedVersions.replace(feedVersion.id, feedVersion); + return gtfsPlusValidation; + } + return null; + } + + /** + * Validate a GTFS+ feed and return a list of issues encountered. + */ + public static GtfsPlusValidation validate(FeedVersion feedVersion) throws Exception { if (!DataManager.isModuleEnabled("gtfsplus")) { throw new IllegalStateException("GTFS+ module must be enabled in server.yml to run GTFS+ validation."); } - LOG.info("Validating GTFS+ for {}", feedVersionId); + GtfsPlusValidation validation = new GtfsPlusValidation(feedVersion.id); + LOG.info("Validating GTFS+ for {}", feedVersion.id); - FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); // Load the main GTFS file. // FIXME: Swap MapDB-backed GTFSFeed for use of SQL data? - File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersionId + ".db"); + File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersion.id + ".db"); String gtfsFeedDbFilePath = gtfsFeedDbFile.getAbsolutePath(); GTFSFeed gtfsFeed; try { @@ -87,7 +112,7 @@ public static GtfsPlusValidation validate(String feedVersionId) throws Exception } // check for saved GTFS+ data - File file = gtfsPlusStore.getFeed(feedVersionId); + File file = gtfsPlusStore.getFeed(feedVersion.id); if (file == null) { validation.published = true; LOG.warn("GTFS+ Validation -- Modified GTFS+ file not found, loading from main version GTFS."); diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java index b835a3996..34c59b578 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java @@ -10,6 +10,10 @@ public class ValidationIssue implements Serializable { public int rowIndex; public String description; + public ValidationIssue() { + // Empty constructor for serialization + } + public ValidationIssue(String tableId, String fieldName, int rowIndex, String description) { this.tableId = tableId; this.fieldName = fieldName; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 52f28f8ba..d73b62851 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -41,7 +41,7 @@ public class ProcessSingleFeedJob extends FeedVersionJob { public static boolean ENABLE_MTC_TRANSFORMATIONS = true; // Used in testing to skip validation and speed up response times. - public static boolean VALIDATE_MOBILITY_DATA = true; + public static boolean ENABLE_ADDITIONAL_VALIDATION = true; /** * Create a job for the given feed version. @@ -131,8 +131,9 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); - if (VALIDATE_MOBILITY_DATA) { + if (ENABLE_ADDITIONAL_VALIDATION) { addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion)); + addNextJob(new ValidateGtfsPlusFeedJob(feedVersion, owner, isNewVersion)); } // We only need to snapshot the feed if there are transformations at the database level. In the case that there diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java new file mode 100644 index 000000000..badfd1f1b --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java @@ -0,0 +1,70 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.FeedVersionJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This job handles the GTFS+ validation of a given feed version. If the version is not new, it will simply + * replace the existing version with the version object that has updated validation info. + */ +public class ValidateGtfsPlusFeedJob extends FeedVersionJob { + public static final Logger LOG = LoggerFactory.getLogger(ValidateGtfsPlusFeedJob.class); + + private final FeedVersion feedVersion; + private final boolean isNewVersion; + + public ValidateGtfsPlusFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) { + super(owner, "Validating GTFS+", JobType.VALIDATE_FEED); + feedVersion = version; + this.isNewVersion = isNewVersion; + status.update("Waiting to begin GTFS+ validation...", 0); + } + + @Override + public void jobLogic () { + LOG.info("Running ValidateGtfsPlusFeedJob for {}", feedVersion.id); + feedVersion.validateGtfsPlus(status); + } + + @Override + public void jobFinished () { + if (!status.error) { + if (parentJobId != null && JobType.PROCESS_FEED.equals(parentJobType)) { + // Validate stage is happening as part of an overall process feed job. + // At this point all GTFS data has been loaded and validated, so we record + // the FeedVersion into mongo. + // This happens here because otherwise we would have to wait for other jobs, + // such as BuildTransportNetwork, to finish. If those subsequent jobs fail, + // the version won't get loaded into MongoDB (even though it exists in postgres). + feedVersion.persistFeedVersionAfterValidation(isNewVersion); + } + status.completeSuccessfully("GTFS+ validation finished!"); + } else { + // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version + // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is + // expected. + feedVersion.delete(); + } + } + + /** + * Getter that allows a client to know the ID of the feed version that will be created as soon as the upload is + * initiated; however, we will not store the FeedVersion in the mongo application database until the upload and + * processing is completed. This prevents clients from manipulating GTFS data before it is entirely imported. + */ + @JsonProperty + public String getFeedVersionId () { + return feedVersion.id; + } + + @JsonProperty + public String getFeedSourceId () { + return feedVersion.parentFeedSource().id; + } + + +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 609ff51a5..1319a69a4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -1,9 +1,8 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; -import com.conveyal.datatools.manager.extensions.ExternalPropertiesRetriever; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.gtfs.validator.ValidationResult; +import com.conveyal.datatools.manager.extensions.ExternalPropertiesRetriever; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -11,12 +10,12 @@ import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.UnwindOptions; import org.bson.Document; import org.bson.conversions.Bson; import java.time.LocalDate; import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -37,9 +36,11 @@ import static com.mongodb.client.model.Aggregates.unwind; import static com.mongodb.client.model.Filters.in; +/** + * For explicit mongo queries (matching the queries defined in this class) see resources/mongo and README.md for + * explanation of use. + */ public class FeedSourceSummary { - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - public String projectId; public String id; @@ -77,6 +78,10 @@ public class FeedSourceSummary { public String organizationId; + public Date latestSentToExternalPublisher; + + public PublishState publishState; + @JsonInclude(JsonInclude.Include.NON_NULL) public Map> externalProperties; @@ -86,24 +91,23 @@ public FeedSourceSummary() { public FeedSourceSummary(String projectId, String organizationId, Document feedSourceDocument) { this.projectId = projectId; this.organizationId = organizationId; - this.id = feedSourceDocument.getString("_id"); - this.name = feedSourceDocument.getString("name"); - this.deployable = feedSourceDocument.getBoolean("deployable"); - this.isPublic = feedSourceDocument.getBoolean("isPublic"); + id = feedSourceDocument.getString("_id"); + name = feedSourceDocument.getString("name"); + deployable = feedSourceDocument.getBoolean("deployable"); + isPublic = feedSourceDocument.getBoolean("isPublic"); List documentLabelIds = feedSourceDocument.getList("labelIds", String.class); if (documentLabelIds != null) { - this.labelIds = documentLabelIds; + labelIds = documentLabelIds; } List documentNoteIds = feedSourceDocument.getList("noteIds", String.class); if (documentNoteIds != null) { - this.noteIds = documentNoteIds; + noteIds = documentNoteIds; } // Convert to local date type for consistency. - this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); - this.url = feedSourceDocument.getString("url"); + lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); + url = feedSourceDocument.getString("url"); // Get optional filename. - this.filename = feedSourceDocument.getString("filename"); - + filename = feedSourceDocument.getString("filename"); // Optional external properties, if enabled by config. if ( isModuleEnabled("gtfsapi") && @@ -115,56 +119,39 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS } /** - * Set the appropriate feed version. For consistency, if no error count is available set the related number of - * issues to null. + * Update the publish and validation state based on the provided feed version summary. */ - public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDeployed) { - if (feedVersionSummary != null) { - if (isDeployed) { - this.deployedFeedVersionId = feedVersionSummary.id; - this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? 0 - : feedVersionSummary.validationResult.errorCount; - } else { - this.latestValidation = new LatestValidationResult(feedVersionSummary); - } + public void updatePublishAndValidationState(FeedVersionSummary feedVersionSummary) { + if (feedVersionSummary == null) { + return; } + latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + publishState = feedVersionSummary.getPublishState(); + latestValidation = new LatestValidationResult(feedVersionSummary); } /** - * Get all feed source summaries matching the project id. + * Set the deployed feed version values. For consistency, if no error count is available set the related number of + * issues to zero. + */ + public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { + if (feedVersionSummary == null) { + return; + } + deployedFeedVersionId = feedVersionSummary.id; + deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? 0 + : feedVersionSummary.validationResult.errorCount; + } + + /** + * Get all feed source summaries matching the project id. For equivalent Mongo query, see + * getFeedSourceSummaries.js. + * For equivalent Mongo query, @see src/main/resources/mongo/getFeedSourceSummaries.js */ public static List getFeedSourceSummaries(String projectId, String organizationId) { - /* - db.getCollection('FeedSource').aggregate([ - { - // Match provided project id. - $match: { - projectId: "" - } - }, - { - $project: { - "_id": 1, - "name": 1, - "deployable": 1, - "isPublic": 1, - "lastUpdated": 1, - "labelIds": 1, - "url": 1, - "filename": 1, - "noteIds": 1 - } - }, - { - $sort: { - "name": 1 - } - } - ]) - */ List stages = Lists.newArrayList( match( in("projectId", projectId) @@ -188,63 +175,30 @@ public static List getFeedSourceSummaries(String projectId, S } /** - * Get the latest feed version from all feed sources for this project. + * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * getLatestFeedVersionForFeedSources.js. */ public static Map getLatestFeedVersionForFeedSources(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('FeedSource').aggregate([ - { - // Match provided project id. - $match: { - projectId: "project-with-latest-deployment" - } - }, - { - $lookup: { - from: "FeedVersion", - localField: "_id", - foreignField: "feedSourceId", - as: "feedVersions" - } - }, - { - $unwind: "$feedVersions" - }, - { - $group: { - _id: "$_id", - doc: { - $max: { - version: "$feedVersions.version", - feedVersionId: "$feedVersions._id", - firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", - lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", - issues: "$feedVersions.validationResult.errorCount" - } - } - } - } - ]) - */ List stages = Lists.newArrayList( match( in("projectId", projectId) ), lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), + lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), + unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + sort(Sorts.descending("feedVersions.version")), group( "$_id", - Accumulators.last("feedVersionId", "$feedVersions._id"), - Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), - Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), - Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount") + Accumulators.first("publishedVersionId", "$publishedVersionId"), + Accumulators.first("feedVersionId", "$feedVersions._id"), + Accumulators.first("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), + Accumulators.first("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), + Accumulators.first("errorCount", "$feedVersions.validationResult.errorCount"), + Accumulators.first("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), + Accumulators.first("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), + Accumulators.first("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation"), + Accumulators.first("namespace", "$feedVersions.namespace") ) ); return extractFeedVersionSummaries( @@ -252,86 +206,15 @@ public static Map getLatestFeedVersionForFeedSources "feedVersionId", "_id", false, - stages); + stages + ); } /** - * Get the deployed feed versions from the latest deployment for this project. + * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see + * getFeedVersionsFromLatestDeployment.js. */ public static Map getFeedVersionsFromLatestDeployment(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-latest-deployment" - } - }, - { - // Get all deployments for this project. - $lookup:{ - from:"Deployment", - localField:"_id", - foreignField:"projectId", - as:"deployment" - } - }, - { - // Deconstruct deployments array to a document for each element. - $unwind: "$deployment" - }, - { - // Make the deployment documents the input/root document. - "$replaceRoot": { - "newRoot": "$deployment" - } - }, - { - // Sort descending. - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest deployment for a project. - $limit: 1 - }, - { - $lookup:{ - from:"FeedVersion", - localField:"feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $project: { - "_id": 1, - "feedSourceId": 1, - "validationResult.firstCalendarDate": 1, - "validationResult.lastCalendarDate": 1, - "validationResult.errorCount": 1 - } - } - ]) - */ List stages = Lists.newArrayList( match( in("_id", projectId) @@ -358,74 +241,15 @@ public static Map getFeedVersionsFromLatestDeploymen "_id", "feedSourceId", true, - stages); + stages + ); } /** - * Get the deployed feed version from the pinned deployment for this feed source. + * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see + * getFeedVersionsFromPinnedDeployment.js. */ public static Map getFeedVersionsFromPinnedDeployment(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-pinned-deployment" - } - }, - { - $project: { - pinnedDeploymentId: 1 - } - }, - { - $lookup:{ - from:"Deployment", - localField:"pinnedDeploymentId", - foreignField:"_id", - as:"deployment" - } - }, - { - $unwind: "$deployment" - }, - { - $lookup:{ - from:"FeedVersion", - localField:"deployment.feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $project: { - "_id": 1, - "feedSourceId": 1, - "validationResult.firstCalendarDate": 1, - "validationResult.lastCalendarDate": 1, - "validationResult.errorCount": 1 - } - } - ]) - */ - List stages = Lists.newArrayList( match( in("_id", projectId) @@ -452,14 +276,19 @@ public static Map getFeedVersionsFromPinnedDeploymen "_id", "feedSourceId", true, - stages); + stages + ); } /** * Produce a list of all feed source summaries for a project. */ - private static List extractFeedSourceSummaries(String projectId, String organizationId, List stages) { + private static List extractFeedSourceSummaries( + String projectId, + String organizationId, + List stages + ) { List feedSourceSummaries = new ArrayList<>(); for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { feedSourceSummaries.add(new FeedSourceSummary(projectId, organizationId, feedSourceDocument)); @@ -479,83 +308,15 @@ private static Map extractFeedVersionSummaries( List stages ) { Map feedVersionSummaries = new HashMap<>(); - for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); - feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); - feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); - feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); + for (Document feedVersion : Persistence.getDocuments(collection, stages)) { + feedVersionSummaries.put( + feedVersion.getString(feedSourceKey), + new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) + ); } return feedVersionSummaries; } - /** - * Build validation result from feed version document. - */ - private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { - ValidationResult validationResult = new ValidationResult(); - validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); - validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); - validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); - return validationResult; - } - - private static LocalDate getValidationResultDate( - boolean hasChildValidationResultDocument, - Document feedVersionDocument, - String key - ) { - return (hasChildValidationResultDocument) - ? getDateFieldFromDocument(feedVersionDocument, key) - : getDateFromString(feedVersionDocument.getString(key)); - } - - /** - * Extract date value from validation result document. - */ - private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { - Document validationResult = getDocumentChild(document, "validationResult"); - return (validationResult != null) - ? getDateFromString(validationResult.getString(dateKey)) - : null; - } - - /** - * Extract the error count from the parent document or child validation result document. If the error count is not - * available, return -1. - */ - private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { - int errorCount; - try { - errorCount = (hasChildValidationResultDocument) - ? getErrorCount(feedVersionDocument) - : feedVersionDocument.getInteger("errorCount"); - } catch (NullPointerException e) { - errorCount = -1; - } - return errorCount; - } - - /** - * Get the child validation result document and extract the error count from this. - */ - private static int getErrorCount(Document document) { - return getDocumentChild(document, "validationResult").getInteger("errorCount"); - } - - /** - * Extract child document matching provided name. - */ - private static Document getDocumentChild(Document document, String name) { - return (Document) document.get(name); - } - - /** - * Convert String date (if not null) into LocalDate. - */ - private static LocalDate getDateFromString(String date) { - return (date == null) ? null : LocalDate.parse(date, formatter); - } - /** * Convert Date object into LocalDate object. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 8452d63d7..ded673c37 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -5,6 +5,7 @@ import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.jobs.validation.RouteTypeValidatorBuilder; @@ -276,6 +277,8 @@ public FeedValidationResultSummary validationSummary() { public Document mobilityDataResult; + public GtfsPlusValidation gtfsPlusValidation; + public String formattedTimestamp() { SimpleDateFormat format = new SimpleDateFormat(HUMAN_READABLE_TIMESTAMP_FORMAT); return format.format(this.updated); @@ -477,6 +480,28 @@ public void validateMobility(MonitorableJob.Status status) { } } + /** + * Produce GTFS+ validation results for this feed version if GTFS+ module is enabled. + */ + public void validateGtfsPlus(MonitorableJob.Status status) { + + // Sometimes this method is called when no status object is available. + if (status == null) status = new MonitorableJob.Status(); + + if (DataManager.isModuleEnabled("gtfsplus")) { + try { + gtfsPlusValidation = GtfsPlusValidation.validate(this); + } catch (Exception e) { + LOG.warn("Unable to validate GTFS+ validation.", e); + status.fail(String.format("Unable to validate feed %s", this.id), e); + validationResult = new ValidationResult(); + validationResult.fatalException = "failure!"; + } + } else { + LOG.warn("GTFS+ module not enabled, skipping GTFS+ validation."); + } + } + public void validate() { validate(null); } @@ -512,6 +537,10 @@ private boolean hasValidationAndLoadErrors() { @JsonIgnore @BsonIgnore public boolean hasExpired() { + return hasExpired(validationResult); + } + + public static boolean hasExpired(ValidationResult validationResult) { return validationResult.lastCalendarDate == null || getNowAsLocalDate().isAfter(validationResult.lastCalendarDate); } @@ -530,14 +559,22 @@ public static void setDateOverrideForTesting(LocalDate value) { */ private boolean hasHighSeverityErrorTypes() { return hasSpecificErrorTypes(Stream.of(NewGTFSErrorType.values()) - .filter(type -> type.priority == Priority.HIGH)); + .filter(type -> type.priority == Priority.HIGH), namespace, name); } /** * Checks for issues that block feed publishing, consistent with UI. */ public boolean hasBlockingIssuesForPublishing() { - if (this.validationResult.fatalException != null) return true; + return hasBlockingIssuesForPublishing(validationResult, namespace, name); + } + + public static boolean hasBlockingIssuesForPublishing( + ValidationResult validationResult, + String namespace, + String name + ) { + if (validationResult.fatalException != null) return true; return hasSpecificErrorTypes(Stream.of( NewGTFSErrorType.ILLEGAL_FIELD_VALUE, @@ -550,13 +587,13 @@ public boolean hasBlockingIssuesForPublishing() { NewGTFSErrorType.TABLE_IN_SUBDIRECTORY, NewGTFSErrorType.TABLE_MISSING_COLUMN_HEADERS, NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS - )); + ), namespace, name); } /** * Determines whether this feed has specific error types. */ - private boolean hasSpecificErrorTypes(Stream errorTypes) { + private static boolean hasSpecificErrorTypes(Stream errorTypes, String namespace, String name) { Set highSeverityErrorTypes = errorTypes .map(NewGTFSErrorType::toString) .collect(Collectors.toSet()); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index be2413a9a..317c152e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -1,20 +1,31 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.bson.Document; import java.io.Serializable; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * Includes summary data (a subset of fields) for a feed version. */ public class FeedVersionSummary extends Model implements Serializable { private static final long serialVersionUID = 1L; + private static final ObjectMapper mapper = new ObjectMapper(); + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + public static Boolean hasBlockingIssueForPublishingForTesting = null; public FeedRetrievalMethod retrievalMethod; public int version; @@ -28,6 +39,10 @@ public class FeedVersionSummary extends Model implements Serializable { @JsonIgnore public ValidationResult validationResult; private PartialValidationSummary validationSummary; + public Date processedByExternalPublisher; + public Date sentToExternalPublisher; + public GtfsPlusValidation gtfsPlusValidation; + public String feedSourcePublishedVersionId; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { @@ -41,6 +56,22 @@ public FeedVersionSummary() { // Do nothing } + public FeedVersionSummary( + String feedVersionKey, + boolean hasChildValidationResultDocument, + Document feedVersionDocument + ) { + id = feedVersionDocument.getString(feedVersionKey); + processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); + sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); + gtfsPlusValidation = getGtfsPlusValidation(id, feedVersionDocument); + namespace = feedVersionDocument.getString("namespace"); + validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + + // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. + feedSourcePublishedVersionId = feedVersionDocument.getString("publishedVersionId"); + } + /** * Holds a subset of fields from {@link:FeedValidationResultSummary} for UI use only. */ @@ -63,4 +94,151 @@ public class PartialValidationSummary { } } } + + /** + * Build GtfsPlusValidation object from feed version document. + */ + private static GtfsPlusValidation getGtfsPlusValidation(String feedVersionId, Document feedVersionDocument) { + Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); + if (gtfsPlusValidationDocument == null) { + return null; + } + List issues = null; + if (gtfsPlusValidationDocument.containsKey("issues") && gtfsPlusValidationDocument.get("issues") != null) { + List issueDocs = gtfsPlusValidationDocument.getList("issues", Document.class); + issues = issueDocs + .stream() + .map(doc -> mapper.convertValue(doc, ValidationIssue.class)) + .collect(Collectors.toList()); + } + boolean published = Boolean.TRUE.equals(gtfsPlusValidationDocument.getBoolean("published")); + return new GtfsPlusValidation(feedVersionId, published, issues); + } + + /** + * Build validation result from feed version document. + */ + private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + ValidationResult validationResult = new ValidationResult(); + validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); + validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); + validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); + return validationResult; + } + + /** + * Convert String date (if not null) into LocalDate. + */ + private static LocalDate getDateFromString(String date) { + return (date == null) ? null : LocalDate.parse(date, formatter); + } + + /** + * Extract child document matching provided name. + */ + private static Document getDocumentChild(Document document, String name) { + return (Document) document.get(name); + } + + /** + * Extract date value from parent document or child validation result document. + */ + private static LocalDate getValidationResultDate( + boolean hasChildValidationResultDocument, + Document feedVersionDocument, + String key + ) { + return (hasChildValidationResultDocument) + ? getDateFieldFromDocument(feedVersionDocument, key) + : getDateFromString(feedVersionDocument.getString(key)); + } + + /** + * Extract date value from validation result document. + */ + private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { + Document validationResult = getDocumentChild(document, "validationResult"); + return (validationResult != null) + ? getDateFromString(validationResult.getString(dateKey)) + : null; + } + + /** + * Extract the error count from the parent document or child validation result document. If the error count is not + * available, return -1. + */ + private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + int errorCount; + try { + errorCount = (hasChildValidationResultDocument) + ? getErrorCount(feedVersionDocument) + : feedVersionDocument.getInteger("errorCount"); + } catch (NullPointerException e) { + errorCount = -1; + } + return errorCount; + } + + /** + * Get the child validation result document and extract the error count from this. + */ + private static int getErrorCount(Document document) { + return getDocumentChild(document, "validationResult").getInteger("errorCount"); + } + + /** + * Determine the published state of the feed version. + */ + public PublishState getPublishState() { + if (isPublished()) { + return PublishState.PUBLISHED; + } else if (isPublishing()) { + return PublishState.PUBLISHING; + } else if (isPublishBlocked()) { + return PublishState.PUBLISH_BLOCKED; + } + return PublishState.READY_TO_PUBLISH; + } + + /** + * Determine the published state of the feed version. + */ + private boolean isPublished() { + return namespace != null && namespace.equals(feedSourcePublishedVersionId); + } + + /** + * Deemed to be publishing if it has been sent to external publisher but not yet processed. + */ + private boolean isPublishing() { + return sentToExternalPublisher != null && processedByExternalPublisher == null; + } + + /** + * Determine if publishing is blocked due to validation, expiration, blocking issues or loading. + */ + private boolean isPublishBlocked() { + return + gtfsPlusValidation == null || + gtfsPlusValidation.issues == null || + !gtfsPlusValidation.issues.isEmpty() || + !gtfsPlusValidation.published || + FeedVersion.hasExpired(validationResult) || + hasBlockingIssuesForPublishing(); + } + + /** + * Determine if there are blocking issues for publishing. + */ + private boolean hasBlockingIssuesForPublishing() { + return Objects.requireNonNullElseGet(hasBlockingIssueForPublishingForTesting, () -> FeedVersion.hasBlockingIssuesForPublishing( + validationResult, + namespace, + name + )); + } + + public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { + hasBlockingIssueForPublishingForTesting = value; + } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 073405297..f6a8b2bc2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -10,8 +10,6 @@ import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Date; @@ -40,7 +38,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class Project extends Model { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(Project.class); /** The name of this project, e.g. NYSDOT. */ public String name; @@ -168,19 +165,28 @@ public Collection retrieveDeploymentSummaries() { */ public Collection retrieveFeedSourceSummaries() { List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); - Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); - Map deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); - if (deployedFeedVersions.isEmpty()) { - // No pinned deployments, instead, get the deployed feed versions from the latest deployment. - deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); - } - for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { - feedSourceSummary.setFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id), false); - feedSourceSummary.setFeedVersion(deployedFeedVersions.get(feedSourceSummary.id), true); - } + assignDeployedVersion(feedSourceSummaries); return feedSourceSummaries; } + /** + * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. + */ + private void assignDeployedVersion(List feedSourceSummaries) { + Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); + Map pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); + Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); + + feedSourceSummaries.forEach(feedSourceSummary -> { + feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( + feedSourceSummary.id, + latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) + ); + feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); + }); + } + // TODO: Does this need to be returned with JSON API response public Organization retrieveOrganization() { if (organizationId != null) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/PublishState.java b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java new file mode 100644 index 000000000..15b28c70c --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java @@ -0,0 +1,11 @@ +package com.conveyal.datatools.manager.models; + +/** + * Enumeration of possible publish states for a FeedVersion. + */ +public enum PublishState { + PUBLISHED, + PUBLISHING, + PUBLISH_BLOCKED, + READY_TO_PUBLISH +} diff --git a/src/main/resources/mongo/README.md b/src/main/resources/mongo/README.md new file mode 100644 index 000000000..ed4a5a05e --- /dev/null +++ b/src/main/resources/mongo/README.md @@ -0,0 +1,11 @@ +## Running the Scripts in this folder using test data from FeedSourceControllerTest + +1. **Comment out** the call to `tearDownDeployedFeedVersion()` in `FeedSourceControllerTest` \-> `tearDown()`. +2. **Run** `FeedSourceControllerTest` to create required objects referenced here. +3. **Delete** documents via MongoDB once complete. +4. **Uncomment** the call to `tearDownDeployedFeedVersion()` in `FeedSourceControllerTest` \-> `tearDown()`. +5. **Re-run** `FeedSourceControllerTest` to confirm deletion of objects. + +## Alternative Approach + +If the appropriate data has already been created (e.g., via the DT UI), the `` tag in each script can be replaced with the actual projectId value. \ No newline at end of file diff --git a/src/main/resources/mongo/getFeedSourceSummaries.js b/src/main/resources/mongo/getFeedSourceSummaries.js new file mode 100644 index 000000000..3e4db025d --- /dev/null +++ b/src/main/resources/mongo/getFeedSourceSummaries.js @@ -0,0 +1,26 @@ +db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "" + } + }, + { + $project: { + "_id": 1, + "name": 1, + "deployable": 1, + "isPublic": 1, + "lastUpdated": 1, + "labelIds": 1, + "url": 1, + "filename": 1, + "noteIds": 1 + } + }, + { + $sort: { + "name": 1 + } + } +]) diff --git a/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js new file mode 100644 index 000000000..7779b6a84 --- /dev/null +++ b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js @@ -0,0 +1,64 @@ +db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "" + } + }, + { + // Get all deployments for this project. + $lookup:{ + from:"Deployment", + localField:"_id", + foreignField:"projectId", + as:"deployment" + } + }, + { + // Deconstruct deployments array to a document for each element. + $unwind: "$deployment" + }, + { + // Make the deployment documents the input/root document. + "$replaceRoot": { + "newRoot": "$deployment" + } + }, + { + // Sort descending. + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest deployment for a project. + $limit: 1 + }, + { + $lookup:{ + from:"FeedVersion", + localField:"feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } +]) diff --git a/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js new file mode 100644 index 000000000..008de3c56 --- /dev/null +++ b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js @@ -0,0 +1,51 @@ +db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "" + } + }, + { + $project: { + pinnedDeploymentId: 1 + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $unwind: "$deployment" + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } +]) diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js new file mode 100644 index 000000000..e4fe3a4bd --- /dev/null +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -0,0 +1,57 @@ +db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "_id", + foreignField: "feedSourceId", + as: "feedVersions" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "publishedVersionId", + foreignField: "namespace", + as: "publishedFeedVersion" + } + }, + { + $unwind: "$feedVersions" + }, + { + $unwind: { + path: "$publishedFeedVersion", + preserveNullAndEmptyArrays: true + } + }, + { + $sort: { + "feedVersions.version": -1 + } + }, + { + $group: { + _id: "$_id", + publishedVersionId: { $first: "$publishedVersionId" }, + feedVersion: { + $first: { + version: "$feedVersions.version", + feedVersionId: "$feedVersions._id", + firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", + lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", + errorCount: "$feedVersions.validationResult.errorCount", + processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", + sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", + gtfsPlusValidation: "$feedVersions.gtfsPlusValidation", + namespace: "$feedVersions.namespace" + } + } + } + } +]) diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index b17a84c67..a610f6e80 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -44,7 +44,7 @@ static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; project = new Project(); project.name = appendDate("Test"); Persistence.projects.create(project); @@ -84,7 +84,7 @@ static void setUp() throws IOException { @AfterAll static void tearDown() { Auth0Connection.setAuthDisabled(false); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; project.delete(); FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionOrphan.id); if (feedVersion != null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 1350fbe52..b87de98d6 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -4,15 +4,20 @@ import com.conveyal.datatools.TestUtils; import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; +import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.FeedVersionSummary; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Note; import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.models.PublishState; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; @@ -21,6 +26,9 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.net.MalformedURLException; @@ -28,10 +36,13 @@ import java.time.LocalDate; import java.time.Month; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.stream.Stream; +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.mongodb.client.model.Filters.eq; import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; import static org.eclipse.jetty.http.HttpStatus.OK_200; @@ -48,35 +59,23 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedSource feedSourceWithInvalidLabels = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; - private static Label feedSourceWithLatestDeploymentAdminOnlyLabel = null; - private static Label feedSourceWithPinnedDeploymentAdminOnlyLabel = null; - private static Note feedSourceWithLatestDeploymentAdminOnlyNote = null; - private static Note feedSourceWithPinnedDeploymentAdminOnlyNote = null; private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; - private static FeedVersion feedVersionFromLatestDeployment = null; - private static Deployment deploymentLatest = null; - private static Deployment deploymentSuperseded = null; + private static FeedVersion feedVersionFromLatestDeploymentVersion2 = null; + private static FeedVersion feedVersionPublishedFromLatestDeployment = null; private static Project projectWithPinnedDeployment = null; private static FeedSource feedSourceWithPinnedDeploymentFeedVersion = null; private static FeedVersion feedVersionFromPinnedDeployment = null; - private static Deployment deploymentPinned = null; @BeforeAll public static void setUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - project = new Project(); - project.name = "ProjectOne"; - project.autoFetchFeeds = true; - Persistence.projects.create(project); - - projectToBeDeleted = new Project(); - projectToBeDeleted.name = "ProjectTwo"; - projectToBeDeleted.autoFetchFeeds = false; - Persistence.projects.create(projectToBeDeleted); + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; + project = createProject("ProjectOne", true); + projectToBeDeleted = createProject("ProjectTwo", false); feedSourceWithUrl = createFeedSource("FeedSourceOne", new URL("http://www.feedsource.com"), project); feedSourceWithNoUrl = createFeedSource("FeedSourceTwo", null, project); @@ -89,7 +88,6 @@ public static void setUp() throws IOException { setUpFeedVersionFromLatestDeployment(); setUpFeedVersionFromPinnedDeployment(); - } /** @@ -101,8 +99,10 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx projectWithLatestDeployment.organizationId = "project-with-latest-deployment-org-id"; Persistence.projects.create(projectWithLatestDeployment); - feedSourceWithLatestDeploymentAdminOnlyLabel = createLabel("label-id-latest-deployment", "Admin Only Label", projectWithLatestDeployment.id); - feedSourceWithLatestDeploymentAdminOnlyNote = createNote("note-id-latest-deployment", "A test note"); + Label feedSourceWithLatestDeploymentAdminOnlyLabel = createLabel( + "label-id-latest-deployment", "Admin Only Label", projectWithLatestDeployment.id + ); + Note feedSourceWithLatestDeploymentAdminOnlyNote = createNote("note-id-latest-deployment", "A test note"); feedSourceWithLatestDeploymentFeedVersion = createFeedSource( "feed-source-with-latest-deployment-feed-version", @@ -117,22 +117,54 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx LocalDate deployedSuperseded = LocalDate.of(2020, Month.MARCH, 12); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); - feedVersionFromLatestDeployment = createFeedVersion( - "feed-version-from-latest-deployment", + // Create feed version 1 to be superseded by feed version 2 to confirm that the newer version is the one retrieved. + createFeedVersion( + "feed-version-from-latest-deployment-1", feedSourceWithLatestDeploymentFeedVersion.id, deployedStartDate, - deployedEndDate + deployedEndDate, + null, + 1 + ); + feedVersionFromLatestDeploymentVersion2 = createFeedVersion( + "feed-version-from-latest-deployment-2", + feedSourceWithLatestDeploymentFeedVersion.id, + deployedStartDate, + deployedEndDate, + null, + 2 ); - deploymentSuperseded = createDeployment( + + FeedVersion feedVersionFromGtfsZip = createFeedVersionFromGtfsZip( + feedSourceWithLatestDeploymentFeedVersion, + "bart_old.zip" + ); + // Update the feed version namespace to match that created from the import. + feedVersionFromLatestDeploymentVersion2.namespace = feedVersionFromGtfsZip.namespace; + + // Remove the imported feed version so it does not conflict with the latest deployment feed version. + Persistence.feedVersions.removeById(feedVersionFromGtfsZip.id); + Persistence.feedVersions.replace(feedVersionFromLatestDeploymentVersion2.id, feedVersionFromLatestDeploymentVersion2); + + feedVersionPublishedFromLatestDeployment = createFeedVersion( + "published-feed-version-from-latest-deployment", + // Set to null so the relationship to feed source is via the published version id. + null, + LocalDate.of(2022, Month.NOVEMBER, 2), + LocalDate.of(2022, Month.NOVEMBER, 3), + feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, + 0 + ); + createDeployment( "deployment-superseded", projectWithLatestDeployment, - feedVersionFromLatestDeployment.id, + feedVersionFromLatestDeploymentVersion2.id, deployedSuperseded ); - deploymentLatest = createDeployment( + createDeployment( "deployment-latest", projectWithLatestDeployment, - feedVersionFromLatestDeployment.id, + feedVersionFromLatestDeploymentVersion2.id, deployedEndDate ); } @@ -146,8 +178,8 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx projectWithPinnedDeployment.organizationId = "project-with-pinned-deployment-org-id"; Persistence.projects.create(projectWithPinnedDeployment); - feedSourceWithPinnedDeploymentAdminOnlyLabel = createLabel("label-id-pinned-deployment", "Admin Only Label", projectWithPinnedDeployment.id); - feedSourceWithPinnedDeploymentAdminOnlyNote = createNote("note-id-pinned-deployment", "A test note"); + Label feedSourceWithPinnedDeploymentAdminOnlyLabel = createLabel("label-id-pinned-deployment", "Admin Only Label", projectWithPinnedDeployment.id); + Note feedSourceWithPinnedDeploymentAdminOnlyNote = createNote("note-id-pinned-deployment", "A test note"); feedSourceWithPinnedDeploymentFeedVersion = createFeedSource( "feed-source-with-pinned-deployment-feed-version", @@ -163,7 +195,7 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx feedSourceWithPinnedDeploymentFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2) ); - deploymentPinned = createDeployment( + Deployment deploymentPinned = createDeployment( "deployment-pinned", projectWithPinnedDeployment, feedVersionFromPinnedDeployment.id, @@ -177,70 +209,30 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx public static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { - Persistence.projects.removeById(project.id); + project.delete(); } if (projectToBeDeleted != null) { - Persistence.projects.removeById(projectToBeDeleted.id); - } - if (feedSourceWithUrl != null) { - Persistence.feedSources.removeById(feedSourceWithUrl.id); - } - if (feedSourceWithNoUrl != null) { - Persistence.feedSources.removeById(feedSourceWithNoUrl.id); - } - if (publicLabel != null) { - Persistence.labels.removeById(publicLabel.id); - } - if (adminOnlyLabel != null) { - Persistence.labels.removeById(adminOnlyLabel.id); + projectToBeDeleted.delete(); } tearDownDeployedFeedVersion(); + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /** - * These entities are removed separately so that if the need arises they can be kept. + * These projects are removed separately so that if the need arises they can be kept. * This would then allow the Mongo queries defined in FeedSource#getFeedVersionFromLatestDeployment and * FeedSource#getFeedVersionFromPinnedDeployment to be tested. */ private static void tearDownDeployedFeedVersion() { if (projectWithPinnedDeployment != null) { - Persistence.projects.removeById(projectWithPinnedDeployment.id); + projectWithPinnedDeployment.delete(); } if (projectWithLatestDeployment != null) { - Persistence.projects.removeById(projectWithLatestDeployment.id); - } - if (feedSourceWithLatestDeploymentFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithLatestDeploymentFeedVersion.id); - } - if (feedSourceWithPinnedDeploymentFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithPinnedDeploymentFeedVersion.id); - } - if (feedVersionFromLatestDeployment != null) { - Persistence.feedVersions.removeById(feedVersionFromLatestDeployment.id); - } - if (feedVersionFromPinnedDeployment != null) { - Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); - } - if (deploymentPinned != null) { - Persistence.deployments.removeById(deploymentPinned.id); - } - if (deploymentLatest != null) { - Persistence.deployments.removeById(deploymentLatest.id); - } - if (deploymentSuperseded != null) { - Persistence.deployments.removeById(deploymentSuperseded.id); + projectWithLatestDeployment.delete(); } - if (feedSourceWithPinnedDeploymentAdminOnlyLabel != null) { - Persistence.labels.removeById(feedSourceWithPinnedDeploymentAdminOnlyLabel.id); - } - if (feedSourceWithLatestDeploymentAdminOnlyLabel != null) { - Persistence.labels.removeById(feedSourceWithLatestDeploymentAdminOnlyLabel.id); - } - if (feedSourceWithPinnedDeploymentAdminOnlyNote != null) { - Persistence.notes.removeById(feedSourceWithPinnedDeploymentAdminOnlyNote.id); - } - if (feedSourceWithLatestDeploymentAdminOnlyNote != null) { - Persistence.notes.removeById(feedSourceWithLatestDeploymentAdminOnlyNote.id); + // Orphan feed version must be explicitly deleted. + if (feedVersionPublishedFromLatestDeployment != null) { + Persistence.feedVersions.removeById(feedVersionPublishedFromLatestDeployment.id); } } @@ -337,8 +329,8 @@ public void createFeedSourceWithLabels() { // Test that they are assigned properly assertEquals(2, labelCountForFeed(feedSourceWithLabels.id)); // Test that project shows only correct labels based on user auth - assertEquals(2, labelCountforProject(feedSourceWithLabels.projectId, true)); - assertEquals(1, labelCountforProject(feedSourceWithLabels.projectId, false)); + assertEquals(2, labelCountForProject(feedSourceWithLabels.projectId, true)); + assertEquals(1, labelCountForProject(feedSourceWithLabels.projectId, false)); // Test that feed source shows only correct labels based on user auth List labelsSeenByAdmin = FeedSourceController.cleanFeedSourceForNonAdmins(feedSourceWithLabels, true).labelIds; @@ -354,7 +346,7 @@ public void createFeedSourceWithLabels() { ); assertEquals(OK_200, deleteSecondLabelResponse.status); assertEquals(1, labelCountForFeed(feedSourceWithLabels.id)); - assertEquals(1, labelCountforProject(feedSourceWithLabels.projectId, true)); + assertEquals(1, labelCountForProject(feedSourceWithLabels.projectId, true)); // Test that labels are removed when deleting project assertEquals(1, Persistence.labels.getFiltered(eq("projectId", projectToBeDeleted.id)).size()); @@ -389,14 +381,16 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedSourceWithLatestDeploymentFeedVersion.url.toString(), feedSourceSummaries.get(0).url); assertEquals(feedSourceWithLatestDeploymentFeedVersion.noteIds, feedSourceSummaries.get(0).noteIds); assertEquals(feedSourceWithLatestDeploymentFeedVersion.organizationId(), feedSourceSummaries.get(0).organizationId); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromLatestDeploymentVersion2.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); + assertEquals(feedVersionFromLatestDeploymentVersion2.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromLatestDeploymentVersion2.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(PublishState.PUBLISH_BLOCKED, feedSourceSummaries.get(0).publishState); } @Test @@ -431,6 +425,82 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(PublishState.PUBLISH_BLOCKED, feedSourceSummaries.get(0).publishState); + } + + @ParameterizedTest + @MethodSource("createPublishStates") + void canDeterminePublishState( + boolean isPublished, + boolean isPublishing, + boolean isPublishBlocked, + boolean isFeedLoading, + PublishState expectedPublishState + ) { + FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + + if (isPublished) { + feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; + } + if (isPublishing) { + feedVersionSummary.sentToExternalPublisher = new Date(); + feedVersionSummary.processedByExternalPublisher = null; + } + if (isPublishBlocked) { + feedVersionSummary.gtfsPlusValidation = null; + } + if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { + // Ready to publish case. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + passPublishBlockedCheck(feedVersionSummary); + + feedVersionSummary.validationResult.errorCount = 1; + feedVersionSummary.id = "feed-version-id"; + feedSourceSummary.id = "feed-source-id"; + } + PublishState publishState = feedVersionSummary.getPublishState(); + assertEquals(expectedPublishState, publishState); + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); + } + + /** + * Set up a feed version summary to pass the publish blocked check. + */ + private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { + feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); + feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); + feedVersionSummary.gtfsPlusValidation.published = true; + feedVersionSummary.validationResult = new ValidationResult(); + feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + } + + private static Stream createPublishStates() { + return Stream.of( + Arguments.of( + true, false, false, false, PublishState.PUBLISHED + ), + Arguments.of( + false, true, false, false, PublishState.PUBLISHING + ), + Arguments.of( + false, false, true, false, PublishState.PUBLISH_BLOCKED + ), + Arguments.of( + false, false, false, false, PublishState.READY_TO_PUBLISH + ) + ); + } + + private static Project createProject(String name, boolean autoFetchFeeds) { + Project project = new Project(); + project.name = name; + project.autoFetchFeeds = autoFetchFeeds; + Persistence.projects.create(project); + return project; } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -461,6 +531,7 @@ private static FeedSource createFeedSource( feedSource.projectId = project.id; feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; + feedSource.publishedVersionId = "published-version-id-1"; if (labels != null) feedSource.labelIds = labels; if (notes != null) feedSource.noteIds = notes; if (persist) Persistence.feedSources.create(feedSource); @@ -489,13 +560,20 @@ private static Deployment createDeployment( * Helper method to create a feed version with no start date. */ private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate) { - return createFeedVersion(id, feedSourceId, null, endDate); + return createFeedVersion(id, feedSourceId, null, endDate, null, 0); } /** * Helper method to create a feed version. */ - private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate) { + private static FeedVersion createFeedVersion( + String id, + String feedSourceId, + LocalDate startDate, + LocalDate endDate, + String namespace, + int version + ) { FeedVersion feedVersion = new FeedVersion(); feedVersion.id = id; feedVersion.feedSourceId = feedSourceId; @@ -504,6 +582,15 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc validationResult.lastCalendarDate = endDate; validationResult.errorCount = 5 + (int)(Math.random() * ((1000 - 5) + 1)); feedVersion.validationResult = validationResult; + feedVersion.processedByExternalPublisher = new Date(); + feedVersion.sentToExternalPublisher = new Date(); + List issues = List.of( + new ValidationIssue("Test issue 1", "stops.txt", 1, "stop_id"), + new ValidationIssue("Test issue 2", "stops.txt", 2, "stop_id") + ); + feedVersion.gtfsPlusValidation = new GtfsPlusValidation(id, true, issues); + feedVersion.namespace = namespace != null ? namespace : "feed-version-namespace"; + feedVersion.version = version; Persistence.feedVersions.create(feedVersion); return feedVersion; } @@ -555,7 +642,7 @@ private int labelCountForFeed(String feedSourceId) { /** * Provide the label count for a given project */ - private int labelCountforProject(String projectId, boolean isAdmin) { + private int labelCountForProject(String projectId, boolean isAdmin) { return Persistence.projects.getById(projectId).retrieveProjectLabels(isAdmin).size(); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 7b383075e..39340905d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -87,7 +87,7 @@ public class MergeFeedsJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; // Enable MTC extension, but disable the transformations. ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = false; DatatoolsTest.enableMTCExtension(); @@ -169,7 +169,7 @@ public static void tearDown() { } DatatoolsTest.resetMTCExtension(); ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = true; - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /**