From a82bed7e1110458c0bc3becd757b77c74f40c88c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 24 Sep 2025 16:24:42 +0100 Subject: [PATCH 01/20] feat(Additional feed summary params): --- .../manager/models/FeedSourceSummary.java | 17 ++++++++++++++--- .../manager/models/FeedVersionSummary.java | 2 ++ .../api/FeedSourceControllerTest.java | 6 ++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 1b88b91a5..f68f97af8 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -33,7 +33,6 @@ public class FeedSourceSummary { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - public String projectId; public String id; @@ -71,6 +70,10 @@ public class FeedSourceSummary { public String organizationId; + public Date latestProcessedByExternalPublisher; + + public Date latestSentToExternalPublisher; + public FeedSourceSummary() { } @@ -111,6 +114,8 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl : feedVersionSummary.validationResult.errorCount; } else { this.latestValidation = new LatestValidationResult(feedVersionSummary); + this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; + this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; } } } @@ -208,7 +213,9 @@ public static Map getLatestFeedVersionForFeedSources feedVersionId: "$feedVersions._id", firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", - issues: "$feedVersions.validationResult.errorCount" + errorCount: "$feedVersions.validationResult.errorCount", + processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", + sentToExternalPublisher: "$feedVersions.sentToExternalPublisher" } } } @@ -226,7 +233,9 @@ public static Map getLatestFeedVersionForFeedSources Accumulators.last("feedVersionId", "$feedVersions._id"), Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), - Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount") + Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), + Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), + Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher") ) ); return extractFeedVersionSummaries( @@ -464,6 +473,8 @@ private static Map extractFeedVersionSummaries( for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); + feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); + feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index be2413a9a..8b8e8de63 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -28,6 +28,8 @@ public class FeedVersionSummary extends Model implements Serializable { @JsonIgnore public ValidationResult validationResult; private PartialValidationSummary validationSummary; + public Date processedByExternalPublisher; + public Date sentToExternalPublisher; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 1350fbe52..b351cf4c6 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -397,6 +397,8 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromLatestDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); + assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); } @Test @@ -431,6 +433,8 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromPinnedDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); + assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -504,6 +508,8 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc validationResult.lastCalendarDate = endDate; validationResult.errorCount = 5 + (int)(Math.random() * ((1000 - 5) + 1)); feedVersion.validationResult = validationResult; + feedVersion.processedByExternalPublisher = new Date(); + feedVersion.sentToExternalPublisher = new Date(); Persistence.feedVersions.create(feedVersion); return feedVersion; } From af7ec62dabdee784dbdf1673fbfb134d1f06a49d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 9 Oct 2025 16:30:10 +0100 Subject: [PATCH 02/20] improvement(Update to include gtfs plus validation): Gen gtfs plus validation and save to feed versi --- .../manager/controllers/DumpController.java | 2 + .../manager/gtfsplus/GtfsPlusValidation.java | 21 +++--- .../manager/jobs/ProcessSingleFeedJob.java | 5 +- .../manager/jobs/ValidateGtfsPlusFeedJob.java | 70 +++++++++++++++++++ .../manager/models/FeedSourceSummary.java | 9 ++- .../datatools/manager/models/FeedVersion.java | 27 +++++++ .../manager/models/FeedVersionSummary.java | 1 + .../conveyal/datatools/DataSanitizerTest.java | 4 +- .../api/FeedSourceControllerTest.java | 3 + .../manager/jobs/MergeFeedsJobTest.java | 4 +- 10 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index 28f4fec28..6336813c2 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -4,6 +4,7 @@ import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; +import com.conveyal.datatools.manager.jobs.ValidateGtfsPlusFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; @@ -365,6 +366,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee } else { JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false)); JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser, false)); + JobUtils.heavyExecutor.execute(new ValidateGtfsPlusFeedJob(version, systemUser, false)); } } // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load); diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 0de656b91..aeb6cde71 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -50,21 +50,26 @@ private GtfsPlusValidation (String feedVersionId) { } /** - * Validate a GTFS+ feed and return a list of issues encountered. - * FIXME: For now this uses the MapDB-backed GTFSFeed class. Which actually suggests that this might - * should be contained within a MonitorableJob. + * Overload method to retrieve the feed version. */ public static GtfsPlusValidation validate(String feedVersionId) throws Exception { - GtfsPlusValidation validation = new GtfsPlusValidation(feedVersionId); + FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); + return validate(feedVersion); + } + + /** + * Validate a GTFS+ feed and return a list of issues encountered. + */ + public static GtfsPlusValidation validate(FeedVersion feedVersion) throws Exception { if (!DataManager.isModuleEnabled("gtfsplus")) { throw new IllegalStateException("GTFS+ module must be enabled in server.yml to run GTFS+ validation."); } - LOG.info("Validating GTFS+ for {}", feedVersionId); + GtfsPlusValidation validation = new GtfsPlusValidation(feedVersion.id); + LOG.info("Validating GTFS+ for {}", feedVersion.id); - FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); // Load the main GTFS file. // FIXME: Swap MapDB-backed GTFSFeed for use of SQL data? - File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersionId + ".db"); + File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersion.id + ".db"); String gtfsFeedDbFilePath = gtfsFeedDbFile.getAbsolutePath(); GTFSFeed gtfsFeed; try { @@ -87,7 +92,7 @@ public static GtfsPlusValidation validate(String feedVersionId) throws Exception } // check for saved GTFS+ data - File file = gtfsPlusStore.getFeed(feedVersionId); + File file = gtfsPlusStore.getFeed(feedVersion.id); if (file == null) { validation.published = true; LOG.warn("GTFS+ Validation -- Modified GTFS+ file not found, loading from main version GTFS."); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 52f28f8ba..6766a50e4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -41,7 +41,7 @@ public class ProcessSingleFeedJob extends FeedVersionJob { public static boolean ENABLE_MTC_TRANSFORMATIONS = true; // Used in testing to skip validation and speed up response times. - public static boolean VALIDATE_MOBILITY_DATA = true; + public static boolean SKIP_ADDITIONAL_VALIDATION = true; /** * Create a job for the given feed version. @@ -131,8 +131,9 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); - if (VALIDATE_MOBILITY_DATA) { + if (SKIP_ADDITIONAL_VALIDATION) { addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion)); + addNextJob(new ValidateGtfsPlusFeedJob(feedVersion, owner, isNewVersion)); } // We only need to snapshot the feed if there are transformations at the database level. In the case that there diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java new file mode 100644 index 000000000..4255f3ae0 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java @@ -0,0 +1,70 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.FeedVersionJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This job handles the Gtfs+ validation of a given feed version. If the version is not new, it will simply + * replace the existing version with the version object that has updated validation info. + */ +public class ValidateGtfsPlusFeedJob extends FeedVersionJob { + public static final Logger LOG = LoggerFactory.getLogger(ValidateGtfsPlusFeedJob.class); + + private final FeedVersion feedVersion; + private final boolean isNewVersion; + + public ValidateGtfsPlusFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) { + super(owner, "Validating Gtfs+", JobType.VALIDATE_FEED); + feedVersion = version; + this.isNewVersion = isNewVersion; + status.update("Waiting to begin Gtfs+ validation...", 0); + } + + @Override + public void jobLogic () { + LOG.info("Running ValidateGtfsPlusFeedJob for {}", feedVersion.id); + feedVersion.validateGtfsPlus(status); + } + + @Override + public void jobFinished () { + if (!status.error) { + if (parentJobId != null && JobType.PROCESS_FEED.equals(parentJobType)) { + // Validate stage is happening as part of an overall process feed job. + // At this point all GTFS data has been loaded and validated, so we record + // the FeedVersion into mongo. + // This happens here because otherwise we would have to wait for other jobs, + // such as BuildTransportNetwork, to finish. If those subsequent jobs fail, + // the version won't get loaded into MongoDB (even though it exists in postgres). + feedVersion.persistFeedVersionAfterValidation(isNewVersion); + } + status.completeSuccessfully("Gtfs+ validation finished!"); + } else { + // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version + // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is + // expected. + feedVersion.delete(); + } + } + + /** + * Getter that allows a client to know the ID of the feed version that will be created as soon as the upload is + * initiated; however, we will not store the FeedVersion in the mongo application database until the upload and + * processing is completed. This prevents clients from manipulating GTFS data before it is entirely imported. + */ + @JsonProperty + public String getFeedVersionId () { + return feedVersion.id; + } + + @JsonProperty + public String getFeedSourceId () { + return feedVersion.parentFeedSource().id; + } + + +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index f68f97af8..5abf45d7b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -74,6 +74,8 @@ public class FeedSourceSummary { public Date latestSentToExternalPublisher; + public boolean latestHasGtfsPlusValidationIssues; + public FeedSourceSummary() { } @@ -116,6 +118,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.latestValidation = new LatestValidationResult(feedVersionSummary); this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + this.latestHasGtfsPlusValidationIssues = feedVersionSummary.hasGtfsPlusValidationIssues; } } } @@ -216,6 +219,7 @@ public static Map getLatestFeedVersionForFeedSources errorCount: "$feedVersions.validationResult.errorCount", processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", sentToExternalPublisher: "$feedVersions.sentToExternalPublisher" + hasGtfsPlusValidationIssues: "$feedVersions.hasGtfsPlusValidationIssues" } } } @@ -235,7 +239,8 @@ public static Map getLatestFeedVersionForFeedSources Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), - Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher") + Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), + Accumulators.last("hasGtfsPlusValidationIssues", "$feedVersions.hasGtfsPlusValidationIssues") ) ); return extractFeedVersionSummaries( @@ -475,6 +480,8 @@ private static Map extractFeedVersionSummaries( feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); + Boolean hasGtfsPlusValidationIssues = feedVersionDocument.getBoolean("hasGtfsPlusValidationIssues"); + feedVersionSummary.hasGtfsPlusValidationIssues = hasGtfsPlusValidationIssues != null && hasGtfsPlusValidationIssues; feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 512eecd32..cf15899f2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -5,6 +5,7 @@ import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.jobs.validation.RouteTypeValidatorBuilder; @@ -276,6 +277,8 @@ public FeedValidationResultSummary validationSummary() { public Document mobilityDataResult; + public boolean hasGtfsPlusValidationIssues; + public String formattedTimestamp() { SimpleDateFormat format = new SimpleDateFormat(HUMAN_READABLE_TIMESTAMP_FORMAT); return format.format(this.updated); @@ -475,6 +478,30 @@ public void validateMobility(MonitorableJob.Status status) { } } + /** + * Produce Gtfs+ validation results for this feed version if GTFS+ module is enabled. + */ + public void validateGtfsPlus(MonitorableJob.Status status) { + + // Sometimes this method is called when no status object is available. + if (status == null) status = new MonitorableJob.Status(); + + if (DataManager.isModuleEnabled("gtfsplus")) { + try { + GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(this); + hasGtfsPlusValidationIssues = !gtfsPlusValidation.issues.isEmpty(); + } catch (Exception e) { + LOG.warn("Unable to validate GTFS+ validation.", e); + status.fail(String.format("Unable to validate feed %s", this.id), e); + hasGtfsPlusValidationIssues = true; + validationResult = new ValidationResult(); + validationResult.fatalException = "failure!"; + } + } else { + LOG.warn("GTFS+ module not enabled, skipping GTFS+ validation."); + } + } + public void validate() { validate(null); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 8b8e8de63..539eca25b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -30,6 +30,7 @@ public class FeedVersionSummary extends Model implements Serializable { private PartialValidationSummary validationSummary; public Date processedByExternalPublisher; public Date sentToExternalPublisher; + public boolean hasGtfsPlusValidationIssues; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 9648d0982..5ae99d4e5 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -47,7 +47,7 @@ static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = false; project = new Project(); project.name = appendDate("Test"); Persistence.projects.create(project); @@ -86,7 +86,7 @@ static void setUp() throws IOException { @AfterAll static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { Auth0Connection.setAuthDisabled(false); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = true; project.delete(); FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionOrphan.id); if (feedVersion != null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index b351cf4c6..168f300ca 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -399,6 +399,7 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); assertEquals(feedVersionFromLatestDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(feedVersionFromLatestDeployment.hasGtfsPlusValidationIssues, feedSourceSummaries.get(0).latestHasGtfsPlusValidationIssues); } @Test @@ -435,6 +436,7 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); assertEquals(feedVersionFromPinnedDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(feedVersionFromPinnedDeployment.hasGtfsPlusValidationIssues, feedSourceSummaries.get(0).latestHasGtfsPlusValidationIssues); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -510,6 +512,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc feedVersion.validationResult = validationResult; feedVersion.processedByExternalPublisher = new Date(); feedVersion.sentToExternalPublisher = new Date(); + feedVersion.hasGtfsPlusValidationIssues = true; Persistence.feedVersions.create(feedVersion); return feedVersion; } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 7b383075e..770374c60 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -87,7 +87,7 @@ public class MergeFeedsJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = false; // Enable MTC extension, but disable the transformations. ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = false; DatatoolsTest.enableMTCExtension(); @@ -169,7 +169,7 @@ public static void tearDown() { } DatatoolsTest.resetMTCExtension(); ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = true; - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = true; } /** From 075cdc72299d4ac30b6cc706c6da2958d8e27a28 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 10 Oct 2025 12:46:43 +0100 Subject: [PATCH 03/20] improvement(Store gtfs plus validation in feed version): Update to feed version and tests --- .../manager/gtfsplus/GtfsPlusValidation.java | 11 ++++- .../manager/gtfsplus/ValidationIssue.java | 4 ++ .../manager/models/FeedSourceSummary.java | 41 +++++++++++++++---- .../datatools/manager/models/FeedVersion.java | 6 +-- .../manager/models/FeedVersionSummary.java | 3 +- .../api/FeedSourceControllerTest.java | 15 +++++-- 6 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index aeb6cde71..33f50476e 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -38,7 +38,7 @@ public class GtfsPlusValidation implements Serializable { private static final String REALTIME_ROUTES_TXT = "realtime_routes.txt"; // Public fields to appear in validation JSON. - public final String feedVersionId; + public String feedVersionId; /** Indicates whether GTFS+ validation applies to user-edited feed or original published GTFS feed */ public boolean published; public long lastModified; @@ -49,6 +49,15 @@ private GtfsPlusValidation (String feedVersionId) { this.feedVersionId = feedVersionId; } + public GtfsPlusValidation(boolean published, List issues) { + this.published = published; + this.issues = issues; + } + + public GtfsPlusValidation() { + // Empty constructor for serialization + } + /** * Overload method to retrieve the feed version. */ diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java index b835a3996..34c59b578 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java @@ -10,6 +10,10 @@ public class ValidationIssue implements Serializable { public int rowIndex; public String description; + public ValidationIssue() { + // Empty constructor for serialization + } + public ValidationIssue(String tableId, String fieldName, int rowIndex, String description) { this.tableId = tableId; this.fieldName = fieldName; diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 5abf45d7b..1789cd489 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -1,8 +1,12 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.validator.ValidationResult; + +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; @@ -32,6 +36,7 @@ import static com.mongodb.client.model.Filters.in; public class FeedSourceSummary { + private static final ObjectMapper mapper = new ObjectMapper(); private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); public String projectId; @@ -74,7 +79,7 @@ public class FeedSourceSummary { public Date latestSentToExternalPublisher; - public boolean latestHasGtfsPlusValidationIssues; + public GtfsPlusValidation latestGtfsPlusValidation; public FeedSourceSummary() { } @@ -118,7 +123,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.latestValidation = new LatestValidationResult(feedVersionSummary); this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; - this.latestHasGtfsPlusValidationIssues = feedVersionSummary.hasGtfsPlusValidationIssues; + this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; } } } @@ -218,8 +223,8 @@ public static Map getLatestFeedVersionForFeedSources lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", errorCount: "$feedVersions.validationResult.errorCount", processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", - sentToExternalPublisher: "$feedVersions.sentToExternalPublisher" - hasGtfsPlusValidationIssues: "$feedVersions.hasGtfsPlusValidationIssues" + sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", + gtfsPlusValidation: "$feedVersions.gtfsPlusValidation" } } } @@ -240,7 +245,7 @@ public static Map getLatestFeedVersionForFeedSources Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), - Accumulators.last("hasGtfsPlusValidationIssues", "$feedVersions.hasGtfsPlusValidationIssues") + Accumulators.last("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation") ) ); return extractFeedVersionSummaries( @@ -480,14 +485,36 @@ private static Map extractFeedVersionSummaries( feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); - Boolean hasGtfsPlusValidationIssues = feedVersionDocument.getBoolean("hasGtfsPlusValidationIssues"); - feedVersionSummary.hasGtfsPlusValidationIssues = hasGtfsPlusValidationIssues != null && hasGtfsPlusValidationIssues; + feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } return feedVersionSummaries; } + /** + * Build GtfsPlusValidation object from feed version document. + */ + private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocument) { + Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); + if (gtfsPlusValidationDocument == null) return null; + List issues = null; + if (gtfsPlusValidationDocument.containsKey("issues") && gtfsPlusValidationDocument.get("issues") != null) { + List issueDocs = gtfsPlusValidationDocument.getList("issues", Document.class); + if (issueDocs != null) { + issues = new ArrayList<>(); + for (Document doc : issueDocs) { + issues.add(mapper.convertValue(doc, ValidationIssue.class)); + } + } + } + boolean published = false; + if (gtfsPlusValidationDocument.getBoolean("published") != null) { + published = gtfsPlusValidationDocument.getBoolean("published"); + } + return new GtfsPlusValidation(published, issues); + } + /** * Build validation result from feed version document. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index cf15899f2..4591fd995 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -277,7 +277,7 @@ public FeedValidationResultSummary validationSummary() { public Document mobilityDataResult; - public boolean hasGtfsPlusValidationIssues; + public GtfsPlusValidation gtfsPlusValidation; public String formattedTimestamp() { SimpleDateFormat format = new SimpleDateFormat(HUMAN_READABLE_TIMESTAMP_FORMAT); @@ -488,12 +488,10 @@ public void validateGtfsPlus(MonitorableJob.Status status) { if (DataManager.isModuleEnabled("gtfsplus")) { try { - GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(this); - hasGtfsPlusValidationIssues = !gtfsPlusValidation.issues.isEmpty(); + gtfsPlusValidation = GtfsPlusValidation.validate(this); } catch (Exception e) { LOG.warn("Unable to validate GTFS+ validation.", e); status.fail(String.format("Unable to validate feed %s", this.id), e); - hasGtfsPlusValidationIssues = true; validationResult = new ValidationResult(); validationResult.fatalException = "failure!"; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 539eca25b..ae9dfd715 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -30,7 +31,7 @@ public class FeedVersionSummary extends Model implements Serializable { private PartialValidationSummary validationSummary; public Date processedByExternalPublisher; public Date sentToExternalPublisher; - public boolean hasGtfsPlusValidationIssues; + public GtfsPlusValidation gtfsPlusValidation; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 168f300ca..87a26f6d0 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -4,6 +4,8 @@ import com.conveyal.datatools.TestUtils; import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -399,7 +401,9 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); assertEquals(feedVersionFromLatestDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); - assertEquals(feedVersionFromLatestDeployment.hasGtfsPlusValidationIssues, feedSourceSummaries.get(0).latestHasGtfsPlusValidationIssues); + assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); + assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); + } @Test @@ -436,7 +440,8 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); assertEquals(feedVersionFromPinnedDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); - assertEquals(feedVersionFromPinnedDeployment.hasGtfsPlusValidationIssues, feedSourceSummaries.get(0).latestHasGtfsPlusValidationIssues); + assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); + assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -512,7 +517,11 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc feedVersion.validationResult = validationResult; feedVersion.processedByExternalPublisher = new Date(); feedVersion.sentToExternalPublisher = new Date(); - feedVersion.hasGtfsPlusValidationIssues = true; + List issues = List.of( + new ValidationIssue("Test issue 1", "stops.txt", 1, "stop_id"), + new ValidationIssue("Test issue 2", "stops.txt", 2, "stop_id") + ); + feedVersion.gtfsPlusValidation = new GtfsPlusValidation(true, issues); Persistence.feedVersions.create(feedVersion); return feedVersion; } From a9fb58db9eb01590246756563e3236a81144ac91 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 10 Oct 2025 13:04:40 +0100 Subject: [PATCH 04/20] improvement(Updates param name): Name is more inline with usage --- .../conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java | 4 ++-- src/test/java/com/conveyal/datatools/DataSanitizerTest.java | 4 ++-- .../conveyal/datatools/manager/jobs/MergeFeedsJobTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 6766a50e4..d73b62851 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -41,7 +41,7 @@ public class ProcessSingleFeedJob extends FeedVersionJob { public static boolean ENABLE_MTC_TRANSFORMATIONS = true; // Used in testing to skip validation and speed up response times. - public static boolean SKIP_ADDITIONAL_VALIDATION = true; + public static boolean ENABLE_ADDITIONAL_VALIDATION = true; /** * Create a job for the given feed version. @@ -131,7 +131,7 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); - if (SKIP_ADDITIONAL_VALIDATION) { + if (ENABLE_ADDITIONAL_VALIDATION) { addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion)); addNextJob(new ValidateGtfsPlusFeedJob(feedVersion, owner, isNewVersion)); } diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 5ae99d4e5..994536d53 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -47,7 +47,7 @@ static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; project = new Project(); project.name = appendDate("Test"); Persistence.projects.create(project); @@ -86,7 +86,7 @@ static void setUp() throws IOException { @AfterAll static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { Auth0Connection.setAuthDisabled(false); - ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; project.delete(); FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionOrphan.id); if (feedVersion != null) { diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 770374c60..39340905d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -87,7 +87,7 @@ public class MergeFeedsJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); - ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; // Enable MTC extension, but disable the transformations. ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = false; DatatoolsTest.enableMTCExtension(); @@ -169,7 +169,7 @@ public static void tearDown() { } DatatoolsTest.resetMTCExtension(); ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = true; - ProcessSingleFeedJob.SKIP_ADDITIONAL_VALIDATION = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /** From a1a056f19c26c5d7c5fef7452f1ae6706ea7e9af Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 15 Oct 2025 15:45:52 +0100 Subject: [PATCH 05/20] improvement(Updated feed source summary): Added published version id --- .../datatools/manager/models/FeedSourceSummary.java | 6 ++++++ .../datatools/manager/models/FeedVersionSummary.java | 1 + .../manager/controllers/api/FeedSourceControllerTest.java | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 1789cd489..a732e2f56 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -81,6 +81,8 @@ public class FeedSourceSummary { public GtfsPlusValidation latestGtfsPlusValidation; + public String latestPublishedVersionId; + public FeedSourceSummary() { } @@ -124,6 +126,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; + this.latestPublishedVersionId = feedVersionSummary.publishedVersionId; } } } @@ -215,6 +218,7 @@ public static Map getLatestFeedVersionForFeedSources { $group: { _id: "$_id", + publishedVersionId: { $first: "$publishedVersionId" }, doc: { $max: { version: "$feedVersions.version", @@ -239,6 +243,7 @@ public static Map getLatestFeedVersionForFeedSources unwind("$feedVersions"), group( "$_id", + Accumulators.first("publishedVersionId", "$publishedVersionId"), Accumulators.last("feedVersionId", "$feedVersions._id"), Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), @@ -486,6 +491,7 @@ private static Map extractFeedVersionSummaries( feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); + feedVersionSummary.publishedVersionId = feedVersionDocument.getString("publishedVersionId"); feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index ae9dfd715..f52c7f116 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -32,6 +32,7 @@ public class FeedVersionSummary extends Model implements Serializable { public Date processedByExternalPublisher; public Date sentToExternalPublisher; public GtfsPlusValidation gtfsPlusValidation; + public String publishedVersionId; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 87a26f6d0..93e885494 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -403,7 +403,7 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); - + assertEquals(feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); } @Test @@ -442,6 +442,7 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -472,6 +473,7 @@ private static FeedSource createFeedSource( feedSource.projectId = project.id; feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; + feedSource.publishedVersionId = "published-version-id-1"; if (labels != null) feedSource.labelIds = labels; if (notes != null) feedSource.noteIds = notes; if (persist) Persistence.feedSources.create(feedSource); From 7a5ce4807d301ba9beb7c5e69ed6c062850807e6 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 16 Oct 2025 16:02:05 +0100 Subject: [PATCH 06/20] improvement(GtfsPlusController): Update to save validation to related feed version --- .../manager/controllers/api/GtfsPlusController.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java index 5c83414b8..8047a52ca 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java @@ -246,13 +246,22 @@ private static String publishGtfsPlusFile(Request req, Response res) { } /** - * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). + * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). If the feed + * version already has GTFS+ validation results, those will be returned instead of re-validating. */ private static GtfsPlusValidation getGtfsPlusValidation(Request req, Response res) { String feedVersionId = req.params("versionid"); GtfsPlusValidation gtfsPlusValidation = null; try { + FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); + if (feedVersion != null && feedVersion.gtfsPlusValidation != null) { + return feedVersion.gtfsPlusValidation; + } gtfsPlusValidation = GtfsPlusValidation.validate(feedVersionId); + if (feedVersion != null) { + feedVersion.gtfsPlusValidation = gtfsPlusValidation; + Persistence.feedVersions.replace(feedVersion.id, feedVersion); + } } catch(Exception e) { logMessageAndHalt(req, 500, "Could not read GTFS+ zip file", e); } From 43756e8cf04e0476c87276221979fefc54073769 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 17 Oct 2025 10:04:14 +0100 Subject: [PATCH 07/20] improvement(maven.yml): Bumped node version from 20.x to 22.x --- .github/workflows/maven.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index daa3a663a..071259596 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,11 +32,11 @@ jobs: with: java-version: 19 distribution: 'temurin' - # Install node 20 for running e2e tests (and for maven-semantic-release). - - name: Use Node.js 20.x + # Install node 22 for running e2e tests (and for maven-semantic-release). + - name: Use Node.js 22.x uses: actions/setup-node@v1 with: - node-version: 20.x + node-version: 22.x - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: @@ -98,10 +98,10 @@ jobs: # Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github. - - name: Use Node.js 20.x + - name: Use Node.js 22.x uses: actions/setup-node@v1 with: - node-version: 20.x + node-version: 22.x - name: Run maven-semantic-release if: env.SAVE_JAR_TO_S3 == 'true' env: From fcdbd93ab40193b36049b2ca97a745d500f68ed1 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 23 Oct 2025 13:51:24 +0100 Subject: [PATCH 08/20] improvement(Added new params): Now includes published feed version and namespace --- .../manager/models/FeedSourceSummary.java | 56 +++++++++++++++++-- .../manager/models/FeedVersionSummary.java | 3 + .../api/FeedSourceControllerTest.java | 34 +++++++++-- 3 files changed, 82 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index a732e2f56..0ec3abab7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -83,6 +83,10 @@ public class FeedSourceSummary { public String latestPublishedVersionId; + public String latestNamespace; + + public FeedValidationResultSummary publishedValidationSummary; + public FeedSourceSummary() { } @@ -126,7 +130,12 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; + this.latestNamespace = feedVersionSummary.namespace; this.latestPublishedVersionId = feedVersionSummary.publishedVersionId; + this.publishedValidationSummary = new FeedValidationResultSummary(); + this.publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; + this.publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; + this.publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; } } } @@ -213,13 +222,25 @@ public static Map getLatestFeedVersionForFeedSources } }, { - $unwind: "$feedVersions" + $lookup: { + from: "FeedVersion", + localField: "publishedVersionId", + foreignField: "namespace", + as: "publishedFeedVersion" + } + }, + { + $unwind: "$feedVersions", + $unwind: "$publishedFeedVersion", }, { $group: { _id: "$_id", publishedVersionId: { $first: "$publishedVersionId" }, - doc: { + publishedFeedVersionErrorCount: { $first: "$publishedFeedVersion.validationResult.errorCount"}, + publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, + publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, + feedVersion: { $max: { version: "$feedVersions.version", feedVersionId: "$feedVersions._id", @@ -228,7 +249,8 @@ public static Map getLatestFeedVersionForFeedSources errorCount: "$feedVersions.validationResult.errorCount", processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", - gtfsPlusValidation: "$feedVersions.gtfsPlusValidation" + gtfsPlusValidation: "$feedVersions.gtfsPlusValidation", + namespace: "$feedVersions.namespace" } } } @@ -240,17 +262,23 @@ public static Map getLatestFeedVersionForFeedSources in("projectId", projectId) ), lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), + lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), + unwind("$publishedFeedVersion"), group( "$_id", Accumulators.first("publishedVersionId", "$publishedVersionId"), + Accumulators.first("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), + Accumulators.first("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), + Accumulators.first("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), Accumulators.last("feedVersionId", "$feedVersions._id"), Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), - Accumulators.last("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation") + Accumulators.last("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation"), + Accumulators.last("namespace", "$feedVersions.namespace") ) ); return extractFeedVersionSummaries( @@ -485,14 +513,30 @@ private static Map extractFeedVersionSummaries( List stages ) { Map feedVersionSummaries = new HashMap<>(); - for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { + Document feedVersionDocument = Persistence.getDocuments(collection, stages).first(); + if (feedVersionDocument != null) { + // Only expected one or zero documents to be returned. FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); - feedVersionSummary.publishedVersionId = feedVersionDocument.getString("publishedVersionId"); + feedVersionSummary.namespace = feedVersionDocument.getString("namespace"); feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + + // The feed source's published version id. + feedVersionSummary.publishedVersionId = feedVersionDocument.getString("publishedVersionId"); + + // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. + feedVersionSummary.publishedFeedVersionErrorCount = feedVersionDocument.get("publishedFeedVersionErrorCount") != null + ? feedVersionDocument.getInteger("publishedFeedVersionErrorCount") + : 0; + feedVersionSummary.publishedFeedVersionStartDate = feedVersionDocument.get("publishedFeedVersionStartDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionStartDate")) + : null; + feedVersionSummary.publishedFeedVersionEndDate = feedVersionDocument.get("publishedFeedVersionEndDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionEndDate")) + : null; feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } return feedVersionSummaries; diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index f52c7f116..f9879a3d1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -33,6 +33,9 @@ public class FeedVersionSummary extends Model implements Serializable { public Date sentToExternalPublisher; public GtfsPlusValidation gtfsPlusValidation; public String publishedVersionId; + public int publishedFeedVersionErrorCount; + public LocalDate publishedFeedVersionStartDate; + public LocalDate publishedFeedVersionEndDate; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 93e885494..7e37debfd 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -57,6 +57,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; private static FeedVersion feedVersionFromLatestDeployment = null; + private static FeedVersion feedVersionPublishedFromLatestDeployment = null; private static Deployment deploymentLatest = null; private static Deployment deploymentSuperseded = null; @@ -69,6 +70,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { public static void setUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); + project = new Project(); project.name = "ProjectOne"; project.autoFetchFeeds = true; @@ -123,7 +125,16 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx "feed-version-from-latest-deployment", feedSourceWithLatestDeploymentFeedVersion.id, deployedStartDate, - deployedEndDate + deployedEndDate, + null + ); + feedVersionPublishedFromLatestDeployment = createFeedVersion( + "published-feed-version-from-latest-deployment", + // Set to null so the relationship to feed source is via the published version id. + null, + LocalDate.of(2022, Month.NOVEMBER, 2), + LocalDate.of(2022, Month.NOVEMBER, 3), + feedSourceWithLatestDeploymentFeedVersion.publishedVersionId ); deploymentSuperseded = createDeployment( "deployment-superseded", @@ -220,6 +231,9 @@ private static void tearDownDeployedFeedVersion() { if (feedVersionFromLatestDeployment != null) { Persistence.feedVersions.removeById(feedVersionFromLatestDeployment.id); } + if (feedVersionPublishedFromLatestDeployment != null) { + Persistence.feedVersions.removeById(feedVersionPublishedFromLatestDeployment.id); + } if (feedVersionFromPinnedDeployment != null) { Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); } @@ -403,7 +417,13 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); - assertEquals(feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); + assertEquals(feedVersionFromLatestDeployment.namespace, feedSourceSummaries.get(0).latestNamespace); + + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); + + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, feedSourceSummaries.get(0).publishedValidationSummary.errorCount); + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.firstCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.startDate); + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.lastCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.endDate); } @Test @@ -442,7 +462,6 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); - assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -502,13 +521,17 @@ private static Deployment createDeployment( * Helper method to create a feed version with no start date. */ private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate) { - return createFeedVersion(id, feedSourceId, null, endDate); + return createFeedVersion(id, feedSourceId, null, endDate, null); + } + + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate, String namespace) { + return createFeedVersion(id, feedSourceId, null, endDate, namespace); } /** * Helper method to create a feed version. */ - private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate) { + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate, String namespace) { FeedVersion feedVersion = new FeedVersion(); feedVersion.id = id; feedVersion.feedSourceId = feedSourceId; @@ -524,6 +547,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc new ValidationIssue("Test issue 2", "stops.txt", 2, "stop_id") ); feedVersion.gtfsPlusValidation = new GtfsPlusValidation(true, issues); + feedVersion.namespace = namespace != null ? namespace : "feed-version-namespace"; Persistence.feedVersions.create(feedVersion); return feedVersion; } From fed3371c9f3a0c011b3c23fef8a573babf76435e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 11 Nov 2025 16:37:45 +0000 Subject: [PATCH 09/20] improvement(Update to include error counts): Feed source summary now includes error counts using exp --- pom.xml | 2 +- .../manager/models/FeedSourceSummary.java | 17 ++++++++++++ .../manager/models/FeedVersionSummary.java | 4 +++ .../api/FeedSourceControllerTest.java | 27 ++++++++++++++++--- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 58dffc704..3f1b815f1 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 5e004388b2473c391412fdd4954068c35186e231 + cc566848ed diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 0ec3abab7..9c2a18b71 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -4,6 +4,8 @@ import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,7 +17,13 @@ import com.mongodb.client.model.Sorts; import org.bson.Document; import org.bson.conversions.Bson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -25,6 +33,7 @@ import java.util.List; import java.util.Map; +import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.mongodb.client.model.Aggregates.group; import static com.mongodb.client.model.Aggregates.limit; import static com.mongodb.client.model.Aggregates.lookup; @@ -36,6 +45,7 @@ import static com.mongodb.client.model.Filters.in; public class FeedSourceSummary { + private static final Logger LOG = LoggerFactory.getLogger(FeedSourceSummary.class); private static final ObjectMapper mapper = new ObjectMapper(); private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); public String projectId; @@ -87,6 +97,8 @@ public class FeedSourceSummary { public FeedValidationResultSummary publishedValidationSummary; + public List latestErrorCounts = new ArrayList<>(); + public FeedSourceSummary() { } @@ -131,6 +143,7 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; this.latestNamespace = feedVersionSummary.namespace; + this.latestErrorCounts = feedVersionSummary.errorCounts; this.latestPublishedVersionId = feedVersionSummary.publishedVersionId; this.publishedValidationSummary = new FeedValidationResultSummary(); this.publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; @@ -522,6 +535,10 @@ private static Map extractFeedVersionSummaries( feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); feedVersionSummary.namespace = feedVersionDocument.getString("namespace"); + if (feedVersionSummary.namespace != null) { + ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); + feedVersionSummary.errorCounts = errorCountFetcher.getErrorCounts(feedVersionSummary.namespace); + } feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); // The feed source's published version id. diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index f9879a3d1..7e6c2e997 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -9,7 +10,9 @@ import java.io.Serializable; import java.time.LocalDate; +import java.util.ArrayList; import java.util.Date; +import java.util.List; /** * Includes summary data (a subset of fields) for a feed version. @@ -36,6 +39,7 @@ public class FeedVersionSummary extends Model implements Serializable { public int publishedFeedVersionErrorCount; public LocalDate publishedFeedVersionStartDate; public LocalDate publishedFeedVersionEndDate; + public List errorCounts = new ArrayList<>(); public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 7e37debfd..4cc64ab1d 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -6,6 +6,7 @@ import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; +import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -19,6 +20,7 @@ import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; import com.conveyal.datatools.manager.utils.json.JsonUtil; +import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -34,12 +36,15 @@ import java.util.Date; import java.util.List; +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.mongodb.client.model.Filters.eq; import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; import static org.eclipse.jetty.http.HttpStatus.OK_200; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; @@ -70,6 +75,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { public static void setUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; project = new Project(); project.name = "ProjectOne"; @@ -128,6 +134,14 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx deployedEndDate, null ); + + FeedVersion feedVersionFromGtfsZip = createFeedVersionFromGtfsZip(feedSourceWithLatestDeploymentFeedVersion, "bart_old.zip"); + // Update the feed version namespace to match that created from the import. + feedVersionFromLatestDeployment.namespace = feedVersionFromGtfsZip.namespace; + + Persistence.feedVersions.removeById(feedVersionFromGtfsZip.id); + Persistence.feedVersions.replace(feedVersionFromLatestDeployment.id, feedVersionFromLatestDeployment); + feedVersionPublishedFromLatestDeployment = createFeedVersion( "published-feed-version-from-latest-deployment", // Set to null so the relationship to feed source is via the published version id. @@ -208,6 +222,7 @@ public static void tearDown() { Persistence.labels.removeById(adminOnlyLabel.id); } tearDownDeployedFeedVersion(); + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /** @@ -216,8 +231,11 @@ public static void tearDown() { * FeedSource#getFeedVersionFromPinnedDeployment to be tested. */ private static void tearDownDeployedFeedVersion() { + if (feedVersionFromLatestDeployment != null) { + feedVersionFromLatestDeployment.delete(); + } if (projectWithPinnedDeployment != null) { - Persistence.projects.removeById(projectWithPinnedDeployment.id); + projectWithPinnedDeployment.delete(); } if (projectWithLatestDeployment != null) { Persistence.projects.removeById(projectWithLatestDeployment.id); @@ -228,9 +246,6 @@ private static void tearDownDeployedFeedVersion() { if (feedSourceWithPinnedDeploymentFeedVersion != null) { Persistence.feedSources.removeById(feedSourceWithPinnedDeploymentFeedVersion.id); } - if (feedVersionFromLatestDeployment != null) { - Persistence.feedVersions.removeById(feedVersionFromLatestDeployment.id); - } if (feedVersionPublishedFromLatestDeployment != null) { Persistence.feedVersions.removeById(feedVersionPublishedFromLatestDeployment.id); } @@ -418,6 +433,10 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); assertEquals(feedVersionFromLatestDeployment.namespace, feedSourceSummaries.get(0).latestNamespace); + assertFalse(feedSourceSummaries.get(0).latestErrorCounts.isEmpty()); + assertEquals(NewGTFSErrorType.CONDITIONALLY_REQUIRED, feedSourceSummaries.get(0).latestErrorCounts.get(0).type); + assertEquals(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED, feedSourceSummaries.get(0).latestErrorCounts.get(1).type); + assertEquals(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS, feedSourceSummaries.get(0).latestErrorCounts.get(2).type); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); From a437f5d1fa7faa9f4ce21eae7eaf56c107a01641 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 12 Nov 2025 12:08:02 +0000 Subject: [PATCH 10/20] improvement(Code refactor): Move feed version extraction into FeedVersionSummary and created separat --- .../manager/models/FeedSourceSummary.java | 426 ++---------------- .../manager/models/FeedVersionSummary.java | 128 +++++- .../datatools/manager/models/Project.java | 5 + src/main/resources/mongo/README.md | 7 + .../resources/mongo/getFeedSourceSummaries.js | 26 ++ .../getFeedVersionsFromLatestDeployment.js | 64 +++ .../getFeedVersionsFromPinnedDeployment.js | 51 +++ .../getLatestFeedVersionForFeedSources.js | 50 ++ .../api/FeedSourceControllerTest.java | 117 ++--- 9 files changed, 405 insertions(+), 469 deletions(-) create mode 100644 src/main/resources/mongo/README.md create mode 100644 src/main/resources/mongo/getFeedSourceSummaries.js create mode 100644 src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js create mode 100644 src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js create mode 100644 src/main/resources/mongo/getLatestFeedVersionForFeedSources.js diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 9c2a18b71..5c644ae84 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -2,13 +2,9 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; -import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; -import com.conveyal.gtfs.validator.ValidationResult; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; @@ -17,23 +13,15 @@ import com.mongodb.client.model.Sorts; import org.bson.Document; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; import java.time.LocalDate; import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.mongodb.client.model.Aggregates.group; import static com.mongodb.client.model.Aggregates.limit; import static com.mongodb.client.model.Aggregates.lookup; @@ -44,10 +32,11 @@ import static com.mongodb.client.model.Aggregates.unwind; import static com.mongodb.client.model.Filters.in; +/** + * For explicit mongo queries (matching the queries defined in this class) see resources/mongo and README.md for + * explanation of use. + */ public class FeedSourceSummary { - private static final Logger LOG = LoggerFactory.getLogger(FeedSourceSummary.class); - private static final ObjectMapper mapper = new ObjectMapper(); - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); public String projectId; public String id; @@ -105,50 +94,49 @@ public FeedSourceSummary() { public FeedSourceSummary(String projectId, String organizationId, Document feedSourceDocument) { this.projectId = projectId; this.organizationId = organizationId; - this.id = feedSourceDocument.getString("_id"); - this.name = feedSourceDocument.getString("name"); - this.deployable = feedSourceDocument.getBoolean("deployable"); - this.isPublic = feedSourceDocument.getBoolean("isPublic"); + id = feedSourceDocument.getString("_id"); + name = feedSourceDocument.getString("name"); + deployable = feedSourceDocument.getBoolean("deployable"); + isPublic = feedSourceDocument.getBoolean("isPublic"); List documentLabelIds = feedSourceDocument.getList("labelIds", String.class); if (documentLabelIds != null) { - this.labelIds = documentLabelIds; + labelIds = documentLabelIds; } List documentNoteIds = feedSourceDocument.getList("noteIds", String.class); if (documentNoteIds != null) { - this.noteIds = documentNoteIds; + noteIds = documentNoteIds; } // Convert to local date type for consistency. - this.lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); - this.url = feedSourceDocument.getString("url"); + lastUpdated = getLocalDateFromDate(feedSourceDocument.getDate("lastUpdated")); + url = feedSourceDocument.getString("url"); // Get optional filename. - this.filename = feedSourceDocument.getString("filename"); + filename = feedSourceDocument.getString("filename"); } /** * Set the appropriate feed version. For consistency, if no error count is available set the related number of - * issues to null. + * issues to zero. */ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDeployed) { if (feedVersionSummary != null) { if (isDeployed) { - this.deployedFeedVersionId = feedVersionSummary.id; - this.deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - this.deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - this.deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + deployedFeedVersionId = feedVersionSummary.id; + deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) ? 0 : feedVersionSummary.validationResult.errorCount; } else { - this.latestValidation = new LatestValidationResult(feedVersionSummary); - this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; - this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; - this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; - this.latestNamespace = feedVersionSummary.namespace; - this.latestErrorCounts = feedVersionSummary.errorCounts; - this.latestPublishedVersionId = feedVersionSummary.publishedVersionId; - this.publishedValidationSummary = new FeedValidationResultSummary(); - this.publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; - this.publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; - this.publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; + latestValidation = new LatestValidationResult(feedVersionSummary); + latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; + latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; + latestNamespace = feedVersionSummary.namespace; + latestPublishedVersionId = feedVersionSummary.publishedVersionId; + publishedValidationSummary = new FeedValidationResultSummary(); + publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; + publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; + publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; } } } @@ -157,34 +145,6 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl * Get all feed source summaries matching the project id. */ public static List getFeedSourceSummaries(String projectId, String organizationId) { - /* - db.getCollection('FeedSource').aggregate([ - { - // Match provided project id. - $match: { - projectId: "" - } - }, - { - $project: { - "_id": 1, - "name": 1, - "deployable": 1, - "isPublic": 1, - "lastUpdated": 1, - "labelIds": 1, - "url": 1, - "filename": 1, - "noteIds": 1 - } - }, - { - $sort: { - "name": 1 - } - } - ]) - */ List stages = Lists.newArrayList( match( in("projectId", projectId) @@ -211,65 +171,6 @@ public static List getFeedSourceSummaries(String projectId, S * Get the latest feed version from all feed sources for this project. */ public static Map getLatestFeedVersionForFeedSources(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('FeedSource').aggregate([ - { - // Match provided project id. - $match: { - projectId: "project-with-latest-deployment" - } - }, - { - $lookup: { - from: "FeedVersion", - localField: "_id", - foreignField: "feedSourceId", - as: "feedVersions" - } - }, - { - $lookup: { - from: "FeedVersion", - localField: "publishedVersionId", - foreignField: "namespace", - as: "publishedFeedVersion" - } - }, - { - $unwind: "$feedVersions", - $unwind: "$publishedFeedVersion", - }, - { - $group: { - _id: "$_id", - publishedVersionId: { $first: "$publishedVersionId" }, - publishedFeedVersionErrorCount: { $first: "$publishedFeedVersion.validationResult.errorCount"}, - publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, - publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, - feedVersion: { - $max: { - version: "$feedVersions.version", - feedVersionId: "$feedVersions._id", - firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", - lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", - errorCount: "$feedVersions.validationResult.errorCount", - processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", - sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", - gtfsPlusValidation: "$feedVersions.gtfsPlusValidation", - namespace: "$feedVersions.namespace" - } - } - } - } - ]) - */ List stages = Lists.newArrayList( match( in("projectId", projectId) @@ -299,86 +200,14 @@ public static Map getLatestFeedVersionForFeedSources "feedVersionId", "_id", false, - stages); + stages + ); } /** * Get the deployed feed versions from the latest deployment for this project. */ public static Map getFeedVersionsFromLatestDeployment(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-latest-deployment" - } - }, - { - // Get all deployments for this project. - $lookup:{ - from:"Deployment", - localField:"_id", - foreignField:"projectId", - as:"deployment" - } - }, - { - // Deconstruct deployments array to a document for each element. - $unwind: "$deployment" - }, - { - // Make the deployment documents the input/root document. - "$replaceRoot": { - "newRoot": "$deployment" - } - }, - { - // Sort descending. - $sort: { - lastUpdated : -1 - } - }, - { - // At this point we will have the latest deployment for a project. - $limit: 1 - }, - { - $lookup:{ - from:"FeedVersion", - localField:"feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $project: { - "_id": 1, - "feedSourceId": 1, - "validationResult.firstCalendarDate": 1, - "validationResult.lastCalendarDate": 1, - "validationResult.errorCount": 1 - } - } - ]) - */ List stages = Lists.newArrayList( match( in("_id", projectId) @@ -405,74 +234,14 @@ public static Map getFeedVersionsFromLatestDeploymen "_id", "feedSourceId", true, - stages); + stages + ); } /** * Get the deployed feed version from the pinned deployment for this feed source. */ public static Map getFeedVersionsFromPinnedDeployment(String projectId) { - /* - Note: To test this script: - 1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 2) Run FeedSourceControllerTest to created required objects referenced here. - 3) Once complete, delete documents via MongoDB. - 4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). - 5) Re-run FeedSourceControllerTest to confirm deletion of objects. - - db.getCollection('Project').aggregate([ - { - // Match provided project id. - $match: { - _id: "project-with-pinned-deployment" - } - }, - { - $project: { - pinnedDeploymentId: 1 - } - }, - { - $lookup:{ - from:"Deployment", - localField:"pinnedDeploymentId", - foreignField:"_id", - as:"deployment" - } - }, - { - $unwind: "$deployment" - }, - { - $lookup:{ - from:"FeedVersion", - localField:"deployment.feedVersionIds", - foreignField:"_id", - as:"feedVersions" - } - }, - { - // Deconstruct feedVersions array to a document for each element. - $unwind: "$feedVersions" - }, - { - // Make the feed version documents the input/root document. - "$replaceRoot": { - "newRoot": "$feedVersions" - } - }, - { - $project: { - "_id": 1, - "feedSourceId": 1, - "validationResult.firstCalendarDate": 1, - "validationResult.lastCalendarDate": 1, - "validationResult.errorCount": 1 - } - } - ]) - */ - List stages = Lists.newArrayList( match( in("_id", projectId) @@ -499,14 +268,19 @@ public static Map getFeedVersionsFromPinnedDeploymen "_id", "feedSourceId", true, - stages); + stages + ); } /** * Produce a list of all feed source summaries for a project. */ - private static List extractFeedSourceSummaries(String projectId, String organizationId, List stages) { + private static List extractFeedSourceSummaries( + String projectId, + String organizationId, + List stages + ) { List feedSourceSummaries = new ArrayList<>(); for (Document feedSourceDocument : Persistence.getDocuments("FeedSource", stages)) { feedSourceSummaries.add(new FeedSourceSummary(projectId, organizationId, feedSourceDocument)); @@ -528,128 +302,16 @@ private static Map extractFeedVersionSummaries( Map feedVersionSummaries = new HashMap<>(); Document feedVersionDocument = Persistence.getDocuments(collection, stages).first(); if (feedVersionDocument != null) { - // Only expected one or zero documents to be returned. - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); - feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); - feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); - feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); - feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); - feedVersionSummary.namespace = feedVersionDocument.getString("namespace"); - if (feedVersionSummary.namespace != null) { - ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); - feedVersionSummary.errorCounts = errorCountFetcher.getErrorCounts(feedVersionSummary.namespace); - } - feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); - - // The feed source's published version id. - feedVersionSummary.publishedVersionId = feedVersionDocument.getString("publishedVersionId"); - - // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. - feedVersionSummary.publishedFeedVersionErrorCount = feedVersionDocument.get("publishedFeedVersionErrorCount") != null - ? feedVersionDocument.getInteger("publishedFeedVersionErrorCount") - : 0; - feedVersionSummary.publishedFeedVersionStartDate = feedVersionDocument.get("publishedFeedVersionStartDate") != null - ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionStartDate")) - : null; - feedVersionSummary.publishedFeedVersionEndDate = feedVersionDocument.get("publishedFeedVersionEndDate") != null - ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionEndDate")) - : null; + FeedVersionSummary feedVersionSummary = new FeedVersionSummary( + feedVersionKey, + hasChildValidationResultDocument, + feedVersionDocument + ); feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } return feedVersionSummaries; } - /** - * Build GtfsPlusValidation object from feed version document. - */ - private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocument) { - Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); - if (gtfsPlusValidationDocument == null) return null; - List issues = null; - if (gtfsPlusValidationDocument.containsKey("issues") && gtfsPlusValidationDocument.get("issues") != null) { - List issueDocs = gtfsPlusValidationDocument.getList("issues", Document.class); - if (issueDocs != null) { - issues = new ArrayList<>(); - for (Document doc : issueDocs) { - issues.add(mapper.convertValue(doc, ValidationIssue.class)); - } - } - } - boolean published = false; - if (gtfsPlusValidationDocument.getBoolean("published") != null) { - published = gtfsPlusValidationDocument.getBoolean("published"); - } - return new GtfsPlusValidation(published, issues); - } - - /** - * Build validation result from feed version document. - */ - private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { - ValidationResult validationResult = new ValidationResult(); - validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); - validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); - validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); - return validationResult; - } - - private static LocalDate getValidationResultDate( - boolean hasChildValidationResultDocument, - Document feedVersionDocument, - String key - ) { - return (hasChildValidationResultDocument) - ? getDateFieldFromDocument(feedVersionDocument, key) - : getDateFromString(feedVersionDocument.getString(key)); - } - - /** - * Extract date value from validation result document. - */ - private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { - Document validationResult = getDocumentChild(document, "validationResult"); - return (validationResult != null) - ? getDateFromString(validationResult.getString(dateKey)) - : null; - } - - /** - * Extract the error count from the parent document or child validation result document. If the error count is not - * available, return -1. - */ - private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { - int errorCount; - try { - errorCount = (hasChildValidationResultDocument) - ? getErrorCount(feedVersionDocument) - : feedVersionDocument.getInteger("errorCount"); - } catch (NullPointerException e) { - errorCount = -1; - } - return errorCount; - } - - /** - * Get the child validation result document and extract the error count from this. - */ - private static int getErrorCount(Document document) { - return getDocumentChild(document, "validationResult").getInteger("errorCount"); - } - - /** - * Extract child document matching provided name. - */ - private static Document getDocumentChild(Document document, String name) { - return (Document) document.get(name); - } - - /** - * Convert String date (if not null) into LocalDate. - */ - private static LocalDate getDateFromString(String date) { - return (date == null) ? null : LocalDate.parse(date, formatter); - } - /** * Convert Date object into LocalDate object. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 7e6c2e997..ac10843d7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -2,23 +2,28 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; -import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.bson.Document; import java.io.Serializable; import java.time.LocalDate; -import java.util.ArrayList; +import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.List; +import java.util.stream.Collectors; /** * Includes summary data (a subset of fields) for a feed version. */ public class FeedVersionSummary extends Model implements Serializable { private static final long serialVersionUID = 1L; + private static final ObjectMapper mapper = new ObjectMapper(); + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); public FeedRetrievalMethod retrievalMethod; public int version; @@ -39,7 +44,6 @@ public class FeedVersionSummary extends Model implements Serializable { public int publishedFeedVersionErrorCount; public LocalDate publishedFeedVersionStartDate; public LocalDate publishedFeedVersionEndDate; - public List errorCounts = new ArrayList<>(); public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { @@ -53,6 +57,33 @@ public FeedVersionSummary() { // Do nothing } + public FeedVersionSummary( + String feedVersionKey, + boolean hasChildValidationResultDocument, + Document feedVersionDocument + ) { + id = feedVersionDocument.getString(feedVersionKey); + processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); + sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); + gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); + namespace = feedVersionDocument.getString("namespace"); + validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + + // The feed source's published version id. + publishedVersionId = feedVersionDocument.getString("publishedVersionId"); + + // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. + publishedFeedVersionErrorCount = feedVersionDocument.get("publishedFeedVersionErrorCount") != null + ? feedVersionDocument.getInteger("publishedFeedVersionErrorCount") + : 0; + publishedFeedVersionStartDate = feedVersionDocument.get("publishedFeedVersionStartDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionStartDate")) + : null; + publishedFeedVersionEndDate = feedVersionDocument.get("publishedFeedVersionEndDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionEndDate")) + : null; + } + /** * Holds a subset of fields from {@link:FeedValidationResultSummary} for UI use only. */ @@ -75,4 +106,95 @@ public class PartialValidationSummary { } } } + + /** + * Build GtfsPlusValidation object from feed version document. + */ + private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocument) { + Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); + if (gtfsPlusValidationDocument == null) { + return null; + } + List issues = null; + if (gtfsPlusValidationDocument.containsKey("issues") && gtfsPlusValidationDocument.get("issues") != null) { + List issueDocs = gtfsPlusValidationDocument.getList("issues", Document.class); + issues = issueDocs + .stream() + .map(doc -> mapper.convertValue(doc, ValidationIssue.class)) + .collect(Collectors.toList()); + } + boolean published = Boolean.TRUE.equals(gtfsPlusValidationDocument.getBoolean("published")); + return new GtfsPlusValidation(published, issues); + } + + /** + * Build validation result from feed version document. + */ + private static ValidationResult getValidationResult(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + ValidationResult validationResult = new ValidationResult(); + validationResult.errorCount = getValidationResultErrorCount(hasChildValidationResultDocument, feedVersionDocument); + validationResult.firstCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "firstCalendarDate"); + validationResult.lastCalendarDate = getValidationResultDate(hasChildValidationResultDocument, feedVersionDocument, "lastCalendarDate"); + return validationResult; + } + + /** + * Convert String date (if not null) into LocalDate. + */ + private static LocalDate getDateFromString(String date) { + return (date == null) ? null : LocalDate.parse(date, formatter); + } + + /** + * Extract child document matching provided name. + */ + private static Document getDocumentChild(Document document, String name) { + return (Document) document.get(name); + } + + /** + * Extract date value from parent document or child validation result document. + */ + private static LocalDate getValidationResultDate( + boolean hasChildValidationResultDocument, + Document feedVersionDocument, + String key + ) { + return (hasChildValidationResultDocument) + ? getDateFieldFromDocument(feedVersionDocument, key) + : getDateFromString(feedVersionDocument.getString(key)); + } + + /** + * Extract date value from validation result document. + */ + private static LocalDate getDateFieldFromDocument(Document document, String dateKey) { + Document validationResult = getDocumentChild(document, "validationResult"); + return (validationResult != null) + ? getDateFromString(validationResult.getString(dateKey)) + : null; + } + + /** + * Extract the error count from the parent document or child validation result document. If the error count is not + * available, return -1. + */ + private static int getValidationResultErrorCount(boolean hasChildValidationResultDocument, Document feedVersionDocument) { + int errorCount; + try { + errorCount = (hasChildValidationResultDocument) + ? getErrorCount(feedVersionDocument) + : feedVersionDocument.getInteger("errorCount"); + } catch (NullPointerException e) { + errorCount = -1; + } + return errorCount; + } + + /** + * Get the child validation result document and extract the error count from this. + */ + private static int getErrorCount(Document document) { + return getDocumentChild(document, "validationResult").getInteger("errorCount"); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 073405297..70340f643 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.manager.jobs.AutoDeployType; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -174,9 +175,13 @@ public Collection retrieveFeedSourceSummaries() { // No pinned deployments, instead, get the deployed feed versions from the latest deployment. deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); } + ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { feedSourceSummary.setFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id), false); feedSourceSummary.setFeedVersion(deployedFeedVersions.get(feedSourceSummary.id), true); + if (feedSourceSummary.latestNamespace != null) { + feedSourceSummary.latestErrorCounts = errorCountFetcher.getErrorCounts(feedSourceSummary.latestNamespace); + } } return feedSourceSummaries; } diff --git a/src/main/resources/mongo/README.md b/src/main/resources/mongo/README.md new file mode 100644 index 000000000..2ed5bad29 --- /dev/null +++ b/src/main/resources/mongo/README.md @@ -0,0 +1,7 @@ +To run the scripts in this folder, follow these steps: + +1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). +2) Run FeedSourceControllerTest to created required objects referenced here. +3) Once complete, delete documents via MongoDB. +4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). +5) Re-run FeedSourceControllerTest to confirm deletion of objects. \ No newline at end of file diff --git a/src/main/resources/mongo/getFeedSourceSummaries.js b/src/main/resources/mongo/getFeedSourceSummaries.js new file mode 100644 index 000000000..3e4db025d --- /dev/null +++ b/src/main/resources/mongo/getFeedSourceSummaries.js @@ -0,0 +1,26 @@ +db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "" + } + }, + { + $project: { + "_id": 1, + "name": 1, + "deployable": 1, + "isPublic": 1, + "lastUpdated": 1, + "labelIds": 1, + "url": 1, + "filename": 1, + "noteIds": 1 + } + }, + { + $sort: { + "name": 1 + } + } +]) diff --git a/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js new file mode 100644 index 000000000..fa829afe9 --- /dev/null +++ b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js @@ -0,0 +1,64 @@ +db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-latest-deployment" + } + }, + { + // Get all deployments for this project. + $lookup:{ + from:"Deployment", + localField:"_id", + foreignField:"projectId", + as:"deployment" + } + }, + { + // Deconstruct deployments array to a document for each element. + $unwind: "$deployment" + }, + { + // Make the deployment documents the input/root document. + "$replaceRoot": { + "newRoot": "$deployment" + } + }, + { + // Sort descending. + $sort: { + lastUpdated : -1 + } + }, + { + // At this point we will have the latest deployment for a project. + $limit: 1 + }, + { + $lookup:{ + from:"FeedVersion", + localField:"feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } +]) diff --git a/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js new file mode 100644 index 000000000..7aef5bf34 --- /dev/null +++ b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js @@ -0,0 +1,51 @@ +db.getCollection('Project').aggregate([ + { + // Match provided project id. + $match: { + _id: "project-with-pinned-deployment" + } + }, + { + $project: { + pinnedDeploymentId: 1 + } + }, + { + $lookup:{ + from:"Deployment", + localField:"pinnedDeploymentId", + foreignField:"_id", + as:"deployment" + } + }, + { + $unwind: "$deployment" + }, + { + $lookup:{ + from:"FeedVersion", + localField:"deployment.feedVersionIds", + foreignField:"_id", + as:"feedVersions" + } + }, + { + // Deconstruct feedVersions array to a document for each element. + $unwind: "$feedVersions" + }, + { + // Make the feed version documents the input/root document. + "$replaceRoot": { + "newRoot": "$feedVersions" + } + }, + { + $project: { + "_id": 1, + "feedSourceId": 1, + "validationResult.firstCalendarDate": 1, + "validationResult.lastCalendarDate": 1, + "validationResult.errorCount": 1 + } + } +]) diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js new file mode 100644 index 000000000..ba19d6193 --- /dev/null +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -0,0 +1,50 @@ +db.getCollection('FeedSource').aggregate([ + { + // Match provided project id. + $match: { + projectId: "project-with-latest-deployment" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "_id", + foreignField: "feedSourceId", + as: "feedVersions" + } + }, + { + $lookup: { + from: "FeedVersion", + localField: "publishedVersionId", + foreignField: "namespace", + as: "publishedFeedVersion" + } + }, + { + $unwind: "$feedVersions", + $unwind: "$publishedFeedVersion", + }, + { + $group: { + _id: "$_id", + publishedVersionId: { $first: "$publishedVersionId" }, + publishedFeedVersionErrorCount: { $first: "$publishedFeedVersion.validationResult.errorCount"}, + publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, + publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, + feedVersion: { + $max: { + version: "$feedVersions.version", + feedVersionId: "$feedVersions._id", + firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", + lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", + errorCount: "$feedVersions.validationResult.errorCount", + processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", + sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", + gtfsPlusValidation: "$feedVersions.gtfsPlusValidation", + namespace: "$feedVersions.namespace" + } + } + } + } +]) diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 4cc64ab1d..17430bb43 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -44,7 +44,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; public class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; @@ -55,21 +54,14 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedSource feedSourceWithInvalidLabels = null; private static Label publicLabel = null; private static Label adminOnlyLabel = null; - private static Label feedSourceWithLatestDeploymentAdminOnlyLabel = null; - private static Label feedSourceWithPinnedDeploymentAdminOnlyLabel = null; - private static Note feedSourceWithLatestDeploymentAdminOnlyNote = null; - private static Note feedSourceWithPinnedDeploymentAdminOnlyNote = null; private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; private static FeedVersion feedVersionFromLatestDeployment = null; private static FeedVersion feedVersionPublishedFromLatestDeployment = null; - private static Deployment deploymentLatest = null; - private static Deployment deploymentSuperseded = null; private static Project projectWithPinnedDeployment = null; private static FeedSource feedSourceWithPinnedDeploymentFeedVersion = null; private static FeedVersion feedVersionFromPinnedDeployment = null; - private static Deployment deploymentPinned = null; @BeforeAll public static void setUp() throws IOException { @@ -77,16 +69,8 @@ public static void setUp() throws IOException { Auth0Connection.setAuthDisabled(true); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; - project = new Project(); - project.name = "ProjectOne"; - project.autoFetchFeeds = true; - Persistence.projects.create(project); - - projectToBeDeleted = new Project(); - projectToBeDeleted.name = "ProjectTwo"; - projectToBeDeleted.autoFetchFeeds = false; - Persistence.projects.create(projectToBeDeleted); - + project = createProject("ProjectOne", true); + projectToBeDeleted = createProject("ProjectTwo", false); feedSourceWithUrl = createFeedSource("FeedSourceOne", new URL("http://www.feedsource.com"), project); feedSourceWithNoUrl = createFeedSource("FeedSourceTwo", null, project); @@ -99,7 +83,6 @@ public static void setUp() throws IOException { setUpFeedVersionFromLatestDeployment(); setUpFeedVersionFromPinnedDeployment(); - } /** @@ -111,8 +94,10 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx projectWithLatestDeployment.organizationId = "project-with-latest-deployment-org-id"; Persistence.projects.create(projectWithLatestDeployment); - feedSourceWithLatestDeploymentAdminOnlyLabel = createLabel("label-id-latest-deployment", "Admin Only Label", projectWithLatestDeployment.id); - feedSourceWithLatestDeploymentAdminOnlyNote = createNote("note-id-latest-deployment", "A test note"); + Label feedSourceWithLatestDeploymentAdminOnlyLabel = createLabel( + "label-id-latest-deployment", "Admin Only Label", projectWithLatestDeployment.id + ); + Note feedSourceWithLatestDeploymentAdminOnlyNote = createNote("note-id-latest-deployment", "A test note"); feedSourceWithLatestDeploymentFeedVersion = createFeedSource( "feed-source-with-latest-deployment-feed-version", @@ -135,10 +120,14 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx null ); - FeedVersion feedVersionFromGtfsZip = createFeedVersionFromGtfsZip(feedSourceWithLatestDeploymentFeedVersion, "bart_old.zip"); + FeedVersion feedVersionFromGtfsZip = createFeedVersionFromGtfsZip( + feedSourceWithLatestDeploymentFeedVersion, + "bart_old.zip" + ); // Update the feed version namespace to match that created from the import. feedVersionFromLatestDeployment.namespace = feedVersionFromGtfsZip.namespace; + // Remove the imported feed version so it does not conflict with the latest deployment feed version. Persistence.feedVersions.removeById(feedVersionFromGtfsZip.id); Persistence.feedVersions.replace(feedVersionFromLatestDeployment.id, feedVersionFromLatestDeployment); @@ -150,13 +139,13 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx LocalDate.of(2022, Month.NOVEMBER, 3), feedSourceWithLatestDeploymentFeedVersion.publishedVersionId ); - deploymentSuperseded = createDeployment( + createDeployment( "deployment-superseded", projectWithLatestDeployment, feedVersionFromLatestDeployment.id, deployedSuperseded ); - deploymentLatest = createDeployment( + createDeployment( "deployment-latest", projectWithLatestDeployment, feedVersionFromLatestDeployment.id, @@ -173,8 +162,8 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx projectWithPinnedDeployment.organizationId = "project-with-pinned-deployment-org-id"; Persistence.projects.create(projectWithPinnedDeployment); - feedSourceWithPinnedDeploymentAdminOnlyLabel = createLabel("label-id-pinned-deployment", "Admin Only Label", projectWithPinnedDeployment.id); - feedSourceWithPinnedDeploymentAdminOnlyNote = createNote("note-id-pinned-deployment", "A test note"); + Label feedSourceWithPinnedDeploymentAdminOnlyLabel = createLabel("label-id-pinned-deployment", "Admin Only Label", projectWithPinnedDeployment.id); + Note feedSourceWithPinnedDeploymentAdminOnlyNote = createNote("note-id-pinned-deployment", "A test note"); feedSourceWithPinnedDeploymentFeedVersion = createFeedSource( "feed-source-with-pinned-deployment-feed-version", @@ -190,7 +179,7 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx feedSourceWithPinnedDeploymentFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2) ); - deploymentPinned = createDeployment( + Deployment deploymentPinned = createDeployment( "deployment-pinned", projectWithPinnedDeployment, feedVersionFromPinnedDeployment.id, @@ -204,75 +193,31 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx public static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { - Persistence.projects.removeById(project.id); + project.delete(); } if (projectToBeDeleted != null) { - Persistence.projects.removeById(projectToBeDeleted.id); - } - if (feedSourceWithUrl != null) { - Persistence.feedSources.removeById(feedSourceWithUrl.id); - } - if (feedSourceWithNoUrl != null) { - Persistence.feedSources.removeById(feedSourceWithNoUrl.id); - } - if (publicLabel != null) { - Persistence.labels.removeById(publicLabel.id); - } - if (adminOnlyLabel != null) { - Persistence.labels.removeById(adminOnlyLabel.id); + projectToBeDeleted.delete(); } tearDownDeployedFeedVersion(); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /** - * These entities are removed separately so that if the need arises they can be kept. + * These projects are removed separately so that if the need arises they can be kept. * This would then allow the Mongo queries defined in FeedSource#getFeedVersionFromLatestDeployment and * FeedSource#getFeedVersionFromPinnedDeployment to be tested. */ private static void tearDownDeployedFeedVersion() { - if (feedVersionFromLatestDeployment != null) { - feedVersionFromLatestDeployment.delete(); - } if (projectWithPinnedDeployment != null) { projectWithPinnedDeployment.delete(); } if (projectWithLatestDeployment != null) { - Persistence.projects.removeById(projectWithLatestDeployment.id); - } - if (feedSourceWithLatestDeploymentFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithLatestDeploymentFeedVersion.id); - } - if (feedSourceWithPinnedDeploymentFeedVersion != null) { - Persistence.feedSources.removeById(feedSourceWithPinnedDeploymentFeedVersion.id); + projectWithLatestDeployment.delete(); } + // Orphan feed version must be explicitly deleted. if (feedVersionPublishedFromLatestDeployment != null) { Persistence.feedVersions.removeById(feedVersionPublishedFromLatestDeployment.id); } - if (feedVersionFromPinnedDeployment != null) { - Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); - } - if (deploymentPinned != null) { - Persistence.deployments.removeById(deploymentPinned.id); - } - if (deploymentLatest != null) { - Persistence.deployments.removeById(deploymentLatest.id); - } - if (deploymentSuperseded != null) { - Persistence.deployments.removeById(deploymentSuperseded.id); - } - if (feedSourceWithPinnedDeploymentAdminOnlyLabel != null) { - Persistence.labels.removeById(feedSourceWithPinnedDeploymentAdminOnlyLabel.id); - } - if (feedSourceWithLatestDeploymentAdminOnlyLabel != null) { - Persistence.labels.removeById(feedSourceWithLatestDeploymentAdminOnlyLabel.id); - } - if (feedSourceWithPinnedDeploymentAdminOnlyNote != null) { - Persistence.notes.removeById(feedSourceWithPinnedDeploymentAdminOnlyNote.id); - } - if (feedSourceWithLatestDeploymentAdminOnlyNote != null) { - Persistence.notes.removeById(feedSourceWithLatestDeploymentAdminOnlyNote.id); - } } /** @@ -368,8 +313,8 @@ public void createFeedSourceWithLabels() { // Test that they are assigned properly assertEquals(2, labelCountForFeed(feedSourceWithLabels.id)); // Test that project shows only correct labels based on user auth - assertEquals(2, labelCountforProject(feedSourceWithLabels.projectId, true)); - assertEquals(1, labelCountforProject(feedSourceWithLabels.projectId, false)); + assertEquals(2, labelCountForProject(feedSourceWithLabels.projectId, true)); + assertEquals(1, labelCountForProject(feedSourceWithLabels.projectId, false)); // Test that feed source shows only correct labels based on user auth List labelsSeenByAdmin = FeedSourceController.cleanFeedSourceForNonAdmins(feedSourceWithLabels, true).labelIds; @@ -385,7 +330,7 @@ public void createFeedSourceWithLabels() { ); assertEquals(OK_200, deleteSecondLabelResponse.status); assertEquals(1, labelCountForFeed(feedSourceWithLabels.id)); - assertEquals(1, labelCountforProject(feedSourceWithLabels.projectId, true)); + assertEquals(1, labelCountForProject(feedSourceWithLabels.projectId, true)); // Test that labels are removed when deleting project assertEquals(1, Persistence.labels.getFiltered(eq("projectId", projectToBeDeleted.id)).size()); @@ -483,6 +428,14 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); } + private static Project createProject(String name, boolean autoFetchFeeds) { + Project project = new Project(); + project.name = name; + project.autoFetchFeeds = autoFetchFeeds; + Persistence.projects.create(project); + return project; + } + private static FeedSource createFeedSource(String name, URL url, Project project) { return createFeedSource(null, name, url, project, false); } @@ -543,10 +496,6 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc return createFeedVersion(id, feedSourceId, null, endDate, null); } - private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate, String namespace) { - return createFeedVersion(id, feedSourceId, null, endDate, namespace); - } - /** * Helper method to create a feed version. */ @@ -618,7 +567,7 @@ private int labelCountForFeed(String feedSourceId) { /** * Provide the label count for a given project */ - private int labelCountforProject(String projectId, boolean isAdmin) { + private int labelCountForProject(String projectId, boolean isAdmin) { return Persistence.projects.getById(projectId).retrieveProjectLabels(isAdmin).size(); } } From 50455b998d435e87d05fa78ceb187f1d086adc6a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 13 Nov 2025 17:09:32 +0000 Subject: [PATCH 11/20] improvement(FeedSourceSummary): Update to mongo query to return result if published feed version not --- .../datatools/manager/models/FeedSourceSummary.java | 3 ++- .../mongo/getLatestFeedVersionForFeedSources.js | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 5c644ae84..587db81b4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -11,6 +11,7 @@ import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; +import com.mongodb.client.model.UnwindOptions; import org.bson.Document; import org.bson.conversions.Bson; @@ -178,7 +179,7 @@ public static Map getLatestFeedVersionForFeedSources lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), - unwind("$publishedFeedVersion"), + unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), group( "$_id", Accumulators.first("publishedVersionId", "$publishedVersionId"), diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index ba19d6193..0d321c8ff 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -22,8 +22,13 @@ db.getCollection('FeedSource').aggregate([ } }, { - $unwind: "$feedVersions", - $unwind: "$publishedFeedVersion", + $unwind: "$feedVersions" + }, + { + $unwind: { + path: "$publishedFeedVersion", + preserveNullAndEmptyArrays: true + } }, { $group: { From e02a53f6fa7e8d8300dc17fa531594016a07e3e6 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 18 Nov 2025 13:03:06 +0000 Subject: [PATCH 12/20] improvement(Addressed bug with extracting feed source summaries): Update to process all returned doc --- .../manager/models/FeedSourceSummary.java | 64 ++++++++++--------- .../datatools/manager/models/Project.java | 20 +++--- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 587db81b4..ef838245a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; +import com.mongodb.client.AggregateIterable; import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; @@ -115,31 +116,38 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS } /** - * Set the appropriate feed version. For consistency, if no error count is available set the related number of + * Set the common feed version values. + */ + public void setFeedVersionValues(FeedVersionSummary feedVersionSummary) { + if (feedVersionSummary == null) { + return; + } + latestValidation = new LatestValidationResult(feedVersionSummary); + latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; + latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; + latestNamespace = feedVersionSummary.namespace; + latestPublishedVersionId = feedVersionSummary.publishedVersionId; + publishedValidationSummary = new FeedValidationResultSummary(); + publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; + publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; + publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; + } + + /** + * Set the deployed feed version values. For consistency, if no error count is available set the related number of * issues to zero. */ - public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDeployed) { - if (feedVersionSummary != null) { - if (isDeployed) { - deployedFeedVersionId = feedVersionSummary.id; - deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; - deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; - deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) - ? 0 - : feedVersionSummary.validationResult.errorCount; - } else { - latestValidation = new LatestValidationResult(feedVersionSummary); - latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; - latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; - latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; - latestNamespace = feedVersionSummary.namespace; - latestPublishedVersionId = feedVersionSummary.publishedVersionId; - publishedValidationSummary = new FeedValidationResultSummary(); - publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; - publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; - publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; - } + public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { + if (feedVersionSummary == null) { + return; } + deployedFeedVersionId = feedVersionSummary.id; + deployedFeedVersionStartDate = feedVersionSummary.validationResult.firstCalendarDate; + deployedFeedVersionEndDate = feedVersionSummary.validationResult.lastCalendarDate; + deployedFeedVersionIssues = (feedVersionSummary.validationResult.errorCount == -1) + ? 0 + : feedVersionSummary.validationResult.errorCount; } /** @@ -301,14 +309,10 @@ private static Map extractFeedVersionSummaries( List stages ) { Map feedVersionSummaries = new HashMap<>(); - Document feedVersionDocument = Persistence.getDocuments(collection, stages).first(); - if (feedVersionDocument != null) { - FeedVersionSummary feedVersionSummary = new FeedVersionSummary( - feedVersionKey, - hasChildValidationResultDocument, - feedVersionDocument - ); - feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); + AggregateIterable feedVersions = Persistence.getDocuments(collection, stages); + for (Document feedVersion : feedVersions) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion); + feedVersionSummaries.put(feedVersion.getString(feedSourceKey), feedVersionSummary); } return feedVersionSummaries; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 70340f643..965f43718 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -168,17 +168,21 @@ public Collection retrieveDeploymentSummaries() { * Get all feed source summaries for this project. */ public Collection retrieveFeedSourceSummaries() { + ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); + List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); - Map deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); - if (deployedFeedVersions.isEmpty()) { - // No pinned deployments, instead, get the deployed feed versions from the latest deployment. - deployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); - } - ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); + Map pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); + Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); + for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { - feedSourceSummary.setFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id), false); - feedSourceSummary.setFeedVersion(deployedFeedVersions.get(feedSourceSummary.id), true); + feedSourceSummary.setFeedVersionValues(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( + feedSourceSummary.id, + latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) + ); + feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); + if (feedSourceSummary.latestNamespace != null) { feedSourceSummary.latestErrorCounts = errorCountFetcher.getErrorCounts(feedSourceSummary.latestNamespace); } From 70eef64bdf0bdb9c3cbc6a7351c442bbf7c975e5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 18 Nov 2025 16:04:20 +0000 Subject: [PATCH 13/20] improvement(FeedSourceSummary): Updated query to get latest feed version for feed sources Added sort --- .../datatools/manager/models/FeedSourceSummary.java | 1 + .../resources/mongo/getLatestFeedVersionForFeedSources.js | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index ef838245a..e583a0629 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -188,6 +188,7 @@ public static Map getLatestFeedVersionForFeedSources lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + sort(Sorts.descending("feedVersions.dateCreated")), group( "$_id", Accumulators.first("publishedVersionId", "$publishedVersionId"), diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index 0d321c8ff..a87f7ab13 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -30,6 +30,11 @@ db.getCollection('FeedSource').aggregate([ preserveNullAndEmptyArrays: true } }, + { + $sort: { + "feedVersions.dateCreated": -1 + } + }, { $group: { _id: "$_id", @@ -38,7 +43,7 @@ db.getCollection('FeedSource').aggregate([ publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, feedVersion: { - $max: { + $last: { version: "$feedVersions.version", feedVersionId: "$feedVersions._id", firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", From 2b6cc83a1d1b6d6549c1139efdd1ff3c611c8073 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 2 Dec 2025 11:16:28 +0000 Subject: [PATCH 14/20] improvement(Feed source error count retrieval): New separate class to obtain feed source error count --- .../datatools/manager/models/Project.java | 37 +++++--- .../utils/FeedSourceErrorCountBroker.java | 92 +++++++++++++++++++ 2 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 965f43718..1543158c8 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.manager.jobs.AutoDeployType; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.FeedSourceErrorCountBroker; import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -11,11 +12,10 @@ import org.bson.Document; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -41,7 +41,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class Project extends Model { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(Project.class); /** The name of this project, e.g. NYSDOT. */ public String name; @@ -168,26 +167,40 @@ public Collection retrieveDeploymentSummaries() { * Get all feed source summaries for this project. */ public Collection retrieveFeedSourceSummaries() { - ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); - List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); + assignDeployedVersion(feedSourceSummaries); + assignErrorCounts(feedSourceSummaries); + return feedSourceSummaries; + } + + /** + * Assign error counts to feed source summaries. This must happen after deployed versions are assigned. + */ + private void assignErrorCounts(List feedSourceSummaries) { + HashMap> feedSourceErrorCounts = FeedSourceErrorCountBroker.getErrorCountsForFeedSources( + feedSourceSummaries + ); + feedSourceSummaries.forEach(feedSourceSummary -> + feedSourceSummary.latestErrorCounts = feedSourceErrorCounts.getOrDefault(feedSourceSummary.id, null) + ); + } + + /** + * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. + */ + private void assignDeployedVersion(List feedSourceSummaries) { Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); Map pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); - for (FeedSourceSummary feedSourceSummary : feedSourceSummaries) { + feedSourceSummaries.forEach(feedSourceSummary -> { feedSourceSummary.setFeedVersionValues(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( feedSourceSummary.id, latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) ); feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); - - if (feedSourceSummary.latestNamespace != null) { - feedSourceSummary.latestErrorCounts = errorCountFetcher.getErrorCounts(feedSourceSummary.latestNamespace); - } - } - return feedSourceSummaries; + }); } // TODO: Does this need to be returned with JSON API response diff --git a/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java b/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java new file mode 100644 index 000000000..4c3a9681d --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java @@ -0,0 +1,92 @@ +package com.conveyal.datatools.manager.utils; + +import com.conveyal.datatools.manager.models.FeedSourceSummary; +import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Broker to fetch error counts for multiple feed sources in parallel. + */ +public class FeedSourceErrorCountBroker { + + private static final Logger LOG = LoggerFactory.getLogger(FeedSourceErrorCountBroker.class); + private static final int ERROR_COUNT_REQUEST_TIMEOUT_IN_SECONDS = 60; + + /** + * Fetch error counts for all qualifying feed sources. + */ + public static HashMap> getErrorCountsForFeedSources( + Collection feedSourceSummaries + ) { + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + try { + ConcurrentMap>> errorCountTasks = assignErrorCountToFeedSource( + feedSourceSummaries, + executor + ); + + HashMap> feedSourceErrorCounts = new HashMap<>(); + + CompletableFuture allDone = CompletableFuture.allOf( + errorCountTasks.values().toArray(new CompletableFuture[0]) + ); + + try { + allDone.get(ERROR_COUNT_REQUEST_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.warn("Timeout or interruption while waiting for error counts.", e); + } + + errorCountTasks.forEach((feedSourceId, future) -> { + try { + List errorCounts = future.getNow(null); + if (errorCounts != null) { + LOG.debug("Error counts for {}: {}", feedSourceId, errorCounts); + feedSourceErrorCounts.put(feedSourceId, errorCounts); + } else { + LOG.error("No error counts for {}.", feedSourceId); + } + } catch (CompletionException e) { + LOG.error("Failed to get error counts for {}.", feedSourceId, e); + } + }); + + return feedSourceErrorCounts; + } catch (Exception e) { + LOG.error("Exception during error count fetching.", e); + return new HashMap<>(); + } finally { + executor.shutdownNow(); + } + } + + /** + * Asynchronously fetch error counts for feed sources that have a 'latest namespace'. + */ + private static ConcurrentMap>> assignErrorCountToFeedSource( + Collection feedSourceSummaries, + ExecutorService executor + ) { + ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); + return feedSourceSummaries + .stream() + .filter(fs -> fs.latestNamespace != null) + .collect(Collectors.toConcurrentMap( + fs -> fs.id, + fs -> CompletableFuture.supplyAsync(() -> errorCountFetcher.getErrorCounts(fs.latestNamespace), executor)) + ); + } +} From ec3a329db7d45179937c28ce5fa443ee31261912 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 11 Dec 2025 08:21:33 +0000 Subject: [PATCH 15/20] feat(Exclude editor schemas from deletion): Data sanitizer update to exclude editor namespaces from --- .../datatools/manager/DataSanitizer.java | 17 +++++++++++++++-- .../conveyal/datatools/DataSanitizerTest.java | 13 +++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java index efd02b294..e19e0d415 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java +++ b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.conveyal.datatools.manager.DataManager.initializeApplication; @@ -231,7 +232,7 @@ public static List feedVersionAudit() { * Group orphaned schemas and optionally delete. */ public static void sanitizeDBSchemas(boolean delete) { - Set orphanedSchemas = getOrphanedDBSchemas(getFieldFromDocument("namespace", "FeedVersion")); + Set orphanedSchemas = getOrphanedDBSchemas(getActiveDBSchemas()); if (orphanedSchemas.isEmpty()) { System.out.println("No orphaned DB schemas found!"); @@ -247,6 +248,18 @@ public static void sanitizeDBSchemas(boolean delete) { } } + /** + * Get all active namespaces/schemas from feed sources and feed versions. + */ + public static Set getActiveDBSchemas() { + return Stream + .concat( + getFieldFromDocument("namespace", "FeedVersion").stream(), + getFieldFromDocument("editorNamespace", "FeedSource").stream() + ) + .collect(Collectors.toSet()); + } + /** * Delete orphaned feed versions. */ @@ -283,7 +296,7 @@ private static List getOrphanedFeedVersions() { } /** - * Get all qualifying schemas that are not associated with a feed version. + * Get all qualifying schemas that are not associated with a feed version or feed source. */ public static Set getOrphanedDBSchemas(Set associatedSchemas) { String whereClause = associatedSchemas.isEmpty() ? "" : String.format(" WHERE nspname NOT IN (%s)", associatedSchemas diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 994536d53..327ea8fb1 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -1,6 +1,5 @@ package com.conveyal.datatools; -import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.DataSanitizer; import com.conveyal.datatools.manager.auth.Auth0Connection; @@ -10,14 +9,12 @@ import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.GTFS; -import com.conveyal.gtfs.util.InvalidNamespaceException; import com.mongodb.client.model.Sorts; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.sql.SQLException; import java.util.Collection; import java.util.List; import java.util.HashSet; @@ -54,6 +51,7 @@ static void setUp() throws IOException { FeedSource feedSource = new FeedSource(appendDate("Test Feed"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSource); feedSourceParent = new FeedSource(appendDate("Test Feed 2"), project.id, MANUALLY_UPLOADED); + feedSourceParent.editorNamespace = "active-editor-namespace"; Persistence.feedSources.create(feedSourceParent); feedSourceWithObsoleteFeedVersion = new FeedSource(appendDate("Test Feed 3"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSourceWithObsoleteFeedVersion); @@ -84,7 +82,7 @@ static void setUp() throws IOException { } @AfterAll - static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { + static void tearDown() { Auth0Connection.setAuthDisabled(false); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; project.delete(); @@ -120,6 +118,13 @@ void canIdentifyOrphanedDBSchemas() { assertTrue(orphanedSchemas.contains(orphanedDBSchema)); } + @Test + void canIdentifyActiveDBSchemas() { + // Other schemas will exist. For this test, just make sure the list contains the feed source editor namespace. + Set activeDBSchemas = DataSanitizer.getActiveDBSchemas(); + assertTrue(activeDBSchemas.contains(feedSourceParent.editorNamespace)); + } + @Test void canRemoveOrphanedDBSchema() { assertEquals(1, DataSanitizer.deleteOrphanedDBSchemas(Set.of(orphanedDBSchema))); From 0572b6a09bbc782dfb90ad379dcc067dd92756d5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 11 Dec 2025 09:02:33 +0000 Subject: [PATCH 16/20] improvement(Restored to previous commit to remove data sanitizer updates): Data sanitizer updates sh --- .../datatools/manager/DataSanitizer.java | 19 +++---------------- .../conveyal/datatools/DataSanitizerTest.java | 15 +++++---------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java index e19e0d415..7e682fb27 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java +++ b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.conveyal.datatools.manager.DataManager.initializeApplication; @@ -171,7 +170,7 @@ public static int deleteObsoleteFeedVersions( int keepCount = 0; int deleteCount = 0; - + for (FeedVersion feedVersion : feedVersions) { if (keepCount < numberOfVersionsToKeep) { keepCount++; @@ -232,7 +231,7 @@ public static List feedVersionAudit() { * Group orphaned schemas and optionally delete. */ public static void sanitizeDBSchemas(boolean delete) { - Set orphanedSchemas = getOrphanedDBSchemas(getActiveDBSchemas()); + Set orphanedSchemas = getOrphanedDBSchemas(getFieldFromDocument("namespace", "FeedVersion")); if (orphanedSchemas.isEmpty()) { System.out.println("No orphaned DB schemas found!"); @@ -248,18 +247,6 @@ public static void sanitizeDBSchemas(boolean delete) { } } - /** - * Get all active namespaces/schemas from feed sources and feed versions. - */ - public static Set getActiveDBSchemas() { - return Stream - .concat( - getFieldFromDocument("namespace", "FeedVersion").stream(), - getFieldFromDocument("editorNamespace", "FeedSource").stream() - ) - .collect(Collectors.toSet()); - } - /** * Delete orphaned feed versions. */ @@ -296,7 +283,7 @@ private static List getOrphanedFeedVersions() { } /** - * Get all qualifying schemas that are not associated with a feed version or feed source. + * Get all qualifying schemas that are not associated with a feed version. */ public static Set getOrphanedDBSchemas(Set associatedSchemas) { String whereClause = associatedSchemas.isEmpty() ? "" : String.format(" WHERE nspname NOT IN (%s)", associatedSchemas diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 327ea8fb1..7f3508cfd 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -1,5 +1,6 @@ package com.conveyal.datatools; +import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.DataSanitizer; import com.conveyal.datatools.manager.auth.Auth0Connection; @@ -9,12 +10,14 @@ import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.GTFS; +import com.conveyal.gtfs.util.InvalidNamespaceException; import com.mongodb.client.model.Sorts; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.sql.SQLException; import java.util.Collection; import java.util.List; import java.util.HashSet; @@ -51,7 +54,6 @@ static void setUp() throws IOException { FeedSource feedSource = new FeedSource(appendDate("Test Feed"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSource); feedSourceParent = new FeedSource(appendDate("Test Feed 2"), project.id, MANUALLY_UPLOADED); - feedSourceParent.editorNamespace = "active-editor-namespace"; Persistence.feedSources.create(feedSourceParent); feedSourceWithObsoleteFeedVersion = new FeedSource(appendDate("Test Feed 3"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSourceWithObsoleteFeedVersion); @@ -82,7 +84,7 @@ static void setUp() throws IOException { } @AfterAll - static void tearDown() { + static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { Auth0Connection.setAuthDisabled(false); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; project.delete(); @@ -93,7 +95,7 @@ static void tearDown() { try { GTFS.delete(orphanedDBSchema, DataManager.GTFS_DATA_SOURCE); } catch (Exception e) { - // Do nothing. Schema removed in unit test. + // Do nothing. Schema removed in unit test. } } @@ -118,13 +120,6 @@ void canIdentifyOrphanedDBSchemas() { assertTrue(orphanedSchemas.contains(orphanedDBSchema)); } - @Test - void canIdentifyActiveDBSchemas() { - // Other schemas will exist. For this test, just make sure the list contains the feed source editor namespace. - Set activeDBSchemas = DataSanitizer.getActiveDBSchemas(); - assertTrue(activeDBSchemas.contains(feedSourceParent.editorNamespace)); - } - @Test void canRemoveOrphanedDBSchema() { assertEquals(1, DataSanitizer.deleteOrphanedDBSchemas(Set.of(orphanedDBSchema))); From 4310c1bc591f251ca5d43df23fcb7ecf20b586b1 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 15 Dec 2025 13:29:29 +0000 Subject: [PATCH 17/20] improvement(Refactor to provide publish state and address PR feedback): --- .../manager/controllers/DumpController.java | 1 - .../controllers/api/GtfsPlusController.java | 15 +-- .../manager/gtfsplus/GtfsPlusValidation.java | 11 +- .../manager/jobs/ValidateGtfsPlusFeedJob.java | 8 +- .../manager/models/FeedSourceSummary.java | 122 +++++++++++++----- .../datatools/manager/models/FeedVersion.java | 22 +++- .../manager/models/FeedVersionSummary.java | 18 +-- .../datatools/manager/models/Project.java | 16 --- .../manager/models/PublishState.java | 12 ++ .../utils/FeedSourceErrorCountBroker.java | 92 ------------- src/main/resources/mongo/README.md | 16 ++- .../getFeedVersionsFromLatestDeployment.js | 2 +- .../getFeedVersionsFromPinnedDeployment.js | 2 +- .../getLatestFeedVersionForFeedSources.js | 5 +- .../api/FeedSourceControllerTest.java | 107 ++++++++++++--- 15 files changed, 238 insertions(+), 211 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/PublishState.java delete mode 100644 src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index 6336813c2..4e656554a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -1,6 +1,5 @@ package com.conveyal.datatools.manager.controllers; -import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java index b49026dcf..e07cd55e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java @@ -253,22 +253,13 @@ private static String publishGtfsPlusFile(Request req, Response res) { * version already has GTFS+ validation results, those will be returned instead of re-validating. */ private static GtfsPlusValidation getGtfsPlusValidation(Request req, Response res) { - String feedVersionId = req.params("versionid"); - GtfsPlusValidation gtfsPlusValidation = null; try { - FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); - if (feedVersion != null && feedVersion.gtfsPlusValidation != null) { - return feedVersion.gtfsPlusValidation; - } - gtfsPlusValidation = GtfsPlusValidation.validate(feedVersionId); - if (feedVersion != null) { - feedVersion.gtfsPlusValidation = gtfsPlusValidation; - Persistence.feedVersions.replace(feedVersion.id, feedVersion); - } + String feedVersionId = req.params("versionid"); + return GtfsPlusValidation.validate(feedVersionId); } catch(Exception e) { logMessageAndHalt(req, 500, "Could not read GTFS+ zip file", e); } - return gtfsPlusValidation; + return null; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 33f50476e..7c33a9426 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -62,8 +62,17 @@ public GtfsPlusValidation() { * Overload method to retrieve the feed version. */ public static GtfsPlusValidation validate(String feedVersionId) throws Exception { + GtfsPlusValidation gtfsPlusValidation = null; FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); - return validate(feedVersion); + if (feedVersion != null) { + if (feedVersion.gtfsPlusValidation != null) { + return feedVersion.gtfsPlusValidation; + } + gtfsPlusValidation = validate(feedVersion); + feedVersion.gtfsPlusValidation = gtfsPlusValidation; + Persistence.feedVersions.replace(feedVersion.id, feedVersion); + } + return gtfsPlusValidation; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java index 4255f3ae0..badfd1f1b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java @@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory; /** - * This job handles the Gtfs+ validation of a given feed version. If the version is not new, it will simply + * This job handles the GTFS+ validation of a given feed version. If the version is not new, it will simply * replace the existing version with the version object that has updated validation info. */ public class ValidateGtfsPlusFeedJob extends FeedVersionJob { @@ -18,10 +18,10 @@ public class ValidateGtfsPlusFeedJob extends FeedVersionJob { private final boolean isNewVersion; public ValidateGtfsPlusFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) { - super(owner, "Validating Gtfs+", JobType.VALIDATE_FEED); + super(owner, "Validating GTFS+", JobType.VALIDATE_FEED); feedVersion = version; this.isNewVersion = isNewVersion; - status.update("Waiting to begin Gtfs+ validation...", 0); + status.update("Waiting to begin GTFS+ validation...", 0); } @Override @@ -42,7 +42,7 @@ public void jobFinished () { // the version won't get loaded into MongoDB (even though it exists in postgres). feedVersion.persistFeedVersionAfterValidation(isNewVersion); } - status.completeSuccessfully("Gtfs+ validation finished!"); + status.completeSuccessfully("GTFS+ validation finished!"); } else { // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 35c0d79e8..76fbdded4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -1,9 +1,7 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; -import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.conveyal.datatools.manager.extensions.ExternalPropertiesRetriever; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -24,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; import static com.conveyal.datatools.manager.DataManager.hasConfigProperty; @@ -81,23 +80,15 @@ public class FeedSourceSummary { public String organizationId; - public Date latestProcessedByExternalPublisher; - public Date latestSentToExternalPublisher; - public GtfsPlusValidation latestGtfsPlusValidation; - - public String latestPublishedVersionId; - - public String latestNamespace; - - public FeedValidationResultSummary publishedValidationSummary; - - public List latestErrorCounts = new ArrayList<>(); + public PublishState publishState; @JsonInclude(JsonInclude.Include.NON_NULL) public Map> externalProperties; + public static Boolean hasBlockingIssueForPublishingForTesting = null; + public FeedSourceSummary() { } @@ -138,16 +129,79 @@ public void setFeedVersionValues(FeedVersionSummary feedVersionSummary) { if (feedVersionSummary == null) { return; } - latestValidation = new LatestValidationResult(feedVersionSummary); - latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; - latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; - latestNamespace = feedVersionSummary.namespace; - latestPublishedVersionId = feedVersionSummary.publishedVersionId; - publishedValidationSummary = new FeedValidationResultSummary(); - publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; - publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; - publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; + publishState = getPublishState(feedVersionSummary); + latestValidation = new LatestValidationResult(feedVersionSummary); + } + + /** + * Determine the published state of the feed version. + */ + public PublishState getPublishState(FeedVersionSummary feedVersionSummary) { + if (isPublished(feedVersionSummary)) { + return PublishState.PUBLISHED; + } else if (isPublishing(feedVersionSummary)) { + return PublishState.PUBLISHING; + } else if (isPublishBlocked(feedVersionSummary)) { + return PublishState.PUBLISH_BLOCKED; + } else if (isFeedLoading(feedVersionSummary)) { + return PublishState.FEED_LOADING; + } + return PublishState.READY_TO_PUBLISH; + } + + /** + * Determine the published state of the feed version. + */ + private boolean isPublished(FeedVersionSummary feedVersionSummary) { + return feedVersionSummary.namespace != null && feedVersionSummary.namespace.equals(feedVersionSummary.feedSourcePublishedVersionId); + } + + /** + * Deemed to be publishing if it has been sent to external publisher but not yet processed. + */ + private boolean isPublishing(FeedVersionSummary feedVersionSummary) { + return feedVersionSummary.sentToExternalPublisher != null && feedVersionSummary.processedByExternalPublisher == null; + } + + /** + * Determine if publishing is blocked due to validation, expiration, or blocking issues. + */ + private boolean isPublishBlocked(FeedVersionSummary feedVersionSummary) { + return + feedVersionSummary.gtfsPlusValidation == null || + feedVersionSummary.gtfsPlusValidation.issues == null || + !feedVersionSummary.gtfsPlusValidation.issues.isEmpty() || + !feedVersionSummary.gtfsPlusValidation.published || + FeedVersion.hasExpired(feedVersionSummary.validationResult) || + hasBlockingIssuesForPublishing(feedVersionSummary); + } + + /** + * Determine if there are blocking issues for publishing. + */ + private boolean hasBlockingIssuesForPublishing(FeedVersionSummary feedVersionSummary) { + return Objects.requireNonNullElseGet(hasBlockingIssueForPublishingForTesting, () -> FeedVersion.hasBlockingIssuesForPublishing( + feedVersionSummary.validationResult, + feedVersionSummary.namespace, + feedVersionSummary.name + )); + } + + public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { + hasBlockingIssueForPublishingForTesting = value; + } + + /** + * Determine if the feed is still being imported/validated. + */ + private boolean isFeedLoading(FeedVersionSummary feedVersionSummary) { + return + feedVersionSummary.gtfsPlusValidation == null || + feedVersionSummary.validationResult == null || + // Job processing this version. + // TODO: This follows the UI, but not sure if the same logic works here. + (feedVersionSummary.id != null && feedVersionSummary.id.equals(id)); } /** @@ -167,7 +221,8 @@ public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) } /** - * Get all feed source summaries matching the project id. + * Get all feed source summaries matching the project id. For equivalent Mongo query, see + * getFeedSourceSummaries.js. */ public static List getFeedSourceSummaries(String projectId, String organizationId) { List stages = Lists.newArrayList( @@ -193,7 +248,8 @@ public static List getFeedSourceSummaries(String projectId, S } /** - * Get the latest feed version from all feed sources for this project. + * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * getLatestFeedVersionForFeedSources.js. */ public static Map getLatestFeedVersionForFeedSources(String projectId) { List stages = Lists.newArrayList( @@ -208,9 +264,6 @@ public static Map getLatestFeedVersionForFeedSources group( "$_id", Accumulators.first("publishedVersionId", "$publishedVersionId"), - Accumulators.first("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), - Accumulators.first("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), - Accumulators.first("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), Accumulators.last("feedVersionId", "$feedVersions._id"), Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), @@ -231,7 +284,8 @@ public static Map getLatestFeedVersionForFeedSources } /** - * Get the deployed feed versions from the latest deployment for this project. + * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see + * getFeedVersionsFromLatestDeployment.js. */ public static Map getFeedVersionsFromLatestDeployment(String projectId) { List stages = Lists.newArrayList( @@ -265,7 +319,8 @@ public static Map getFeedVersionsFromLatestDeploymen } /** - * Get the deployed feed version from the pinned deployment for this feed source. + * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see + * getFeedVersionsFromPinnedDeployment.js. */ public static Map getFeedVersionsFromPinnedDeployment(String projectId) { List stages = Lists.newArrayList( @@ -326,10 +381,11 @@ private static Map extractFeedVersionSummaries( List stages ) { Map feedVersionSummaries = new HashMap<>(); - AggregateIterable feedVersions = Persistence.getDocuments(collection, stages); - for (Document feedVersion : feedVersions) { - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion); - feedVersionSummaries.put(feedVersion.getString(feedSourceKey), feedVersionSummary); + for (Document feedVersion : Persistence.getDocuments(collection, stages)) { + feedVersionSummaries.put( + feedVersion.getString(feedSourceKey), + new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) + ); } return feedVersionSummaries; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 93095f204..ded673c37 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -481,7 +481,7 @@ public void validateMobility(MonitorableJob.Status status) { } /** - * Produce Gtfs+ validation results for this feed version if GTFS+ module is enabled. + * Produce GTFS+ validation results for this feed version if GTFS+ module is enabled. */ public void validateGtfsPlus(MonitorableJob.Status status) { @@ -537,6 +537,10 @@ private boolean hasValidationAndLoadErrors() { @JsonIgnore @BsonIgnore public boolean hasExpired() { + return hasExpired(validationResult); + } + + public static boolean hasExpired(ValidationResult validationResult) { return validationResult.lastCalendarDate == null || getNowAsLocalDate().isAfter(validationResult.lastCalendarDate); } @@ -555,14 +559,22 @@ public static void setDateOverrideForTesting(LocalDate value) { */ private boolean hasHighSeverityErrorTypes() { return hasSpecificErrorTypes(Stream.of(NewGTFSErrorType.values()) - .filter(type -> type.priority == Priority.HIGH)); + .filter(type -> type.priority == Priority.HIGH), namespace, name); } /** * Checks for issues that block feed publishing, consistent with UI. */ public boolean hasBlockingIssuesForPublishing() { - if (this.validationResult.fatalException != null) return true; + return hasBlockingIssuesForPublishing(validationResult, namespace, name); + } + + public static boolean hasBlockingIssuesForPublishing( + ValidationResult validationResult, + String namespace, + String name + ) { + if (validationResult.fatalException != null) return true; return hasSpecificErrorTypes(Stream.of( NewGTFSErrorType.ILLEGAL_FIELD_VALUE, @@ -575,13 +587,13 @@ public boolean hasBlockingIssuesForPublishing() { NewGTFSErrorType.TABLE_IN_SUBDIRECTORY, NewGTFSErrorType.TABLE_MISSING_COLUMN_HEADERS, NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS - )); + ), namespace, name); } /** * Determines whether this feed has specific error types. */ - private boolean hasSpecificErrorTypes(Stream errorTypes) { + private static boolean hasSpecificErrorTypes(Stream errorTypes, String namespace, String name) { Set highSeverityErrorTypes = errorTypes .map(NewGTFSErrorType::toString) .collect(Collectors.toSet()); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index ac10843d7..baa21b9f3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -40,10 +40,7 @@ public class FeedVersionSummary extends Model implements Serializable { public Date processedByExternalPublisher; public Date sentToExternalPublisher; public GtfsPlusValidation gtfsPlusValidation; - public String publishedVersionId; - public int publishedFeedVersionErrorCount; - public LocalDate publishedFeedVersionStartDate; - public LocalDate publishedFeedVersionEndDate; + public String feedSourcePublishedVersionId; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { @@ -69,19 +66,8 @@ public FeedVersionSummary( namespace = feedVersionDocument.getString("namespace"); validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); - // The feed source's published version id. - publishedVersionId = feedVersionDocument.getString("publishedVersionId"); - // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. - publishedFeedVersionErrorCount = feedVersionDocument.get("publishedFeedVersionErrorCount") != null - ? feedVersionDocument.getInteger("publishedFeedVersionErrorCount") - : 0; - publishedFeedVersionStartDate = feedVersionDocument.get("publishedFeedVersionStartDate") != null - ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionStartDate")) - : null; - publishedFeedVersionEndDate = feedVersionDocument.get("publishedFeedVersionEndDate") != null - ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionEndDate")) - : null; + feedSourcePublishedVersionId = feedVersionDocument.getString("publishedVersionId"); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 1543158c8..0bc45b337 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,8 +2,6 @@ import com.conveyal.datatools.manager.jobs.AutoDeployType; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.datatools.manager.utils.FeedSourceErrorCountBroker; -import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -15,7 +13,6 @@ import java.util.Collection; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -169,22 +166,9 @@ public Collection retrieveDeploymentSummaries() { public Collection retrieveFeedSourceSummaries() { List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); assignDeployedVersion(feedSourceSummaries); - assignErrorCounts(feedSourceSummaries); return feedSourceSummaries; } - /** - * Assign error counts to feed source summaries. This must happen after deployed versions are assigned. - */ - private void assignErrorCounts(List feedSourceSummaries) { - HashMap> feedSourceErrorCounts = FeedSourceErrorCountBroker.getErrorCountsForFeedSources( - feedSourceSummaries - ); - feedSourceSummaries.forEach(feedSourceSummary -> - feedSourceSummary.latestErrorCounts = feedSourceErrorCounts.getOrDefault(feedSourceSummary.id, null) - ); - } - /** * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/PublishState.java b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java new file mode 100644 index 000000000..bf2ab9c87 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java @@ -0,0 +1,12 @@ +package com.conveyal.datatools.manager.models; + +/** + * Enumeration of possible publish states for a FeedVersion. + */ +public enum PublishState { + PUBLISHED, + PUBLISHING, + PUBLISH_BLOCKED, + FEED_LOADING, + READY_TO_PUBLISH +} diff --git a/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java b/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java deleted file mode 100644 index 4c3a9681d..000000000 --- a/src/main/java/com/conveyal/datatools/manager/utils/FeedSourceErrorCountBroker.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.conveyal.datatools.manager.utils; - -import com.conveyal.datatools.manager.models.FeedSourceSummary; -import com.conveyal.gtfs.graphql.fetchers.ErrorCountFetcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -/** - * Broker to fetch error counts for multiple feed sources in parallel. - */ -public class FeedSourceErrorCountBroker { - - private static final Logger LOG = LoggerFactory.getLogger(FeedSourceErrorCountBroker.class); - private static final int ERROR_COUNT_REQUEST_TIMEOUT_IN_SECONDS = 60; - - /** - * Fetch error counts for all qualifying feed sources. - */ - public static HashMap> getErrorCountsForFeedSources( - Collection feedSourceSummaries - ) { - ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - try { - ConcurrentMap>> errorCountTasks = assignErrorCountToFeedSource( - feedSourceSummaries, - executor - ); - - HashMap> feedSourceErrorCounts = new HashMap<>(); - - CompletableFuture allDone = CompletableFuture.allOf( - errorCountTasks.values().toArray(new CompletableFuture[0]) - ); - - try { - allDone.get(ERROR_COUNT_REQUEST_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS); - } catch (Exception e) { - LOG.warn("Timeout or interruption while waiting for error counts.", e); - } - - errorCountTasks.forEach((feedSourceId, future) -> { - try { - List errorCounts = future.getNow(null); - if (errorCounts != null) { - LOG.debug("Error counts for {}: {}", feedSourceId, errorCounts); - feedSourceErrorCounts.put(feedSourceId, errorCounts); - } else { - LOG.error("No error counts for {}.", feedSourceId); - } - } catch (CompletionException e) { - LOG.error("Failed to get error counts for {}.", feedSourceId, e); - } - }); - - return feedSourceErrorCounts; - } catch (Exception e) { - LOG.error("Exception during error count fetching.", e); - return new HashMap<>(); - } finally { - executor.shutdownNow(); - } - } - - /** - * Asynchronously fetch error counts for feed sources that have a 'latest namespace'. - */ - private static ConcurrentMap>> assignErrorCountToFeedSource( - Collection feedSourceSummaries, - ExecutorService executor - ) { - ErrorCountFetcher errorCountFetcher = new ErrorCountFetcher(); - return feedSourceSummaries - .stream() - .filter(fs -> fs.latestNamespace != null) - .collect(Collectors.toConcurrentMap( - fs -> fs.id, - fs -> CompletableFuture.supplyAsync(() -> errorCountFetcher.getErrorCounts(fs.latestNamespace), executor)) - ); - } -} diff --git a/src/main/resources/mongo/README.md b/src/main/resources/mongo/README.md index 2ed5bad29..ed4a5a05e 100644 --- a/src/main/resources/mongo/README.md +++ b/src/main/resources/mongo/README.md @@ -1,7 +1,11 @@ -To run the scripts in this folder, follow these steps: +## Running the Scripts in this folder using test data from FeedSourceControllerTest -1) Comment out the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). -2) Run FeedSourceControllerTest to created required objects referenced here. -3) Once complete, delete documents via MongoDB. -4) Uncomment the call to tearDownDeployedFeedVersion() in FeedSourceControllerTest -> tearDown(). -5) Re-run FeedSourceControllerTest to confirm deletion of objects. \ No newline at end of file +1. **Comment out** the call to `tearDownDeployedFeedVersion()` in `FeedSourceControllerTest` \-> `tearDown()`. +2. **Run** `FeedSourceControllerTest` to create required objects referenced here. +3. **Delete** documents via MongoDB once complete. +4. **Uncomment** the call to `tearDownDeployedFeedVersion()` in `FeedSourceControllerTest` \-> `tearDown()`. +5. **Re-run** `FeedSourceControllerTest` to confirm deletion of objects. + +## Alternative Approach + +If the appropriate data has already been created (e.g., via the DT UI), the `` tag in each script can be replaced with the actual projectId value. \ No newline at end of file diff --git a/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js index fa829afe9..7779b6a84 100644 --- a/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js +++ b/src/main/resources/mongo/getFeedVersionsFromLatestDeployment.js @@ -2,7 +2,7 @@ db.getCollection('Project').aggregate([ { // Match provided project id. $match: { - _id: "project-with-latest-deployment" + _id: "" } }, { diff --git a/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js index 7aef5bf34..008de3c56 100644 --- a/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js +++ b/src/main/resources/mongo/getFeedVersionsFromPinnedDeployment.js @@ -2,7 +2,7 @@ db.getCollection('Project').aggregate([ { // Match provided project id. $match: { - _id: "project-with-pinned-deployment" + _id: "" } }, { diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index a87f7ab13..363096268 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -2,7 +2,7 @@ db.getCollection('FeedSource').aggregate([ { // Match provided project id. $match: { - projectId: "project-with-latest-deployment" + projectId: "" } }, { @@ -39,9 +39,6 @@ db.getCollection('FeedSource').aggregate([ $group: { _id: "$_id", publishedVersionId: { $first: "$publishedVersionId" }, - publishedFeedVersionErrorCount: { $first: "$publishedFeedVersion.validationResult.errorCount"}, - publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, - publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, feedVersion: { $last: { version: "$feedVersions.version", diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 17430bb43..2a427f4a4 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -12,19 +12,23 @@ import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.FeedVersionSummary; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Note; import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.models.PublishState; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; import com.conveyal.datatools.manager.utils.json.JsonUtil; -import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.net.MalformedURLException; @@ -32,16 +36,17 @@ import java.time.LocalDate; import java.time.Month; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.stream.Stream; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.mongodb.client.model.Filters.eq; import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; import static org.eclipse.jetty.http.HttpStatus.OK_200; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -373,21 +378,8 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); - assertEquals(feedVersionFromLatestDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); - assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); - assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); - assertEquals(feedVersionFromLatestDeployment.namespace, feedSourceSummaries.get(0).latestNamespace); - assertFalse(feedSourceSummaries.get(0).latestErrorCounts.isEmpty()); - assertEquals(NewGTFSErrorType.CONDITIONALLY_REQUIRED, feedSourceSummaries.get(0).latestErrorCounts.get(0).type); - assertEquals(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED, feedSourceSummaries.get(0).latestErrorCounts.get(1).type); - assertEquals(NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS, feedSourceSummaries.get(0).latestErrorCounts.get(2).type); - - assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); - - assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, feedSourceSummaries.get(0).publishedValidationSummary.errorCount); - assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.firstCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.startDate); - assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.lastCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.endDate); + assertEquals(PublishState.PUBLISH_BLOCKED, feedSourceSummaries.get(0).publishState); } @Test @@ -422,10 +414,87 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); - assertEquals(feedVersionFromPinnedDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); - assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); - assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); + assertEquals(PublishState.PUBLISH_BLOCKED, feedSourceSummaries.get(0).publishState); + } + + @ParameterizedTest + @MethodSource("createPublishStates") + void canDeterminePublishState( + boolean isPublished, + boolean isPublishing, + boolean isPublishBlocked, + boolean isFeedLoading, + PublishState expectedPublishState + ) { + FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + FeedSourceSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + + if (isPublished) { + feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; + } + if (isPublishing) { + feedVersionSummary.sentToExternalPublisher = new Date(); + feedVersionSummary.processedByExternalPublisher = null; + } + if (isPublishBlocked) { + feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); + feedVersionSummary.gtfsPlusValidation.issues = List.of(new ValidationIssue()); + feedVersionSummary.gtfsPlusValidation.published = false; + feedVersionSummary.validationResult = new ValidationResult(); + FeedSourceSummary.setHasBlockingIssueForPublishingOverrideForTesting(true); + } + if (isFeedLoading) { + passPublishBlockedCheck(feedVersionSummary); + + feedVersionSummary.validationResult.errorCount = 1; + feedVersionSummary.id = "feed-source-id"; + feedSourceSummary.id = "feed-source-id"; + } + if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { + // Ready to publish case. + passPublishBlockedCheck(feedVersionSummary); + + feedVersionSummary.validationResult.errorCount = 1; + feedVersionSummary.id = "feed-version-id"; + feedSourceSummary.id = "feed-source-id"; + } + PublishState publishState = feedSourceSummary.getPublishState(feedVersionSummary); + assertEquals(expectedPublishState, publishState); + } + + /** + * Set up a feed version summary to pass the publish blocked check. + */ + private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { + feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); + feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); + feedVersionSummary.gtfsPlusValidation.published = true; + feedVersionSummary.validationResult = new ValidationResult(); + feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + } + + private static Stream createPublishStates() { + return Stream.of( + Arguments.of( + true, false, false, false, PublishState.PUBLISHED + ), + Arguments.of( + false, true, false, false, PublishState.PUBLISHING + ), + Arguments.of( + false, false, true, false, PublishState.PUBLISH_BLOCKED + ), + Arguments.of( + false, false, false, true, PublishState.FEED_LOADING + ), + Arguments.of( + false, false, false, false, PublishState.READY_TO_PUBLISH + ) + ); } private static Project createProject(String name, boolean autoFetchFeeds) { From cb06d5f25bc5b2c7881ca6df642a8ad1ebb7dc9e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 15 Dec 2025 14:46:47 +0000 Subject: [PATCH 18/20] improvement(pom.xml): Restored gtfs-lib library reference Newer reference is no longer needed --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f1b815f1..58dffc704 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - cc566848ed + 5e004388b2473c391412fdd4954068c35186e231 From ac2f2c7b8a8c5374bfc8939e86e7f91d9d760bad Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 17 Dec 2025 12:16:03 +0000 Subject: [PATCH 19/20] improvement(Addressed PR feedback): --- .../manager/gtfsplus/GtfsPlusValidation.java | 12 +-- .../manager/models/FeedSourceSummary.java | 81 +------------------ .../manager/models/FeedVersionSummary.java | 64 ++++++++++++++- .../datatools/manager/models/Project.java | 2 +- .../manager/models/PublishState.java | 1 - .../api/FeedSourceControllerTest.java | 23 ++---- 6 files changed, 78 insertions(+), 105 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 7c33a9426..da7c81dfc 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -38,7 +38,7 @@ public class GtfsPlusValidation implements Serializable { private static final String REALTIME_ROUTES_TXT = "realtime_routes.txt"; // Public fields to appear in validation JSON. - public String feedVersionId; + public final String feedVersionId; /** Indicates whether GTFS+ validation applies to user-edited feed or original published GTFS feed */ public boolean published; public long lastModified; @@ -49,12 +49,14 @@ private GtfsPlusValidation (String feedVersionId) { this.feedVersionId = feedVersionId; } - public GtfsPlusValidation(boolean published, List issues) { + public GtfsPlusValidation(String feedVersionId, boolean published, List issues) { + this(feedVersionId); this.published = published; this.issues = issues; } public GtfsPlusValidation() { + this(null); // Empty constructor for serialization } @@ -62,17 +64,17 @@ public GtfsPlusValidation() { * Overload method to retrieve the feed version. */ public static GtfsPlusValidation validate(String feedVersionId) throws Exception { - GtfsPlusValidation gtfsPlusValidation = null; FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); if (feedVersion != null) { if (feedVersion.gtfsPlusValidation != null) { return feedVersion.gtfsPlusValidation; } - gtfsPlusValidation = validate(feedVersion); + GtfsPlusValidation gtfsPlusValidation = validate(feedVersion); feedVersion.gtfsPlusValidation = gtfsPlusValidation; Persistence.feedVersions.replace(feedVersion.id, feedVersion); + return gtfsPlusValidation; } - return gtfsPlusValidation; + return null; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 76fbdded4..563b6a21f 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; -import com.mongodb.client.AggregateIterable; import com.mongodb.client.model.Accumulators; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; @@ -22,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import static com.conveyal.datatools.manager.DataManager.getConfigPropertyAsText; import static com.conveyal.datatools.manager.DataManager.hasConfigProperty; @@ -87,8 +85,6 @@ public class FeedSourceSummary { @JsonInclude(JsonInclude.Include.NON_NULL) public Map> externalProperties; - public static Boolean hasBlockingIssueForPublishingForTesting = null; - public FeedSourceSummary() { } @@ -123,87 +119,17 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS } /** - * Set the common feed version values. + * Update the publish and validation state based on the provided feed version summary. */ - public void setFeedVersionValues(FeedVersionSummary feedVersionSummary) { + public void updatePublishAndValidationState(FeedVersionSummary feedVersionSummary) { if (feedVersionSummary == null) { return; } latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; - publishState = getPublishState(feedVersionSummary); + publishState = feedVersionSummary.getPublishState(); latestValidation = new LatestValidationResult(feedVersionSummary); } - /** - * Determine the published state of the feed version. - */ - public PublishState getPublishState(FeedVersionSummary feedVersionSummary) { - if (isPublished(feedVersionSummary)) { - return PublishState.PUBLISHED; - } else if (isPublishing(feedVersionSummary)) { - return PublishState.PUBLISHING; - } else if (isPublishBlocked(feedVersionSummary)) { - return PublishState.PUBLISH_BLOCKED; - } else if (isFeedLoading(feedVersionSummary)) { - return PublishState.FEED_LOADING; - } - return PublishState.READY_TO_PUBLISH; - } - - /** - * Determine the published state of the feed version. - */ - private boolean isPublished(FeedVersionSummary feedVersionSummary) { - return feedVersionSummary.namespace != null && feedVersionSummary.namespace.equals(feedVersionSummary.feedSourcePublishedVersionId); - } - - /** - * Deemed to be publishing if it has been sent to external publisher but not yet processed. - */ - private boolean isPublishing(FeedVersionSummary feedVersionSummary) { - return feedVersionSummary.sentToExternalPublisher != null && feedVersionSummary.processedByExternalPublisher == null; - } - - /** - * Determine if publishing is blocked due to validation, expiration, or blocking issues. - */ - private boolean isPublishBlocked(FeedVersionSummary feedVersionSummary) { - return - feedVersionSummary.gtfsPlusValidation == null || - feedVersionSummary.gtfsPlusValidation.issues == null || - !feedVersionSummary.gtfsPlusValidation.issues.isEmpty() || - !feedVersionSummary.gtfsPlusValidation.published || - FeedVersion.hasExpired(feedVersionSummary.validationResult) || - hasBlockingIssuesForPublishing(feedVersionSummary); - } - - /** - * Determine if there are blocking issues for publishing. - */ - private boolean hasBlockingIssuesForPublishing(FeedVersionSummary feedVersionSummary) { - return Objects.requireNonNullElseGet(hasBlockingIssueForPublishingForTesting, () -> FeedVersion.hasBlockingIssuesForPublishing( - feedVersionSummary.validationResult, - feedVersionSummary.namespace, - feedVersionSummary.name - )); - } - - public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { - hasBlockingIssueForPublishingForTesting = value; - } - - /** - * Determine if the feed is still being imported/validated. - */ - private boolean isFeedLoading(FeedVersionSummary feedVersionSummary) { - return - feedVersionSummary.gtfsPlusValidation == null || - feedVersionSummary.validationResult == null || - // Job processing this version. - // TODO: This follows the UI, but not sure if the same logic works here. - (feedVersionSummary.id != null && feedVersionSummary.id.equals(id)); - } - /** * Set the deployed feed version values. For consistency, if no error count is available set the related number of * issues to zero. @@ -223,6 +149,7 @@ public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) /** * Get all feed source summaries matching the project id. For equivalent Mongo query, see * getFeedSourceSummaries.js. + * For equivalent Mongo query, @see src/main/resources/mongo/getFeedSourceSummaries.js */ public static List getFeedSourceSummaries(String projectId, String organizationId) { List stages = Lists.newArrayList( diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index baa21b9f3..317c152e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -15,6 +15,7 @@ import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -24,6 +25,7 @@ public class FeedVersionSummary extends Model implements Serializable { private static final long serialVersionUID = 1L; private static final ObjectMapper mapper = new ObjectMapper(); private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + public static Boolean hasBlockingIssueForPublishingForTesting = null; public FeedRetrievalMethod retrievalMethod; public int version; @@ -62,7 +64,7 @@ public FeedVersionSummary( id = feedVersionDocument.getString(feedVersionKey); processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); - gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); + gtfsPlusValidation = getGtfsPlusValidation(id, feedVersionDocument); namespace = feedVersionDocument.getString("namespace"); validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); @@ -96,7 +98,7 @@ public class PartialValidationSummary { /** * Build GtfsPlusValidation object from feed version document. */ - private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocument) { + private static GtfsPlusValidation getGtfsPlusValidation(String feedVersionId, Document feedVersionDocument) { Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); if (gtfsPlusValidationDocument == null) { return null; @@ -110,7 +112,7 @@ private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocu .collect(Collectors.toList()); } boolean published = Boolean.TRUE.equals(gtfsPlusValidationDocument.getBoolean("published")); - return new GtfsPlusValidation(published, issues); + return new GtfsPlusValidation(feedVersionId, published, issues); } /** @@ -183,4 +185,60 @@ private static int getValidationResultErrorCount(boolean hasChildValidationResul private static int getErrorCount(Document document) { return getDocumentChild(document, "validationResult").getInteger("errorCount"); } + + /** + * Determine the published state of the feed version. + */ + public PublishState getPublishState() { + if (isPublished()) { + return PublishState.PUBLISHED; + } else if (isPublishing()) { + return PublishState.PUBLISHING; + } else if (isPublishBlocked()) { + return PublishState.PUBLISH_BLOCKED; + } + return PublishState.READY_TO_PUBLISH; + } + + /** + * Determine the published state of the feed version. + */ + private boolean isPublished() { + return namespace != null && namespace.equals(feedSourcePublishedVersionId); + } + + /** + * Deemed to be publishing if it has been sent to external publisher but not yet processed. + */ + private boolean isPublishing() { + return sentToExternalPublisher != null && processedByExternalPublisher == null; + } + + /** + * Determine if publishing is blocked due to validation, expiration, blocking issues or loading. + */ + private boolean isPublishBlocked() { + return + gtfsPlusValidation == null || + gtfsPlusValidation.issues == null || + !gtfsPlusValidation.issues.isEmpty() || + !gtfsPlusValidation.published || + FeedVersion.hasExpired(validationResult) || + hasBlockingIssuesForPublishing(); + } + + /** + * Determine if there are blocking issues for publishing. + */ + private boolean hasBlockingIssuesForPublishing() { + return Objects.requireNonNullElseGet(hasBlockingIssueForPublishingForTesting, () -> FeedVersion.hasBlockingIssuesForPublishing( + validationResult, + namespace, + name + )); + } + + public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { + hasBlockingIssueForPublishingForTesting = value; + } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 0bc45b337..f6a8b2bc2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -178,7 +178,7 @@ private void assignDeployedVersion(List feedSourceSummaries) Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); feedSourceSummaries.forEach(feedSourceSummary -> { - feedSourceSummary.setFeedVersionValues(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( feedSourceSummary.id, latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) diff --git a/src/main/java/com/conveyal/datatools/manager/models/PublishState.java b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java index bf2ab9c87..15b28c70c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/PublishState.java +++ b/src/main/java/com/conveyal/datatools/manager/models/PublishState.java @@ -7,6 +7,5 @@ public enum PublishState { PUBLISHED, PUBLISHING, PUBLISH_BLOCKED, - FEED_LOADING, READY_TO_PUBLISH } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 2a427f4a4..e8a8fb53b 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -430,7 +430,6 @@ void canDeterminePublishState( FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); FeedVersion.setDateOverrideForTesting(LocalDate.now()); - FeedSourceSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); if (isPublished) { feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; @@ -440,29 +439,20 @@ void canDeterminePublishState( feedVersionSummary.processedByExternalPublisher = null; } if (isPublishBlocked) { - feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); - feedVersionSummary.gtfsPlusValidation.issues = List.of(new ValidationIssue()); - feedVersionSummary.gtfsPlusValidation.published = false; - feedVersionSummary.validationResult = new ValidationResult(); - FeedSourceSummary.setHasBlockingIssueForPublishingOverrideForTesting(true); - } - if (isFeedLoading) { - passPublishBlockedCheck(feedVersionSummary); - - feedVersionSummary.validationResult.errorCount = 1; - feedVersionSummary.id = "feed-source-id"; - feedSourceSummary.id = "feed-source-id"; + feedVersionSummary.gtfsPlusValidation = null; } if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { // Ready to publish case. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); passPublishBlockedCheck(feedVersionSummary); feedVersionSummary.validationResult.errorCount = 1; feedVersionSummary.id = "feed-version-id"; feedSourceSummary.id = "feed-source-id"; } - PublishState publishState = feedSourceSummary.getPublishState(feedVersionSummary); + PublishState publishState = feedVersionSummary.getPublishState(); assertEquals(expectedPublishState, publishState); + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); } /** @@ -488,9 +478,6 @@ private static Stream createPublishStates() { Arguments.of( false, false, true, false, PublishState.PUBLISH_BLOCKED ), - Arguments.of( - false, false, false, true, PublishState.FEED_LOADING - ), Arguments.of( false, false, false, false, PublishState.READY_TO_PUBLISH ) @@ -583,7 +570,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc new ValidationIssue("Test issue 1", "stops.txt", 1, "stop_id"), new ValidationIssue("Test issue 2", "stops.txt", 2, "stop_id") ); - feedVersion.gtfsPlusValidation = new GtfsPlusValidation(true, issues); + feedVersion.gtfsPlusValidation = new GtfsPlusValidation(id, true, issues); feedVersion.namespace = namespace != null ? namespace : "feed-version-namespace"; Persistence.feedVersions.create(feedVersion); return feedVersion; From 3f5512417d754e194d9584f34bd5a5f3d4a0a7b6 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 7 Jan 2026 11:09:49 +0000 Subject: [PATCH 20/20] improvement(Return correct feed version): Return the latest feed version from latest deployment --- .../manager/models/FeedSourceSummary.java | 18 +++--- .../getLatestFeedVersionForFeedSources.js | 4 +- .../api/FeedSourceControllerTest.java | 59 ++++++++++++------- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 563b6a21f..1319a69a4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -187,18 +187,18 @@ public static Map getLatestFeedVersionForFeedSources lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - sort(Sorts.descending("feedVersions.dateCreated")), + sort(Sorts.descending("feedVersions.version")), group( "$_id", Accumulators.first("publishedVersionId", "$publishedVersionId"), - Accumulators.last("feedVersionId", "$feedVersions._id"), - Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), - Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), - Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), - Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), - Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), - Accumulators.last("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation"), - Accumulators.last("namespace", "$feedVersions.namespace") + Accumulators.first("feedVersionId", "$feedVersions._id"), + Accumulators.first("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), + Accumulators.first("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), + Accumulators.first("errorCount", "$feedVersions.validationResult.errorCount"), + Accumulators.first("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), + Accumulators.first("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), + Accumulators.first("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation"), + Accumulators.first("namespace", "$feedVersions.namespace") ) ); return extractFeedVersionSummaries( diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index 363096268..e4fe3a4bd 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -32,7 +32,7 @@ db.getCollection('FeedSource').aggregate([ }, { $sort: { - "feedVersions.dateCreated": -1 + "feedVersions.version": -1 } }, { @@ -40,7 +40,7 @@ db.getCollection('FeedSource').aggregate([ _id: "$_id", publishedVersionId: { $first: "$publishedVersionId" }, feedVersion: { - $last: { + $first: { version: "$feedVersions.version", feedVersionId: "$feedVersions._id", firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index e8a8fb53b..b87de98d6 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -61,7 +61,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static Label adminOnlyLabel = null; private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; - private static FeedVersion feedVersionFromLatestDeployment = null; + private static FeedVersion feedVersionFromLatestDeploymentVersion2 = null; private static FeedVersion feedVersionPublishedFromLatestDeployment = null; private static Project projectWithPinnedDeployment = null; @@ -117,12 +117,22 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx LocalDate deployedSuperseded = LocalDate.of(2020, Month.MARCH, 12); LocalDate deployedEndDate = LocalDate.of(2021, Month.MARCH, 12); LocalDate deployedStartDate = LocalDate.of(2021, Month.MARCH, 1); - feedVersionFromLatestDeployment = createFeedVersion( - "feed-version-from-latest-deployment", + // Create feed version 1 to be superseded by feed version 2 to confirm that the newer version is the one retrieved. + createFeedVersion( + "feed-version-from-latest-deployment-1", feedSourceWithLatestDeploymentFeedVersion.id, deployedStartDate, deployedEndDate, - null + null, + 1 + ); + feedVersionFromLatestDeploymentVersion2 = createFeedVersion( + "feed-version-from-latest-deployment-2", + feedSourceWithLatestDeploymentFeedVersion.id, + deployedStartDate, + deployedEndDate, + null, + 2 ); FeedVersion feedVersionFromGtfsZip = createFeedVersionFromGtfsZip( @@ -130,11 +140,11 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx "bart_old.zip" ); // Update the feed version namespace to match that created from the import. - feedVersionFromLatestDeployment.namespace = feedVersionFromGtfsZip.namespace; + feedVersionFromLatestDeploymentVersion2.namespace = feedVersionFromGtfsZip.namespace; // Remove the imported feed version so it does not conflict with the latest deployment feed version. Persistence.feedVersions.removeById(feedVersionFromGtfsZip.id); - Persistence.feedVersions.replace(feedVersionFromLatestDeployment.id, feedVersionFromLatestDeployment); + Persistence.feedVersions.replace(feedVersionFromLatestDeploymentVersion2.id, feedVersionFromLatestDeploymentVersion2); feedVersionPublishedFromLatestDeployment = createFeedVersion( "published-feed-version-from-latest-deployment", @@ -142,18 +152,19 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx null, LocalDate.of(2022, Month.NOVEMBER, 2), LocalDate.of(2022, Month.NOVEMBER, 3), - feedSourceWithLatestDeploymentFeedVersion.publishedVersionId + feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, + 0 ); createDeployment( "deployment-superseded", projectWithLatestDeployment, - feedVersionFromLatestDeployment.id, + feedVersionFromLatestDeploymentVersion2.id, deployedSuperseded ); createDeployment( "deployment-latest", projectWithLatestDeployment, - feedVersionFromLatestDeployment.id, + feedVersionFromLatestDeploymentVersion2.id, deployedEndDate ); } @@ -370,15 +381,15 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedSourceWithLatestDeploymentFeedVersion.url.toString(), feedSourceSummaries.get(0).url); assertEquals(feedSourceWithLatestDeploymentFeedVersion.noteIds, feedSourceSummaries.get(0).noteIds); assertEquals(feedSourceWithLatestDeploymentFeedVersion.organizationId(), feedSourceSummaries.get(0).organizationId); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).deployedFeedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); - assertEquals(feedVersionFromLatestDeployment.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); - assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); - assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); - assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(feedVersionFromLatestDeploymentVersion2.id, feedSourceSummaries.get(0).deployedFeedVersionId); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().startDate, feedSourceSummaries.get(0).deployedFeedVersionStartDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().endDate, feedSourceSummaries.get(0).deployedFeedVersionEndDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().errorCount, feedSourceSummaries.get(0).deployedFeedVersionIssues); + assertEquals(feedVersionFromLatestDeploymentVersion2.id, feedSourceSummaries.get(0).latestValidation.feedVersionId); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); + assertEquals(feedVersionFromLatestDeploymentVersion2.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromLatestDeploymentVersion2.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); assertEquals(PublishState.PUBLISH_BLOCKED, feedSourceSummaries.get(0).publishState); } @@ -549,13 +560,20 @@ private static Deployment createDeployment( * Helper method to create a feed version with no start date. */ private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate) { - return createFeedVersion(id, feedSourceId, null, endDate, null); + return createFeedVersion(id, feedSourceId, null, endDate, null, 0); } /** * Helper method to create a feed version. */ - private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate, String namespace) { + private static FeedVersion createFeedVersion( + String id, + String feedSourceId, + LocalDate startDate, + LocalDate endDate, + String namespace, + int version + ) { FeedVersion feedVersion = new FeedVersion(); feedVersion.id = id; feedVersion.feedSourceId = feedSourceId; @@ -572,6 +590,7 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc ); feedVersion.gtfsPlusValidation = new GtfsPlusValidation(id, true, issues); feedVersion.namespace = namespace != null ? namespace : "feed-version-namespace"; + feedVersion.version = version; Persistence.feedVersions.create(feedVersion); return feedVersion; }