From 642d048447a841309aca57731a20073a13126f6f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 24 Sep 2026 16:07:26 -0400 Subject: [PATCH 01/24] refactor(FeedVersion): Add isFuture method. --- .../datatools/manager/models/FeedVersion.java | 15 ++++++ .../manager/models/FeedVersionTest.java | 52 +++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 6f8586bc8..7c62e20fa 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -551,6 +551,21 @@ public static boolean hasExpired(ValidationResult validationResult) { getNowAsLocalDate().isAfter(validationResult.lastCalendarDate); } + /** + * Whether the Has this feed expired? + * @return If the validation result last calendar date is null or has expired return true, else return false. + */ + @JsonIgnore + @BsonIgnore + public boolean isFuture() { + return isFuture(validationResult); + } + + public static boolean isFuture(ValidationResult validationResult) { + return validationResult.firstCalendarDate != null && + getNowAsLocalDate().isBefore(validationResult.firstCalendarDate); + } + private static LocalDate getNowAsLocalDate() { return dateOverrideForTesting == null ? LocalDate.now() : dateOverrideForTesting; } diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index e0550c8eb..c7464bb0a 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -13,11 +13,15 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import java.sql.Connection; import java.sql.SQLException; +import java.time.LocalDate; import java.util.Date; +import java.util.stream.Stream; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; @@ -26,14 +30,17 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class FeedVersionTest extends UnitTest { +class FeedVersionTest extends UnitTest { private static Project project; private static FeedSource feedSource; /** Initialize application for tests to run. */ @BeforeAll - public static void setUp() throws Exception { + static void setUp() throws Exception { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); @@ -49,13 +56,52 @@ public static void setUp() throws Exception { } @AfterAll - public static void tearDown() { + static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } } + /** + * Future feed detection. + */ + @ParameterizedTest + @MethodSource("futureFeedCases") + void canDetermineFutureFeed(int daysOffsetFromToday, boolean isFuture, String caseName) { + FeedVersion feedVersion1 = new FeedVersion(feedSource); + feedVersion1.validationResult = new ValidationResult(); + LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); + feedVersion1.validationResult.firstCalendarDate = date; + feedVersion1.validationResult.lastCalendarDate = date.plusDays(3); + + assertEquals( + isFuture, + feedVersion1.isFuture(), + String.format("Feed starting '%s' is %s in the future.", caseName, isFuture ? "indeed" : "not") + ); + } + + private static Stream futureFeedCases() { + return Stream.of( + Arguments.of(0, false, "today"), + Arguments.of(-1, false, "yesterday"), + Arguments.of(-2, false, "the day before yesterday"), + Arguments.of(1, true, "tomorrow"), + Arguments.of(2, true, "the day after tomorrow") + ); + } + + /** + * Future feed detection. + */ + @Test + void canDetermineFutureFeedNullFirstCalendarDate() { + FeedVersion feedVersion1 = new FeedVersion(feedSource); + feedVersion1.validationResult = new ValidationResult(); + assertFalse(feedVersion1.isFuture(), "null firstCalendarDate is not future."); + } + /** * Make sure FeedVersionIDs are always unique, even if created at the same second. * See https://github.com/ibi-group/datatools-server/issues/251 From 4cd818b7d28f5c1cd76ffbdc1a86016d7867dbb2 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 24 Sep 2026 16:54:13 -0400 Subject: [PATCH 02/24] test(FeedVersion): Remove unused import. --- .../com/conveyal/datatools/manager/models/FeedVersionTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index c7464bb0a..dd628160d 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -32,7 +32,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; class FeedVersionTest extends UnitTest { private static Project project; From a6d1f7d2fde7005154186e7ac717f008bc8e2f38 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 24 Sep 2026 17:45:11 -0400 Subject: [PATCH 03/24] fix(FeedVersionSummary): Deem feeds in future blocked for publishing. --- .../manager/models/FeedVersionSummary.java | 1 + .../api/FeedSourceControllerTest.java | 56 ++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 27d6096b6..0aa8bb9a3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -232,6 +232,7 @@ private boolean isPublishBlocked() { !gtfsPlusValidation.issues.isEmpty() || !gtfsPlusValidation.published || FeedVersion.hasExpired(validationResult) || + FeedVersion.isFuture(validationResult) || hasBlockingIssuesForPublishing(); } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index f8ad6b0ae..f26371e39 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -25,6 +25,7 @@ import com.conveyal.datatools.manager.utils.json.JsonUtil; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -51,7 +52,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -public class FeedSourceControllerTest extends DatatoolsTest { +class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; private static Project projectToBeDeleted = null; private static FeedSource feedSourceWithUrl = null; @@ -70,7 +71,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedVersion feedVersionFromPinnedDeployment = null; @BeforeAll - public static void setUp() throws IOException { + static void settingUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; @@ -91,6 +92,11 @@ public static void setUp() throws IOException { setUpFeedVersionFromPinnedDeployment(); } + @AfterEach + void afterEach() { + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); + } + /** * Create all the required objects to test a feed version from the latest deployment. */ @@ -206,7 +212,7 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx } @AfterAll - public static void tearDown() { + static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); @@ -243,7 +249,7 @@ private static void tearDownDeployedFeedVersion() { * 3. Update the same feed source turning auth fetch back on and confirm it is scheduled once more. */ @Test - public void createFeedSourceWithUrlTest() { + void createFeedSourceWithUrlTest() { // create a feed source. SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource", JsonUtil.toJson(feedSourceWithUrl), @@ -278,7 +284,7 @@ public void createFeedSourceWithUrlTest() { * Create a feed source without defining the feed source url. Confirm that the feed source is not scheduled. */ @Test - public void createFeedSourceWithNoUrlTest() { + void createFeedSourceWithNoUrlTest() { SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource", JsonUtil.toJson(feedSourceWithNoUrl), HttpUtils.REQUEST_METHOD.POST @@ -292,7 +298,7 @@ public void createFeedSourceWithNoUrlTest() { * Create some labels, add them to the feed source make them admin only, and check that they don't appear if not an admin */ @Test - public void createFeedSourceWithLabels() { + void createFeedSourceWithLabels() { // Create labels SimpleHttpResponse createFirstLabelResponse = TestUtils.makeRequest("/api/manager/secure/label", JsonUtil.toJson(publicLabel), @@ -473,7 +479,6 @@ void canDeterminePublishState( } PublishState publishState = feedVersionSummary.getPublishState(); assertEquals(expectedPublishState, publishState); - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); } /** @@ -505,6 +510,43 @@ private static Stream createPublishStates() { ); } + /** + * Additional cases besides canDeterminePublishState + */ + @ParameterizedTest + @MethodSource("publishStateTimeCases") + void canGetPublishState(int daysOffsetFromToday, PublishState expectedState, String caseName) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + passPublishBlockedCheck(feedVersionSummary); + LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); + feedVersionSummary.validationResult.firstCalendarDate = date; + feedVersionSummary.validationResult.lastCalendarDate = date.plusDays(3); + + // Ready to publish case. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + assertEquals(expectedState, feedVersionSummary.getPublishState(), caseName); + } + + private static Stream publishStateTimeCases() { + return Stream.of( + Arguments.of( + 1, + PublishState.PUBLISH_BLOCKED, + "future feed" + ), + Arguments.of( + 0, + PublishState.READY_TO_PUBLISH, + "present feed" + ), + Arguments.of( + -5, + PublishState.PUBLISH_BLOCKED, + "expired feed" + ) + ); + } + private static Project createProject(String name, boolean autoFetchFeeds) { Project project = new Project(); project.name = name; From 8ce8265764e72a2f88bff81cf533e10f774fb98e Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 24 Sep 2026 17:56:28 -0400 Subject: [PATCH 04/24] test: Extract FeedVersionSummaryTest from FeedSourceControllerTest. --- .../api/FeedSourceControllerTest.java | 114 --------------- .../models/FeedVersionSummaryTest.java | 134 ++++++++++++++++++ 2 files changed, 134 insertions(+), 114 deletions(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index f26371e39..c3d0a16f4 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -13,7 +13,6 @@ import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedValidationResultSummary; import com.conveyal.datatools.manager.models.FeedVersion; -import com.conveyal.datatools.manager.models.FeedVersionSummary; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Note; @@ -25,12 +24,8 @@ import com.conveyal.datatools.manager.utils.json.JsonUtil; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.net.MalformedURLException; @@ -38,11 +33,9 @@ import java.time.LocalDate; import java.time.Month; import java.time.ZoneId; -import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.stream.Stream; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.mongodb.client.model.Filters.eq; @@ -92,11 +85,6 @@ static void settingUp() throws IOException { setUpFeedVersionFromPinnedDeployment(); } - @AfterEach - void afterEach() { - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); - } - /** * Create all the required objects to test a feed version from the latest deployment. */ @@ -445,108 +433,6 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId); } - @ParameterizedTest - @MethodSource("createPublishStates") - void canDeterminePublishState( - boolean isPublished, - boolean isPublishing, - boolean isPublishBlocked, - boolean isFeedLoading, - PublishState expectedPublishState - ) { - FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); - FeedVersion.setDateOverrideForTesting(LocalDate.now()); - - if (isPublished) { - feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; - } - if (isPublishing) { - feedVersionSummary.sentToExternalPublisher = new Date(); - feedVersionSummary.processedByExternalPublisher = null; - } - if (isPublishBlocked) { - feedVersionSummary.gtfsPlusValidation = null; - } - if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { - // Ready to publish case. - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); - passPublishBlockedCheck(feedVersionSummary); - - feedVersionSummary.validationResult.errorCount = 1; - feedVersionSummary.id = "feed-version-id"; - feedSourceSummary.id = "feed-source-id"; - } - PublishState publishState = feedVersionSummary.getPublishState(); - assertEquals(expectedPublishState, publishState); - } - - /** - * Set up a feed version summary to pass the publish blocked check. - */ - private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { - feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); - feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); - feedVersionSummary.gtfsPlusValidation.published = true; - feedVersionSummary.validationResult = new ValidationResult(); - feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); - FeedVersion.setDateOverrideForTesting(LocalDate.now()); - } - - private static Stream createPublishStates() { - return Stream.of( - Arguments.of( - true, false, false, false, PublishState.PUBLISHED - ), - Arguments.of( - false, true, false, false, PublishState.PUBLISHING - ), - Arguments.of( - false, false, true, false, PublishState.PUBLISH_BLOCKED - ), - Arguments.of( - false, false, false, false, PublishState.READY_TO_PUBLISH - ) - ); - } - - /** - * Additional cases besides canDeterminePublishState - */ - @ParameterizedTest - @MethodSource("publishStateTimeCases") - void canGetPublishState(int daysOffsetFromToday, PublishState expectedState, String caseName) { - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); - passPublishBlockedCheck(feedVersionSummary); - LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); - feedVersionSummary.validationResult.firstCalendarDate = date; - feedVersionSummary.validationResult.lastCalendarDate = date.plusDays(3); - - // Ready to publish case. - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); - assertEquals(expectedState, feedVersionSummary.getPublishState(), caseName); - } - - private static Stream publishStateTimeCases() { - return Stream.of( - Arguments.of( - 1, - PublishState.PUBLISH_BLOCKED, - "future feed" - ), - Arguments.of( - 0, - PublishState.READY_TO_PUBLISH, - "present feed" - ), - Arguments.of( - -5, - PublishState.PUBLISH_BLOCKED, - "expired feed" - ) - ); - } - private static Project createProject(String name, boolean autoFetchFeeds) { Project project = new Project(); project.name = name; diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java new file mode 100644 index 000000000..0c4c01e7a --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java @@ -0,0 +1,134 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.gtfs.validator.ValidationResult; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Date; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FeedVersionSummaryTest extends DatatoolsTest { + @BeforeAll + static void settingUp() throws IOException { + DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); + } + + @AfterEach + void afterEach() { + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); + } + + @ParameterizedTest + @MethodSource("createPublishStates") + void canDeterminePublishState( + boolean isPublished, + boolean isPublishing, + boolean isPublishBlocked, + boolean isFeedLoading, + PublishState expectedPublishState + ) { + FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + + if (isPublished) { + feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; + } + if (isPublishing) { + feedVersionSummary.sentToExternalPublisher = new Date(); + feedVersionSummary.processedByExternalPublisher = null; + } + if (isPublishBlocked) { + feedVersionSummary.gtfsPlusValidation = null; + } + if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { + // Ignore blocking issues. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + passPublishBlockedCheck(feedVersionSummary); + + feedVersionSummary.validationResult.errorCount = 1; + feedVersionSummary.id = "feed-version-id"; + feedSourceSummary.id = "feed-source-id"; + } + PublishState publishState = feedVersionSummary.getPublishState(); + assertEquals(expectedPublishState, publishState); + } + + /** + * Set up a feed version summary to pass the publish blocked check. + */ + private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { + feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); + feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); + feedVersionSummary.gtfsPlusValidation.published = true; + feedVersionSummary.validationResult = new ValidationResult(); + feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + } + + private static Stream createPublishStates() { + return Stream.of( + Arguments.of( + true, false, false, false, PublishState.PUBLISHED + ), + Arguments.of( + false, true, false, false, PublishState.PUBLISHING + ), + Arguments.of( + false, false, true, false, PublishState.PUBLISH_BLOCKED + ), + Arguments.of( + false, false, false, false, PublishState.READY_TO_PUBLISH + ) + ); + } + + /** + * Additional cases for future and expired feeds. + */ + @ParameterizedTest + @MethodSource("publishStateTimeCases") + void canGetPublishState(int daysOffsetFromToday, PublishState expectedState, String caseName) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + passPublishBlockedCheck(feedVersionSummary); + LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); + feedVersionSummary.validationResult.firstCalendarDate = date; + feedVersionSummary.validationResult.lastCalendarDate = date.plusDays(3); + + // Ignore blocking issues. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + assertEquals(expectedState, feedVersionSummary.getPublishState(), caseName); + } + + private static Stream publishStateTimeCases() { + return Stream.of( + Arguments.of( + 1, + PublishState.PUBLISH_BLOCKED, + "future feed" + ), + Arguments.of( + 0, + PublishState.READY_TO_PUBLISH, + "present feed" + ), + Arguments.of( + -5, + PublishState.PUBLISH_BLOCKED, + "expired feed" + ) + ); + } +} From 6a3e6b55e741b05124d1c57b7540da18e5d5e1f1 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 24 Sep 2026 17:59:15 -0400 Subject: [PATCH 05/24] test(FeedVersionSummary): Clean up test. --- .../models/FeedVersionSummaryTest.java | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java index 0c4c01e7a..05dbc74b5 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java @@ -1,7 +1,6 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.DatatoolsTest; -import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterEach; @@ -22,7 +21,6 @@ class FeedVersionSummaryTest extends DatatoolsTest { @BeforeAll static void settingUp() throws IOException { DatatoolsTest.setUp(); - Auth0Connection.setAuthDisabled(true); } @AfterEach @@ -80,18 +78,10 @@ private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummar private static Stream createPublishStates() { return Stream.of( - Arguments.of( - true, false, false, false, PublishState.PUBLISHED - ), - Arguments.of( - false, true, false, false, PublishState.PUBLISHING - ), - Arguments.of( - false, false, true, false, PublishState.PUBLISH_BLOCKED - ), - Arguments.of( - false, false, false, false, PublishState.READY_TO_PUBLISH - ) + Arguments.of(true, false, false, false, PublishState.PUBLISHED), + Arguments.of(false, true, false, false, PublishState.PUBLISHING), + Arguments.of(false, false, true, false, PublishState.PUBLISH_BLOCKED), + Arguments.of(false, false, false, false, PublishState.READY_TO_PUBLISH) ); } @@ -114,21 +104,9 @@ void canGetPublishState(int daysOffsetFromToday, PublishState expectedState, Str private static Stream publishStateTimeCases() { return Stream.of( - Arguments.of( - 1, - PublishState.PUBLISH_BLOCKED, - "future feed" - ), - Arguments.of( - 0, - PublishState.READY_TO_PUBLISH, - "present feed" - ), - Arguments.of( - -5, - PublishState.PUBLISH_BLOCKED, - "expired feed" - ) + Arguments.of(1, PublishState.PUBLISH_BLOCKED, "future feed"), + Arguments.of(0, PublishState.READY_TO_PUBLISH, "present feed"), + Arguments.of(-5, PublishState.PUBLISH_BLOCKED, "expired feed") ); } } From 1808f22d0e78cad0000757861cd86c36e26d3053 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 25 Sep 2026 14:29:01 -0400 Subject: [PATCH 06/24] docs(FeedVersion): Update isFuture JavaDoc --- .../com/conveyal/datatools/manager/models/FeedVersion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 7c62e20fa..62c382bd4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -552,8 +552,8 @@ public static boolean hasExpired(ValidationResult validationResult) { } /** - * Whether the Has this feed expired? - * @return If the validation result last calendar date is null or has expired return true, else return false. + * Whether this feed starts in the future + * @return true if the validation result first calendar date is not null and in the future true, false otherwise. */ @JsonIgnore @BsonIgnore From e9fad83f821ba38d5635badb214e408aa9a163d3 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 25 Sep 2026 17:27:09 -0400 Subject: [PATCH 07/24] refactor(Project): Move populating FeedVersionSummary to that class. --- .../manager/models/FeedSourceSummary.java | 32 +++++++++++++++---- .../datatools/manager/models/Project.java | 23 +------------ 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 90d9943fb..3d26c9b9d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -128,10 +128,28 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS } } + /** + * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. + */ + private static void assignDeployedVersion(String projectId, List feedSourceSummaries) { + Map latestFeedVersionForFeedSources = getLatestFeedVersionForFeedSources(projectId); + Map pinnedDeploymentFeedVersions = getFeedVersionsFromPinnedDeployment(projectId); + Map latestDeploymentDeployedFeedVersions = getFeedVersionsFromLatestDeployment(projectId); + + feedSourceSummaries.forEach(feedSourceSummary -> { + feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( + feedSourceSummary.id, + latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) + ); + feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); + }); + } + /** * Update the publish and validation state based on the provided feed version summary. */ - public void updatePublishAndValidationState(FeedVersionSummary feedVersionSummary) { + private void updatePublishAndValidationState(FeedVersionSummary feedVersionSummary) { if (feedVersionSummary == null) { return; } @@ -148,7 +166,7 @@ public void updatePublishAndValidationState(FeedVersionSummary feedVersionSummar * Set the deployed feed version values. For consistency, if no error count is available set the related number of * issues to zero. */ - public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { + private void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { if (feedVersionSummary == null) { return; } @@ -190,7 +208,9 @@ public static List getFeedSourceSummaries(String projectId, S sort(Sorts.ascending("name")) ); - return extractFeedSourceSummaries(projectId, organizationId, stages); + List feedSourceSummaries = extractFeedSourceSummaries(projectId, organizationId, stages); + FeedSourceSummary.assignDeployedVersion(projectId, feedSourceSummaries); + return feedSourceSummaries; } /** @@ -198,7 +218,7 @@ public static List getFeedSourceSummaries(String projectId, S * getLatestFeedVersionForFeedSources.js. * If this is updated, be sure to also update the matching Mongo query. */ - public static Map getLatestFeedVersionForFeedSources(String projectId) { + private static Map getLatestFeedVersionForFeedSources(String projectId) { List feedVersionPipeline = Arrays.asList( // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. match( @@ -321,7 +341,7 @@ public static Map getLatestFeedVersionForFeedSources * getFeedVersionsFromLatestDeployment.js. * If this is updated, be sure to also update the matching Mongo query. */ - public static Map getFeedVersionsFromLatestDeployment(String projectId) { + private static Map getFeedVersionsFromLatestDeployment(String projectId) { List stages = new ArrayList<>(); stages.add(match(in("_id", projectId))); @@ -392,7 +412,7 @@ public static Map getFeedVersionsFromLatestDeploymen * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see * getFeedVersionsFromPinnedDeployment.js. */ - public static Map getFeedVersionsFromPinnedDeployment(String projectId) { + private static Map getFeedVersionsFromPinnedDeployment(String projectId) { List stages = new ArrayList<>(); // Match projects by projectId. diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index f6a8b2bc2..fc1c48135 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -15,7 +15,6 @@ import java.util.Date; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -164,27 +163,7 @@ public Collection retrieveDeploymentSummaries() { * Get all feed source summaries for this project. */ public Collection retrieveFeedSourceSummaries() { - List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); - assignDeployedVersion(feedSourceSummaries); - return feedSourceSummaries; - } - - /** - * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. - */ - private void assignDeployedVersion(List feedSourceSummaries) { - Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); - Map pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); - Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); - - feedSourceSummaries.forEach(feedSourceSummary -> { - feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); - FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( - feedSourceSummary.id, - latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) - ); - feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); - }); + return FeedSourceSummary.getFeedSourceSummaries(id, organizationId); } // TODO: Does this need to be returned with JSON API response From ed27f27eb8a808943b50cce6b42f9cdb53bb861f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 25 Sep 2026 17:37:23 -0400 Subject: [PATCH 08/24] refactor(FeedSourceSummary): Move populating FeedVersionSummary to that class. --- .../manager/models/FeedSourceSummary.java | 289 +---------------- .../manager/models/FeedVersionSummary.java | 293 ++++++++++++++++++ 2 files changed, 296 insertions(+), 286 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 3d26c9b9d..66a299e1a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -8,17 +8,13 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; import com.mongodb.client.model.Sorts; -import com.mongodb.client.model.UnwindOptions; -import com.mongodb.client.model.Variable; import org.bson.Document; import org.bson.conversions.Bson; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,19 +22,11 @@ import static com.conveyal.datatools.manager.DataManager.hasConfigProperty; import static com.conveyal.datatools.manager.DataManager.isExtensionEnabled; import static com.conveyal.datatools.manager.DataManager.isModuleEnabled; -import static com.mongodb.client.model.Aggregates.limit; -import static com.mongodb.client.model.Aggregates.lookup; import static com.mongodb.client.model.Aggregates.match; import static com.mongodb.client.model.Aggregates.project; -import static com.mongodb.client.model.Aggregates.replaceRoot; import static com.mongodb.client.model.Aggregates.sort; -import static com.mongodb.client.model.Aggregates.unwind; -import static com.mongodb.client.model.Filters.expr; import static com.mongodb.client.model.Filters.in; -import static com.mongodb.client.model.Projections.computed; -import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; -import static com.mongodb.client.model.Sorts.descending; import static java.util.Objects.requireNonNullElse; /** @@ -132,9 +120,9 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. */ private static void assignDeployedVersion(String projectId, List feedSourceSummaries) { - Map latestFeedVersionForFeedSources = getLatestFeedVersionForFeedSources(projectId); - Map pinnedDeploymentFeedVersions = getFeedVersionsFromPinnedDeployment(projectId); - Map latestDeploymentDeployedFeedVersions = getFeedVersionsFromLatestDeployment(projectId); + Map latestFeedVersionForFeedSources = FeedVersionSummary.getLatestFeedVersionForFeedSources(projectId); + Map pinnedDeploymentFeedVersions = FeedVersionSummary.getFeedVersionsFromPinnedDeployment(projectId); + Map latestDeploymentDeployedFeedVersions = FeedVersionSummary.getFeedVersionsFromLatestDeployment(projectId); feedSourceSummaries.forEach(feedSourceSummary -> { feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); @@ -213,256 +201,6 @@ public static List getFeedSourceSummaries(String projectId, S return feedSourceSummaries; } - /** - * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see - * getLatestFeedVersionForFeedSources.js. - * If this is updated, be sure to also update the matching Mongo query. - */ - private static Map getLatestFeedVersionForFeedSources(String projectId) { - List feedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - sort(descending("version")), - limit(1), - // Project only the fields needed from the FeedVersion to reduce payload size. - project( - include( - "version", - "_id", - "validationResult", - "processedByExternalPublisher", - "sentToExternalPublisher", - "gtfsPlusValidation", - "namespace" - ) - ) - ); - - // Define the variable passed into the lookup pipeline. - Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); - List> feedSourceId = List.of(feedSourceIdVariable); - - // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). - Bson lookupLatestFeedVersion = lookup( - "FeedVersion", - feedSourceId, - feedVersionPipeline, - "latestFeedVersion" - ); - - // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) - List publishedFeedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where namespace equals the outer document's - // feedSourceId (important when dealing with many FeedVersions) - // and publishedVersionId. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - match( - expr( - new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) - ) - ), - limit(1), - // Project only the validationResult because that's all that is needed later. - project(include("validationResult")) - ); - - // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. - List> feedSourceIdAndPublishedVersionId = List.of( - feedSourceIdVariable, - new Variable<>("publishedVersionId", "$publishedVersionId") - ); - - // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). - Bson lookupPublishedFeedVersion = lookup( - "FeedVersion", - feedSourceIdAndPublishedVersionId, - publishedFeedVersionPipeline, - "publishedFeedVersion" - ); - - // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. - List stages = Arrays.asList( - // Start by filtering documents by projectId (reduces the number of input documents early). - match(in("projectId", projectId)), - - // Attach the latest FeedVersion (as an array "latestFeedVersion"). - lookupLatestFeedVersion, - - // Attach the published FeedVersion (as an array "publishedFeedVersion"). - lookupPublishedFeedVersion, - - // Unwind the latestFeedVersion array into a single document. - unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Unwind the publishedFeedVersion array into a single document. - unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Final projection: select and compute only the fields needed for the output to minimize size. - project(fields( - // keep the raw publishedVersionId field for reference. - include("publishedVersionId"), - - // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). - computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), - computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), - computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), - - // Latest feed version fields (mapped from the nested latestFeedVersion). - computed("feedVersionId", "$latestFeedVersion._id"), - computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), - computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), - computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), - computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), - computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), - computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), - computed("namespace", "$latestFeedVersion.namespace") - )) - ); - - return extractFeedVersionSummaries( - "FeedSource", - "feedVersionId", - "_id", - false, - stages - ); - } - - /** - * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see - * getFeedVersionsFromLatestDeployment.js. - * If this is updated, be sure to also update the matching Mongo query. - */ - private static Map getFeedVersionsFromLatestDeployment(String projectId) { - List stages = new ArrayList<>(); - stages.add(match(in("_id", projectId))); - - // Lookup Deployments for the project. - stages.add(lookup( - "Deployment", - "_id", - "projectId", - "deployments" - )); - - // Unwind deployments array to get individual deployment documents. - stages.add(unwind("$deployments")); - - // Project only fields needed from deployment to reduce doc size before sorting. - stages.add(project(fields( - computed("deployment", "$deployments._id"), - computed("lastUpdated", "$deployments.lastUpdated"), - computed("feedVersionIds", "$deployments.feedVersionIds") - ))); - - // Sort deployments by lastUpdated descending. - stages.add(sort(descending("lastUpdated"))); - stages.add(limit(1)); - - List feedVersionPipeline = Arrays.asList( - match(expr(new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")))), - project( - include( - "feedSourceId", - "validationResult.firstCalendarDate", - "validationResult.lastCalendarDate", - "validationResult.errorCount" - ) - ) - ); - - // Use pipeline form of lookup to fetch FeedVersions matching deployment’s feedVersionIds - List> feedVersionIds = List.of(new Variable<>("feedVersionIds", "$feedVersionIds")); - - stages.add(lookup( - "FeedVersion", - feedVersionIds, - feedVersionPipeline, - "feedVersions" - )); - stages.add(unwind("$feedVersions", new UnwindOptions().preserveNullAndEmptyArrays(false))); - stages.add(replaceRoot("$feedVersions")); - // Final projection: select and compute only the fields needed for the output to minimize size. - stages.add(project( - include( - "_id", - "feedSourceId", - "validationResult" - ) - )); - - return extractFeedVersionSummaries( - "Project", - "_id", - "feedSourceId", - true, - stages - ); - } - - /** - * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see - * getFeedVersionsFromPinnedDeployment.js. - */ - private static Map getFeedVersionsFromPinnedDeployment(String projectId) { - List stages = new ArrayList<>(); - - // Match projects by projectId. - stages.add(match(in("_id", projectId))); - - // Project only pinnedDeploymentId to keep doc small. - stages.add(project(include("pinnedDeploymentId"))); - - // Lookup Deployment documents by pinnedDeploymentId. - stages.add(lookup("Deployment", "pinnedDeploymentId", "_id", "deployment")); - - // Unwind deployment array (assuming single deployment per project). - stages.add(unwind("$deployment")); - - // Define pipeline in $lookup to filter and project FeedVersion docs. - List feedVersionPipeline = Arrays.asList( - match( - expr( - new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")) - ) - ), - project( - include( - "_id", - "feedSourceId", - "validationResult.firstCalendarDate", - "validationResult.lastCalendarDate", - "validationResult.errorCount" - ) - ) - ); - - // Define variable for correlated lookup on FeedVersion collection. - List> feedVersionIds = List.of( - new Variable<>("feedVersionIds", "$deployment.feedVersionIds") - ); - - // Lookup FeedVersion docs with pipeline and store as feedVersions array. - stages.add(lookup("FeedVersion", feedVersionIds, feedVersionPipeline, "feedVersions")); - - return extractFeedVersionSummaries( - "Project", - "_id", - "feedSourceId", - true, - stages - ); - } - - /** * Produce a list of all feed source summaries for a project. */ @@ -478,27 +216,6 @@ private static List extractFeedSourceSummaries( return feedSourceSummaries; } - /** - * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed - * source. - */ - private static Map extractFeedVersionSummaries( - String collection, - String feedVersionKey, - String feedSourceKey, - boolean hasChildValidationResultDocument, - List stages - ) { - Map feedVersionSummaries = new HashMap<>(); - for (Document feedVersion : Persistence.getDocuments(collection, stages)) { - feedVersionSummaries.put( - feedVersion.getString(feedSourceKey), - new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) - ); - } - return feedVersionSummaries; - } - /** * Convert Date object into LocalDate object. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 0aa8bb9a3..a89555c89 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -3,21 +3,43 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; +import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.mongodb.client.model.UnwindOptions; +import com.mongodb.client.model.Variable; import org.bson.Document; +import org.bson.conversions.Bson; import java.io.Serializable; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.expr; +import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Projections.computed; +import static com.mongodb.client.model.Projections.fields; +import static com.mongodb.client.model.Projections.include; +import static com.mongodb.client.model.Sorts.descending; + /** * Includes summary data (a subset of fields) for a feed version. */ @@ -250,4 +272,275 @@ private boolean hasBlockingIssuesForPublishing() { public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { hasBlockingIssueForPublishingForTesting = value; } + + + /** + * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * getLatestFeedVersionForFeedSources.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getLatestFeedVersionForFeedSources(String projectId) { + List feedVersionPipeline = Arrays.asList( + // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. + match( + expr( + new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) + ) + ), + sort(descending("version")), + limit(1), + // Project only the fields needed from the FeedVersion to reduce payload size. + project( + include( + "version", + "_id", + "validationResult", + "processedByExternalPublisher", + "sentToExternalPublisher", + "gtfsPlusValidation", + "namespace" + ) + ) + ); + + // Define the variable passed into the lookup pipeline. + Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); + List> feedSourceId = List.of(feedSourceIdVariable); + + // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). + Bson lookupLatestFeedVersion = lookup( + "FeedVersion", + feedSourceId, + feedVersionPipeline, + "latestFeedVersion" + ); + + // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) + List publishedFeedVersionPipeline = Arrays.asList( + // Match FeedVersion documents where namespace equals the outer document's + // feedSourceId (important when dealing with many FeedVersions) + // and publishedVersionId. + match( + expr( + new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) + ) + ), + match( + expr( + new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) + ) + ), + limit(1), + // Project only the validationResult because that's all that is needed later. + project(include("validationResult")) + ); + + // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. + List> feedSourceIdAndPublishedVersionId = List.of( + feedSourceIdVariable, + new Variable<>("publishedVersionId", "$publishedVersionId") + ); + + // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). + Bson lookupPublishedFeedVersion = lookup( + "FeedVersion", + feedSourceIdAndPublishedVersionId, + publishedFeedVersionPipeline, + "publishedFeedVersion" + ); + + // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. + List stages = Arrays.asList( + // Start by filtering documents by projectId (reduces the number of input documents early). + match(in("projectId", projectId)), + + // Attach the latest FeedVersion (as an array "latestFeedVersion"). + lookupLatestFeedVersion, + + // Attach the published FeedVersion (as an array "publishedFeedVersion"). + lookupPublishedFeedVersion, + + // Unwind the latestFeedVersion array into a single document. + unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Unwind the publishedFeedVersion array into a single document. + unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Final projection: select and compute only the fields needed for the output to minimize size. + project(fields( + // keep the raw publishedVersionId field for reference. + include("publishedVersionId"), + + // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). + computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), + computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), + computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), + + // Latest feed version fields (mapped from the nested latestFeedVersion). + computed("feedVersionId", "$latestFeedVersion._id"), + computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), + computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), + computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), + computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), + computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), + computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), + computed("namespace", "$latestFeedVersion.namespace") + )) + ); + + return extractFeedVersionSummaries( + "FeedSource", + "feedVersionId", + "_id", + false, + stages + ); + } + + /** + * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see + * getFeedVersionsFromLatestDeployment.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getFeedVersionsFromLatestDeployment(String projectId) { + List stages = new ArrayList<>(); + stages.add(match(in("_id", projectId))); + + // Lookup Deployments for the project. + stages.add(lookup( + "Deployment", + "_id", + "projectId", + "deployments" + )); + + // Unwind deployments array to get individual deployment documents. + stages.add(unwind("$deployments")); + + // Project only fields needed from deployment to reduce doc size before sorting. + stages.add(project(fields( + computed("deployment", "$deployments._id"), + computed("lastUpdated", "$deployments.lastUpdated"), + computed("feedVersionIds", "$deployments.feedVersionIds") + ))); + + // Sort deployments by lastUpdated descending. + stages.add(sort(descending("lastUpdated"))); + stages.add(limit(1)); + + List feedVersionPipeline = Arrays.asList( + match(expr(new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")))), + project( + include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount" + ) + ) + ); + + // Use pipeline form of lookup to fetch FeedVersions matching deployment’s feedVersionIds + List> feedVersionIds = List.of(new Variable<>("feedVersionIds", "$feedVersionIds")); + + stages.add(lookup( + "FeedVersion", + feedVersionIds, + feedVersionPipeline, + "feedVersions" + )); + stages.add(unwind("$feedVersions", new UnwindOptions().preserveNullAndEmptyArrays(false))); + stages.add(replaceRoot("$feedVersions")); + // Final projection: select and compute only the fields needed for the output to minimize size. + stages.add(project( + include( + "_id", + "feedSourceId", + "validationResult" + ) + )); + + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages + ); + } + + /** + * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see + * getFeedVersionsFromPinnedDeployment.js. + */ + static Map getFeedVersionsFromPinnedDeployment(String projectId) { + List stages = new ArrayList<>(); + + // Match projects by projectId. + stages.add(match(in("_id", projectId))); + + // Project only pinnedDeploymentId to keep doc small. + stages.add(project(include("pinnedDeploymentId"))); + + // Lookup Deployment documents by pinnedDeploymentId. + stages.add(lookup("Deployment", "pinnedDeploymentId", "_id", "deployment")); + + // Unwind deployment array (assuming single deployment per project). + stages.add(unwind("$deployment")); + + // Define pipeline in $lookup to filter and project FeedVersion docs. + List feedVersionPipeline = Arrays.asList( + match( + expr( + new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")) + ) + ), + project( + include( + "_id", + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount" + ) + ) + ); + + // Define variable for correlated lookup on FeedVersion collection. + List> feedVersionIds = List.of( + new Variable<>("feedVersionIds", "$deployment.feedVersionIds") + ); + + // Lookup FeedVersion docs with pipeline and store as feedVersions array. + stages.add(lookup("FeedVersion", feedVersionIds, feedVersionPipeline, "feedVersions")); + + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages + ); + } + + /** + * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed + * source. + */ + private static Map extractFeedVersionSummaries( + String collection, + String feedVersionKey, + String feedSourceKey, + boolean hasChildValidationResultDocument, + List stages + ) { + Map feedVersionSummaries = new HashMap<>(); + for (Document feedVersion : Persistence.getDocuments(collection, stages)) { + feedVersionSummaries.put( + feedVersion.getString(feedSourceKey), + new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) + ); + } + return feedVersionSummaries; + } } From 26a06d762f7f544df52eec0cd88cadad8f8e3c4c Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 10:55:50 -0400 Subject: [PATCH 09/24] feat(FeedVersionSummary): Add query for last active feed version. --- .../manager/models/FeedVersionSummary.java | 139 +++++++++++++++++ ...etLatestActiveFeedVersionForFeedSources.js | 75 +++++++++ .../models/FeedVersionSummaryQueryTest.java | 146 ++++++++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js create mode 100644 src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index a89555c89..0c9b14eec 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -33,8 +33,10 @@ import static com.mongodb.client.model.Aggregates.replaceRoot; import static com.mongodb.client.model.Aggregates.sort; import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.expr; import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Filters.ne; import static com.mongodb.client.model.Projections.computed; import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; @@ -397,6 +399,143 @@ static Map getLatestFeedVersionForFeedSources(String ); } + /** + * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * getLatestActiveFeedVersionForFeedSources.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getLatestActiveFeedVersionForFeedSources(String projectId) { + List feedVersionPipeline = Arrays.asList( + // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. + match( + expr( + new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) + ) + ), + match( + expr( + and( + new Document("$ne", Arrays.asList("$validationResult", null)), + new Document("$lte", Arrays.asList("$validationResult.firstCalendarDate", "$$referenceDate")), + new Document("$gte", Arrays.asList("$validationResult.lastCalendarDate", "$$referenceDate")) + ) + ) + ), + sort(descending("version")), + limit(1), + // Project only the fields needed from the FeedVersion to reduce payload size. + project( + include( + "version", + "_id", + "validationResult", + "processedByExternalPublisher", + "sentToExternalPublisher", + "gtfsPlusValidation", + "namespace" + ) + ) + ); + + // Define the variables passed into the lookup pipeline. + Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); + String todayFormatted = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + System.out.println(todayFormatted); + Variable referenceDateVariable = new Variable<>("referenceDate", todayFormatted); + List> feedSourceIdAndToday = List.of(feedSourceIdVariable, referenceDateVariable); + + // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). + Bson lookupLatestFeedVersion = lookup( + "FeedVersion", + feedSourceIdAndToday, + feedVersionPipeline, + "latestFeedVersion" + ); + + // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) + List publishedFeedVersionPipeline = Arrays.asList( + // Match FeedVersion documents where namespace equals the outer document's + // feedSourceId (important when dealing with many FeedVersions) + // and publishedVersionId. + match( + expr( + new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) + ) + ), + match( + expr( + new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) + ) + ), + limit(1), + // Project only the validationResult because that's all that is needed later. + project(include("validationResult")) + ); + + // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. + List> feedSourceIdAndPublishedVersionId = List.of( + feedSourceIdVariable, + new Variable<>("publishedVersionId", "$publishedVersionId") + ); + + // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). + Bson lookupPublishedFeedVersion = lookup( + "FeedVersion", + feedSourceIdAndPublishedVersionId, + publishedFeedVersionPipeline, + "publishedFeedVersion" + ); + + // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. + List stages = Arrays.asList( + // Start by filtering documents by projectId (reduces the number of input documents early). + match(in("projectId", projectId)), + + // Attach the latest FeedVersion (as an array "latestFeedVersion"). + lookupLatestFeedVersion, + + // Attach the published FeedVersion (as an array "publishedFeedVersion"). + lookupPublishedFeedVersion, + + // Unwind the latestFeedVersion array into a single document. + unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Unwind the publishedFeedVersion array into a single document. + unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Final projection: select and compute only the fields needed for the output to minimize size. + project(fields( + // keep the raw publishedVersionId field for reference. + include("publishedVersionId"), + + // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). + computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), + computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), + computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), + + // Latest feed version fields (mapped from the nested latestFeedVersion). + computed("feedVersionId", "$latestFeedVersion._id"), + computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), + computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), + computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), + computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), + computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), + computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), + computed("namespace", "$latestFeedVersion.namespace") + )), + + match(ne("feedVersionId", null)) + ); + + return extractFeedVersionSummaries( + "FeedSource", + "feedVersionId", + "_id", + false, + stages + ); + } + /** * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see * getFeedVersionsFromLatestDeployment.js. diff --git a/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js new file mode 100644 index 000000000..6b351aff7 --- /dev/null +++ b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js @@ -0,0 +1,75 @@ +db.FeedSource.aggregate([ + { $match: { projectId: "" } }, + { + $lookup: { + from: "FeedVersion", + let: { + feedSourceId: "$_id", + referenceDate: "" + }, + pipeline: [ + { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } }, + { + $match: { + $expr: { + $and: [ + { $ne: ["$validationResult", null ] }, + { $gte: ["$validationResult.firstCalendarDate", "$$referenceDate"] }, + { $lte: ["$validationResult.lastCalendarDate", "$$referenceDate"] } + ] + } + } + }, + { $sort: { version: -1 } }, + { $limit: 1 }, + { + $project: { + version: 1, + validationResult: 1, + processedByExternalPublisher: 1, + sentToExternalPublisher: 1, + gtfsPlusValidation: 1, + namespace: 1 + } + } + ], + as: "latestFeedVersion" + } + }, + { + $lookup: { + from: "FeedVersion", + let: { feedSourceId: "$_id", publishedVersionId: "$publishedVersionId" }, + pipeline: [ + // Filtering by feedSourceId helps by 100x when dealing with large number of feed versions. + { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } }, + { $match: { $expr: { $eq: ["$namespace", "$$publishedVersionId"] } } }, + { $limit: 1 }, + { $project: { validationResult: 1 } } + ], + as: "publishedFeedVersion" + } + }, + { $unwind: { path: "$latestFeedVersion", preserveNullAndEmptyArrays: true } }, + { $unwind: { path: "$publishedFeedVersion", preserveNullAndEmptyArrays: true } }, + { + $project: { + publishedVersionId: 1, + publishedFeedVersionErrorCount: "$publishedFeedVersion.validationResult.errorCount", + publishedFeedVersionStartDate: "$publishedFeedVersion.validationResult.firstCalendarDate", + publishedFeedVersionEndDate: "$publishedFeedVersion.validationResult.lastCalendarDate", + version: "$latestFeedVersion.version", + feedVersionId: "$latestFeedVersion._id", + firstCalendarDate: "$latestFeedVersion.validationResult.firstCalendarDate", + lastCalendarDate: "$latestFeedVersion.validationResult.lastCalendarDate", + errorCount: "$latestFeedVersion.validationResult.errorCount", + processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher", + sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher", + gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation", + namespace: "$latestFeedVersion.namespace" + } + }, + { + $match: { feedVersionId: {$ne: null} } + }, +]) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java new file mode 100644 index 000000000..a3b7e2334 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -0,0 +1,146 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.validator.ValidationResult; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.util.Date; +import java.util.Map; +import java.util.Set; + +import static com.mongodb.client.model.Filters.eq; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FeedVersionSummaryQueryTest extends DatatoolsTest { + private static Project project; + private static FeedSource feedSource1; + private static FeedSource feedSource2; + private static FeedSource feedSource3; + + /** Initialize application for tests to run. */ + @BeforeAll + static void settingUp() throws Exception { + // start server if it isn't already running + DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); + + // set up project and feed source + project = new Project(); + project.name = String.format("Test project %s", new Date()); + Persistence.projects.create(project); + + feedSource1 = new FeedSource("Test feed source 1"); + feedSource1.projectId = project.id; + Persistence.feedSources.create(feedSource1); + + feedSource2 = new FeedSource("Test feed source 2"); + feedSource2.projectId = project.id; + Persistence.feedSources.create(feedSource2); + + feedSource3 = new FeedSource("Test feed source 3"); + feedSource3.projectId = project.id; + Persistence.feedSources.create(feedSource3); + + // Add some feed versions + } + + @AfterAll + static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + if (project != null) { + project.delete(); + } + } + + /** + * TODO: Make sure the correct version is obtained for the latest feed version. + */ + @Test + void canObtainLatestVersion() { + // Create a project, feed sources, and feed versions to merge. + // create two feedVersions immediately after each other which should end up having unique IDs + FeedVersion feedVersion1 = new FeedVersion(feedSource1); + FeedVersion feedVersion2 = new FeedVersion(feedSource1); + assertThat(feedVersion1.id, not(equalTo(feedVersion2.id))); + } + + /** + * TODO: Make sure the correct version is obtained for the latest active feed version. + */ + @Test + void canObtainLatestActiveVersion() { + LocalDate nowDate = LocalDate.now(); + FeedVersion feedVersionActiveOlder1 = createFeedVersion( + 1, + feedSource1, + nowDate.minusDays(30), + nowDate.plusDays(10) + ); + FeedVersion feedVersionActive1 = createFeedVersion( + 2, + feedSource1, + nowDate.minusDays(5), + nowDate.plusDays(0) + ); + FeedVersion feedVersionFuture1 = createFeedVersion( + 3, + feedSource1, + nowDate.plusDays(1), + nowDate.plusDays(60) + ); + FeedVersion feedVersionActive2 = createFeedVersion( + 1, + feedSource2, + nowDate.minusDays(0), + nowDate.plusDays(20) + ); + FeedVersion feedVersionFuture2 = createFeedVersion( + 2, + feedSource2, + nowDate.plusDays(1), + nowDate.plusDays(60) + ); + + FeedVersion feedVersionActiveOlder3 = createFeedVersion( + 1, + feedSource3, + nowDate.minusDays(30), + nowDate.minusDays(10) + ); + + Map activeSummaries = FeedVersionSummary.getLatestActiveFeedVersionForFeedSources(project.id); + + // feedSource3 should not appear in the results because it has no active feeds. + assertEquals(Set.of(feedSource1.id, feedSource2.id), activeSummaries.keySet()); + assertEquals(feedVersionActive1.id, activeSummaries.get(feedSource1.id).id); + assertEquals(feedVersionActive2.id, activeSummaries.get(feedSource2.id).id); + } + + /** + * Helper method to create a feed version. + */ + private static FeedVersion createFeedVersion( + int version, + FeedSource feedSource, + LocalDate startDate, + LocalDate endDate + ) { + FeedVersion feedVersion = new FeedVersion(feedSource); + ValidationResult validationResult = new ValidationResult(); + validationResult.firstCalendarDate = startDate; + validationResult.lastCalendarDate = endDate; + feedVersion.validationResult = validationResult; + feedVersion.version = version; + Persistence.feedVersions.create(feedVersion); + return feedVersion; + } +} From cad76c9c9602e97acc785028fb820fe58d7c0d94 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:16:31 -0400 Subject: [PATCH 10/24] test(FeedVersionSummaryQuery): Refactor test. --- .../models/FeedVersionSummaryQueryTest.java | 83 +++++++------------ 1 file changed, 32 insertions(+), 51 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index a3b7e2334..96891bc5a 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -5,19 +5,21 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.time.LocalDate; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Set; -import static com.mongodb.client.model.Filters.eq; +import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestActiveFeedVersionForFeedSources; +import static com.mongodb.client.model.Filters.in; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; class FeedVersionSummaryQueryTest extends DatatoolsTest { @@ -25,6 +27,7 @@ class FeedVersionSummaryQueryTest extends DatatoolsTest { private static FeedSource feedSource1; private static FeedSource feedSource2; private static FeedSource feedSource3; + private static final LocalDate TODAY = LocalDate.now(); /** Initialize application for tests to run. */ @BeforeAll @@ -61,6 +64,14 @@ static void tearDown() { } } + @AfterEach + void afterEach() { + // Delete feed versions created for each source. + Persistence.feedVersions.removeFiltered( + in("feedSourceId", List.of(feedSource1.id, feedSource2.id, feedSource3.id)) + ); + } + /** * TODO: Make sure the correct version is obtained for the latest feed version. */ @@ -74,73 +85,43 @@ void canObtainLatestVersion() { } /** - * TODO: Make sure the correct version is obtained for the latest active feed version. + * Make sure the latest active feed versions are correct for each feed source. */ @Test void canObtainLatestActiveVersion() { - LocalDate nowDate = LocalDate.now(); - FeedVersion feedVersionActiveOlder1 = createFeedVersion( - 1, - feedSource1, - nowDate.minusDays(30), - nowDate.plusDays(10) - ); - FeedVersion feedVersionActive1 = createFeedVersion( - 2, - feedSource1, - nowDate.minusDays(5), - nowDate.plusDays(0) - ); - FeedVersion feedVersionFuture1 = createFeedVersion( - 3, - feedSource1, - nowDate.plusDays(1), - nowDate.plusDays(60) - ); - FeedVersion feedVersionActive2 = createFeedVersion( - 1, - feedSource2, - nowDate.minusDays(0), - nowDate.plusDays(20) - ); - FeedVersion feedVersionFuture2 = createFeedVersion( - 2, - feedSource2, - nowDate.plusDays(1), - nowDate.plusDays(60) - ); + createFeedVersion("1-active-older", feedSource1, -30, 10); + createFeedVersion("1-active-current", feedSource1, -5, 0); + createFeedVersion("1-future", feedSource1, 1, 60); - FeedVersion feedVersionActiveOlder3 = createFeedVersion( - 1, - feedSource3, - nowDate.minusDays(30), - nowDate.minusDays(10) - ); + createFeedVersion("2-active-current", feedSource2, 0, 20); + createFeedVersion("2-future", feedSource2, 1, 60); + + createFeedVersion("3-expired", feedSource3, -30, -10); - Map activeSummaries = FeedVersionSummary.getLatestActiveFeedVersionForFeedSources(project.id); + Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); // feedSource3 should not appear in the results because it has no active feeds. assertEquals(Set.of(feedSource1.id, feedSource2.id), activeSummaries.keySet()); - assertEquals(feedVersionActive1.id, activeSummaries.get(feedSource1.id).id); - assertEquals(feedVersionActive2.id, activeSummaries.get(feedSource2.id).id); + assertEquals("1-active-current", activeSummaries.get(feedSource1.id).id); + assertEquals("2-active-current", activeSummaries.get(feedSource2.id).id); } /** * Helper method to create a feed version. + * id serves as description for each version. */ - private static FeedVersion createFeedVersion( - int version, + private static void createFeedVersion( + String id, FeedSource feedSource, - LocalDate startDate, - LocalDate endDate + int startOffsetDaysFromToday, + int endOffsetDaysFromToday ) { FeedVersion feedVersion = new FeedVersion(feedSource); + feedVersion.id = id; ValidationResult validationResult = new ValidationResult(); - validationResult.firstCalendarDate = startDate; - validationResult.lastCalendarDate = endDate; + validationResult.firstCalendarDate = TODAY.plusDays(startOffsetDaysFromToday); + validationResult.lastCalendarDate = TODAY.plusDays(endOffsetDaysFromToday); feedVersion.validationResult = validationResult; - feedVersion.version = version; Persistence.feedVersions.create(feedVersion); - return feedVersion; } } From d6f508200d2e9bc525d5fd5f608619ee6c6cb7f9 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:21:40 -0400 Subject: [PATCH 11/24] test(FeedVersionSummaryQuery): Add back version number. --- .../models/FeedVersionSummaryQueryTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 96891bc5a..86d69ee0f 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -89,14 +89,14 @@ void canObtainLatestVersion() { */ @Test void canObtainLatestActiveVersion() { - createFeedVersion("1-active-older", feedSource1, -30, 10); - createFeedVersion("1-active-current", feedSource1, -5, 0); - createFeedVersion("1-future", feedSource1, 1, 60); + createFeedVersion("1-active-older", 1, feedSource1, -30, 10); + createFeedVersion("1-active-current", 2, feedSource1, -5, 0); + createFeedVersion("1-future", 3, feedSource1, 1, 60); - createFeedVersion("2-active-current", feedSource2, 0, 20); - createFeedVersion("2-future", feedSource2, 1, 60); + createFeedVersion("2-active-current", 1, feedSource2, 0, 20); + createFeedVersion("2-future", 2, feedSource2, 1, 60); - createFeedVersion("3-expired", feedSource3, -30, -10); + createFeedVersion("3-expired", 1, feedSource3, -30, -10); Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); @@ -112,12 +112,14 @@ void canObtainLatestActiveVersion() { */ private static void createFeedVersion( String id, + int version, FeedSource feedSource, int startOffsetDaysFromToday, int endOffsetDaysFromToday ) { FeedVersion feedVersion = new FeedVersion(feedSource); feedVersion.id = id; + feedVersion.version = version; ValidationResult validationResult = new ValidationResult(); validationResult.firstCalendarDate = TODAY.plusDays(startOffsetDaysFromToday); validationResult.lastCalendarDate = TODAY.plusDays(endOffsetDaysFromToday); From b8ee0312cc00ec8f5acb76a6a7c9821b9cc7e9dd Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:25:39 -0400 Subject: [PATCH 12/24] test(FeedVersionSummaryQuery): Add test for last feed version. --- .../models/FeedVersionSummaryQueryTest.java | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 86d69ee0f..27c9ad704 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -16,10 +16,8 @@ import java.util.Set; import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestActiveFeedVersionForFeedSources; +import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestFeedVersionForFeedSources; import static com.mongodb.client.model.Filters.in; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; import static org.junit.jupiter.api.Assertions.assertEquals; class FeedVersionSummaryQueryTest extends DatatoolsTest { @@ -73,15 +71,26 @@ void afterEach() { } /** - * TODO: Make sure the correct version is obtained for the latest feed version. + * Make sure the latest feed versions, whether expired, future, or current, are correct for each feed source. */ @Test void canObtainLatestVersion() { - // Create a project, feed sources, and feed versions to merge. - // create two feedVersions immediately after each other which should end up having unique IDs - FeedVersion feedVersion1 = new FeedVersion(feedSource1); - FeedVersion feedVersion2 = new FeedVersion(feedSource1); - assertThat(feedVersion1.id, not(equalTo(feedVersion2.id))); + createFeedVersion("1-active-older", 1, feedSource1, -30, 10); + createFeedVersion("1-active-current", 2, feedSource1, -5, 0); + createFeedVersion("1-future", 3, feedSource1, 1, 60); + + createFeedVersion("2-active-current", 1, feedSource2, 0, 20); + createFeedVersion("2-future", 2, feedSource2, 1, 60); + + createFeedVersion("3-expired", 1, feedSource3, -30, -10); + + Map activeSummaries = getLatestFeedVersionForFeedSources(project.id); + + // feedSource3 should not appear in the results because it has no active feeds. + assertEquals(Set.of(feedSource1.id, feedSource2.id, feedSource3.id), activeSummaries.keySet()); + assertEquals("1-future", activeSummaries.get(feedSource1.id).id); + assertEquals("2-future", activeSummaries.get(feedSource2.id).id); + assertEquals("3-expired", activeSummaries.get(feedSource3.id).id); } /** From facc125f581503086bf39dfebe5953b480553270 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:29:38 -0400 Subject: [PATCH 13/24] test(FeedVersionSummaryQuery): Extract createFeedVersions. --- .../models/FeedVersionSummaryQueryTest.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 27c9ad704..755e971ea 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -75,18 +75,9 @@ void afterEach() { */ @Test void canObtainLatestVersion() { - createFeedVersion("1-active-older", 1, feedSource1, -30, 10); - createFeedVersion("1-active-current", 2, feedSource1, -5, 0); - createFeedVersion("1-future", 3, feedSource1, 1, 60); - - createFeedVersion("2-active-current", 1, feedSource2, 0, 20); - createFeedVersion("2-future", 2, feedSource2, 1, 60); - - createFeedVersion("3-expired", 1, feedSource3, -30, -10); + createFeedVersions(); Map activeSummaries = getLatestFeedVersionForFeedSources(project.id); - - // feedSource3 should not appear in the results because it has no active feeds. assertEquals(Set.of(feedSource1.id, feedSource2.id, feedSource3.id), activeSummaries.keySet()); assertEquals("1-future", activeSummaries.get(feedSource1.id).id); assertEquals("2-future", activeSummaries.get(feedSource2.id).id); @@ -98,6 +89,17 @@ void canObtainLatestVersion() { */ @Test void canObtainLatestActiveVersion() { + createFeedVersions(); + + Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); + + // feedSource3 should not appear in the results because it has no active feeds. + assertEquals(Set.of(feedSource1.id, feedSource2.id), activeSummaries.keySet()); + assertEquals("1-active-current", activeSummaries.get(feedSource1.id).id); + assertEquals("2-active-current", activeSummaries.get(feedSource2.id).id); + } + + private static void createFeedVersions() { createFeedVersion("1-active-older", 1, feedSource1, -30, 10); createFeedVersion("1-active-current", 2, feedSource1, -5, 0); createFeedVersion("1-future", 3, feedSource1, 1, 60); From 29b50973d5afc256df0d327d6489ceafe953e861 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:29:51 -0400 Subject: [PATCH 14/24] test(FeedVersionSummaryQuery): Fix some comments. --- .../manager/models/FeedVersionSummaryQueryTest.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 755e971ea..4e10cdc78 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -108,18 +108,11 @@ private static void createFeedVersions() { createFeedVersion("2-future", 2, feedSource2, 1, 60); createFeedVersion("3-expired", 1, feedSource3, -30, -10); - - Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); - - // feedSource3 should not appear in the results because it has no active feeds. - assertEquals(Set.of(feedSource1.id, feedSource2.id), activeSummaries.keySet()); - assertEquals("1-active-current", activeSummaries.get(feedSource1.id).id); - assertEquals("2-active-current", activeSummaries.get(feedSource2.id).id); } /** * Helper method to create a feed version. - * id serves as description for each version. + * The id field serves as description for the purpose of each version. */ private static void createFeedVersion( String id, From 1c4d6dd07ee3e01194c5caf72fe632e9a34b9ee7 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 11:57:10 -0400 Subject: [PATCH 15/24] test: Extract basic Project/FeedSource creation. --- .../com/conveyal/datatools/TestUtils.java | 15 ++++++++++++ .../models/FeedVersionSummaryQueryTest.java | 24 +++++-------------- .../manager/models/FeedVersionTest.java | 12 ++++------ 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/TestUtils.java b/src/test/java/com/conveyal/datatools/TestUtils.java index 71c786036..86cef57b9 100644 --- a/src/test/java/com/conveyal/datatools/TestUtils.java +++ b/src/test/java/com/conveyal/datatools/TestUtils.java @@ -5,6 +5,8 @@ import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; import com.fasterxml.jackson.databind.JsonNode; @@ -303,4 +305,17 @@ public static int countSchemaInDb() { return count; } + public static Project createProject(String name) { + Project project = new Project(); + project.name = name; + Persistence.projects.create(project); + return project; + } + + public static FeedSource createFeedSource(String name, Project project) { + FeedSource feedSource = new FeedSource(name); + feedSource.projectId = project.id; + Persistence.feedSources.create(feedSource); + return feedSource; + } } diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 4e10cdc78..8f93212b6 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -15,6 +15,8 @@ import java.util.Map; import java.util.Set; +import static com.conveyal.datatools.TestUtils.createFeedSource; +import static com.conveyal.datatools.TestUtils.createProject; import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestActiveFeedVersionForFeedSources; import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestFeedVersionForFeedSources; import static com.mongodb.client.model.Filters.in; @@ -34,24 +36,10 @@ static void settingUp() throws Exception { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - // set up project and feed source - project = new Project(); - project.name = String.format("Test project %s", new Date()); - Persistence.projects.create(project); - - feedSource1 = new FeedSource("Test feed source 1"); - feedSource1.projectId = project.id; - Persistence.feedSources.create(feedSource1); - - feedSource2 = new FeedSource("Test feed source 2"); - feedSource2.projectId = project.id; - Persistence.feedSources.create(feedSource2); - - feedSource3 = new FeedSource("Test feed source 3"); - feedSource3.projectId = project.id; - Persistence.feedSources.create(feedSource3); - - // Add some feed versions + project = createProject(String.format("Test project %s", new Date())); + feedSource1 = createFeedSource("Test feed source 1", project); + feedSource2 = createFeedSource("Test feed source 2", project); + feedSource3 = createFeedSource("Test feed source 3", project); } @AfterAll diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index dd628160d..76a047daf 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -23,7 +23,9 @@ import java.util.Date; import java.util.stream.Stream; +import static com.conveyal.datatools.TestUtils.createFeedSource; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.TestUtils.createProject; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.mongodb.client.model.Filters.eq; import static org.hamcrest.MatcherAssert.assertThat; @@ -44,14 +46,8 @@ static void setUp() throws Exception { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - // set up project - project = new Project(); - project.name = String.format("Test project %s", new Date()); - Persistence.projects.create(project); - - feedSource = new FeedSource("Test feed source"); - feedSource.projectId = project.id; - Persistence.feedSources.create(feedSource); + project = createProject(String.format("Test project %s", new Date())); + feedSource = createFeedSource("Test feed source", project); } @AfterAll From 738e71fb5e4e64fba61c998c4f9c00bf12ed7fd5 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 14:52:18 -0400 Subject: [PATCH 16/24] refactor(FeedVersionSummary): Reuse common Mongo pipeline parts. --- .../manager/models/FeedVersionSummary.java | 198 +++++------------- 1 file changed, 47 insertions(+), 151 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 0c9b14eec..63fb96722 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -282,165 +282,65 @@ public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean va * If this is updated, be sure to also update the matching Mongo query. */ static Map getLatestFeedVersionForFeedSources(String projectId) { - List feedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - sort(descending("version")), - limit(1), - // Project only the fields needed from the FeedVersion to reduce payload size. - project( - include( - "version", - "_id", - "validationResult", - "processedByExternalPublisher", - "sentToExternalPublisher", - "gtfsPlusValidation", - "namespace" - ) - ) - ); - - // Define the variable passed into the lookup pipeline. - Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); - List> feedSourceId = List.of(feedSourceIdVariable); - - // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). - Bson lookupLatestFeedVersion = lookup( - "FeedVersion", - feedSourceId, - feedVersionPipeline, - "latestFeedVersion" - ); - - // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) - List publishedFeedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where namespace equals the outer document's - // feedSourceId (important when dealing with many FeedVersions) - // and publishedVersionId. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - match( - expr( - new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) - ) - ), - limit(1), - // Project only the validationResult because that's all that is needed later. - project(include("validationResult")) - ); - - // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. - List> feedSourceIdAndPublishedVersionId = List.of( - feedSourceIdVariable, - new Variable<>("publishedVersionId", "$publishedVersionId") - ); - - // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). - Bson lookupPublishedFeedVersion = lookup( - "FeedVersion", - feedSourceIdAndPublishedVersionId, - publishedFeedVersionPipeline, - "publishedFeedVersion" - ); - - // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. - List stages = Arrays.asList( - // Start by filtering documents by projectId (reduces the number of input documents early). - match(in("projectId", projectId)), - - // Attach the latest FeedVersion (as an array "latestFeedVersion"). - lookupLatestFeedVersion, - - // Attach the published FeedVersion (as an array "publishedFeedVersion"). - lookupPublishedFeedVersion, - - // Unwind the latestFeedVersion array into a single document. - unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Unwind the publishedFeedVersion array into a single document. - unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Final projection: select and compute only the fields needed for the output to minimize size. - project(fields( - // keep the raw publishedVersionId field for reference. - include("publishedVersionId"), - - // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). - computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), - computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), - computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), - - // Latest feed version fields (mapped from the nested latestFeedVersion). - computed("feedVersionId", "$latestFeedVersion._id"), - computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), - computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), - computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), - computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), - computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), - computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), - computed("namespace", "$latestFeedVersion.namespace") - )) - ); - - return extractFeedVersionSummaries( - "FeedSource", - "feedVersionId", - "_id", - false, - stages - ); + return getLatestFeedVersionForFeedSources(projectId, null); } /** - * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * Get the latest active feed version (not expired, not future) from all feed sources for this project. + * For equivalent Mongo query, see * getLatestActiveFeedVersionForFeedSources.js. * If this is updated, be sure to also update the matching Mongo query. */ static Map getLatestActiveFeedVersionForFeedSources(String projectId) { - List feedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - match( - expr( - and( - new Document("$ne", Arrays.asList("$validationResult", null)), - new Document("$lte", Arrays.asList("$validationResult.firstCalendarDate", "$$referenceDate")), - new Document("$gte", Arrays.asList("$validationResult.lastCalendarDate", "$$referenceDate")) - ) - ) - ), - sort(descending("version")), - limit(1), - // Project only the fields needed from the FeedVersion to reduce payload size. - project( - include( - "version", - "_id", - "validationResult", - "processedByExternalPublisher", - "sentToExternalPublisher", - "gtfsPlusValidation", - "namespace" + Bson matchActiveVersions = match( + expr( + and( + new Document("$ne", Arrays.asList("$validationResult", null)), + // This string-based date comparison works because the strings are fixed-length with numbers only! + new Document("$lte", Arrays.asList("$validationResult.firstCalendarDate", "$$referenceDate")), + new Document("$gte", Arrays.asList("$validationResult.lastCalendarDate", "$$referenceDate")) ) ) ); + return getLatestFeedVersionForFeedSources(projectId, List.of(matchActiveVersions)); + } + + /** + * Get the latest feed version from all feed sources for this project. + */ + static Map getLatestFeedVersionForFeedSources( + String projectId, + List extraFeedVersionFilters + ) { + Bson matchFeedSourceId = match( + expr( + new Document("$eq", List.of("$feedSourceId", "$$feedSourceId")) + ) + ); + + List feedVersionPipeline = new ArrayList<>(); + feedVersionPipeline.add(matchFeedSourceId); + if (extraFeedVersionFilters != null) { + feedVersionPipeline.addAll(extraFeedVersionFilters); + } + feedVersionPipeline.add(sort(descending("version"))); + feedVersionPipeline.add(limit(1)); + feedVersionPipeline.add(project( + include( + "version", + "_id", + "validationResult", + "processedByExternalPublisher", + "sentToExternalPublisher", + "gtfsPlusValidation", + "namespace" + ) + )); + // Define the variables passed into the lookup pipeline. - Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); String todayFormatted = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - System.out.println(todayFormatted); + Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); Variable referenceDateVariable = new Variable<>("referenceDate", todayFormatted); List> feedSourceIdAndToday = List.of(feedSourceIdVariable, referenceDateVariable); @@ -457,11 +357,7 @@ static Map getLatestActiveFeedVersionForFeedSources( // Match FeedVersion documents where namespace equals the outer document's // feedSourceId (important when dealing with many FeedVersions) // and publishedVersionId. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), + matchFeedSourceId, match( expr( new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) From 0084e5a53c3a19e892bf14df88253f12a41782d0 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 14:56:01 -0400 Subject: [PATCH 17/24] refactor(FeedVersionSummary): Try to use List.of(). --- .../manager/models/FeedVersionSummary.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 63fb96722..cc56a51c1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -295,10 +295,11 @@ static Map getLatestActiveFeedVersionForFeedSources( Bson matchActiveVersions = match( expr( and( + // Using Arrays.asList to pass a null value. new Document("$ne", Arrays.asList("$validationResult", null)), // This string-based date comparison works because the strings are fixed-length with numbers only! - new Document("$lte", Arrays.asList("$validationResult.firstCalendarDate", "$$referenceDate")), - new Document("$gte", Arrays.asList("$validationResult.lastCalendarDate", "$$referenceDate")) + new Document("$lte", List.of("$validationResult.firstCalendarDate", "$$referenceDate")), + new Document("$gte", List.of("$validationResult.lastCalendarDate", "$$referenceDate")) ) ) ); @@ -353,14 +354,14 @@ static Map getLatestFeedVersionForFeedSources( ); // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) - List publishedFeedVersionPipeline = Arrays.asList( + List publishedFeedVersionPipeline = List.of( // Match FeedVersion documents where namespace equals the outer document's // feedSourceId (important when dealing with many FeedVersions) // and publishedVersionId. matchFeedSourceId, match( expr( - new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) + new Document("$eq", List.of("$namespace", "$$publishedVersionId")) ) ), limit(1), @@ -383,7 +384,7 @@ static Map getLatestFeedVersionForFeedSources( ); // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. - List stages = Arrays.asList( + List stages = List.of( // Start by filtering documents by projectId (reduces the number of input documents early). match(in("projectId", projectId)), @@ -463,8 +464,8 @@ static Map getFeedVersionsFromLatestDeployment(Strin stages.add(sort(descending("lastUpdated"))); stages.add(limit(1)); - List feedVersionPipeline = Arrays.asList( - match(expr(new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")))), + List feedVersionPipeline = List.of( + match(expr(new Document("$in", List.of("$_id", "$$feedVersionIds")))), project( include( "feedSourceId", @@ -524,10 +525,10 @@ static Map getFeedVersionsFromPinnedDeployment(Strin stages.add(unwind("$deployment")); // Define pipeline in $lookup to filter and project FeedVersion docs. - List feedVersionPipeline = Arrays.asList( + List feedVersionPipeline = List.of( match( expr( - new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")) + new Document("$in", List.of("$_id", "$$feedVersionIds")) ) ), project( From ff36dee2458fd49ef549c5f6096176a11d5e8db7 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Thu, 1 Oct 2026 15:04:46 -0400 Subject: [PATCH 18/24] test(FeedVersionSummaryQuery): Add empty feed source to tests. --- .../models/FeedVersionSummaryQueryTest.java | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java index 8f93212b6..f7f2798fd 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.Set; @@ -24,9 +23,11 @@ class FeedVersionSummaryQueryTest extends DatatoolsTest { private static Project project; - private static FeedSource feedSource1; - private static FeedSource feedSource2; - private static FeedSource feedSource3; + private static FeedSource sourceWithFutureAnd2ActiveFeeds; + private static FeedSource sourceWithFutureAndActiveFeed; + private static FeedSource sourceWithFutureFeed; + private static Set allNonEmptyFeedSourceIds; + private static final LocalDate TODAY = LocalDate.now(); /** Initialize application for tests to run. */ @@ -37,9 +38,16 @@ static void settingUp() throws Exception { Auth0Connection.setAuthDisabled(true); project = createProject(String.format("Test project %s", new Date())); - feedSource1 = createFeedSource("Test feed source 1", project); - feedSource2 = createFeedSource("Test feed source 2", project); - feedSource3 = createFeedSource("Test feed source 3", project); + sourceWithFutureAnd2ActiveFeeds = createFeedSource("Test feed source 1", project); + sourceWithFutureAndActiveFeed = createFeedSource("Test feed source 2", project); + sourceWithFutureFeed = createFeedSource("Test feed source 3", project); + createFeedSource("Feed source without feed versions", project); + + allNonEmptyFeedSourceIds = Set.of( + sourceWithFutureAnd2ActiveFeeds.id, + sourceWithFutureAndActiveFeed.id, + sourceWithFutureFeed.id + ); } @AfterAll @@ -53,9 +61,7 @@ static void tearDown() { @AfterEach void afterEach() { // Delete feed versions created for each source. - Persistence.feedVersions.removeFiltered( - in("feedSourceId", List.of(feedSource1.id, feedSource2.id, feedSource3.id)) - ); + Persistence.feedVersions.removeFiltered(in("feedSourceId", allNonEmptyFeedSourceIds)); } /** @@ -64,12 +70,11 @@ void afterEach() { @Test void canObtainLatestVersion() { createFeedVersions(); - Map activeSummaries = getLatestFeedVersionForFeedSources(project.id); - assertEquals(Set.of(feedSource1.id, feedSource2.id, feedSource3.id), activeSummaries.keySet()); - assertEquals("1-future", activeSummaries.get(feedSource1.id).id); - assertEquals("2-future", activeSummaries.get(feedSource2.id).id); - assertEquals("3-expired", activeSummaries.get(feedSource3.id).id); + assertEquals(allNonEmptyFeedSourceIds, activeSummaries.keySet()); + assertEquals("1-future", activeSummaries.get(sourceWithFutureAnd2ActiveFeeds.id).id); + assertEquals("2-future", activeSummaries.get(sourceWithFutureAndActiveFeed.id).id); + assertEquals("3-expired", activeSummaries.get(sourceWithFutureFeed.id).id); } /** @@ -82,20 +87,20 @@ void canObtainLatestActiveVersion() { Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); // feedSource3 should not appear in the results because it has no active feeds. - assertEquals(Set.of(feedSource1.id, feedSource2.id), activeSummaries.keySet()); - assertEquals("1-active-current", activeSummaries.get(feedSource1.id).id); - assertEquals("2-active-current", activeSummaries.get(feedSource2.id).id); + assertEquals(Set.of(sourceWithFutureAnd2ActiveFeeds.id, sourceWithFutureAndActiveFeed.id), activeSummaries.keySet()); + assertEquals("1-active-current", activeSummaries.get(sourceWithFutureAnd2ActiveFeeds.id).id); + assertEquals("2-active-current", activeSummaries.get(sourceWithFutureAndActiveFeed.id).id); } private static void createFeedVersions() { - createFeedVersion("1-active-older", 1, feedSource1, -30, 10); - createFeedVersion("1-active-current", 2, feedSource1, -5, 0); - createFeedVersion("1-future", 3, feedSource1, 1, 60); + createFeedVersion("1-active-older", 1, sourceWithFutureAnd2ActiveFeeds, -30, 10); + createFeedVersion("1-active-current", 2, sourceWithFutureAnd2ActiveFeeds, -5, 0); + createFeedVersion("1-future", 3, sourceWithFutureAnd2ActiveFeeds, 1, 60); - createFeedVersion("2-active-current", 1, feedSource2, 0, 20); - createFeedVersion("2-future", 2, feedSource2, 1, 60); + createFeedVersion("2-active-current", 1, sourceWithFutureAndActiveFeed, 0, 20); + createFeedVersion("2-future", 2, sourceWithFutureAndActiveFeed, 1, 60); - createFeedVersion("3-expired", 1, feedSource3, -30, -10); + createFeedVersion("3-expired", 1, sourceWithFutureFeed, -30, -10); } /** From 21565a6684089401fbf605e590618e0c315ca72e Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 2 Oct 2026 18:39:00 -0400 Subject: [PATCH 19/24] feat(FeedSourceSummary): Add fields for active feed versions. --- .../manager/models/FeedSourceSummary.java | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 66a299e1a..0dd375ab4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -65,6 +65,11 @@ public class FeedSourceSummary { public LatestValidationResult latestValidation; + public LatestValidationResult latestActiveFeedValidation; + + public FeedVersionSummary latestVersion; + + public FeedVersionSummary latestActiveVersion; public String url; public List noteIds = new ArrayList<>(); @@ -121,11 +126,13 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS */ private static void assignDeployedVersion(String projectId, List feedSourceSummaries) { Map latestFeedVersionForFeedSources = FeedVersionSummary.getLatestFeedVersionForFeedSources(projectId); + Map latestActiveFeedVersionForFeedSources = FeedVersionSummary.getLatestActiveFeedVersionForFeedSources(projectId); Map pinnedDeploymentFeedVersions = FeedVersionSummary.getFeedVersionsFromPinnedDeployment(projectId); Map latestDeploymentDeployedFeedVersions = FeedVersionSummary.getFeedVersionsFromLatestDeployment(projectId); feedSourceSummaries.forEach(feedSourceSummary -> { - feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.setLatestFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.setLatestActiveFeedVersion(latestActiveFeedVersionForFeedSources.get(feedSourceSummary.id)); FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( feedSourceSummary.id, latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) @@ -135,19 +142,32 @@ private static void assignDeployedVersion(String projectId, List Date: Mon, 5 Oct 2026 11:08:45 -0400 Subject: [PATCH 20/24] refactor(FeedSourceSummary): Remove unused latestSentToExternalPublisher. --- .../conveyal/datatools/manager/models/FeedSourceSummary.java | 4 +--- .../manager/controllers/api/FeedSourceControllerTest.java | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 0dd375ab4..46cdb8464 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -70,14 +70,13 @@ public class FeedSourceSummary { public FeedVersionSummary latestVersion; public FeedVersionSummary latestActiveVersion; + public String url; public List noteIds = new ArrayList<>(); public String organizationId; - public Date latestSentToExternalPublisher; - public FeedValidationResultSummary publishedValidationSummary; public String publishedVersionId; @@ -160,7 +159,6 @@ private void setLatestFeedVersion(FeedVersionSummary latest) { return; } - latestSentToExternalPublisher = latest.sentToExternalPublisher; publishedValidationSummary = new FeedValidationResultSummary(); publishedValidationSummary.errorCount = requireNonNullElse(latest.publishedFeedVersionErrorCount, -1); publishedValidationSummary.startDate = latest.publishedFeedVersionStartDate; diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index c3d0a16f4..ba5287758 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -385,7 +385,6 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate); assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate); assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount); - assertEquals(feedVersionFromLatestDeploymentVersion2.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher); assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState); assertEquals(feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId); assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, firstSummary.publishedValidationSummary.errorCount); @@ -428,7 +427,6 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate); assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate); assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount); - assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher); assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId); } From bab5c4e494c4dbadd45618084251e59be7a5ac91 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Tue, 6 Oct 2026 14:06:15 -0400 Subject: [PATCH 21/24] fix(FeedVersionSummary): Populate lastUpdated. --- .../datatools/manager/models/FeedVersionSummary.java | 7 +++++-- .../mongo/getLatestActiveFeedVersionForFeedSources.js | 6 ++++-- .../resources/mongo/getLatestFeedVersionForFeedSources.js | 6 ++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index cc56a51c1..830437fdd 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -94,6 +94,7 @@ public FeedVersionSummary( sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); gtfsPlusValidation = getGtfsPlusValidation(id, feedVersionDocument); namespace = feedVersionDocument.getString("namespace"); + lastUpdated = feedVersionDocument.getDate("lastUpdated"); validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. @@ -335,7 +336,8 @@ static Map getLatestFeedVersionForFeedSources( "processedByExternalPublisher", "sentToExternalPublisher", "gtfsPlusValidation", - "namespace" + "namespace", + "lastUpdated" ) )); @@ -418,7 +420,8 @@ static Map getLatestFeedVersionForFeedSources( computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), - computed("namespace", "$latestFeedVersion.namespace") + computed("namespace", "$latestFeedVersion.namespace"), + computed("lastUpdated", "$latestFeedVersion.lastUpdated") )), match(ne("feedVersionId", null)) diff --git a/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js index 6b351aff7..4b083ea30 100644 --- a/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js @@ -29,7 +29,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: 1, sentToExternalPublisher: 1, gtfsPlusValidation: 1, - namespace: 1 + namespace: 1, + lastUpdated: 1 } } ], @@ -66,7 +67,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher", sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher", gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation", - namespace: "$latestFeedVersion.namespace" + namespace: "$latestFeedVersion.namespace", + lastUpdated: "$latestFeedVersion.lastUpdated" } }, { diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index de63cc179..5d4808ee0 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -15,7 +15,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: 1, sentToExternalPublisher: 1, gtfsPlusValidation: 1, - namespace: 1 + namespace: 1, + lastUpdated: 1 } } ], @@ -52,7 +53,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher", sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher", gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation", - namespace: "$latestFeedVersion.namespace" + namespace: "$latestFeedVersion.namespace", + lastUpdated: "$latestFeedVersion.lastUpdated" } } ]) \ No newline at end of file From 08fa395d46dae44e3be77a08c69b0243149e7844 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Tue, 6 Oct 2026 17:23:38 -0400 Subject: [PATCH 22/24] refactor(FeedSourceSummary): Remove publishState --- .../datatools/manager/models/FeedSourceSummary.java | 11 ++++------- .../controllers/api/FeedSourceControllerTest.java | 3 --- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 46cdb8464..7d6d375b2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -81,8 +81,6 @@ public class FeedSourceSummary { public String publishedVersionId; - public PublishState publishState; - @JsonInclude(JsonInclude.Include.NON_NULL) public Map> externalProperties; @@ -130,8 +128,8 @@ private static void assignDeployedVersion(String projectId, List latestDeploymentDeployedFeedVersions = FeedVersionSummary.getFeedVersionsFromLatestDeployment(projectId); feedSourceSummaries.forEach(feedSourceSummary -> { - feedSourceSummary.setLatestFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); - feedSourceSummary.setLatestActiveFeedVersion(latestActiveFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.assignLatestFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.assignLatestActiveFeedVersion(latestActiveFeedVersionForFeedSources.get(feedSourceSummary.id)); FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( feedSourceSummary.id, latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) @@ -143,7 +141,7 @@ private static void assignDeployedVersion(String projectId, List Date: Tue, 6 Oct 2026 19:02:43 -0400 Subject: [PATCH 23/24] refactor(Feed...Summary): Fix JSON serialization!! --- .../manager/models/FeedSourceSummary.java | 3 +++ .../manager/models/FeedVersionSummary.java | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 7d6d375b2..6f2946ed7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.extensions.ExternalPropertiesRetriever; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -67,8 +68,10 @@ public class FeedSourceSummary { public LatestValidationResult latestActiveFeedValidation; + @JsonIgnoreProperties(ignoreUnknown = true) public FeedVersionSummary latestVersion; + @JsonIgnoreProperties(ignoreUnknown = true) public FeedVersionSummary latestActiveVersion; public String url; diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 830437fdd..c1e348e01 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -69,12 +69,16 @@ public class FeedVersionSummary extends Model implements Serializable { public String feedSourcePublishedVersionId; public Integer publishedFeedVersionErrorCount; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate publishedFeedVersionStartDate; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate publishedFeedVersionEndDate; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { - validationSummary = new PartialValidationSummary(); + validationSummary = new PartialValidationSummary(validationResult); } return validationSummary; } @@ -108,7 +112,7 @@ public FeedVersionSummary( /** * Holds a subset of fields from {@link:FeedValidationResultSummary} for UI use only. */ - public class PartialValidationSummary { + public static class PartialValidationSummary { /** Copied from FeedVersion */ @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) @@ -119,7 +123,11 @@ public class PartialValidationSummary { @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate endDate; - PartialValidationSummary() { + public PartialValidationSummary() { + + } + + public PartialValidationSummary(ValidationResult validationResult) { // Older feeds created in datatools may not have validationResult if (validationResult != null) { this.startDate = validationResult.firstCalendarDate; From 4027216f4fb960b8aad71b60e03ecf4e58f0a14d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Wed, 7 Oct 2026 13:24:08 -0400 Subject: [PATCH 24/24] refactor(FeedVersionSummary): Update isPublishBlocked JavaDoc --- .../conveyal/datatools/manager/models/FeedVersionSummary.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index c1e348e01..bc9a75de1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -256,7 +256,8 @@ private boolean isPublishing() { } /** - * Determine if publishing is blocked due to validation, expiration, blocking issues or loading. + * Determine if publishing is blocked due to validation, expiration, blocking issues, loading, + * or the feed being in the future. */ private boolean isPublishBlocked() { return