Handle future feeds

src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java

@@ -551,6 +551,21 @@ public static boolean hasExpired(ValidationResult validationResult) {
             getNowAsLocalDate().isAfter(validationResult.lastCalendarDate);
     }
 
+    /**
+     * Whether this feed starts in the future
+     * @return true if the validation result first calendar date is not null and in the future true, false otherwise.
+     */
+    @JsonIgnore
+    @BsonIgnore
+    public boolean isFuture() {
+        return isFuture(validationResult);
+    }
+
+    public static boolean isFuture(ValidationResult validationResult) {
+        return validationResult.firstCalendarDate != null &&
+            getNowAsLocalDate().isBefore(validationResult.firstCalendarDate);
+    }
+
     private static LocalDate getNowAsLocalDate() {
         return dateOverrideForTesting == null ? LocalDate.now() : dateOverrideForTesting;
     }

src/main/java/com/conveyal/datatools/manager/models/Project.java

@@ -15,7 +15,6 @@
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -164,27 +163,7 @@ public Collection<DeploymentSummary> retrieveDeploymentSummaries() {
      * Get all feed source summaries for this project.
      */
     public Collection<FeedSourceSummary> retrieveFeedSourceSummaries() {
-        List<FeedSourceSummary> feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId);
-        assignDeployedVersion(feedSourceSummaries);
-        return feedSourceSummaries;
-    }
-
-    /**
-     * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version.
-     */
-    private void assignDeployedVersion(List<FeedSourceSummary> feedSourceSummaries) {
-        Map<String, FeedVersionSummary> latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id);
-        Map<String, FeedVersionSummary> pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id);
-        Map<String, FeedVersionSummary> latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id);
-
-        feedSourceSummaries.forEach(feedSourceSummary -> {
-            feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id));
-            FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault(
-                feedSourceSummary.id,
-                latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id)
-            );
-            feedSourceSummary.setDeployedFeedVersionValues(deployedVersion);
-        });
+        return FeedSourceSummary.getFeedSourceSummaries(id, organizationId);
     }
 
     // TODO: Does this need to be returned with JSON API response

src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js

@@ -0,0 +1,77 @@
+db.FeedSource.aggregate([
+    { $match: { projectId: "<project-id>" } },
+    {
+        $lookup: {
+            from: "FeedVersion",
+            let: {
+                feedSourceId: "$_id",
+                referenceDate: "<yyyyMMdd>"
+            },
+            pipeline: [
+                { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } },
+                {
+                    $match: {
+                        $expr: {
+                            $and: [
+                                { $ne: ["$validationResult", null ] },
+                                { $gte: ["$validationResult.firstCalendarDate", "$$referenceDate"] },
+                                { $lte: ["$validationResult.lastCalendarDate", "$$referenceDate"] }
+                            ]
+                        }
+                    }
+                },
+                { $sort: { version: -1 } },
+                { $limit: 1 },
+                {
+                    $project: {
+                        version: 1,
+                        validationResult: 1,
+                        processedByExternalPublisher: 1,
+                        sentToExternalPublisher: 1,
+                        gtfsPlusValidation: 1,
+                        namespace: 1,
+                        lastUpdated: 1
+                    }
+                }
+            ],
+            as: "latestFeedVersion"
+        }
+    },
+    {
+        $lookup: {
+            from: "FeedVersion",
+            let: { feedSourceId: "$_id", publishedVersionId: "$publishedVersionId" },
+            pipeline: [
+                // Filtering by feedSourceId helps by 100x when dealing with large number of feed versions.
+                { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } },
+                { $match: { $expr: { $eq: ["$namespace", "$$publishedVersionId"] } } },
+                { $limit: 1 },
+                { $project: { validationResult: 1 } }
+            ],
+            as: "publishedFeedVersion"
+        }
+    },
+    { $unwind: { path: "$latestFeedVersion", preserveNullAndEmptyArrays: true } },
+    { $unwind: { path: "$publishedFeedVersion", preserveNullAndEmptyArrays: true } },
+    {
+        $project: {
+            publishedVersionId: 1,
+            publishedFeedVersionErrorCount: "$publishedFeedVersion.validationResult.errorCount",
+            publishedFeedVersionStartDate: "$publishedFeedVersion.validationResult.firstCalendarDate",
+            publishedFeedVersionEndDate: "$publishedFeedVersion.validationResult.lastCalendarDate",
+            version: "$latestFeedVersion.version",
+            feedVersionId: "$latestFeedVersion._id",
+            firstCalendarDate: "$latestFeedVersion.validationResult.firstCalendarDate",
+            lastCalendarDate: "$latestFeedVersion.validationResult.lastCalendarDate",
+            errorCount: "$latestFeedVersion.validationResult.errorCount",
+            processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher",
+            sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher",
+            gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation",
+            namespace: "$latestFeedVersion.namespace",
+            lastUpdated: "$latestFeedVersion.lastUpdated"
+        }
+    },
+    {
+        $match: { feedVersionId: {$ne: null} }
+    },
+])

src/main/resources/mongo/getLatestFeedVersionForFeedSources.js

@@ -15,7 +15,8 @@ db.FeedSource.aggregate([
                         processedByExternalPublisher: 1,
                         sentToExternalPublisher: 1,
                         gtfsPlusValidation: 1,
-                        namespace: 1
+                        namespace: 1,
+                        lastUpdated: 1
                     }
                 }
             ],
@@ -52,7 +53,8 @@ db.FeedSource.aggregate([
             processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher",
             sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher",
             gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation",
-            namespace: "$latestFeedVersion.namespace"
+            namespace: "$latestFeedVersion.namespace",
+            lastUpdated: "$latestFeedVersion.lastUpdated"
         }
     }
 ])

src/test/java/com/conveyal/datatools/TestUtils.java

@@ -5,6 +5,8 @@
 import com.conveyal.datatools.manager.models.FeedRetrievalMethod;
 import com.conveyal.datatools.manager.models.FeedSource;
 import com.conveyal.datatools.manager.models.FeedVersion;
+import com.conveyal.datatools.manager.models.Project;
+import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.HttpUtils;
 import com.conveyal.datatools.manager.utils.SimpleHttpResponse;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -303,4 +305,17 @@ public static int countSchemaInDb() {
         return count;
     }
 
+    public static Project createProject(String name) {
+        Project project = new Project();
+        project.name = name;
+        Persistence.projects.create(project);
+        return project;
+    }
+
+    public static FeedSource createFeedSource(String name, Project project) {
+        FeedSource feedSource = new FeedSource(name);
+        feedSource.projectId = project.id;
+        Persistence.feedSources.create(feedSource);
+        return feedSource;
+    }
 }

src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java

@@ -13,12 +13,10 @@
 import com.conveyal.datatools.manager.models.FeedSourceSummary;
 import com.conveyal.datatools.manager.models.FeedValidationResultSummary;
 import com.conveyal.datatools.manager.models.FeedVersion;
-import com.conveyal.datatools.manager.models.FeedVersionSummary;
 import com.conveyal.datatools.manager.models.FetchFrequency;
 import com.conveyal.datatools.manager.models.Label;
 import com.conveyal.datatools.manager.models.Note;
 import com.conveyal.datatools.manager.models.Project;
-import com.conveyal.datatools.manager.models.PublishState;
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.HttpUtils;
 import com.conveyal.datatools.manager.utils.SimpleHttpResponse;
@@ -27,21 +25,16 @@
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.time.LocalDate;
 import java.time.Month;
 import java.time.ZoneId;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import java.util.stream.Stream;
 
 import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip;
 import static com.mongodb.client.model.Filters.eq;
@@ -51,7 +44,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
-public class FeedSourceControllerTest extends DatatoolsTest {
+class FeedSourceControllerTest extends DatatoolsTest {
     private static Project project = null;
     private static Project projectToBeDeleted = null;
     private static FeedSource feedSourceWithUrl = null;
@@ -70,7 +63,7 @@ public class FeedSourceControllerTest extends DatatoolsTest {
     private static FeedVersion feedVersionFromPinnedDeployment = null;
 
     @BeforeAll
-    public static void setUp() throws IOException {
+    static void settingUp() throws IOException {
         DatatoolsTest.setUp();
         Auth0Connection.setAuthDisabled(true);
         ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false;
@@ -206,7 +199,7 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx
     }
 
     @AfterAll
-    public static void tearDown() {
+    static void tearDown() {
         Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled());
         if (project != null) {
             project.delete();
@@ -243,7 +236,7 @@ private static void tearDownDeployedFeedVersion() {
      *  3. Update the same feed source turning auth fetch back on and confirm it is scheduled once more.
      */
     @Test
-    public void createFeedSourceWithUrlTest() {
+    void createFeedSourceWithUrlTest() {
         // create a feed source.
         SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource",
             JsonUtil.toJson(feedSourceWithUrl),
@@ -278,7 +271,7 @@ public void createFeedSourceWithUrlTest() {
      * Create a feed source without defining the feed source url. Confirm that the feed source is not scheduled.
      */
     @Test
-    public void createFeedSourceWithNoUrlTest() {
+    void createFeedSourceWithNoUrlTest() {
         SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource",
             JsonUtil.toJson(feedSourceWithNoUrl),
             HttpUtils.REQUEST_METHOD.POST
@@ -292,7 +285,7 @@ public void createFeedSourceWithNoUrlTest() {
      * Create some labels, add them to the feed source make them admin only, and check that they don't appear if not an admin
      */
     @Test
-    public void createFeedSourceWithLabels() {
+    void createFeedSourceWithLabels() {
         // Create labels
         SimpleHttpResponse createFirstLabelResponse = TestUtils.makeRequest("/api/manager/secure/label",
                 JsonUtil.toJson(publicLabel),
@@ -391,8 +384,6 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException {
         assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate);
         assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate);
         assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount);
-        assertEquals(feedVersionFromLatestDeploymentVersion2.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher);
-        assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState);
         assertEquals(feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId);
         assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, firstSummary.publishedValidationSummary.errorCount);
         assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.firstCalendarDate, firstSummary.publishedValidationSummary.startDate);
@@ -434,77 +425,9 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException {
         assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate);
         assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate);
         assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount);
-        assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher);
-        assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState);
         assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId);
     }
 
-    @ParameterizedTest
-    @MethodSource("createPublishStates")
-    void canDeterminePublishState(
-        boolean isPublished,
-        boolean isPublishing,
-        boolean isPublishBlocked,
-        boolean isFeedLoading,
-        PublishState expectedPublishState
-    ) {
-        FeedSourceSummary feedSourceSummary = new FeedSourceSummary();
-        FeedVersionSummary feedVersionSummary = new FeedVersionSummary();
-        FeedVersion.setDateOverrideForTesting(LocalDate.now());
-
-        if (isPublished) {
-            feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace";
-        }
-        if (isPublishing) {
-            feedVersionSummary.sentToExternalPublisher = new Date();
-            feedVersionSummary.processedByExternalPublisher = null;
-        }
-        if (isPublishBlocked) {
-            feedVersionSummary.gtfsPlusValidation = null;
-        }
-        if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) {
-            // Ready to publish case.
-            FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false);
-            passPublishBlockedCheck(feedVersionSummary);
-
-            feedVersionSummary.validationResult.errorCount = 1;
-            feedVersionSummary.id = "feed-version-id";
-            feedSourceSummary.id = "feed-source-id";
-        }
-        PublishState publishState = feedVersionSummary.getPublishState();
-        assertEquals(expectedPublishState, publishState);
-        FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null);
-    }
-
-    /**
-     * Set up a feed version summary to pass the publish blocked check.
-     */
-    private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) {
-        feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation();
-        feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>();
-        feedVersionSummary.gtfsPlusValidation.published = true;
-        feedVersionSummary.validationResult = new ValidationResult();
-        feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1);
-        FeedVersion.setDateOverrideForTesting(LocalDate.now());
-    }
-
-    private static Stream<Arguments> createPublishStates() {
-        return Stream.of(
-            Arguments.of(
-                true, false, false, false, PublishState.PUBLISHED
-            ),
-            Arguments.of(
-                false, true, false, false, PublishState.PUBLISHING
-            ),
-            Arguments.of(
-                false, false, true, false, PublishState.PUBLISH_BLOCKED
-            ),
-            Arguments.of(
-                false, false, false, false, PublishState.READY_TO_PUBLISH
-            )
-        );
-    }
-
     private static Project createProject(String name, boolean autoFetchFeeds) {
         Project project = new Project();
         project.name = name;