diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 90d9943fb..6f2946ed7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -3,22 +3,19 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.extensions.ExternalPropertiesRetriever; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; import com.mongodb.client.model.Sorts; -import com.mongodb.client.model.UnwindOptions; -import com.mongodb.client.model.Variable; import org.bson.Document; import org.bson.conversions.Bson; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,19 +23,11 @@ import static com.conveyal.datatools.manager.DataManager.hasConfigProperty; import static com.conveyal.datatools.manager.DataManager.isExtensionEnabled; import static com.conveyal.datatools.manager.DataManager.isModuleEnabled; -import static com.mongodb.client.model.Aggregates.limit; -import static com.mongodb.client.model.Aggregates.lookup; import static com.mongodb.client.model.Aggregates.match; import static com.mongodb.client.model.Aggregates.project; -import static com.mongodb.client.model.Aggregates.replaceRoot; import static com.mongodb.client.model.Aggregates.sort; -import static com.mongodb.client.model.Aggregates.unwind; -import static com.mongodb.client.model.Filters.expr; import static com.mongodb.client.model.Filters.in; -import static com.mongodb.client.model.Projections.computed; -import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; -import static com.mongodb.client.model.Sorts.descending; import static java.util.Objects.requireNonNullElse; /** @@ -77,20 +66,24 @@ public class FeedSourceSummary { public LatestValidationResult latestValidation; + public LatestValidationResult latestActiveFeedValidation; + + @JsonIgnoreProperties(ignoreUnknown = true) + public FeedVersionSummary latestVersion; + + @JsonIgnoreProperties(ignoreUnknown = true) + public FeedVersionSummary latestActiveVersion; + public String url; public List noteIds = new ArrayList<>(); public String organizationId; - public Date latestSentToExternalPublisher; - public FeedValidationResultSummary publishedValidationSummary; public String publishedVersionId; - public PublishState publishState; - @JsonInclude(JsonInclude.Include.NON_NULL) public Map> externalProperties; @@ -129,26 +122,57 @@ public FeedSourceSummary(String projectId, String organizationId, Document feedS } /** - * Update the publish and validation state based on the provided feed version summary. + * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. */ - public void updatePublishAndValidationState(FeedVersionSummary feedVersionSummary) { - if (feedVersionSummary == null) { + private static void assignDeployedVersion(String projectId, List feedSourceSummaries) { + Map latestFeedVersionForFeedSources = FeedVersionSummary.getLatestFeedVersionForFeedSources(projectId); + Map latestActiveFeedVersionForFeedSources = FeedVersionSummary.getLatestActiveFeedVersionForFeedSources(projectId); + Map pinnedDeploymentFeedVersions = FeedVersionSummary.getFeedVersionsFromPinnedDeployment(projectId); + Map latestDeploymentDeployedFeedVersions = FeedVersionSummary.getFeedVersionsFromLatestDeployment(projectId); + + feedSourceSummaries.forEach(feedSourceSummary -> { + feedSourceSummary.assignLatestFeedVersion(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); + feedSourceSummary.assignLatestActiveFeedVersion(latestActiveFeedVersionForFeedSources.get(feedSourceSummary.id)); + FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( + feedSourceSummary.id, + latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) + ); + feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); + }); + } + + /** + * Sets the latest active feed version and related variables. + */ + private void assignLatestActiveFeedVersion(FeedVersionSummary active) { + if (active == null) { + return; + } + latestActiveFeedValidation = new LatestValidationResult(active); + latestActiveVersion = active; + } + + /** + * Sets the latest feed version (whether active, expired, or future) and related variables. + */ + private void assignLatestFeedVersion(FeedVersionSummary latest) { + if (latest == null) { return; } - latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + publishedValidationSummary = new FeedValidationResultSummary(); - publishedValidationSummary.errorCount = requireNonNullElse(feedVersionSummary.publishedFeedVersionErrorCount, -1); - publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; - publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; - publishState = feedVersionSummary.getPublishState(); - latestValidation = new LatestValidationResult(feedVersionSummary); + publishedValidationSummary.errorCount = requireNonNullElse(latest.publishedFeedVersionErrorCount, -1); + publishedValidationSummary.startDate = latest.publishedFeedVersionStartDate; + publishedValidationSummary.endDate = latest.publishedFeedVersionEndDate; + latestValidation = new LatestValidationResult(latest); + latestVersion = latest; } /** * Set the deployed feed version values. For consistency, if no error count is available set the related number of * issues to zero. */ - public void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { + private void setDeployedFeedVersionValues(FeedVersionSummary feedVersionSummary) { if (feedVersionSummary == null) { return; } @@ -190,259 +214,11 @@ public static List getFeedSourceSummaries(String projectId, S sort(Sorts.ascending("name")) ); - return extractFeedSourceSummaries(projectId, organizationId, stages); - } - - /** - * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see - * getLatestFeedVersionForFeedSources.js. - * If this is updated, be sure to also update the matching Mongo query. - */ - public static Map getLatestFeedVersionForFeedSources(String projectId) { - List feedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where feedSourceId equals the feedSourceId passed from the outer document. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - sort(descending("version")), - limit(1), - // Project only the fields needed from the FeedVersion to reduce payload size. - project( - include( - "version", - "_id", - "validationResult", - "processedByExternalPublisher", - "sentToExternalPublisher", - "gtfsPlusValidation", - "namespace" - ) - ) - ); - - // Define the variable passed into the lookup pipeline. - Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); - List> feedSourceId = List.of(feedSourceIdVariable); - - // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). - Bson lookupLatestFeedVersion = lookup( - "FeedVersion", - feedSourceId, - feedVersionPipeline, - "latestFeedVersion" - ); - - // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) - List publishedFeedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where namespace equals the outer document's - // feedSourceId (important when dealing with many FeedVersions) - // and publishedVersionId. - match( - expr( - new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) - ) - ), - match( - expr( - new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) - ) - ), - limit(1), - // Project only the validationResult because that's all that is needed later. - project(include("validationResult")) - ); - - // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. - List> feedSourceIdAndPublishedVersionId = List.of( - feedSourceIdVariable, - new Variable<>("publishedVersionId", "$publishedVersionId") - ); - - // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). - Bson lookupPublishedFeedVersion = lookup( - "FeedVersion", - feedSourceIdAndPublishedVersionId, - publishedFeedVersionPipeline, - "publishedFeedVersion" - ); - - // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. - List stages = Arrays.asList( - // Start by filtering documents by projectId (reduces the number of input documents early). - match(in("projectId", projectId)), - - // Attach the latest FeedVersion (as an array "latestFeedVersion"). - lookupLatestFeedVersion, - - // Attach the published FeedVersion (as an array "publishedFeedVersion"). - lookupPublishedFeedVersion, - - // Unwind the latestFeedVersion array into a single document. - unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Unwind the publishedFeedVersion array into a single document. - unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), - - // Final projection: select and compute only the fields needed for the output to minimize size. - project(fields( - // keep the raw publishedVersionId field for reference. - include("publishedVersionId"), - - // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). - computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), - computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), - computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), - - // Latest feed version fields (mapped from the nested latestFeedVersion). - computed("feedVersionId", "$latestFeedVersion._id"), - computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), - computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), - computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), - computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), - computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), - computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), - computed("namespace", "$latestFeedVersion.namespace") - )) - ); - - return extractFeedVersionSummaries( - "FeedSource", - "feedVersionId", - "_id", - false, - stages - ); - } - - /** - * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see - * getFeedVersionsFromLatestDeployment.js. - * If this is updated, be sure to also update the matching Mongo query. - */ - public static Map getFeedVersionsFromLatestDeployment(String projectId) { - List stages = new ArrayList<>(); - stages.add(match(in("_id", projectId))); - - // Lookup Deployments for the project. - stages.add(lookup( - "Deployment", - "_id", - "projectId", - "deployments" - )); - - // Unwind deployments array to get individual deployment documents. - stages.add(unwind("$deployments")); - - // Project only fields needed from deployment to reduce doc size before sorting. - stages.add(project(fields( - computed("deployment", "$deployments._id"), - computed("lastUpdated", "$deployments.lastUpdated"), - computed("feedVersionIds", "$deployments.feedVersionIds") - ))); - - // Sort deployments by lastUpdated descending. - stages.add(sort(descending("lastUpdated"))); - stages.add(limit(1)); - - List feedVersionPipeline = Arrays.asList( - match(expr(new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")))), - project( - include( - "feedSourceId", - "validationResult.firstCalendarDate", - "validationResult.lastCalendarDate", - "validationResult.errorCount" - ) - ) - ); - - // Use pipeline form of lookup to fetch FeedVersions matching deployment’s feedVersionIds - List> feedVersionIds = List.of(new Variable<>("feedVersionIds", "$feedVersionIds")); - - stages.add(lookup( - "FeedVersion", - feedVersionIds, - feedVersionPipeline, - "feedVersions" - )); - stages.add(unwind("$feedVersions", new UnwindOptions().preserveNullAndEmptyArrays(false))); - stages.add(replaceRoot("$feedVersions")); - // Final projection: select and compute only the fields needed for the output to minimize size. - stages.add(project( - include( - "_id", - "feedSourceId", - "validationResult" - ) - )); - - return extractFeedVersionSummaries( - "Project", - "_id", - "feedSourceId", - true, - stages - ); - } - - /** - * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see - * getFeedVersionsFromPinnedDeployment.js. - */ - public static Map getFeedVersionsFromPinnedDeployment(String projectId) { - List stages = new ArrayList<>(); - - // Match projects by projectId. - stages.add(match(in("_id", projectId))); - - // Project only pinnedDeploymentId to keep doc small. - stages.add(project(include("pinnedDeploymentId"))); - - // Lookup Deployment documents by pinnedDeploymentId. - stages.add(lookup("Deployment", "pinnedDeploymentId", "_id", "deployment")); - - // Unwind deployment array (assuming single deployment per project). - stages.add(unwind("$deployment")); - - // Define pipeline in $lookup to filter and project FeedVersion docs. - List feedVersionPipeline = Arrays.asList( - match( - expr( - new Document("$in", Arrays.asList("$_id", "$$feedVersionIds")) - ) - ), - project( - include( - "_id", - "feedSourceId", - "validationResult.firstCalendarDate", - "validationResult.lastCalendarDate", - "validationResult.errorCount" - ) - ) - ); - - // Define variable for correlated lookup on FeedVersion collection. - List> feedVersionIds = List.of( - new Variable<>("feedVersionIds", "$deployment.feedVersionIds") - ); - - // Lookup FeedVersion docs with pipeline and store as feedVersions array. - stages.add(lookup("FeedVersion", feedVersionIds, feedVersionPipeline, "feedVersions")); - - return extractFeedVersionSummaries( - "Project", - "_id", - "feedSourceId", - true, - stages - ); + List feedSourceSummaries = extractFeedSourceSummaries(projectId, organizationId, stages); + FeedSourceSummary.assignDeployedVersion(projectId, feedSourceSummaries); + return feedSourceSummaries; } - /** * Produce a list of all feed source summaries for a project. */ @@ -458,27 +234,6 @@ private static List extractFeedSourceSummaries( return feedSourceSummaries; } - /** - * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed - * source. - */ - private static Map extractFeedVersionSummaries( - String collection, - String feedVersionKey, - String feedSourceKey, - boolean hasChildValidationResultDocument, - List stages - ) { - Map feedVersionSummaries = new HashMap<>(); - for (Document feedVersion : Persistence.getDocuments(collection, stages)) { - feedVersionSummaries.put( - feedVersion.getString(feedSourceKey), - new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) - ); - } - return feedVersionSummaries; - } - /** * Convert Date object into LocalDate object. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 6f8586bc8..62c382bd4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -551,6 +551,21 @@ public static boolean hasExpired(ValidationResult validationResult) { getNowAsLocalDate().isAfter(validationResult.lastCalendarDate); } + /** + * Whether this feed starts in the future + * @return true if the validation result first calendar date is not null and in the future true, false otherwise. + */ + @JsonIgnore + @BsonIgnore + public boolean isFuture() { + return isFuture(validationResult); + } + + public static boolean isFuture(ValidationResult validationResult) { + return validationResult.firstCalendarDate != null && + getNowAsLocalDate().isBefore(validationResult.firstCalendarDate); + } + private static LocalDate getNowAsLocalDate() { return dateOverrideForTesting == null ? LocalDate.now() : dateOverrideForTesting; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index 27d6096b6..bc9a75de1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -3,21 +3,45 @@ import com.conveyal.datatools.editor.utils.JacksonSerializers; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; +import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.mongodb.client.model.UnwindOptions; +import com.mongodb.client.model.Variable; import org.bson.Document; +import org.bson.conversions.Bson; import java.io.Serializable; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static com.mongodb.client.model.Aggregates.limit; +import static com.mongodb.client.model.Aggregates.lookup; +import static com.mongodb.client.model.Aggregates.match; +import static com.mongodb.client.model.Aggregates.project; +import static com.mongodb.client.model.Aggregates.replaceRoot; +import static com.mongodb.client.model.Aggregates.sort; +import static com.mongodb.client.model.Aggregates.unwind; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.expr; +import static com.mongodb.client.model.Filters.in; +import static com.mongodb.client.model.Filters.ne; +import static com.mongodb.client.model.Projections.computed; +import static com.mongodb.client.model.Projections.fields; +import static com.mongodb.client.model.Projections.include; +import static com.mongodb.client.model.Sorts.descending; + /** * Includes summary data (a subset of fields) for a feed version. */ @@ -45,12 +69,16 @@ public class FeedVersionSummary extends Model implements Serializable { public String feedSourcePublishedVersionId; public Integer publishedFeedVersionErrorCount; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate publishedFeedVersionStartDate; + @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) + @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate publishedFeedVersionEndDate; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { - validationSummary = new PartialValidationSummary(); + validationSummary = new PartialValidationSummary(validationResult); } return validationSummary; } @@ -70,6 +98,7 @@ public FeedVersionSummary( sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); gtfsPlusValidation = getGtfsPlusValidation(id, feedVersionDocument); namespace = feedVersionDocument.getString("namespace"); + lastUpdated = feedVersionDocument.getDate("lastUpdated"); validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. @@ -83,7 +112,7 @@ public FeedVersionSummary( /** * Holds a subset of fields from {@link:FeedValidationResultSummary} for UI use only. */ - public class PartialValidationSummary { + public static class PartialValidationSummary { /** Copied from FeedVersion */ @JsonSerialize(using = JacksonSerializers.LocalDateIsoSerializer.class) @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) @@ -94,7 +123,11 @@ public class PartialValidationSummary { @JsonDeserialize(using = JacksonSerializers.LocalDateIsoDeserializer.class) public LocalDate endDate; - PartialValidationSummary() { + public PartialValidationSummary() { + + } + + public PartialValidationSummary(ValidationResult validationResult) { // Older feeds created in datatools may not have validationResult if (validationResult != null) { this.startDate = validationResult.firstCalendarDate; @@ -223,7 +256,8 @@ private boolean isPublishing() { } /** - * Determine if publishing is blocked due to validation, expiration, blocking issues or loading. + * Determine if publishing is blocked due to validation, expiration, blocking issues, loading, + * or the feed being in the future. */ private boolean isPublishBlocked() { return @@ -232,6 +266,7 @@ private boolean isPublishBlocked() { !gtfsPlusValidation.issues.isEmpty() || !gtfsPlusValidation.published || FeedVersion.hasExpired(validationResult) || + FeedVersion.isFuture(validationResult) || hasBlockingIssuesForPublishing(); } @@ -249,4 +284,311 @@ private boolean hasBlockingIssuesForPublishing() { public static void setHasBlockingIssueForPublishingOverrideForTesting(Boolean value) { hasBlockingIssueForPublishingForTesting = value; } + + + /** + * Get the latest feed version from all feed sources for this project. For equivalent Mongo query, see + * getLatestFeedVersionForFeedSources.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getLatestFeedVersionForFeedSources(String projectId) { + return getLatestFeedVersionForFeedSources(projectId, null); + } + + /** + * Get the latest active feed version (not expired, not future) from all feed sources for this project. + * For equivalent Mongo query, see + * getLatestActiveFeedVersionForFeedSources.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getLatestActiveFeedVersionForFeedSources(String projectId) { + Bson matchActiveVersions = match( + expr( + and( + // Using Arrays.asList to pass a null value. + new Document("$ne", Arrays.asList("$validationResult", null)), + // This string-based date comparison works because the strings are fixed-length with numbers only! + new Document("$lte", List.of("$validationResult.firstCalendarDate", "$$referenceDate")), + new Document("$gte", List.of("$validationResult.lastCalendarDate", "$$referenceDate")) + ) + ) + ); + + return getLatestFeedVersionForFeedSources(projectId, List.of(matchActiveVersions)); + } + + /** + * Get the latest feed version from all feed sources for this project. + */ + static Map getLatestFeedVersionForFeedSources( + String projectId, + List extraFeedVersionFilters + ) { + Bson matchFeedSourceId = match( + expr( + new Document("$eq", List.of("$feedSourceId", "$$feedSourceId")) + ) + ); + + List feedVersionPipeline = new ArrayList<>(); + feedVersionPipeline.add(matchFeedSourceId); + if (extraFeedVersionFilters != null) { + feedVersionPipeline.addAll(extraFeedVersionFilters); + } + feedVersionPipeline.add(sort(descending("version"))); + feedVersionPipeline.add(limit(1)); + feedVersionPipeline.add(project( + include( + "version", + "_id", + "validationResult", + "processedByExternalPublisher", + "sentToExternalPublisher", + "gtfsPlusValidation", + "namespace", + "lastUpdated" + ) + )); + + // Define the variables passed into the lookup pipeline. + String todayFormatted = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); + Variable referenceDateVariable = new Variable<>("referenceDate", todayFormatted); + List> feedSourceIdAndToday = List.of(feedSourceIdVariable, referenceDateVariable); + + // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). + Bson lookupLatestFeedVersion = lookup( + "FeedVersion", + feedSourceIdAndToday, + feedVersionPipeline, + "latestFeedVersion" + ); + + // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) + List publishedFeedVersionPipeline = List.of( + // Match FeedVersion documents where namespace equals the outer document's + // feedSourceId (important when dealing with many FeedVersions) + // and publishedVersionId. + matchFeedSourceId, + match( + expr( + new Document("$eq", List.of("$namespace", "$$publishedVersionId")) + ) + ), + limit(1), + // Project only the validationResult because that's all that is needed later. + project(include("validationResult")) + ); + + // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. + List> feedSourceIdAndPublishedVersionId = List.of( + feedSourceIdVariable, + new Variable<>("publishedVersionId", "$publishedVersionId") + ); + + // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). + Bson lookupPublishedFeedVersion = lookup( + "FeedVersion", + feedSourceIdAndPublishedVersionId, + publishedFeedVersionPipeline, + "publishedFeedVersion" + ); + + // Top-level aggregation stages that combine the lookups and map required fields into a slimmed down result. + List stages = List.of( + // Start by filtering documents by projectId (reduces the number of input documents early). + match(in("projectId", projectId)), + + // Attach the latest FeedVersion (as an array "latestFeedVersion"). + lookupLatestFeedVersion, + + // Attach the published FeedVersion (as an array "publishedFeedVersion"). + lookupPublishedFeedVersion, + + // Unwind the latestFeedVersion array into a single document. + unwind("$latestFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Unwind the publishedFeedVersion array into a single document. + unwind("$publishedFeedVersion", new UnwindOptions().preserveNullAndEmptyArrays(true)), + + // Final projection: select and compute only the fields needed for the output to minimize size. + project(fields( + // keep the raw publishedVersionId field for reference. + include("publishedVersionId"), + + // Published feed version fields (mapped from the nested publishedFeedVersion.validationResult). + computed("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), + computed("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), + computed("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), + + // Latest feed version fields (mapped from the nested latestFeedVersion). + computed("feedVersionId", "$latestFeedVersion._id"), + computed("firstCalendarDate", "$latestFeedVersion.validationResult.firstCalendarDate"), + computed("lastCalendarDate", "$latestFeedVersion.validationResult.lastCalendarDate"), + computed("errorCount", "$latestFeedVersion.validationResult.errorCount"), + computed("processedByExternalPublisher", "$latestFeedVersion.processedByExternalPublisher"), + computed("sentToExternalPublisher", "$latestFeedVersion.sentToExternalPublisher"), + computed("gtfsPlusValidation", "$latestFeedVersion.gtfsPlusValidation"), + computed("namespace", "$latestFeedVersion.namespace"), + computed("lastUpdated", "$latestFeedVersion.lastUpdated") + )), + + match(ne("feedVersionId", null)) + ); + + return extractFeedVersionSummaries( + "FeedSource", + "feedVersionId", + "_id", + false, + stages + ); + } + + /** + * Get the deployed feed versions from the latest deployment for this project. For equivalent Mongo query, see + * getFeedVersionsFromLatestDeployment.js. + * If this is updated, be sure to also update the matching Mongo query. + */ + static Map getFeedVersionsFromLatestDeployment(String projectId) { + List stages = new ArrayList<>(); + stages.add(match(in("_id", projectId))); + + // Lookup Deployments for the project. + stages.add(lookup( + "Deployment", + "_id", + "projectId", + "deployments" + )); + + // Unwind deployments array to get individual deployment documents. + stages.add(unwind("$deployments")); + + // Project only fields needed from deployment to reduce doc size before sorting. + stages.add(project(fields( + computed("deployment", "$deployments._id"), + computed("lastUpdated", "$deployments.lastUpdated"), + computed("feedVersionIds", "$deployments.feedVersionIds") + ))); + + // Sort deployments by lastUpdated descending. + stages.add(sort(descending("lastUpdated"))); + stages.add(limit(1)); + + List feedVersionPipeline = List.of( + match(expr(new Document("$in", List.of("$_id", "$$feedVersionIds")))), + project( + include( + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount" + ) + ) + ); + + // Use pipeline form of lookup to fetch FeedVersions matching deployment’s feedVersionIds + List> feedVersionIds = List.of(new Variable<>("feedVersionIds", "$feedVersionIds")); + + stages.add(lookup( + "FeedVersion", + feedVersionIds, + feedVersionPipeline, + "feedVersions" + )); + stages.add(unwind("$feedVersions", new UnwindOptions().preserveNullAndEmptyArrays(false))); + stages.add(replaceRoot("$feedVersions")); + // Final projection: select and compute only the fields needed for the output to minimize size. + stages.add(project( + include( + "_id", + "feedSourceId", + "validationResult" + ) + )); + + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages + ); + } + + /** + * Get the deployed feed version from the pinned deployment for this feed source. For equivalent Mongo query, see + * getFeedVersionsFromPinnedDeployment.js. + */ + static Map getFeedVersionsFromPinnedDeployment(String projectId) { + List stages = new ArrayList<>(); + + // Match projects by projectId. + stages.add(match(in("_id", projectId))); + + // Project only pinnedDeploymentId to keep doc small. + stages.add(project(include("pinnedDeploymentId"))); + + // Lookup Deployment documents by pinnedDeploymentId. + stages.add(lookup("Deployment", "pinnedDeploymentId", "_id", "deployment")); + + // Unwind deployment array (assuming single deployment per project). + stages.add(unwind("$deployment")); + + // Define pipeline in $lookup to filter and project FeedVersion docs. + List feedVersionPipeline = List.of( + match( + expr( + new Document("$in", List.of("$_id", "$$feedVersionIds")) + ) + ), + project( + include( + "_id", + "feedSourceId", + "validationResult.firstCalendarDate", + "validationResult.lastCalendarDate", + "validationResult.errorCount" + ) + ) + ); + + // Define variable for correlated lookup on FeedVersion collection. + List> feedVersionIds = List.of( + new Variable<>("feedVersionIds", "$deployment.feedVersionIds") + ); + + // Lookup FeedVersion docs with pipeline and store as feedVersions array. + stages.add(lookup("FeedVersion", feedVersionIds, feedVersionPipeline, "feedVersions")); + + return extractFeedVersionSummaries( + "Project", + "_id", + "feedSourceId", + true, + stages + ); + } + + /** + * Extract feed version summaries from feed version documents. Each feed version is held against the matching feed + * source. + */ + private static Map extractFeedVersionSummaries( + String collection, + String feedVersionKey, + String feedSourceKey, + boolean hasChildValidationResultDocument, + List stages + ) { + Map feedVersionSummaries = new HashMap<>(); + for (Document feedVersion : Persistence.getDocuments(collection, stages)) { + feedVersionSummaries.put( + feedVersion.getString(feedSourceKey), + new FeedVersionSummary(feedVersionKey, hasChildValidationResultDocument, feedVersion) + ); + } + return feedVersionSummaries; + } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index f6a8b2bc2..fc1c48135 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -15,7 +15,6 @@ import java.util.Date; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -164,27 +163,7 @@ public Collection retrieveDeploymentSummaries() { * Get all feed source summaries for this project. */ public Collection retrieveFeedSourceSummaries() { - List feedSourceSummaries = FeedSourceSummary.getFeedSourceSummaries(id, organizationId); - assignDeployedVersion(feedSourceSummaries); - return feedSourceSummaries; - } - - /** - * Assign deployed feed version. Prioritise pinned deployment feed version over latest deployment deployed feed version. - */ - private void assignDeployedVersion(List feedSourceSummaries) { - Map latestFeedVersionForFeedSources = FeedSourceSummary.getLatestFeedVersionForFeedSources(id); - Map pinnedDeploymentFeedVersions = FeedSourceSummary.getFeedVersionsFromPinnedDeployment(id); - Map latestDeploymentDeployedFeedVersions = FeedSourceSummary.getFeedVersionsFromLatestDeployment(id); - - feedSourceSummaries.forEach(feedSourceSummary -> { - feedSourceSummary.updatePublishAndValidationState(latestFeedVersionForFeedSources.get(feedSourceSummary.id)); - FeedVersionSummary deployedVersion = pinnedDeploymentFeedVersions.getOrDefault( - feedSourceSummary.id, - latestDeploymentDeployedFeedVersions.get(feedSourceSummary.id) - ); - feedSourceSummary.setDeployedFeedVersionValues(deployedVersion); - }); + return FeedSourceSummary.getFeedSourceSummaries(id, organizationId); } // TODO: Does this need to be returned with JSON API response diff --git a/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js new file mode 100644 index 000000000..4b083ea30 --- /dev/null +++ b/src/main/resources/mongo/getLatestActiveFeedVersionForFeedSources.js @@ -0,0 +1,77 @@ +db.FeedSource.aggregate([ + { $match: { projectId: "" } }, + { + $lookup: { + from: "FeedVersion", + let: { + feedSourceId: "$_id", + referenceDate: "" + }, + pipeline: [ + { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } }, + { + $match: { + $expr: { + $and: [ + { $ne: ["$validationResult", null ] }, + { $gte: ["$validationResult.firstCalendarDate", "$$referenceDate"] }, + { $lte: ["$validationResult.lastCalendarDate", "$$referenceDate"] } + ] + } + } + }, + { $sort: { version: -1 } }, + { $limit: 1 }, + { + $project: { + version: 1, + validationResult: 1, + processedByExternalPublisher: 1, + sentToExternalPublisher: 1, + gtfsPlusValidation: 1, + namespace: 1, + lastUpdated: 1 + } + } + ], + as: "latestFeedVersion" + } + }, + { + $lookup: { + from: "FeedVersion", + let: { feedSourceId: "$_id", publishedVersionId: "$publishedVersionId" }, + pipeline: [ + // Filtering by feedSourceId helps by 100x when dealing with large number of feed versions. + { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } }, + { $match: { $expr: { $eq: ["$namespace", "$$publishedVersionId"] } } }, + { $limit: 1 }, + { $project: { validationResult: 1 } } + ], + as: "publishedFeedVersion" + } + }, + { $unwind: { path: "$latestFeedVersion", preserveNullAndEmptyArrays: true } }, + { $unwind: { path: "$publishedFeedVersion", preserveNullAndEmptyArrays: true } }, + { + $project: { + publishedVersionId: 1, + publishedFeedVersionErrorCount: "$publishedFeedVersion.validationResult.errorCount", + publishedFeedVersionStartDate: "$publishedFeedVersion.validationResult.firstCalendarDate", + publishedFeedVersionEndDate: "$publishedFeedVersion.validationResult.lastCalendarDate", + version: "$latestFeedVersion.version", + feedVersionId: "$latestFeedVersion._id", + firstCalendarDate: "$latestFeedVersion.validationResult.firstCalendarDate", + lastCalendarDate: "$latestFeedVersion.validationResult.lastCalendarDate", + errorCount: "$latestFeedVersion.validationResult.errorCount", + processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher", + sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher", + gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation", + namespace: "$latestFeedVersion.namespace", + lastUpdated: "$latestFeedVersion.lastUpdated" + } + }, + { + $match: { feedVersionId: {$ne: null} } + }, +]) diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index de63cc179..5d4808ee0 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -15,7 +15,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: 1, sentToExternalPublisher: 1, gtfsPlusValidation: 1, - namespace: 1 + namespace: 1, + lastUpdated: 1 } } ], @@ -52,7 +53,8 @@ db.FeedSource.aggregate([ processedByExternalPublisher: "$latestFeedVersion.processedByExternalPublisher", sentToExternalPublisher: "$latestFeedVersion.sentToExternalPublisher", gtfsPlusValidation: "$latestFeedVersion.gtfsPlusValidation", - namespace: "$latestFeedVersion.namespace" + namespace: "$latestFeedVersion.namespace", + lastUpdated: "$latestFeedVersion.lastUpdated" } } ]) \ No newline at end of file diff --git a/src/test/java/com/conveyal/datatools/TestUtils.java b/src/test/java/com/conveyal/datatools/TestUtils.java index 71c786036..86cef57b9 100644 --- a/src/test/java/com/conveyal/datatools/TestUtils.java +++ b/src/test/java/com/conveyal/datatools/TestUtils.java @@ -5,6 +5,8 @@ import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; import com.fasterxml.jackson.databind.JsonNode; @@ -303,4 +305,17 @@ public static int countSchemaInDb() { return count; } + public static Project createProject(String name) { + Project project = new Project(); + project.name = name; + Persistence.projects.create(project); + return project; + } + + public static FeedSource createFeedSource(String name, Project project) { + FeedSource feedSource = new FeedSource(name); + feedSource.projectId = project.id; + Persistence.feedSources.create(feedSource); + return feedSource; + } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index f8ad6b0ae..3dab6bcff 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -13,12 +13,10 @@ import com.conveyal.datatools.manager.models.FeedSourceSummary; import com.conveyal.datatools.manager.models.FeedValidationResultSummary; import com.conveyal.datatools.manager.models.FeedVersion; -import com.conveyal.datatools.manager.models.FeedVersionSummary; import com.conveyal.datatools.manager.models.FetchFrequency; import com.conveyal.datatools.manager.models.Label; import com.conveyal.datatools.manager.models.Note; import com.conveyal.datatools.manager.models.Project; -import com.conveyal.datatools.manager.models.PublishState; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; @@ -27,9 +25,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.net.MalformedURLException; @@ -37,11 +32,9 @@ import java.time.LocalDate; import java.time.Month; import java.time.ZoneId; -import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.stream.Stream; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.mongodb.client.model.Filters.eq; @@ -51,7 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -public class FeedSourceControllerTest extends DatatoolsTest { +class FeedSourceControllerTest extends DatatoolsTest { private static Project project = null; private static Project projectToBeDeleted = null; private static FeedSource feedSourceWithUrl = null; @@ -70,7 +63,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static FeedVersion feedVersionFromPinnedDeployment = null; @BeforeAll - public static void setUp() throws IOException { + static void settingUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; @@ -206,7 +199,7 @@ private static void setUpFeedVersionFromPinnedDeployment() throws MalformedURLEx } @AfterAll - public static void tearDown() { + static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); @@ -243,7 +236,7 @@ private static void tearDownDeployedFeedVersion() { * 3. Update the same feed source turning auth fetch back on and confirm it is scheduled once more. */ @Test - public void createFeedSourceWithUrlTest() { + void createFeedSourceWithUrlTest() { // create a feed source. SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource", JsonUtil.toJson(feedSourceWithUrl), @@ -278,7 +271,7 @@ public void createFeedSourceWithUrlTest() { * Create a feed source without defining the feed source url. Confirm that the feed source is not scheduled. */ @Test - public void createFeedSourceWithNoUrlTest() { + void createFeedSourceWithNoUrlTest() { SimpleHttpResponse createFeedSourceResponse = TestUtils.makeRequest("/api/manager/secure/feedsource", JsonUtil.toJson(feedSourceWithNoUrl), HttpUtils.REQUEST_METHOD.POST @@ -292,7 +285,7 @@ public void createFeedSourceWithNoUrlTest() { * Create some labels, add them to the feed source make them admin only, and check that they don't appear if not an admin */ @Test - public void createFeedSourceWithLabels() { + void createFeedSourceWithLabels() { // Create labels SimpleHttpResponse createFirstLabelResponse = TestUtils.makeRequest("/api/manager/secure/label", JsonUtil.toJson(publicLabel), @@ -391,8 +384,6 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate); assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate); assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount); - assertEquals(feedVersionFromLatestDeploymentVersion2.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher); - assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState); assertEquals(feedSourceWithLatestDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId); assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, firstSummary.publishedValidationSummary.errorCount); assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.firstCalendarDate, firstSummary.publishedValidationSummary.startDate); @@ -434,77 +425,9 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(expectedValidationSummary.startDate, firstSummary.latestValidation.startDate); assertEquals(expectedValidationSummary.endDate, firstSummary.latestValidation.endDate); assertEquals(expectedValidationSummary.errorCount, firstSummary.latestValidation.errorCount); - assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, firstSummary.latestSentToExternalPublisher); - assertEquals(PublishState.PUBLISH_BLOCKED, firstSummary.publishState); assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, firstSummary.publishedVersionId); } - @ParameterizedTest - @MethodSource("createPublishStates") - void canDeterminePublishState( - boolean isPublished, - boolean isPublishing, - boolean isPublishBlocked, - boolean isFeedLoading, - PublishState expectedPublishState - ) { - FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); - FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); - FeedVersion.setDateOverrideForTesting(LocalDate.now()); - - if (isPublished) { - feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; - } - if (isPublishing) { - feedVersionSummary.sentToExternalPublisher = new Date(); - feedVersionSummary.processedByExternalPublisher = null; - } - if (isPublishBlocked) { - feedVersionSummary.gtfsPlusValidation = null; - } - if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { - // Ready to publish case. - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); - passPublishBlockedCheck(feedVersionSummary); - - feedVersionSummary.validationResult.errorCount = 1; - feedVersionSummary.id = "feed-version-id"; - feedSourceSummary.id = "feed-source-id"; - } - PublishState publishState = feedVersionSummary.getPublishState(); - assertEquals(expectedPublishState, publishState); - FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); - } - - /** - * Set up a feed version summary to pass the publish blocked check. - */ - private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { - feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); - feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); - feedVersionSummary.gtfsPlusValidation.published = true; - feedVersionSummary.validationResult = new ValidationResult(); - feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); - FeedVersion.setDateOverrideForTesting(LocalDate.now()); - } - - private static Stream createPublishStates() { - return Stream.of( - Arguments.of( - true, false, false, false, PublishState.PUBLISHED - ), - Arguments.of( - false, true, false, false, PublishState.PUBLISHING - ), - Arguments.of( - false, false, true, false, PublishState.PUBLISH_BLOCKED - ), - Arguments.of( - false, false, false, false, PublishState.READY_TO_PUBLISH - ) - ); - } - private static Project createProject(String name, boolean autoFetchFeeds) { Project project = new Project(); project.name = name; diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java new file mode 100644 index 000000000..f7f2798fd --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryQueryTest.java @@ -0,0 +1,126 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.validator.ValidationResult; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.time.LocalDate; +import java.util.Date; +import java.util.Map; +import java.util.Set; + +import static com.conveyal.datatools.TestUtils.createFeedSource; +import static com.conveyal.datatools.TestUtils.createProject; +import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestActiveFeedVersionForFeedSources; +import static com.conveyal.datatools.manager.models.FeedVersionSummary.getLatestFeedVersionForFeedSources; +import static com.mongodb.client.model.Filters.in; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FeedVersionSummaryQueryTest extends DatatoolsTest { + private static Project project; + private static FeedSource sourceWithFutureAnd2ActiveFeeds; + private static FeedSource sourceWithFutureAndActiveFeed; + private static FeedSource sourceWithFutureFeed; + private static Set allNonEmptyFeedSourceIds; + + private static final LocalDate TODAY = LocalDate.now(); + + /** Initialize application for tests to run. */ + @BeforeAll + static void settingUp() throws Exception { + // start server if it isn't already running + DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); + + project = createProject(String.format("Test project %s", new Date())); + sourceWithFutureAnd2ActiveFeeds = createFeedSource("Test feed source 1", project); + sourceWithFutureAndActiveFeed = createFeedSource("Test feed source 2", project); + sourceWithFutureFeed = createFeedSource("Test feed source 3", project); + createFeedSource("Feed source without feed versions", project); + + allNonEmptyFeedSourceIds = Set.of( + sourceWithFutureAnd2ActiveFeeds.id, + sourceWithFutureAndActiveFeed.id, + sourceWithFutureFeed.id + ); + } + + @AfterAll + static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + if (project != null) { + project.delete(); + } + } + + @AfterEach + void afterEach() { + // Delete feed versions created for each source. + Persistence.feedVersions.removeFiltered(in("feedSourceId", allNonEmptyFeedSourceIds)); + } + + /** + * Make sure the latest feed versions, whether expired, future, or current, are correct for each feed source. + */ + @Test + void canObtainLatestVersion() { + createFeedVersions(); + Map activeSummaries = getLatestFeedVersionForFeedSources(project.id); + assertEquals(allNonEmptyFeedSourceIds, activeSummaries.keySet()); + assertEquals("1-future", activeSummaries.get(sourceWithFutureAnd2ActiveFeeds.id).id); + assertEquals("2-future", activeSummaries.get(sourceWithFutureAndActiveFeed.id).id); + assertEquals("3-expired", activeSummaries.get(sourceWithFutureFeed.id).id); + } + + /** + * Make sure the latest active feed versions are correct for each feed source. + */ + @Test + void canObtainLatestActiveVersion() { + createFeedVersions(); + + Map activeSummaries = getLatestActiveFeedVersionForFeedSources(project.id); + + // feedSource3 should not appear in the results because it has no active feeds. + assertEquals(Set.of(sourceWithFutureAnd2ActiveFeeds.id, sourceWithFutureAndActiveFeed.id), activeSummaries.keySet()); + assertEquals("1-active-current", activeSummaries.get(sourceWithFutureAnd2ActiveFeeds.id).id); + assertEquals("2-active-current", activeSummaries.get(sourceWithFutureAndActiveFeed.id).id); + } + + private static void createFeedVersions() { + createFeedVersion("1-active-older", 1, sourceWithFutureAnd2ActiveFeeds, -30, 10); + createFeedVersion("1-active-current", 2, sourceWithFutureAnd2ActiveFeeds, -5, 0); + createFeedVersion("1-future", 3, sourceWithFutureAnd2ActiveFeeds, 1, 60); + + createFeedVersion("2-active-current", 1, sourceWithFutureAndActiveFeed, 0, 20); + createFeedVersion("2-future", 2, sourceWithFutureAndActiveFeed, 1, 60); + + createFeedVersion("3-expired", 1, sourceWithFutureFeed, -30, -10); + } + + /** + * Helper method to create a feed version. + * The id field serves as description for the purpose of each version. + */ + private static void createFeedVersion( + String id, + int version, + FeedSource feedSource, + int startOffsetDaysFromToday, + int endOffsetDaysFromToday + ) { + FeedVersion feedVersion = new FeedVersion(feedSource); + feedVersion.id = id; + feedVersion.version = version; + ValidationResult validationResult = new ValidationResult(); + validationResult.firstCalendarDate = TODAY.plusDays(startOffsetDaysFromToday); + validationResult.lastCalendarDate = TODAY.plusDays(endOffsetDaysFromToday); + feedVersion.validationResult = validationResult; + Persistence.feedVersions.create(feedVersion); + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java new file mode 100644 index 000000000..05dbc74b5 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionSummaryTest.java @@ -0,0 +1,112 @@ +package com.conveyal.datatools.manager.models; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.gtfs.validator.ValidationResult; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Date; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FeedVersionSummaryTest extends DatatoolsTest { + @BeforeAll + static void settingUp() throws IOException { + DatatoolsTest.setUp(); + } + + @AfterEach + void afterEach() { + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(null); + } + + @ParameterizedTest + @MethodSource("createPublishStates") + void canDeterminePublishState( + boolean isPublished, + boolean isPublishing, + boolean isPublishBlocked, + boolean isFeedLoading, + PublishState expectedPublishState + ) { + FeedSourceSummary feedSourceSummary = new FeedSourceSummary(); + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + + if (isPublished) { + feedVersionSummary.namespace = feedVersionSummary.feedSourcePublishedVersionId = "namespace"; + } + if (isPublishing) { + feedVersionSummary.sentToExternalPublisher = new Date(); + feedVersionSummary.processedByExternalPublisher = null; + } + if (isPublishBlocked) { + feedVersionSummary.gtfsPlusValidation = null; + } + if (!isPublished && !isPublishing && !isPublishBlocked && !isFeedLoading) { + // Ignore blocking issues. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + passPublishBlockedCheck(feedVersionSummary); + + feedVersionSummary.validationResult.errorCount = 1; + feedVersionSummary.id = "feed-version-id"; + feedSourceSummary.id = "feed-source-id"; + } + PublishState publishState = feedVersionSummary.getPublishState(); + assertEquals(expectedPublishState, publishState); + } + + /** + * Set up a feed version summary to pass the publish blocked check. + */ + private static void passPublishBlockedCheck(FeedVersionSummary feedVersionSummary) { + feedVersionSummary.gtfsPlusValidation = new GtfsPlusValidation(); + feedVersionSummary.gtfsPlusValidation.issues = new ArrayList<>(); + feedVersionSummary.gtfsPlusValidation.published = true; + feedVersionSummary.validationResult = new ValidationResult(); + feedVersionSummary.validationResult.lastCalendarDate = LocalDate.now().plusDays(1); + FeedVersion.setDateOverrideForTesting(LocalDate.now()); + } + + private static Stream createPublishStates() { + return Stream.of( + Arguments.of(true, false, false, false, PublishState.PUBLISHED), + Arguments.of(false, true, false, false, PublishState.PUBLISHING), + Arguments.of(false, false, true, false, PublishState.PUBLISH_BLOCKED), + Arguments.of(false, false, false, false, PublishState.READY_TO_PUBLISH) + ); + } + + /** + * Additional cases for future and expired feeds. + */ + @ParameterizedTest + @MethodSource("publishStateTimeCases") + void canGetPublishState(int daysOffsetFromToday, PublishState expectedState, String caseName) { + FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); + passPublishBlockedCheck(feedVersionSummary); + LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); + feedVersionSummary.validationResult.firstCalendarDate = date; + feedVersionSummary.validationResult.lastCalendarDate = date.plusDays(3); + + // Ignore blocking issues. + FeedVersionSummary.setHasBlockingIssueForPublishingOverrideForTesting(false); + assertEquals(expectedState, feedVersionSummary.getPublishState(), caseName); + } + + private static Stream publishStateTimeCases() { + return Stream.of( + Arguments.of(1, PublishState.PUBLISH_BLOCKED, "future feed"), + Arguments.of(0, PublishState.READY_TO_PUBLISH, "present feed"), + Arguments.of(-5, PublishState.PUBLISH_BLOCKED, "expired feed") + ); + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index e0550c8eb..76a047daf 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -13,49 +13,90 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import java.sql.Connection; import java.sql.SQLException; +import java.time.LocalDate; import java.util.Date; +import java.util.stream.Stream; +import static com.conveyal.datatools.TestUtils.createFeedSource; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.TestUtils.createProject; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.mongodb.client.model.Filters.eq; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; -public class FeedVersionTest extends UnitTest { +class FeedVersionTest extends UnitTest { private static Project project; private static FeedSource feedSource; /** Initialize application for tests to run. */ @BeforeAll - public static void setUp() throws Exception { + static void setUp() throws Exception { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - // set up project - project = new Project(); - project.name = String.format("Test project %s", new Date()); - Persistence.projects.create(project); - - feedSource = new FeedSource("Test feed source"); - feedSource.projectId = project.id; - Persistence.feedSources.create(feedSource); + project = createProject(String.format("Test project %s", new Date())); + feedSource = createFeedSource("Test feed source", project); } @AfterAll - public static void tearDown() { + static void tearDown() { Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } } + /** + * Future feed detection. + */ + @ParameterizedTest + @MethodSource("futureFeedCases") + void canDetermineFutureFeed(int daysOffsetFromToday, boolean isFuture, String caseName) { + FeedVersion feedVersion1 = new FeedVersion(feedSource); + feedVersion1.validationResult = new ValidationResult(); + LocalDate date = LocalDate.now().plusDays(daysOffsetFromToday); + feedVersion1.validationResult.firstCalendarDate = date; + feedVersion1.validationResult.lastCalendarDate = date.plusDays(3); + + assertEquals( + isFuture, + feedVersion1.isFuture(), + String.format("Feed starting '%s' is %s in the future.", caseName, isFuture ? "indeed" : "not") + ); + } + + private static Stream futureFeedCases() { + return Stream.of( + Arguments.of(0, false, "today"), + Arguments.of(-1, false, "yesterday"), + Arguments.of(-2, false, "the day before yesterday"), + Arguments.of(1, true, "tomorrow"), + Arguments.of(2, true, "the day after tomorrow") + ); + } + + /** + * Future feed detection. + */ + @Test + void canDetermineFutureFeedNullFirstCalendarDate() { + FeedVersion feedVersion1 = new FeedVersion(feedSource); + feedVersion1.validationResult = new ValidationResult(); + assertFalse(feedVersion1.isFuture(), "null firstCalendarDate is not future."); + } + /** * Make sure FeedVersionIDs are always unique, even if created at the same second. * See https://github.com/ibi-group/datatools-server/issues/251