diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a3ebe75cb..0666bc85f 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -3,7 +3,7 @@ name: Create and publish a Docker image on: push: - branches: ['dev', 'master', 'dev-flex', 'mtc-deploy'] + branches: ['dev', 'master', 'dev-flex', 'mtc-deploy', 'flex-fix-project-list-perf'] env: REGISTRY: ghcr.io diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index ab5883f2e..eb2fe6c8d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -219,7 +219,8 @@ public static Map getLatestFeedVersionForFeedSources ); // Define the variable passed into the lookup pipeline. - List> feedSourceId = List.of(new Variable<>("feedSourceId", "$_id")); + Variable feedSourceIdVariable = new Variable<>("feedSourceId", "$_id"); + List> feedSourceId = List.of(feedSourceIdVariable); // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element). Bson lookupLatestFeedVersion = lookup( @@ -231,7 +232,14 @@ public static Map getLatestFeedVersionForFeedSources // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId) List publishedFeedVersionPipeline = Arrays.asList( - // Match FeedVersion documents where namespace equals the outer document's publishedVersionId. + // Match FeedVersion documents where namespace equals the outer document's + // feedSourceId (important when dealing with many FeedVersions) + // and publishedVersionId. + match( + expr( + new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId")) + ) + ), match( expr( new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId")) @@ -242,13 +250,16 @@ public static Map getLatestFeedVersionForFeedSources project(include("validationResult")) ); - // Pass publishedVersionId from the local document into the lookup pipeline. - List> publishedVersionId = List.of(new Variable<>("publishedVersionId", "$publishedVersionId")); + // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline. + List> feedSourceIdAndPublishedVersionId = List.of( + feedSourceIdVariable, + new Variable<>("publishedVersionId", "$publishedVersionId") + ); // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element). Bson lookupPublishedFeedVersion = lookup( "FeedVersion", - publishedVersionId, + feedSourceIdAndPublishedVersionId, publishedFeedVersionPipeline, "publishedFeedVersion" ); diff --git a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js index 21a563012..de63cc179 100644 --- a/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js +++ b/src/main/resources/mongo/getLatestFeedVersionForFeedSources.js @@ -25,8 +25,10 @@ db.FeedSource.aggregate([ { $lookup: { from: "FeedVersion", - let: { publishedVersionId: "$publishedVersionId" }, + let: { feedSourceId: "$_id", publishedVersionId: "$publishedVersionId" }, pipeline: [ + // Filtering by feedSourceId helps by 100x when dealing with large number of feed versions. + { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } }, { $match: { $expr: { $eq: ["$namespace", "$$publishedVersionId"] } } }, { $limit: 1 }, { $project: { validationResult: 1 } } diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index b5a9585e3..2bb8423bd 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -148,8 +148,7 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx feedVersionPublishedFromLatestDeployment = createFeedVersion( "published-feed-version-from-latest-deployment", - // Set to null so the relationship to feed source is via the published version id. - null, + feedSourceWithLatestDeploymentFeedVersion.id, LocalDate.of(2022, Month.NOVEMBER, 2), LocalDate.of(2022, Month.NOVEMBER, 3), feedSourceWithLatestDeploymentFeedVersion.publishedVersionId,