Improve Fetch Speed for Feed List

.github/workflows/build-docker.yml

@@ -3,7 +3,7 @@ name: Create and publish a Docker image
 
 on:
   push:
-    branches: ['dev', 'master', 'dev-flex', 'mtc-deploy']
+    branches: ['dev', 'master', 'dev-flex', 'mtc-deploy', 'flex-fix-project-list-perf']
 
 env:
   REGISTRY: ghcr.io

src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java

@@ -219,7 +219,8 @@ public static Map<String, FeedVersionSummary> getLatestFeedVersionForFeedSources
         );
 
         // Define the variable passed into the lookup pipeline.
-        List<Variable<String>> feedSourceId = List.of(new Variable<>("feedSourceId", "$_id"));
+        Variable<String> feedSourceIdVariable = new Variable<>("feedSourceId", "$_id");
+        List<Variable<String>> feedSourceId = List.of(feedSourceIdVariable);
 
         // $lookup that uses the above pipeline to produce "latestFeedVersion" (an array with at most one element).
         Bson lookupLatestFeedVersion = lookup(
@@ -231,7 +232,14 @@ public static Map<String, FeedVersionSummary> getLatestFeedVersionForFeedSources
 
         // Pipeline to find the published FeedVersion by namespace (or identifier stored in publishedVersionId)
         List<Bson> publishedFeedVersionPipeline = Arrays.asList(
-            // Match FeedVersion documents where namespace equals the outer document's publishedVersionId.
+            // Match FeedVersion documents where namespace equals the outer document's
+            // feedSourceId (important when dealing with many FeedVersions)
+            // and publishedVersionId.
+            match(
+                expr(
+                    new Document("$eq", Arrays.asList("$feedSourceId", "$$feedSourceId"))
+                )
+            ),
             match(
                 expr(
                     new Document("$eq", Arrays.asList("$namespace", "$$publishedVersionId"))
@@ -242,13 +250,16 @@ public static Map<String, FeedVersionSummary> getLatestFeedVersionForFeedSources
             project(include("validationResult"))
         );
 
-        // Pass publishedVersionId from the local document into the lookup pipeline.
-        List<Variable<String>> publishedVersionId = List.of(new Variable<>("publishedVersionId", "$publishedVersionId"));
+        // Pass feedSourceId and publishedVersionId from the local document into the lookup pipeline.
+        List<Variable<String>> feedSourceIdAndPublishedVersionId = List.of(
+            feedSourceIdVariable,
+            new Variable<>("publishedVersionId", "$publishedVersionId")
+        );
 
         // $lookup that uses the above pipeline to produce "publishedFeedVersion" (an array with at most one element).
         Bson lookupPublishedFeedVersion = lookup(
             "FeedVersion",
-            publishedVersionId,
+            feedSourceIdAndPublishedVersionId,
             publishedFeedVersionPipeline,
             "publishedFeedVersion"
         );

src/main/resources/mongo/getLatestFeedVersionForFeedSources.js

@@ -25,8 +25,10 @@ db.FeedSource.aggregate([
     {
         $lookup: {
             from: "FeedVersion",
-            let: { publishedVersionId: "$publishedVersionId" },
+            let: { feedSourceId: "$_id", publishedVersionId: "$publishedVersionId" },
             pipeline: [
+                // Filtering by feedSourceId helps by 100x when dealing with large number of feed versions.
+                { $match: { $expr: { $eq: ["$feedSourceId", "$$feedSourceId"] } } },
                 { $match: { $expr: { $eq: ["$namespace", "$$publishedVersionId"] } } },
                 { $limit: 1 },
                 { $project: { validationResult: 1 } }

src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java

@@ -148,8 +148,7 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx
 
         feedVersionPublishedFromLatestDeployment = createFeedVersion(
             "published-feed-version-from-latest-deployment",
-            // Set to null so the relationship to feed source is via the published version id.
-            null,
+            feedSourceWithLatestDeploymentFeedVersion.id,
             LocalDate.of(2022, Month.NOVEMBER, 2),
             LocalDate.of(2022, Month.NOVEMBER, 3),
             feedSourceWithLatestDeploymentFeedVersion.publishedVersionId,