From 949a56759d0ef9a8d86a413aa9aabaf7550cfaf7 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 11:16:38 +0100 Subject: [PATCH 1/7] chore(ci): attempt at optimising slow actions --- .github/workflows/e2e-webapp.yml | 2 +- .github/workflows/e2e.yml | 2 +- .github/workflows/release.yml | 4 +- .github/workflows/typecheck.yml | 2 +- .github/workflows/unit-tests-internal.yml | 85 ++++++++++------------- .github/workflows/unit-tests-webapp.yml | 10 ++- 6 files changed, 47 insertions(+), 58 deletions(-) diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index 4a995a08ed9..c795b3e642a 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -14,7 +14,7 @@ on: jobs: e2eTests: name: "๐Ÿงช E2E Tests: Webapp" - runs-on: warp-ubuntu-latest-x64-8x + runs-on: warp-ubuntu-latest-x64-16x timeout-minutes: 20 env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f2ae36243e2..f2015e0d39d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [warp-ubuntu-latest-x64-4x, warp-windows-latest-x64-4x] + os: [warp-ubuntu-latest-x64-4x, warp-windows-latest-x64-8x] package-manager: ["npm", "pnpm"] steps: - name: โฌ‡๏ธ Checkout repo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6bef7ac5c0..b7f8b14c991 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: release: name: ๐Ÿš€ Release npm packages - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: write @@ -281,7 +281,7 @@ jobs: # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims. prerelease: name: ๐Ÿงช Prerelease - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: read diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 05d3a1496e5..d4c6cd2a5eb 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -8,7 +8,7 @@ permissions: jobs: typecheck: - runs-on: warp-ubuntu-latest-x64-8x + runs-on: warp-ubuntu-latest-x64-16x steps: - name: โฌ‡๏ธ Checkout repo diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml index 6869fe8a5f5..acbe8e05dd6 100644 --- a/.github/workflows/unit-tests-internal.yml +++ b/.github/workflows/unit-tests-internal.yml @@ -14,17 +14,14 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Internal" - runs-on: warp-ubuntu-latest-x64-8x - strategy: - # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard - fail-fast: false - matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - shardTotal: [12] + # Single big machine instead of a 12-job matrix: the internal suites are serial + # (fileParallelism: false) and container-wait-bound, so 12 in-machine shard processes + # fit comfortably in 32 vCPUs while paying the setup cost (install, prisma generate, + # image pulls) once instead of 12 times. + runs-on: warp-ubuntu-latest-x64-32x env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - SHARD_INDEX: ${{ matrix.shardIndex }} - SHARD_TOTAL: ${{ matrix.shardTotal }} + SHARD_TOTAL: 12 steps: - name: ๐Ÿ”ง Disable IPv6 run: | @@ -108,8 +105,34 @@ jobs: - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate - - name: ๐Ÿงช Run Internal Unit Tests - run: pnpm run test:internal --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --passWithNoTests + - name: ๐Ÿ—๏ธ Build test dependencies + # Build once up-front so the parallel shard runs below (turbo --only) never race + # to build or cache-restore the same outputs concurrently. + run: pnpm exec turbo run build --filter "@internal/*..." + + - name: ๐Ÿงช Run Internal Unit Tests (${{ env.SHARD_TOTAL }} in-machine shards) + run: | + # Same shard partitioning as the old 12-job matrix (DurationShardingSequencer + # keys off --shard=i/N), but as parallel local processes. --only skips the + # ^build dependency handled by the step above. + status=0 + declare -a pids + for i in $(seq 1 "$SHARD_TOTAL"); do + pnpm exec turbo run test --only --concurrency=1 --filter "@internal/*" -- \ + --run --reporter=default --reporter=blob --shard="$i/$SHARD_TOTAL" --passWithNoTests \ + > "/tmp/internal-shard-$i.log" 2>&1 & + pids[$i]=$! + done + for i in $(seq 1 "$SHARD_TOTAL"); do + if ! wait "${pids[$i]}"; then + status=1 + echo "::error::internal unit test shard $i/$SHARD_TOTAL failed" + fi + echo "::group::๐Ÿงช shard $i/$SHARD_TOTAL" + cat "/tmp/internal-shard-$i.log" + echo "::endgroup::" + done + exit "$status" - name: Gather all reports if: ${{ !cancelled() }} @@ -118,44 +141,6 @@ jobs: find . -type f -path '*/.vitest-reports/blob-*.json' \ -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \; - - name: Upload blob reports to GitHub Actions Artifacts + - name: ๐Ÿ“Š Merge reports if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: internal-blob-report-${{ matrix.shardIndex }} - path: .vitest-reports/* - include-hidden-files: true - retention-days: 1 - - merge-reports: - name: "๐Ÿ“Š Merge Reports" - if: ${{ !cancelled() }} - needs: [unitTests] - runs-on: warp-ubuntu-latest-x64-2x - steps: - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 1 - persist-credentials: false - - - name: โŽ” Setup pnpm - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - with: - version: 10.33.2 - - - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 22.23.1 - # no cache enabled, we're not installing deps - - - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: .vitest-reports - pattern: internal-blob-report-* - merge-multiple: true - - - name: Merge reports run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index eb1cd424c4b..c9407c6e4ba 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -14,13 +14,17 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - runs-on: warp-ubuntu-latest-x64-8x + # 3 shards on 32x machines instead of 10 on 8x: webapp tests run files in parallel + # (forks pool scales with cores) and are genuinely CPU-hungry, so keep aggregate + # compute similar while paying the setup cost (install, prisma generate, image pulls) + # 3 times instead of 10. + runs-on: warp-ubuntu-latest-x64-32x strategy: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - shardTotal: [10] + shardIndex: [1, 2, 3] + shardTotal: [3] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} From 3d1dee004d479ce90877c7eb6019baa261df6c75 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 11:26:45 +0100 Subject: [PATCH 2/7] fix actionlint error --- .github/workflows/unit-tests-internal.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml index acbe8e05dd6..068d9e47a92 100644 --- a/.github/workflows/unit-tests-internal.yml +++ b/.github/workflows/unit-tests-internal.yml @@ -121,10 +121,10 @@ jobs: pnpm exec turbo run test --only --concurrency=1 --filter "@internal/*" -- \ --run --reporter=default --reporter=blob --shard="$i/$SHARD_TOTAL" --passWithNoTests \ > "/tmp/internal-shard-$i.log" 2>&1 & - pids[$i]=$! + pids[i]=$! done for i in $(seq 1 "$SHARD_TOTAL"); do - if ! wait "${pids[$i]}"; then + if ! wait "${pids[i]}"; then status=1 echo "::error::internal unit test shard $i/$SHARD_TOTAL failed" fi From 2ed22b0a36d8b475cee1fd0d0314449262114f8a Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 11:45:10 +0100 Subject: [PATCH 3/7] 10 times x16 --- .github/workflows/unit-tests-webapp.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index c9407c6e4ba..f3cb07ed30b 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -14,17 +14,18 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - # 3 shards on 32x machines instead of 10 on 8x: webapp tests run files in parallel - # (forks pool scales with cores) and are genuinely CPU-hungry, so keep aggregate - # compute similar while paying the setup cost (install, prisma generate, image pulls) - # 3 times instead of 10. - runs-on: warp-ubuntu-latest-x64-32x + # 10 shards on 16x machines: webapp test throughput is limited per-machine (one + # docker daemon + disk absorbing all the per-file Postgres/ClickHouse container + # spin-up), so many machines beats few big ones - fewer/bigger (3x32) measured + # SLOWER than 10x8. The 16x (vs 8x) gives the fork pool the CPU headroom the 8x + # runners lacked. Setup overhead per machine is ~1 min on warm runners. + runs-on: warp-ubuntu-latest-x64-16x strategy: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3] - shardTotal: [3] + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shardTotal: [10] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} From d84fa471ce1d2a34c86a1b4c9badc76cd213dd84 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 12:12:37 +0100 Subject: [PATCH 4/7] update test-timings.json --- test-timings.json | 695 +++++++++++++++++++++++++++++++--------------- 1 file changed, 477 insertions(+), 218 deletions(-) diff --git a/test-timings.json b/test-timings.json index 9f6cdfbbe77..c294dba73e1 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,227 +1,486 @@ { - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 10249, - "apps/webapp/test/GCRARateLimiter.test.ts": 4984, + "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 1945, + "apps/webapp/app/runEngine/services/triggerTask.server.test.ts": 294497, + "apps/webapp/app/utils/friendlyId.test.ts": 26, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 22, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3199, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 25, + "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 486, + "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 27, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 10361, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 22, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 498, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 485, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 10337, + "apps/webapp/app/v3/runStore.server.test.ts": 8121, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 9867, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 116, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 3847, + "apps/webapp/test/GCRARateLimiter.test.ts": 4735, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 10078, + "apps/webapp/test/activitySeries.server.test.ts": 22, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 663, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 92382, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174393, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5273, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 10127, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11165, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3001, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8581, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12372, + "apps/webapp/test/apiRunListPresenter.test.ts": 142529, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 7138, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3184, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 13383, "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 3, - "apps/webapp/test/calculateNextSchedule.test.ts": 345, - "apps/webapp/test/chat-snapshot-integration.test.ts": 2326, - "apps/webapp/test/clickhouseFactory.test.ts": 13885, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 361, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 17889, - "apps/webapp/test/detectbadJsonStrings.test.ts": 98, - "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, - "apps/webapp/test/environmentVariableRules.test.ts": 3, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 10355, - "apps/webapp/test/environmentVariablesRepository.test.ts": 18320, - "apps/webapp/test/errorFingerprinting.test.ts": 16, - "apps/webapp/test/errorGroupWebhook.test.ts": 7, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 11236, + "apps/webapp/test/batchPresenter.test.ts": 12536, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 2021, + "apps/webapp/test/batchRunAccess.test.ts": 8380, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6380, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1472, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5515, + "apps/webapp/test/billingAlertsFormat.test.ts": 34, + "apps/webapp/test/billingLimit.schemas.test.ts": 29, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 15295, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3409, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 20, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 211, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 565, + "apps/webapp/test/billingLimitHit.test.ts": 29, + "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 37, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12481, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 231, + "apps/webapp/test/billingLimitReconciliation.test.ts": 544, + "apps/webapp/test/billingLimitResolve.test.ts": 20, + "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 20, + "apps/webapp/test/billingLimitsRoute.test.ts": 1013, + "apps/webapp/test/branchableEnvironment.test.ts": 23, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 22, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6047, + "apps/webapp/test/calculateNextSchedule.test.ts": 208, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 35, + "apps/webapp/test/chartXAxisTicks.test.ts": 51, + "apps/webapp/test/chartZoomRange.test.ts": 27, + "apps/webapp/test/chat-snapshot-integration.test.ts": 1654, + "apps/webapp/test/checkPermissions.test.ts": 20, + "apps/webapp/test/clickhouseFactory.test.ts": 3652, + "apps/webapp/test/components/DateTime.test.ts": 801, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 161, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 268, + "apps/webapp/test/components/runs/v3/RunTag.test.ts": 146, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 373, + "apps/webapp/test/computeBucket.test.ts": 106, + "apps/webapp/test/computeMigration.test.ts": 25, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 976, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8757, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 5319, + "apps/webapp/test/dependentAttemptScope.test.ts": 16, + "apps/webapp/test/detectQueryTables.test.ts": 233, + "apps/webapp/test/detectbadJsonStrings.test.ts": 73, + "apps/webapp/test/devBranchServices.test.ts": 4024, + "apps/webapp/test/devPresenceRecency.test.ts": 1168, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2737, + "apps/webapp/test/duplicateTaskIds.test.ts": 19, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1598, + "apps/webapp/test/emailPattern.test.ts": 20, + "apps/webapp/test/engine/batchPayloads.test.ts": 5358, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10952, + "apps/webapp/test/engine/streamBatchItems.test.ts": 20294, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 3590, + "apps/webapp/test/engine/triggerFailedTask.test.ts": 214324, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 133586, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133409, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 171472, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 172402, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 172353, + "apps/webapp/test/engine/triggerTask.test.ts": 91815, + "apps/webapp/test/environmentSort.test.ts": 25, + "apps/webapp/test/environmentVariableDeduplication.test.ts": 35, + "apps/webapp/test/environmentVariableRules.test.ts": 25, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3239, + "apps/webapp/test/environmentVariablesRepository.test.ts": 3696, + "apps/webapp/test/errorFingerprinting.test.ts": 31, + "apps/webapp/test/errorGroupWebhook.test.ts": 58, "apps/webapp/test/fairDequeuingStrategy.test.ts": 5705, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 17442, - "apps/webapp/test/getDeploymentImageRef.test.ts": 7, - "apps/webapp/test/httpErrors.test.ts": 25, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3820, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5478, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8693, + "apps/webapp/test/getDeploymentImageRef.test.ts": 324, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6427, + "apps/webapp/test/googleEmailVerification.test.ts": 19, + "apps/webapp/test/httpErrors.test.ts": 52, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 9844, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 10956, + "apps/webapp/test/inviteRoleLadder.test.ts": 17, "apps/webapp/test/marqsKeyProducer.test.ts": 7, - "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 6, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 497, - "apps/webapp/test/mollifierClaimResolution.test.ts": 6, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 21, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 5, - "apps/webapp/test/mollifierDrainingGauge.test.ts": 449, - "apps/webapp/test/mollifierGate.test.ts": 16, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 13, - "apps/webapp/test/mollifierMollify.test.ts": 5, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 18, - "apps/webapp/test/mollifierReadFallback.test.ts": 15, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 2, - "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 9, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 7, - "apps/webapp/test/mollifierStaleSweep.test.ts": 969, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 5, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 5, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 805, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 2, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 4, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 8, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 6, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 621, - "apps/webapp/test/objectStore.test.ts": 15979, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 16732, - "apps/webapp/test/otlpExporter.test.ts": 13, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 6540, + "apps/webapp/test/member.server.test.ts": 5081, + "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 246, + "apps/webapp/test/mfaRateLimiter.test.ts": 602, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 748, + "apps/webapp/test/mollifierClaimResolution.test.ts": 435, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 43, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 470, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 2073, + "apps/webapp/test/mollifierDrainingGauge.test.ts": 700, + "apps/webapp/test/mollifierGate.test.ts": 444, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 442, + "apps/webapp/test/mollifierMollify.test.ts": 243, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 489, + "apps/webapp/test/mollifierReadFallback.test.ts": 533, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 260, + "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 498, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 614, + "apps/webapp/test/mollifierStaleSweep.test.ts": 975, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 609, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 34, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 753, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 19, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 23, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 234, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 342, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 684, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 24812, + "apps/webapp/test/objectStore.test.ts": 4979, + "apps/webapp/test/orgBanner.test.ts": 22, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 4856, + "apps/webapp/test/otlpExporter.test.ts": 174, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 4773, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 332, + "apps/webapp/test/pauseEnvironment.server.test.ts": 9290, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6662, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 9059, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 5369, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36728, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 20, + "apps/webapp/test/prismaErrors.test.ts": 230, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3465, + "apps/webapp/test/promptOverrideSource.test.ts": 17, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 732, + "apps/webapp/test/queueListPagination.test.ts": 17, + "apps/webapp/test/rbacFallbackBranch.test.ts": 3647, + "apps/webapp/test/realtime/boundedTtlCache.test.ts": 17, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 37321, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 61, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 1180, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4133, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 317, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 601, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1600, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 577, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3546, + "apps/webapp/test/realtime/runReaderProjection.test.ts": 56, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7537, + "apps/webapp/test/realtime/shadowCompare.test.ts": 27, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5719, "apps/webapp/test/realtimeClient.test.ts": 1, - "apps/webapp/test/redisRealtimeStreams.test.ts": 6214, - "apps/webapp/test/registryConfig.test.ts": 652, - "apps/webapp/test/replay-after-crash.test.ts": 2233, - "apps/webapp/test/runsBackfiller.test.ts": 15478, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5733, + "apps/webapp/test/registryConfig.test.ts": 301, + "apps/webapp/test/reloadingRegistry.test.ts": 369, + "apps/webapp/test/removeTeamMember.test.ts": 9388, + "apps/webapp/test/replay-after-crash.test.ts": 1676, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4150, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5723, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6657, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5774, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 5620, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 586, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 6211, + "apps/webapp/test/runEngineHandlers.test.ts": 14676, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 370, + "apps/webapp/test/runOpsDbTopology.test.ts": 4175, + "apps/webapp/test/runOpsMintCutover.test.ts": 3733, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3409, + "apps/webapp/test/runOpsSplitMode.test.ts": 4141, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 503, + "apps/webapp/test/runOpsSplitReadGate.test.ts": 24, + "apps/webapp/test/runPresenterReadRoute.test.ts": 4068, + "apps/webapp/test/runsBackfiller.test.ts": 8145, "apps/webapp/test/runsReplicationBenchmark.test.ts": 0, - "apps/webapp/test/runsRepository.part1.test.ts": 53000, - "apps/webapp/test/runsRepository.part2.test.ts": 57000, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 8, - "apps/webapp/test/sentryTenantContext.test.ts": 5, - "apps/webapp/test/sentryTraceContext.server.test.ts": 12, - "apps/webapp/test/sessionDuration.test.ts": 18416, - "apps/webapp/test/sessionsReplicationService.test.ts": 30000, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 5, + "apps/webapp/test/runsReplicationInstance.test.ts": 23928, + "apps/webapp/test/runsReplicationService.part1.test.ts": 26040, + "apps/webapp/test/runsReplicationService.part2.test.ts": 22148, + "apps/webapp/test/runsReplicationService.part3.test.ts": 12273, + "apps/webapp/test/runsReplicationService.part4.test.ts": 27353, + "apps/webapp/test/runsReplicationService.part5.test.ts": 10249, + "apps/webapp/test/runsReplicationService.part6.test.ts": 12512, + "apps/webapp/test/runsReplicationService.part7.test.ts": 69936, + "apps/webapp/test/runsReplicationService.part8.test.ts": 25504, + "apps/webapp/test/runsReplicationService.part9.test.ts": 12873, + "apps/webapp/test/runsRepository.part1.test.ts": 23966, + "apps/webapp/test/runsRepository.part2.test.ts": 25074, + "apps/webapp/test/runsRepository.part3.test.ts": 21260, + "apps/webapp/test/runsRepository.part4.test.ts": 23884, + "apps/webapp/test/runsRepository.readthrough.test.ts": 36385, + "apps/webapp/test/runsRepositoryCpres.test.ts": 8383, + "apps/webapp/test/runsRepositoryCursor.test.ts": 29311, + "apps/webapp/test/safeEnvironmentLog.test.ts": 29, + "apps/webapp/test/safeIntegrationLog.test.ts": 20, + "apps/webapp/test/safeRequestLogContext.test.ts": 25, + "apps/webapp/test/safeWebhookFetch.test.ts": 230, + "apps/webapp/test/safeWebhookUrl.test.ts": 25, + "apps/webapp/test/sameOriginNavigation.test.ts": 34, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 22, + "apps/webapp/test/sanitizeUrl.test.ts": 30, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 283, + "apps/webapp/test/sentryTenantContext.test.ts": 24, + "apps/webapp/test/sentryTraceContext.server.test.ts": 92, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5811, + "apps/webapp/test/services/organizationAccessToken.test.ts": 255, + "apps/webapp/test/services/personalAccessToken.test.ts": 229, + "apps/webapp/test/sessionDuration.test.ts": 9116, + "apps/webapp/test/sessions.readthrough.test.ts": 6117, + "apps/webapp/test/sessionsReplicationService.test.ts": 16991, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 23, "apps/webapp/test/slackErrorAlerts.test.ts": 0, - "apps/webapp/test/tenantContext.test.ts": 26, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 2, - "apps/webapp/test/tenantContextResolver.test.ts": 19, - "apps/webapp/test/timeGranularity.test.ts": 3, - "apps/webapp/test/timelineSpanEvents.test.ts": 6, - "apps/webapp/test/updateMetadata.test.ts": 26380, - "apps/webapp/test/validateGitBranchName.test.ts": 7, - "apps/webapp/test/vercelUrls.test.ts": 3, - "apps/webapp/test/webhookErrorAlerts.test.ts": 5, - "apps/webapp/test/workerQueueSplit.test.ts": 3, - "apps/webapp/test/components/DateTime.test.ts": 24, - "apps/webapp/test/engine/batchPayloads.test.ts": 5018, - "apps/webapp/test/engine/streamBatchItems.test.ts": 45000, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 13152, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.test.ts": 4125, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 3, - "apps/webapp/test/services/organizationAccessToken.test.ts": 9, - "apps/webapp/test/services/personalAccessToken.test.ts": 8, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 10, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 237, - "apps/webapp/test/components/runs/v3/RunTag.test.ts": 5, - "packages/trigger-sdk/test/chat-snapshot.test.ts": 22, - "packages/trigger-sdk/test/chatHandover.test.ts": 1658, - "packages/trigger-sdk/test/merge-by-id.test.ts": 9, - "packages/trigger-sdk/test/mockChatAgent.test.ts": 2254, - "packages/trigger-sdk/test/recovery-boot.test.ts": 671, - "packages/trigger-sdk/test/replay-session-in.test.ts": 12, - "packages/trigger-sdk/test/replay-session-out.test.ts": 40, - "packages/trigger-sdk/test/skill.test.ts": 16, - "packages/trigger-sdk/test/skillsRuntime.test.ts": 131, - "packages/trigger-sdk/test/wire-shape.test.ts": 15, - "packages/trigger-sdk/src/v3/chat-server.test.ts": 185, - "packages/trigger-sdk/src/v3/chat-tab-coordinator.test.ts": 14, - "packages/trigger-sdk/src/v3/chat.test.ts": 77, - "packages/trigger-sdk/src/v3/createStartSessionAction.test.ts": 5, - "packages/trigger-sdk/src/v3/sessions.test.ts": 31, - "packages/trigger-sdk/src/v3/shared.test.ts": 68, - "packages/trigger-sdk/src/v3/streams.test.ts": 6, - "packages/trigger-sdk/src/v3/triggerClient.test.ts": 66, - "packages/trigger-sdk/src/v3/triggerClient.types.test.ts": 11, - "packages/redis-worker/src/cron.test.ts": 27371, - "packages/redis-worker/src/queue.test.ts": 3435, - "packages/redis-worker/src/worker.test.ts": 32870, - "packages/redis-worker/src/mollifier/buffer.test.ts": 3091, - "packages/redis-worker/src/mollifier/drainer.test.ts": 8403, - "packages/redis-worker/src/fair-queue/tests/concurrency.test.ts": 1341, - "packages/redis-worker/src/fair-queue/tests/drr.test.ts": 1203, - "packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts": 7728, - "packages/redis-worker/src/fair-queue/tests/raceConditions.test.ts": 14961, - "packages/redis-worker/src/fair-queue/tests/retry.test.ts": 10, - "packages/redis-worker/src/fair-queue/tests/tenantDispatch.test.ts": 5769, - "packages/redis-worker/src/fair-queue/tests/visibility.test.ts": 3783, - "packages/redis-worker/src/fair-queue/tests/workerQueue.test.ts": 1116, - "packages/core/test/duration.test.ts": 42, - "packages/core/test/errors.test.ts": 51, - "packages/core/test/eventFilterMatches.test.ts": 38, - "packages/core/test/externalSpanExporterWrapper.test.ts": 13, - "packages/core/test/flattenAttributes.test.ts": 59, - "packages/core/test/ioSerialization.test.ts": 306, - "packages/core/test/jumpHash.test.ts": 385, - "packages/core/test/mockTaskContext.test.ts": 61, - "packages/core/test/recordSpanException.test.ts": 65, - "packages/core/test/resourceCatalog.test.ts": 71, - "packages/core/test/runStream.test.ts": 245, - "packages/core/test/skillCatalog.test.ts": 17, - "packages/core/test/standardMetadataManager.test.ts": 456, - "packages/core/test/streamsWriterV1.test.ts": 84112, - "packages/core/test/taskExecutor.test.ts": 364, - "packages/core/test/utils.test.ts": 15, - "packages/core/src/v3/apiClient/runStream.test.ts": 1893, - "packages/core/src/v3/apiClient/streamBatchItems.test.ts": 278, - "packages/core/src/v3/build/flags.test.ts": 12, + "apps/webapp/test/slackOAuthResultLog.test.ts": 20, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5397, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 5239, + "apps/webapp/test/tenantContext.test.ts": 44, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 17, + "apps/webapp/test/tenantContextResolver.test.ts": 39, + "apps/webapp/test/timeGranularity.test.ts": 25, + "apps/webapp/test/timelineSpanEvents.test.ts": 28, + "apps/webapp/test/traceExport.test.ts": 32, + "apps/webapp/test/updateMetadata.test.ts": 18934, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 8027, + "apps/webapp/test/utils/timezones.test.ts": 29, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 6456, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 7447, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 6225, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4283, + "apps/webapp/test/validateGitBranchName.test.ts": 20, + "apps/webapp/test/vercelUrls.test.ts": 18, + "apps/webapp/test/verifyDeploymentImage.test.ts": 928, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 6413, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8929, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 4918, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8786, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 5354, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 6103, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 32457, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5498, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5732, + "apps/webapp/test/webhookErrorAlerts.test.ts": 86, + "apps/webapp/test/workerGroupAccess.test.ts": 34, + "apps/webapp/test/workerQueueSplit.server.test.ts": 38, + "apps/webapp/test/workerQueueSplit.test.ts": 26, + "apps/webapp/test/workerRegions.test.ts": 508, + "internal-packages/cache/src/stores/lruMemory.test.ts": 65, + "internal-packages/clickhouse/src/client/client.test.ts": 7547, + "internal-packages/clickhouse/src/taskRuns.test.ts": 6768, + "internal-packages/clickhouse/src/tsql.test.ts": 8254, + "internal-packages/clickhouse/src/tsqlFunctions.test.ts": 9360, + "internal-packages/dashboard-agent/src/dashboard-agent.test.ts": 1429, + "internal-packages/dashboard-agent/src/repo-tools.test.ts": 116, + "internal-packages/llm-model-catalog/src/registry.test.ts": 84, + "internal-packages/llm-model-catalog/src/sync.test.ts": 3585, + "internal-packages/rbac/src/ability.test.ts": 55, + "internal-packages/rbac/src/loader.test.ts": 77, + "internal-packages/rbac/src/require-plugins.test.ts": 84, + "internal-packages/replication/src/client.test.ts": 33067, + "internal-packages/run-engine/src/batch-queue/tests/index.test.ts": 4370, + "internal-packages/run-engine/src/engine/systems/batchSystem.test.ts": 35042, + "internal-packages/run-engine/src/engine/systems/debounceSystem.test.ts": 14129, + "internal-packages/run-engine/src/engine/systems/delayedRunSystem.test.ts": 12805, + "internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts": 9585, + "internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts": 12815, + "internal-packages/run-engine/src/engine/systems/pendingVersionSystem.test.ts": 12791, + "internal-packages/run-engine/src/engine/systems/runAttemptSystem.test.ts": 17101, + "internal-packages/run-engine/src/engine/systems/ttlSystem.test.ts": 12916, + "internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts": 31249, + "internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts": 14624, + "internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts": 17248, + "internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts": 19939, + "internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts": 13594, + "internal-packages/run-engine/src/engine/tests/blockEdgeResidency.test.ts": 6962, + "internal-packages/run-engine/src/engine/tests/cancelling.test.ts": 11125, + "internal-packages/run-engine/src/engine/tests/checkpointSystem.controlPlaneResolver.test.ts": 12189, + "internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts": 15100, + "internal-packages/run-engine/src/engine/tests/checkpoints.test.ts": 19226, + "internal-packages/run-engine/src/engine/tests/clearBlockingWaitpointsResidency.test.ts": 6030, + "internal-packages/run-engine/src/engine/tests/completeWaitpointCrossSeamGuard.test.ts": 9881, + "internal-packages/run-engine/src/engine/tests/completeWaitpointReadResidency.test.ts": 7270, + "internal-packages/run-engine/src/engine/tests/controlPlaneResolverInjectability.test.ts": 9790, + "internal-packages/run-engine/src/engine/tests/createCancelledRun.test.ts": 10516, + "internal-packages/run-engine/src/engine/tests/createFailedTaskRun.test.ts": 8961, + "internal-packages/run-engine/src/engine/tests/crossVersionCompat.test.ts": 6563, + "internal-packages/run-engine/src/engine/tests/datetimeWaitpointColocation.test.ts": 10600, + "internal-packages/run-engine/src/engine/tests/debounce.test.ts": 31044, + "internal-packages/run-engine/src/engine/tests/delayedRunSystem.controlPlaneResolver.test.ts": 12846, + "internal-packages/run-engine/src/engine/tests/delays.test.ts": 32798, + "internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts": 13496, + "internal-packages/run-engine/src/engine/tests/dequeueSystem.recovery.controlPlaneResolver.test.ts": 4805, + "internal-packages/run-engine/src/engine/tests/dequeuing.test.ts": 15045, + "internal-packages/run-engine/src/engine/tests/engineResidualInversions.controlPlaneResolver.test.ts": 5341, + "internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts": 222657, + "internal-packages/run-engine/src/engine/tests/getSnapshotsSinceResidency.test.ts": 5582, + "internal-packages/run-engine/src/engine/tests/heartbeats.test.ts": 29022, + "internal-packages/run-engine/src/engine/tests/heteroPostgresFixture.test.ts": 5790, + "internal-packages/run-engine/src/engine/tests/lazyWaitpoint.test.ts": 20398, + "internal-packages/run-engine/src/engine/tests/legacyRunNewTokenResumeFlow.test.ts": 6757, + "internal-packages/run-engine/src/engine/tests/lifecycleRouter.test.ts": 16360, + "internal-packages/run-engine/src/engine/tests/locking.test.ts": 50934, + "internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts": 13218, + "internal-packages/run-engine/src/engine/tests/priority.test.ts": 11463, + "internal-packages/run-engine/src/engine/tests/runAttemptSystem.controlPlaneResolver.test.ts": 15028, + "internal-packages/run-engine/src/engine/tests/runStoreInjectability.test.ts": 9726, + "internal-packages/run-engine/src/engine/tests/startRunAttemptReadResidency.test.ts": 6902, + "internal-packages/run-engine/src/engine/tests/trigger.test.ts": 10838, + "internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts": 14414, + "internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts": 13971, + "internal-packages/run-engine/src/engine/tests/ttl.test.ts": 51947, + "internal-packages/run-engine/src/engine/tests/waitpointPublicRouter.test.ts": 14940, + "internal-packages/run-engine/src/engine/tests/waitpointRace.test.ts": 10633, + "internal-packages/run-engine/src/engine/tests/waitpointReplicaLag.test.ts": 5858, + "internal-packages/run-engine/src/engine/tests/waitpointSystem.controlPlaneResolver.test.ts": 11913, + "internal-packages/run-engine/src/engine/tests/waitpoints.test.ts": 29161, + "internal-packages/run-engine/src/engine/tests/workerQueueObservation.test.ts": 8582, + "internal-packages/run-engine/src/run-queue/index.test.ts": 82145, + "internal-packages/run-engine/src/run-queue/tests/ack.test.ts": 2823, + "internal-packages/run-engine/src/run-queue/tests/ckCounters.test.ts": 15673, + "internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts": 8611, + "internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts": 6162, + "internal-packages/run-engine/src/run-queue/tests/dequeueMessageFromWorkerQueue.test.ts": 67581, + "internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts": 8763, + "internal-packages/run-engine/src/run-queue/tests/fairQueueSelectionStrategy.test.ts": 3513, + "internal-packages/run-engine/src/run-queue/tests/keyProducer.test.ts": 80, + "internal-packages/run-engine/src/run-queue/tests/migrateLegacyMasterQueue.test.ts": 1609, + "internal-packages/run-engine/src/run-queue/tests/nack.test.ts": 5927, + "internal-packages/run-engine/src/run-queue/tests/releaseConcurrency.test.ts": 2649, + "internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts": 239, + "internal-packages/run-ops-database/prisma/schema.parity.test.ts": 43, + "internal-packages/run-store/src/PostgresRunStore.batchProbeReadClient.test.ts": 6303, + "internal-packages/run-store/src/PostgresRunStore.connectedRunsBounded.test.ts": 5104, + "internal-packages/run-store/src/PostgresRunStore.connectedRunsIncludeBounded.test.ts": 2362, + "internal-packages/run-store/src/PostgresRunStore.controlPlaneAlertFk.test.ts": 4777, + "internal-packages/run-store/src/PostgresRunStore.crossGenerationError.test.ts": 2646, + "internal-packages/run-store/src/PostgresRunStore.dedicatedRepro.test.ts": 12323, + "internal-packages/run-store/src/PostgresRunStore.dedicatedSelect.test.ts": 5884, + "internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts": 5443, + "internal-packages/run-store/src/PostgresRunStore.findRunsByIds.test.ts": 2707, + "internal-packages/run-store/src/PostgresRunStore.groupedRelations.test.ts": 2561, + "internal-packages/run-store/src/PostgresRunStore.hydratorSelectPushdown.test.ts": 2428, + "internal-packages/run-store/src/PostgresRunStore.sharedHydratedTarget.test.ts": 2289, + "internal-packages/run-store/src/PostgresRunStore.test.ts": 11219, + "internal-packages/run-store/src/PostgresRunStore.writeAtomicity.test.ts": 7791, + "internal-packages/run-store/src/batchCompletionResidency.test.ts": 6379, + "internal-packages/run-store/src/checkpointBatchReplicaLag.test.ts": 2667, + "internal-packages/run-store/src/clientCompat.test.ts": 32, + "internal-packages/run-store/src/runOpsStore.batchItemMisroute.test.ts": 5285, + "internal-packages/run-store/src/runOpsStore.crossDbCompletedWaitpoint.test.ts": 4837, + "internal-packages/run-store/src/runOpsStore.crossDbTokenBlock.test.ts": 4787, + "internal-packages/run-store/src/runOpsStore.flipWindowDuplicate.test.ts": 4844, + "internal-packages/run-store/src/runOpsStore.forWaitpointCompletion.test.ts": 96, + "internal-packages/run-store/src/runOpsStore.groupedReads.test.ts": 5767, + "internal-packages/run-store/src/runOpsStore.idempotencyDedup.test.ts": 6729, + "internal-packages/run-store/src/runOpsStore.mixedResidency.test.ts": 9524, + "internal-packages/run-store/src/runOpsStore.readAfterWrite.test.ts": 5519, + "internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts": 5564, + "internal-packages/run-store/src/runOpsStore.routedReadPrimary.test.ts": 6475, + "internal-packages/run-store/src/runOpsStore.snapshotCompletedWaitpoints.test.ts": 4889, + "internal-packages/run-store/src/runOpsStore.snapshots.test.ts": 5938, + "internal-packages/run-store/src/runOpsStore.test.ts": 22578, + "internal-packages/run-store/src/runOpsStore.waitpoints.test.ts": 6504, + "internal-packages/run-store/src/scheduleSeam.test.ts": 5180, + "internal-packages/schedule-engine/test/scheduleEngine.test.ts": 125260, + "internal-packages/schedule-engine/test/scheduleEngine2.test.ts": 9011, + "internal-packages/schedule-engine/test/scheduleRecovery.test.ts": 10259, + "internal-packages/sdk-compat-tests/src/tests/bundler.test.ts": 170, + "internal-packages/sdk-compat-tests/src/tests/import.test.ts": 2455, + "internal-packages/sso/src/loader.test.ts": 61, + "internal-packages/tsql/src/grammar/parser.test.ts": 423, + "internal-packages/tsql/src/index.test.ts": 561, + "internal-packages/tsql/src/query/escape.test.ts": 50, + "internal-packages/tsql/src/query/parser.test.ts": 518, + "internal-packages/tsql/src/query/printer.test.ts": 872, + "internal-packages/tsql/src/query/results.test.ts": 59, + "internal-packages/tsql/src/query/schema.test.ts": 78, + "internal-packages/tsql/src/query/security.test.ts": 627, + "internal-packages/tsql/src/query/time_buckets.test.ts": 39, + "internal-packages/tsql/src/query/validator.test.ts": 557, + "packages/build/src/extensions/core/syncEnvVars.test.ts": 33, + "packages/core/src/v3/apiClient/bulkActions.test.ts": 214, + "packages/core/src/v3/apiClient/runStream.test.ts": 2319, + "packages/core/src/v3/apiClient/streamBatchItems.test.ts": 328, + "packages/core/src/v3/apps/exec.test.ts": 24, + "packages/core/src/v3/build/flags.test.ts": 19, + "packages/core/src/v3/errors.test.ts": 51, + "packages/core/src/v3/idempotency-key-catalog/inMemoryIdempotencyKeyCatalog.test.ts": 19, "packages/core/src/v3/idempotency-key-catalog/lruIdempotencyKeyCatalog.test.ts": 21, - "packages/core/src/v3/machines/max-old-space.test.ts": 18, - "packages/core/src/v3/realtimeStreams/manager.test.ts": 28, - "packages/core/src/v3/realtimeStreams/streamsWriterV2.test.ts": 91, - "packages/core/src/v3/schemas/api-type.test.ts": 33, - "packages/core/src/v3/schemas/batchItemNDJSON.test.ts": 15, - "packages/core/src/v3/schemas/idempotencyKey.test.ts": 68, - "packages/core/src/v3/sessionStreams/manager.test.ts": 126, - "packages/core/src/v3/serverOnly/shutdownManager.test.ts": 57, - "packages/core/src/v3/taskContext/index.test.ts": 23, - "packages/core/src/v3/utils/reconnectBackoff.test.ts": 45, - "packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts": 123, - "packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts": 98, - "packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts": 92, - "packages/schema-to-json/tests/index.test.ts": 17, - "internal-packages/run-engine/src/run-queue/index.test.ts": 82296, - "internal-packages/run-engine/src/batch-queue/tests/index.test.ts": 5462, - "internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts": 35471, - "internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts": 33127, - "internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts": 33681, - "internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts": 28159, - "internal-packages/run-engine/src/engine/tests/cancelling.test.ts": 26240, - "internal-packages/run-engine/src/engine/tests/checkpoints.test.ts": 29420, - "internal-packages/run-engine/src/engine/tests/createCancelledRun.test.ts": 31807, - "internal-packages/run-engine/src/engine/tests/createFailedTaskRun.test.ts": 23573, - "internal-packages/run-engine/src/engine/tests/debounce.test.ts": 58554, - "internal-packages/run-engine/src/engine/tests/delays.test.ts": 42748, - "internal-packages/run-engine/src/engine/tests/dequeuing.test.ts": 25542, - "internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts": 32158, - "internal-packages/run-engine/src/engine/tests/heartbeats.test.ts": 39634, - "internal-packages/run-engine/src/engine/tests/lazyWaitpoint.test.ts": 34757, - "internal-packages/run-engine/src/engine/tests/locking.test.ts": 51090, - "internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts": 28762, - "internal-packages/run-engine/src/engine/tests/priority.test.ts": 28808, - "internal-packages/run-engine/src/engine/tests/trigger.test.ts": 30601, - "internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts": 28893, - "internal-packages/run-engine/src/engine/tests/ttl.test.ts": 61981, - "internal-packages/run-engine/src/engine/tests/waitpointRace.test.ts": 22855, - "internal-packages/run-engine/src/engine/tests/waitpoints.test.ts": 39521, - "internal-packages/run-engine/src/run-queue/tests/ack.test.ts": 3156, - "internal-packages/run-engine/src/run-queue/tests/ckCounters.test.ts": 16715, - "internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts": 8916, - "internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts": 6379, - "internal-packages/run-engine/src/run-queue/tests/dequeueMessageFromWorkerQueue.test.ts": 66701, - "internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts": 9108, - "internal-packages/run-engine/src/run-queue/tests/fairQueueSelectionStrategy.test.ts": 7997, - "internal-packages/run-engine/src/run-queue/tests/keyProducer.test.ts": 19, - "internal-packages/run-engine/src/run-queue/tests/migrateLegacyMasterQueue.test.ts": 1728, - "internal-packages/run-engine/src/run-queue/tests/nack.test.ts": 6184, - "internal-packages/run-engine/src/run-queue/tests/releaseConcurrency.test.ts": 3018, - "internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts": 38, - "internal-packages/cache/src/stores/lruMemory.test.ts": 44, - "internal-packages/schedule-engine/test/scheduleEngine.test.ts": 43000, - "internal-packages/schedule-engine/test/scheduleRecovery.test.ts": 17396, - "internal-packages/replication/src/client.test.ts": 31306, - "internal-packages/tsql/src/index.test.ts": 246, - "internal-packages/tsql/src/grammar/parser.test.ts": 150, - "internal-packages/tsql/src/query/escape.test.ts": 9, - "internal-packages/tsql/src/query/parser.test.ts": 368, - "internal-packages/tsql/src/query/printer.test.ts": 942, - "internal-packages/tsql/src/query/results.test.ts": 4, - "internal-packages/tsql/src/query/schema.test.ts": 18, - "internal-packages/tsql/src/query/security.test.ts": 487, - "internal-packages/tsql/src/query/time_buckets.test.ts": 5, - "internal-packages/tsql/src/query/validator.test.ts": 250, - "internal-packages/rbac/src/ability.test.ts": 6, - "internal-packages/rbac/src/loader.test.ts": 3, - "internal-packages/llm-model-catalog/src/registry.test.ts": 15, - "internal-packages/llm-model-catalog/src/sync.test.ts": 15852, - "internal-packages/clickhouse/src/taskRuns.test.ts": 6813, - "internal-packages/clickhouse/src/tsql.test.ts": 9021, - "internal-packages/clickhouse/src/tsqlFunctions.test.ts": 12971, - "internal-packages/clickhouse/src/client/client.test.ts": 9138, - "internal-packages/sdk-compat-tests/src/tests/bundler.test.ts": 348, - "internal-packages/sdk-compat-tests/src/tests/import.test.ts": 4742, - "apps/webapp/test/runsReplicationService.part1.test.ts": 74000, - "apps/webapp/test/runsReplicationService.part2.test.ts": 64000, - "apps/webapp/test/runsReplicationService.part3.test.ts": 30000, - "apps/webapp/test/runsReplicationService.part4.test.ts": 70000, - "apps/webapp/test/runsReplicationService.part5.test.ts": 43000, - "apps/webapp/test/runsReplicationService.part6.test.ts": 32000, - "apps/webapp/test/runsRepository.part3.test.ts": 43000, - "apps/webapp/test/runsRepository.part4.test.ts": 57000, - "apps/webapp/test/runsReplicationService.part7.test.ts": 43000, - "internal-packages/schedule-engine/test/scheduleEngine2.test.ts": 43000 + "packages/core/src/v3/idempotencyKeys.test.ts": 409, + "packages/core/src/v3/isomorphic/friendlyId.test.ts": 371, + "packages/core/src/v3/isomorphic/runOpsResidency.test.ts": 877, + "packages/core/src/v3/machines/max-old-space.test.ts": 20, + "packages/core/src/v3/realtimeStreams/manager.test.ts": 119, + "packages/core/src/v3/realtimeStreams/streamsWriterV2.test.ts": 109, + "packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts": 226, + "packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.test.ts": 65, + "packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts": 125, + "packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts": 57, + "packages/core/src/v3/schemas/api-type.test.ts": 55, + "packages/core/src/v3/schemas/batchItemNDJSON.test.ts": 91, + "packages/core/src/v3/schemas/idempotencyKey.test.ts": 64, + "packages/core/src/v3/serverOnly/shutdownManager.test.ts": 53, + "packages/core/src/v3/sessionStreams/manager.test.ts": 130, + "packages/core/src/v3/taskContext/index.test.ts": 37, + "packages/core/src/v3/telnetLogServer.test.ts": 233, + "packages/core/src/v3/utils/reconnectBackoff.test.ts": 17, + "packages/core/test/duration.test.ts": 28, + "packages/core/test/errors.test.ts": 60, + "packages/core/test/eventFilterMatches.test.ts": 63, + "packages/core/test/externalSpanExporterWrapper.test.ts": 285, + "packages/core/test/flattenAttributes.test.ts": 166, + "packages/core/test/ioSerialization.test.ts": 559, + "packages/core/test/jumpHash.test.ts": 297, + "packages/core/test/mockTaskContext.test.ts": 607, + "packages/core/test/recordSpanException.test.ts": 75, + "packages/core/test/resourceCatalog.test.ts": 48, + "packages/core/test/runStream.test.ts": 304, + "packages/core/test/skillCatalog.test.ts": 24, + "packages/core/test/standardMetadataManager.test.ts": 719, + "packages/core/test/streamsWriterV1.test.ts": 86772, + "packages/core/test/taskExecutor.test.ts": 1120, + "packages/core/test/utils.test.ts": 17, + "packages/redis-worker/src/cron.test.ts": 27511, + "packages/redis-worker/src/fair-queue/tests/concurrency.test.ts": 460, + "packages/redis-worker/src/fair-queue/tests/drr.test.ts": 437, + "packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts": 6821, + "packages/redis-worker/src/fair-queue/tests/raceConditions.test.ts": 13720, + "packages/redis-worker/src/fair-queue/tests/retry.test.ts": 215, + "packages/redis-worker/src/fair-queue/tests/tenantDispatch.test.ts": 4820, + "packages/redis-worker/src/fair-queue/tests/visibility.test.ts": 1428, + "packages/redis-worker/src/fair-queue/tests/workerQueue.test.ts": 432, + "packages/redis-worker/src/mollifier/buffer.test.ts": 1354, + "packages/redis-worker/src/mollifier/drainer.test.ts": 7797, + "packages/redis-worker/src/queue.test.ts": 2866, + "packages/redis-worker/src/worker.test.ts": 16743, + "packages/schema-to-json/tests/index.test.ts": 395, + "packages/trigger-sdk/src/v3/chat-server.test.ts": 858, + "packages/trigger-sdk/src/v3/chat-tab-coordinator.test.ts": 30, + "packages/trigger-sdk/src/v3/chat.test.ts": 848, + "packages/trigger-sdk/src/v3/createStartSessionAction.test.ts": 276, + "packages/trigger-sdk/src/v3/runs-bulk.test.ts": 224, + "packages/trigger-sdk/src/v3/sessions.test.ts": 218, + "packages/trigger-sdk/src/v3/shared.test.ts": 266, + "packages/trigger-sdk/src/v3/streams.test.ts": 193, + "packages/trigger-sdk/src/v3/triggerClient.test.ts": 841, + "packages/trigger-sdk/src/v3/triggerClient.types.test.ts": 218, + "packages/trigger-sdk/test/chat-snapshot.test.ts": 1300, + "packages/trigger-sdk/test/chat-transport-events.test.ts": 292, + "packages/trigger-sdk/test/chat-turn-correlation.test.ts": 385, + "packages/trigger-sdk/test/chatHandover.test.ts": 2984, + "packages/trigger-sdk/test/chatHandoverBackends.test.ts": 1455, + "packages/trigger-sdk/test/declaration-emit.test.ts": 1699, + "packages/trigger-sdk/test/merge-by-id.test.ts": 364, + "packages/trigger-sdk/test/mockChatAgent.test.ts": 4622, + "packages/trigger-sdk/test/pending-message-drain.test.ts": 4563, + "packages/trigger-sdk/test/promptCaching.test.ts": 513, + "packages/trigger-sdk/test/recovery-boot.test.ts": 1893, + "packages/trigger-sdk/test/replay-session-in.test.ts": 514, + "packages/trigger-sdk/test/replay-session-out.test.ts": 526, + "packages/trigger-sdk/test/skill.test.ts": 194, + "packages/trigger-sdk/test/skillsRuntime.test.ts": 532, + "packages/trigger-sdk/test/wire-shape.test.ts": 1262 } From 178c4900f063ad8ca72f60f45757bd6a0b721ac6 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 13:17:47 +0100 Subject: [PATCH 5/7] split long tests --- .github/workflows/unit-tests-webapp.yml | 4 +- .../services/triggerTask.server.test.ts | 825 ------------------ .../test/engine/triggerFailedTask.test.ts | 264 ------ test-timings.json | 7 +- 4 files changed, 7 insertions(+), 1093 deletions(-) delete mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.test.ts delete mode 100644 apps/webapp/test/engine/triggerFailedTask.test.ts diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index f3cb07ed30b..0ba248ca7db 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -24,8 +24,8 @@ jobs: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - shardTotal: [10] + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + shardTotal: [12] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.test.ts deleted file mode 100644 index 31c624a3864..00000000000 --- a/apps/webapp/app/runEngine/services/triggerTask.server.test.ts +++ /dev/null @@ -1,825 +0,0 @@ -import { describe, expect, vi } from "vitest"; - -// Mock the db prisma client. The service is constructed against a real -// testcontainer prisma instead โ€” these empty singletons only satisfy the -// module-level imports of the production wiring (infrastructure boundary). -vi.mock("~/db.server", () => ({ - prisma: {}, - $replica: {}, - runOpsNewPrisma: {}, - runOpsLegacyPrisma: {}, - runOpsNewReplica: {}, - runOpsLegacyReplica: {}, -})); -// Inherited harness boilerplate. The parent read under test takes the -// findRun(where, client) overload with this.prisma, so it does not consult this -// flag; the mock only satisfies other wiring imported transitively. -vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); - -vi.mock("~/services/platform.v3.server", async (importOriginal) => { - const actual = (await importOriginal()) as Record; - return { - ...actual, - getEntitlement: vi.fn(), - }; -}); - -import { RunEngine } from "@internal/run-engine"; -import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; -import { assertNonNullable, containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; -import type { IOPacket } from "@trigger.dev/core/v3"; -import type { TaskRun } from "@trigger.dev/database"; -import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; -import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; -import type { - EntitlementValidationParams, - MaxAttemptsValidationParams, - ParentRunValidationParams, - PayloadProcessor, - TagValidationParams, - TracedEventSpan, - TraceEventConcern, - TriggerTaskRequest, - TriggerTaskValidator, - ValidationResult, -} from "~/runEngine/types"; -import { RunEngineTriggerTaskService } from "./triggerTask.server"; - -vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout - -class MockPayloadProcessor implements PayloadProcessor { - async process(request: TriggerTaskRequest): Promise { - return { - data: JSON.stringify(request.body.payload), - dataType: "application/json", - }; - } -} - -// Captures the `parentRun` the service resolved (via runStore.findRun) and -// passed into validation, so a test can assert on the resolved parent without -// mocking the read itself. Returns ok so the child triggers regardless. -class CapturingParentRunValidator implements TriggerTaskValidator { - public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset"; - - validateTags(_params: TagValidationParams): ValidationResult { - return { ok: true }; - } - validateEntitlement(_params: EntitlementValidationParams): Promise { - return Promise.resolve({ ok: true }); - } - validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult { - return { ok: true }; - } - validateParentRun(params: ParentRunValidationParams): ValidationResult { - this.capturedParentRun = params.parentRun; - return { ok: true }; - } -} - -class MockTraceEventConcern implements TraceEventConcern { - async traceRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } - - async traceIdempotentRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - _options: { - existingRun: TaskRun; - idempotencyKey: string; - incomplete: boolean; - isError: boolean; - }, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } - - async traceDebouncedRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - _options: { - existingRun: TaskRun; - debounceKey: string; - incomplete: boolean; - isError: boolean; - }, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } -} - -function buildEngine(prisma: any, redisOptions: any) { - return new RunEngine({ - prisma, - worker: { - redis: redisOptions, - workers: 1, - tasksPerWorker: 10, - pollIntervalMs: 100, - }, - queue: { - redis: redisOptions, - }, - runLock: { - redis: redisOptions, - }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { - name: "small-1x" as const, - cpu: 0.5, - memory: 0.5, - centsPerMs: 0.0001, - }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); -} - -describe("RunEngineTriggerTaskService parent + locked-worker reads", () => { - containerTest( - "resolves the parent run through the run-ops store by minted run id", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger a ROOT run first to create a real parent TaskRun. - const parentResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // Trigger a CHILD pointing at the parent's friendlyId. The service must - // resolve the parent via runStore.findRun (minted RunId, env-scoped). - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "child" }, - options: { parentRunId: parentResult.run.friendlyId }, - }, - }); - assertNonNullable(childResult); - - // The capturing validator observed the resolved parent โ€” proving the - // read ran (against the container DB) and returned the right row. - expect(validator.capturedParentRun).not.toBe("unset"); - const capturedParent = validator.capturedParentRun; - assertNonNullable(capturedParent); - expect(capturedParent.id).toBe(parentResult.run.id); - expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId); - - // depth and root carry through โ€” proving parentRun.depth and the parent - // id were read off the resolved row and threaded into the child. - const parentRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: parentResult.run.id }, - }); - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - - expect(childRow.depth).toBe(parentRow.depth + 1); - expect(childRow.parentTaskRunId).toBe(parentRow.id); - expect(childRow.rootTaskRunId).toBe(parentRow.id); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "scopes the parent lookup to the run's environment (cross-env parent is not resolved)", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - // Two independent authenticated environments. The setup helper hardcodes - // several globally-unique fields (org/project slug, env apiKey/pkApiKey, - // worker-group token hash), so rename envA's before the second call to - // avoid unique-constraint collisions. - const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await prisma.organization.update({ - where: { id: envA.organizationId }, - data: { slug: `${envA.organization.slug}-a` }, - }); - await prisma.project.update({ - where: { id: envA.projectId }, - data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, - }); - await prisma.runtimeEnvironment.update({ - where: { id: envA.id }, - data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, - }); - await prisma.workerGroupToken.updateMany({ - where: { tokenHash: "token_hash" }, - data: { tokenHash: "token_hash_a" }, - }); - await prisma.workerInstanceGroup.updateMany({ - where: { masterQueue: "default" }, - data: { masterQueue: "default_a" }, - }); - const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - expect(envA.id).not.toBe(envB.id); - expect(envA.organizationId).not.toBe(envB.organizationId); - - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, envA, taskIdentifier); - await setupBackgroundWorker(engine, envB, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // A real parent run in envA. - const parentResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envA, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // Trigger a child in envB pointing at the envA parent's friendlyId. The - // env guard in runStore.findRun's `where` rejects the cross-env parent - // in a single query, so the resolved parentRun is null. - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envB, - body: { - payload: { kind: "child" }, - options: { parentRunId: parentResult.run.friendlyId }, - }, - }); - assertNonNullable(childResult); - - // validateParentRun was called with no resolved parent. - expect(validator.capturedParentRun).not.toBe("unset"); - expect(validator.capturedParentRun ?? null).toBeNull(); - - // The child still triggered, at the root depth with no parent linkage โ€” - // confirming the cross-env parent was dropped, not silently joined. - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - expect(childRow.depth).toBe(0); - expect(childRow.parentTaskRunId).toBeNull(); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "resolves the locked background worker on the control-plane client with no cross-DB join", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); - - // Read the seeded worker row to get its real version/id. - const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: worker.id }, - }); - - // Counting proxy over the control-plane client. `this.prisma` is ALWAYS - // the control-plane client; the locked-worker lookup is a DIRECT - // backgroundWorker.findFirst on it. The parent read uses a DIFFERENT - // call (runStore.findRun โ†’ taskRun), so a single call() issues two - // separate single-table reads โ€” never one cross-seam join. Here we count - // the findFirst calls and capture their args to assert no include/join. - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - // The queue manager gets the real (unproxied) prisma so the counting - // proxy only observes reads issued by the service itself. - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: workerRow.version }, - }, - }); - assertNonNullable(result); - - // Observable proof the locked worker was resolved on the control-plane - // client: the created run records the worker id in lockedToVersionId. - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - expect(runRow.lockedToVersionId).toBe(workerRow.id); - expect(runRow.taskVersion).toBe(workerRow.version); - - // Exactly one backgroundWorker.findFirst fired for the locked-worker read. - expect(backgroundWorkerFindFirstCalls).toBe(1); - - // NO-JOIN assertion: the read referenced ONLY the backgroundWorker table. - // No `include` (which would join into another table); the `select` lists - // only backgroundWorker scalar columns. - const args = findFirstArgs[0]; - assertNonNullable(args); - expect(args.include).toBeUndefined(); - expect(Object.keys(args.select ?? {}).sort()).toEqual([ - "cliVersion", - "id", - "sdkVersion", - "version", - ]); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "issues two independent single-table reads when one call supplies both parentRunId and lockToVersion", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: worker.id }, - }); - - // Count BOTH reads issued by the service on the control-plane client: - // the parent read (runStore.findRun โ†’ taskRun.findFirst) and the - // locked-worker read (backgroundWorker.findFirst). Capture every - // findFirst arg so we can assert no read carries a cross-seam include. - let taskRunFindFirstCalls = 0; - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - if (prop === "taskRun") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(trTarget, trProp, trReceiver) { - if (trProp === "findFirst") { - return async (args: any) => { - taskRunFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(trTarget, trProp, trReceiver); - return typeof value === "function" ? value.bind(trTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - // queueConcern/idempotency get the real unproxied prisma so the - // counting proxy only observes reads issued by the service itself. - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // ROOT parent first (uses the unproxied prisma via a separate service so - // its internal reads don't pollute the child's counts). - const parentService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - const parentResult = await parentService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // CHILD supplying BOTH parentRunId AND lockToVersion in one call. - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "child" }, - options: { - parentRunId: parentResult.run.friendlyId, - lockToVersion: workerRow.version, - }, - }, - }); - assertNonNullable(childResult); - - const parentRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: parentResult.run.id }, - }); - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - - // Child resolved the parent (single-table parent read). - expect(childRow.parentTaskRunId).toBe(parentRow.id); - expect(childRow.depth).toBe(parentRow.depth + 1); - - // Child locked to the worker (single-table worker read). - expect(childRow.lockedToVersionId).toBe(workerRow.id); - expect(childRow.taskVersion).toBe(workerRow.version); - - // Exactly one backgroundWorker.findFirst fired for the locked-worker read, - // and at least one taskRun.findFirst fired for the parent read. - expect(backgroundWorkerFindFirstCalls).toBe(1); - expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1); - - // NO-JOIN proof: no captured read carried an `include` joining - // taskRun <-> backgroundWorker. Every findFirst arg has include undefined. - for (const args of findFirstArgs) { - expect(args?.include).toBeUndefined(); - } - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "lockToVersion matching no worker rejects the trigger after a single scalar-only worker read", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - const bogusVersion = "v-does-not-exist-0000"; - // The no-match worker read returns null; the queue concern then rejects - // the trigger rather than silently locking the run to a phantom version. - await expect( - triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: bogusVersion }, - }, - }) - ).rejects.toThrow(/no worker found with that version/); - - // No run was locked to the bogus version (none was created). - const lockedRuns = await prisma.taskRun.findMany({ - where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion }, - }); - expect(lockedRuns).toEqual([]); - - // The lone worker read fired exactly once with the scalar-only select and - // no cross-seam include. - expect(backgroundWorkerFindFirstCalls).toBe(1); - const args = findFirstArgs[0]; - assertNonNullable(args); - expect(args.include).toBeUndefined(); - expect(Object.keys(args.select ?? {}).sort()).toEqual([ - "cliVersion", - "id", - "sdkVersion", - "version", - ]); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "does not resolve a locked worker from a different environment", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - // Two independent authenticated environments. Rename envA's globally-unique - // fields before the second setup call to avoid unique-constraint collisions. - const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await prisma.organization.update({ - where: { id: envA.organizationId }, - data: { slug: `${envA.organization.slug}-a` }, - }); - await prisma.project.update({ - where: { id: envA.projectId }, - data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, - }); - await prisma.runtimeEnvironment.update({ - where: { id: envA.id }, - data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, - }); - await prisma.workerGroupToken.updateMany({ - where: { tokenHash: "token_hash" }, - data: { tokenHash: "token_hash_a" }, - }); - await prisma.workerInstanceGroup.updateMany({ - where: { masterQueue: "default" }, - data: { masterQueue: "default_a" }, - }); - const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - expect(envA.id).not.toBe(envB.id); - expect(envA.organizationId).not.toBe(envB.organizationId); - - const taskIdentifier = "test-task"; - const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier); - const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier); - - const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: workerA.id }, - }); - const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: workerB.id }, - }); - // Both seeded workers share the same version string. - expect(workerARow.version).toBe(workerBRow.version); - expect(workerARow.id).not.toBe(workerBRow.id); - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger in envB locking to the shared version string. - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envB, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: workerBRow.version }, - }, - }); - assertNonNullable(result); - - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - // The projectId + runtimeEnvironmentId guard in the single-table worker - // read resolves envB's worker, never envA's same-version worker. - expect(runRow.lockedToVersionId).toBe(workerBRow.id); - expect(runRow.lockedToVersionId).not.toBe(workerARow.id); - expect(runRow.taskVersion).toBe(workerBRow.version); - } finally { - await engine.quit(); - } - } - ); - - containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger with NO parentRunId. - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "root" } }, - }); - assertNonNullable(result); - - // The validator ran but received no resolved parent: the parent read was - // skipped because no parentRunId was supplied. - expect(validator.capturedParentRun).not.toBe("unset"); - expect(validator.capturedParentRun).toBeUndefined(); - - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - expect(runRow.depth).toBe(0); - expect(runRow.parentTaskRunId).toBeNull(); - } finally { - await engine.quit(); - } - }); -}); diff --git a/apps/webapp/test/engine/triggerFailedTask.test.ts b/apps/webapp/test/engine/triggerFailedTask.test.ts deleted file mode 100644 index 498312956aa..00000000000 --- a/apps/webapp/test/engine/triggerFailedTask.test.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { describe, expect } from "vitest"; - -import { RunEngine } from "@internal/run-engine"; -import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; -import { containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; -import { RunId, classifyKind, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; -import { TriggerFailedTaskService } from "../../app/runEngine/services/triggerFailedTask.server"; -import { EventRepository } from "../../app/v3/eventRepository/eventRepository.server"; - -vi.setConfig?.({ testTimeout: 60_000 }); - -// Bind the service's trace-event writes to the testcontainer DB. Without this, -// call() resolves the repository via getEventRepository โ†’ global prisma, which -// points at a database that doesn't exist in CI. -function makeService(prisma: any, engine: RunEngine) { - return new TriggerFailedTaskService({ - prisma, - engine, - // Read the parent through the same store the engine wrote it to. - runStore: engine.runStore, - eventRepository: { - repository: new EventRepository(prisma, prisma, { - batchSize: 100, - batchInterval: 1000, - retentionInDays: 30, - partitioningEnabled: false, - }), - store: "taskEvent", - }, - }); -} - -function makeEngine(prisma: any, redisOptions: any) { - return new RunEngine({ - prisma, - worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, - queue: { redis: redisOptions }, - runLock: { redis: redisOptions }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { - name: "small-1x" as const, - cpu: 0.5, - memory: 0.5, - centsPerMs: 0.0001, - }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); -} - -describe("TriggerFailedTaskService โ€” failed run residency", () => { - containerTest( - "root failed run mints cuid when split is off (call)", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const service = makeService(prisma, engine); - - const friendlyId = await service.call({ - taskId: taskIdentifier, - environment, - payload: { test: "root" }, - errorMessage: "boom", - }); - - expect(friendlyId).toBeTruthy(); - expect(classifyKind(friendlyId!)).toBe("cuid"); - - // The failed run write must land (persistence) with no parent linkage. - const persisted = await prisma.taskRun.findFirst({ where: { friendlyId: friendlyId! } }); - expect(persisted).not.toBeNull(); - expect(persisted!.status).toBe("SYSTEM_FAILURE"); - expect(persisted!.depth).toBe(0); - expect(persisted!.parentTaskRunId).toBeNull(); - - await engine.quit(); - } - ); - - containerTest( - "failed child of a NEW (run-ops id) parent mints run-ops id (call)", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); - expect(classifyKind(parentFriendlyId)).toBe("runOpsId"); - await engine.trigger( - { - friendlyId: parentFriendlyId, - environment, - taskIdentifier, - payload: "{}", - payloadType: "application/json", - traceId: "00000000000000000000000000000000", - spanId: "0000000000000000", - workerQueue: "main", - queue: `task/${taskIdentifier}`, - isTest: false, - tags: [], - } as any, - prisma - ); - - const service = makeService(prisma, engine); - - const friendlyId = await service.call({ - taskId: taskIdentifier, - environment, - payload: { test: "child" }, - errorMessage: "boom", - parentRunId: parentFriendlyId, - }); - - expect(classifyKind(friendlyId!)).toBe("runOpsId"); - - // The failed run write must land (persistence) and link to the resolved parent. - const persisted = await prisma.taskRun.findFirst({ where: { friendlyId: friendlyId! } }); - expect(persisted).not.toBeNull(); - expect(persisted!.status).toBe("SYSTEM_FAILURE"); - - const parent = await prisma.taskRun.findFirst({ where: { friendlyId: parentFriendlyId } }); - expect(persisted!.parentTaskRunId).toBe(parent!.id); - expect(persisted!.depth).toBe(parent!.depth + 1); - expect(persisted!.rootTaskRunId).toBe(parent!.rootTaskRunId ?? parent!.id); - - await engine.quit(); - } - ); - - containerTest( - "failed child of a LEGACY (cuid) parent mints cuid (call)", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const parentFriendlyId = RunId.generate().friendlyId; // cuid โ†’ LEGACY - expect(classifyKind(parentFriendlyId)).toBe("cuid"); - await engine.trigger( - { - friendlyId: parentFriendlyId, - environment, - taskIdentifier, - payload: "{}", - payloadType: "application/json", - traceId: "00000000000000000000000000000000", - spanId: "0000000000000000", - workerQueue: "main", - queue: `task/${taskIdentifier}`, - isTest: false, - tags: [], - } as any, - prisma - ); - - const service = makeService(prisma, engine); - - const friendlyId = await service.call({ - taskId: taskIdentifier, - environment, - payload: { test: "child" }, - errorMessage: "boom", - parentRunId: parentFriendlyId, - }); - - expect(classifyKind(friendlyId!)).toBe("cuid"); - - await engine.quit(); - } - ); - - containerTest( - "failed child of a NEW parent mints run-ops id (callWithoutTraceEvents)", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); - await engine.trigger( - { - friendlyId: parentFriendlyId, - environment, - taskIdentifier, - payload: "{}", - payloadType: "application/json", - traceId: "00000000000000000000000000000000", - spanId: "0000000000000000", - workerQueue: "main", - queue: `task/${taskIdentifier}`, - isTest: false, - tags: [], - } as any, - prisma - ); - - const service = makeService(prisma, engine); - - const friendlyId = await service.callWithoutTraceEvents({ - environmentId: environment.id, - environmentType: environment.type, - projectId: environment.projectId, - organizationId: environment.organizationId, - taskId: taskIdentifier, - payload: { test: "child" }, - errorMessage: "boom", - parentRunId: parentFriendlyId, - }); - - expect(classifyKind(friendlyId!)).toBe("runOpsId"); - - await engine.quit(); - } - ); - - containerTest( - "callWithoutTraceEvents returns null (best-effort) when the derived parent row is absent", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const service = makeService(prisma, engine); - - // A well-formed run-ops parent friendlyId that was NEVER triggered โ†’ no row. - // Exercises the missing-parent fallback in callWithoutTraceEvents. - const absentParentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); - - const friendlyId = await service.callWithoutTraceEvents({ - environmentId: environment.id, - environmentType: environment.type, - projectId: environment.projectId, - organizationId: environment.organizationId, - taskId: taskIdentifier, - payload: { test: "absent-parent" }, - errorMessage: "boom", - parentRunId: absentParentFriendlyId, - }); - - // Fallback derives parentTaskRunId from an id with no row; the parentTaskRunId FK rejects the create, so the method returns null instead of throwing. - expect(friendlyId).toBeNull(); - const orphan = await prisma.taskRun.findFirst({ - where: { parentTaskRunId: RunId.fromFriendlyId(absentParentFriendlyId) }, - }); - expect(orphan).toBeNull(); - - await engine.quit(); - } - ); -}); diff --git a/test-timings.json b/test-timings.json index c294dba73e1..85588499aa6 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,7 +1,9 @@ { "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 1945, - "apps/webapp/app/runEngine/services/triggerTask.server.test.ts": 294497, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 42000, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 126000, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 126000, "apps/webapp/app/utils/friendlyId.test.ts": 26, "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 22, "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3199, @@ -89,7 +91,8 @@ "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10952, "apps/webapp/test/engine/streamBatchItems.test.ts": 20294, "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 3590, - "apps/webapp/test/engine/triggerFailedTask.test.ts": 214324, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 129000, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 86000, "apps/webapp/test/engine/triggerTask.debounce.test.ts": 133586, "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133409, "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 171472, From c9d7f7e2c666f6bea6121a546605b7b4ac62571d Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 13:35:43 +0100 Subject: [PATCH 6/7] test(webapp): add the split test files missing from the previous commit The split of triggerTask.server.test.ts and triggerFailedTask.test.ts deleted the originals but the replacement files were never staged, so their 12 container tests silently stopped running. --- .../triggerTask.server.combinedReads.test.ts | 191 +++++++++++ .../triggerTask.server.lockedWorker.test.ts | 324 ++++++++++++++++++ .../triggerTask.server.parentReads.test.ts | 259 ++++++++++++++ .../triggerTask.server.test.helpers.ts | 156 +++++++++ .../engine/triggerFailedTask.call.test.ts | 139 ++++++++ ...iggerFailedTask.withoutTraceEvents.test.ts | 93 +++++ .../engine/triggerFailedTaskTestHelpers.ts | 50 +++ 7 files changed, 1212 insertions(+) create mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts create mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts create mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts create mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts create mode 100644 apps/webapp/test/engine/triggerFailedTask.call.test.ts create mode 100644 apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts create mode 100644 apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts new file mode 100644 index 00000000000..a200d2f9e66 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts @@ -0,0 +1,191 @@ +// Split from triggerTask.server.test.ts (combined parent + locked-worker read) +// so CI's duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService combined parent + locked-worker reads", () => { + containerTest( + "issues two independent single-table reads when one call supplies both parentRunId and lockToVersion", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: worker.id }, + }); + + // Count BOTH reads issued by the service on the control-plane client: + // the parent read (runStore.findRun โ†’ taskRun.findFirst) and the + // locked-worker read (backgroundWorker.findFirst). Capture every + // findFirst arg so we can assert no read carries a cross-seam include. + let taskRunFindFirstCalls = 0; + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + if (prop === "taskRun") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(trTarget, trProp, trReceiver) { + if (trProp === "findFirst") { + return async (args: any) => { + taskRunFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(trTarget, trProp, trReceiver); + return typeof value === "function" ? value.bind(trTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + // queueConcern/idempotency get the real unproxied prisma so the + // counting proxy only observes reads issued by the service itself. + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // ROOT parent first (uses the unproxied prisma via a separate service so + // its internal reads don't pollute the child's counts). + const parentService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + const parentResult = await parentService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // CHILD supplying BOTH parentRunId AND lockToVersion in one call. + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "child" }, + options: { + parentRunId: parentResult.run.friendlyId, + lockToVersion: workerRow.version, + }, + }, + }); + assertNonNullable(childResult); + + const parentRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: parentResult.run.id }, + }); + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + + // Child resolved the parent (single-table parent read). + expect(childRow.parentTaskRunId).toBe(parentRow.id); + expect(childRow.depth).toBe(parentRow.depth + 1); + + // Child locked to the worker (single-table worker read). + expect(childRow.lockedToVersionId).toBe(workerRow.id); + expect(childRow.taskVersion).toBe(workerRow.version); + + // Exactly one backgroundWorker.findFirst fired for the locked-worker read, + // and at least one taskRun.findFirst fired for the parent read. + expect(backgroundWorkerFindFirstCalls).toBe(1); + expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1); + + // NO-JOIN proof: no captured read carried an `include` joining + // taskRun <-> backgroundWorker. Every findFirst arg has include undefined. + for (const args of findFirstArgs) { + expect(args?.include).toBeUndefined(); + } + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts new file mode 100644 index 00000000000..af0b335bfae --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts @@ -0,0 +1,324 @@ +// Split from triggerTask.server.test.ts (locked-worker read concerns) so CI's +// duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService locked-worker reads", () => { + containerTest( + "resolves the locked background worker on the control-plane client with no cross-DB join", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); + + // Read the seeded worker row to get its real version/id. + const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: worker.id }, + }); + + // Counting proxy over the control-plane client. `this.prisma` is ALWAYS + // the control-plane client; the locked-worker lookup is a DIRECT + // backgroundWorker.findFirst on it. The parent read uses a DIFFERENT + // call (runStore.findRun โ†’ taskRun), so a single call() issues two + // separate single-table reads โ€” never one cross-seam join. Here we count + // the findFirst calls and capture their args to assert no include/join. + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + // The queue manager gets the real (unproxied) prisma so the counting + // proxy only observes reads issued by the service itself. + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: workerRow.version }, + }, + }); + assertNonNullable(result); + + // Observable proof the locked worker was resolved on the control-plane + // client: the created run records the worker id in lockedToVersionId. + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + expect(runRow.lockedToVersionId).toBe(workerRow.id); + expect(runRow.taskVersion).toBe(workerRow.version); + + // Exactly one backgroundWorker.findFirst fired for the locked-worker read. + expect(backgroundWorkerFindFirstCalls).toBe(1); + + // NO-JOIN assertion: the read referenced ONLY the backgroundWorker table. + // No `include` (which would join into another table); the `select` lists + // only backgroundWorker scalar columns. + const args = findFirstArgs[0]; + assertNonNullable(args); + expect(args.include).toBeUndefined(); + expect(Object.keys(args.select ?? {}).sort()).toEqual([ + "cliVersion", + "id", + "sdkVersion", + "version", + ]); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "lockToVersion matching no worker rejects the trigger after a single scalar-only worker read", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + const bogusVersion = "v-does-not-exist-0000"; + // The no-match worker read returns null; the queue concern then rejects + // the trigger rather than silently locking the run to a phantom version. + await expect( + triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: bogusVersion }, + }, + }) + ).rejects.toThrow(/no worker found with that version/); + + // No run was locked to the bogus version (none was created). + const lockedRuns = await prisma.taskRun.findMany({ + where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion }, + }); + expect(lockedRuns).toEqual([]); + + // The lone worker read fired exactly once with the scalar-only select and + // no cross-seam include. + expect(backgroundWorkerFindFirstCalls).toBe(1); + const args = findFirstArgs[0]; + assertNonNullable(args); + expect(args.include).toBeUndefined(); + expect(Object.keys(args.select ?? {}).sort()).toEqual([ + "cliVersion", + "id", + "sdkVersion", + "version", + ]); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "does not resolve a locked worker from a different environment", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + // Two independent authenticated environments. Rename envA's globally-unique + // fields before the second setup call to avoid unique-constraint collisions. + const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await prisma.organization.update({ + where: { id: envA.organizationId }, + data: { slug: `${envA.organization.slug}-a` }, + }); + await prisma.project.update({ + where: { id: envA.projectId }, + data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, + }); + await prisma.runtimeEnvironment.update({ + where: { id: envA.id }, + data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, + }); + await prisma.workerGroupToken.updateMany({ + where: { tokenHash: "token_hash" }, + data: { tokenHash: "token_hash_a" }, + }); + await prisma.workerInstanceGroup.updateMany({ + where: { masterQueue: "default" }, + data: { masterQueue: "default_a" }, + }); + const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + expect(envA.id).not.toBe(envB.id); + expect(envA.organizationId).not.toBe(envB.organizationId); + + const taskIdentifier = "test-task"; + const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier); + const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier); + + const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: workerA.id }, + }); + const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: workerB.id }, + }); + // Both seeded workers share the same version string. + expect(workerARow.version).toBe(workerBRow.version); + expect(workerARow.id).not.toBe(workerBRow.id); + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger in envB locking to the shared version string. + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envB, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: workerBRow.version }, + }, + }); + assertNonNullable(result); + + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + // The projectId + runtimeEnvironmentId guard in the single-table worker + // read resolves envB's worker, never envA's same-version worker. + expect(runRow.lockedToVersionId).toBe(workerBRow.id); + expect(runRow.lockedToVersionId).not.toBe(workerARow.id); + expect(runRow.taskVersion).toBe(workerBRow.version); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts new file mode 100644 index 00000000000..7e8b3cd5310 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts @@ -0,0 +1,259 @@ +// Split from triggerTask.server.test.ts (parent-read concerns) so CI's +// duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService parent reads", () => { + containerTest( + "resolves the parent run through the run-ops store by minted run id", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger a ROOT run first to create a real parent TaskRun. + const parentResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // Trigger a CHILD pointing at the parent's friendlyId. The service must + // resolve the parent via runStore.findRun (minted RunId, env-scoped). + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "child" }, + options: { parentRunId: parentResult.run.friendlyId }, + }, + }); + assertNonNullable(childResult); + + // The capturing validator observed the resolved parent โ€” proving the + // read ran (against the container DB) and returned the right row. + expect(validator.capturedParentRun).not.toBe("unset"); + const capturedParent = validator.capturedParentRun; + assertNonNullable(capturedParent); + expect(capturedParent.id).toBe(parentResult.run.id); + expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId); + + // depth and root carry through โ€” proving parentRun.depth and the parent + // id were read off the resolved row and threaded into the child. + const parentRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: parentResult.run.id }, + }); + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + + expect(childRow.depth).toBe(parentRow.depth + 1); + expect(childRow.parentTaskRunId).toBe(parentRow.id); + expect(childRow.rootTaskRunId).toBe(parentRow.id); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "scopes the parent lookup to the run's environment (cross-env parent is not resolved)", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + // Two independent authenticated environments. The setup helper hardcodes + // several globally-unique fields (org/project slug, env apiKey/pkApiKey, + // worker-group token hash), so rename envA's before the second call to + // avoid unique-constraint collisions. + const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await prisma.organization.update({ + where: { id: envA.organizationId }, + data: { slug: `${envA.organization.slug}-a` }, + }); + await prisma.project.update({ + where: { id: envA.projectId }, + data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, + }); + await prisma.runtimeEnvironment.update({ + where: { id: envA.id }, + data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, + }); + await prisma.workerGroupToken.updateMany({ + where: { tokenHash: "token_hash" }, + data: { tokenHash: "token_hash_a" }, + }); + await prisma.workerInstanceGroup.updateMany({ + where: { masterQueue: "default" }, + data: { masterQueue: "default_a" }, + }); + const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + expect(envA.id).not.toBe(envB.id); + expect(envA.organizationId).not.toBe(envB.organizationId); + + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, envA, taskIdentifier); + await setupBackgroundWorker(engine, envB, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // A real parent run in envA. + const parentResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envA, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // Trigger a child in envB pointing at the envA parent's friendlyId. The + // env guard in runStore.findRun's `where` rejects the cross-env parent + // in a single query, so the resolved parentRun is null. + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envB, + body: { + payload: { kind: "child" }, + options: { parentRunId: parentResult.run.friendlyId }, + }, + }); + assertNonNullable(childResult); + + // validateParentRun was called with no resolved parent. + expect(validator.capturedParentRun).not.toBe("unset"); + expect(validator.capturedParentRun ?? null).toBeNull(); + + // The child still triggered, at the root depth with no parent linkage โ€” + // confirming the cross-env parent was dropped, not silently joined. + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + expect(childRow.depth).toBe(0); + expect(childRow.parentTaskRunId).toBeNull(); + } finally { + await engine.quit(); + } + } + ); + + containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger with NO parentRunId. + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "root" } }, + }); + assertNonNullable(result); + + // The validator ran but received no resolved parent: the parent read was + // skipped because no parentRunId was supplied. + expect(validator.capturedParentRun).not.toBe("unset"); + expect(validator.capturedParentRun).toBeUndefined(); + + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + expect(runRow.depth).toBe(0); + expect(runRow.parentTaskRunId).toBeNull(); + } finally { + await engine.quit(); + } + }); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts b/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts new file mode 100644 index 00000000000..fed87481c06 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts @@ -0,0 +1,156 @@ +// Shared harness for the triggerTask.server.*.test.ts family. The suite is split +// across per-concern files (parentReads / lockedWorker / combinedReads) so CI's +// duration-based sharding can spread the container-heavy tests over shards; these +// helpers hold the pure mocks and engine wiring they all share. +// +// NOTE: this is not a test file (vitest's include only matches *.test.ts) and it +// must not import from "vitest" โ€” vi.mock declarations live in each test file +// because vitest hoists them per test module. +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { IOPacket } from "@trigger.dev/core/v3"; +import type { TaskRun } from "@trigger.dev/database"; +import type { + EntitlementValidationParams, + MaxAttemptsValidationParams, + ParentRunValidationParams, + PayloadProcessor, + TagValidationParams, + TracedEventSpan, + TraceEventConcern, + TriggerTaskRequest, + TriggerTaskValidator, + ValidationResult, +} from "~/runEngine/types"; + +export class MockPayloadProcessor implements PayloadProcessor { + async process(request: TriggerTaskRequest): Promise { + return { + data: JSON.stringify(request.body.payload), + dataType: "application/json", + }; + } +} + +// Captures the `parentRun` the service resolved (via runStore.findRun) and +// passed into validation, so a test can assert on the resolved parent without +// mocking the read itself. Returns ok so the child triggers regardless. +export class CapturingParentRunValidator implements TriggerTaskValidator { + public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset"; + + validateTags(_params: TagValidationParams): ValidationResult { + return { ok: true }; + } + validateEntitlement(_params: EntitlementValidationParams): Promise { + return Promise.resolve({ ok: true }); + } + validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult { + return { ok: true }; + } + validateParentRun(params: ParentRunValidationParams): ValidationResult { + this.capturedParentRun = params.parentRun; + return { ok: true }; + } +} + +export class MockTraceEventConcern implements TraceEventConcern { + async traceRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } + + async traceIdempotentRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + _options: { + existingRun: TaskRun; + idempotencyKey: string; + incomplete: boolean; + isError: boolean; + }, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } + + async traceDebouncedRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + _options: { + existingRun: TaskRun; + debounceKey: string; + incomplete: boolean; + isError: boolean; + }, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } +} + +export function buildEngine(prisma: any, redisOptions: any) { + return new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} diff --git a/apps/webapp/test/engine/triggerFailedTask.call.test.ts b/apps/webapp/test/engine/triggerFailedTask.call.test.ts new file mode 100644 index 00000000000..94b95d18786 --- /dev/null +++ b/apps/webapp/test/engine/triggerFailedTask.call.test.ts @@ -0,0 +1,139 @@ +// Split from triggerFailedTask.test.ts (the call() path) so CI's +// duration-based sharding can balance the container-heavy tests. +import { describe, expect } from "vitest"; + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { containerTest } from "@internal/testcontainers"; +import { RunId, classifyKind, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { makeEngine, makeService } from "./triggerFailedTaskTestHelpers"; + +vi.setConfig?.({ testTimeout: 60_000 }); + +describe("TriggerFailedTaskService โ€” failed run residency (call)", () => { + containerTest( + "root failed run mints cuid when split is off (call)", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const service = makeService(prisma, engine); + + const friendlyId = await service.call({ + taskId: taskIdentifier, + environment, + payload: { test: "root" }, + errorMessage: "boom", + }); + + expect(friendlyId).toBeTruthy(); + expect(classifyKind(friendlyId!)).toBe("cuid"); + + // The failed run write must land (persistence) with no parent linkage. + const persisted = await prisma.taskRun.findFirst({ where: { friendlyId: friendlyId! } }); + expect(persisted).not.toBeNull(); + expect(persisted!.status).toBe("SYSTEM_FAILURE"); + expect(persisted!.depth).toBe(0); + expect(persisted!.parentTaskRunId).toBeNull(); + + await engine.quit(); + } + ); + + containerTest( + "failed child of a NEW (run-ops id) parent mints run-ops id (call)", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + expect(classifyKind(parentFriendlyId)).toBe("runOpsId"); + await engine.trigger( + { + friendlyId: parentFriendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], + } as any, + prisma + ); + + const service = makeService(prisma, engine); + + const friendlyId = await service.call({ + taskId: taskIdentifier, + environment, + payload: { test: "child" }, + errorMessage: "boom", + parentRunId: parentFriendlyId, + }); + + expect(classifyKind(friendlyId!)).toBe("runOpsId"); + + // The failed run write must land (persistence) and link to the resolved parent. + const persisted = await prisma.taskRun.findFirst({ where: { friendlyId: friendlyId! } }); + expect(persisted).not.toBeNull(); + expect(persisted!.status).toBe("SYSTEM_FAILURE"); + + const parent = await prisma.taskRun.findFirst({ where: { friendlyId: parentFriendlyId } }); + expect(persisted!.parentTaskRunId).toBe(parent!.id); + expect(persisted!.depth).toBe(parent!.depth + 1); + expect(persisted!.rootTaskRunId).toBe(parent!.rootTaskRunId ?? parent!.id); + + await engine.quit(); + } + ); + + containerTest( + "failed child of a LEGACY (cuid) parent mints cuid (call)", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const parentFriendlyId = RunId.generate().friendlyId; // cuid โ†’ LEGACY + expect(classifyKind(parentFriendlyId)).toBe("cuid"); + await engine.trigger( + { + friendlyId: parentFriendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], + } as any, + prisma + ); + + const service = makeService(prisma, engine); + + const friendlyId = await service.call({ + taskId: taskIdentifier, + environment, + payload: { test: "child" }, + errorMessage: "boom", + parentRunId: parentFriendlyId, + }); + + expect(classifyKind(friendlyId!)).toBe("cuid"); + + await engine.quit(); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts new file mode 100644 index 00000000000..a0be900fb82 --- /dev/null +++ b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts @@ -0,0 +1,93 @@ +// Split from triggerFailedTask.test.ts (the callWithoutTraceEvents() path) so +// CI's duration-based sharding can balance the container-heavy tests. +import { describe, expect } from "vitest"; + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { containerTest } from "@internal/testcontainers"; +import { RunId, classifyKind, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { makeEngine, makeService } from "./triggerFailedTaskTestHelpers"; + +vi.setConfig?.({ testTimeout: 60_000 }); + +describe("TriggerFailedTaskService โ€” failed run residency (callWithoutTraceEvents)", () => { + containerTest( + "failed child of a NEW parent mints run-ops id (callWithoutTraceEvents)", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + await engine.trigger( + { + friendlyId: parentFriendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], + } as any, + prisma + ); + + const service = makeService(prisma, engine); + + const friendlyId = await service.callWithoutTraceEvents({ + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.projectId, + organizationId: environment.organizationId, + taskId: taskIdentifier, + payload: { test: "child" }, + errorMessage: "boom", + parentRunId: parentFriendlyId, + }); + + expect(classifyKind(friendlyId!)).toBe("runOpsId"); + + await engine.quit(); + } + ); + + containerTest( + "callWithoutTraceEvents returns null (best-effort) when the derived parent row is absent", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const service = makeService(prisma, engine); + + // A well-formed run-ops parent friendlyId that was NEVER triggered โ†’ no row. + // Exercises the missing-parent fallback in callWithoutTraceEvents. + const absentParentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + + const friendlyId = await service.callWithoutTraceEvents({ + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.projectId, + organizationId: environment.organizationId, + taskId: taskIdentifier, + payload: { test: "absent-parent" }, + errorMessage: "boom", + parentRunId: absentParentFriendlyId, + }); + + // Fallback derives parentTaskRunId from an id with no row; the parentTaskRunId FK rejects the create, so the method returns null instead of throwing. + expect(friendlyId).toBeNull(); + const orphan = await prisma.taskRun.findFirst({ + where: { parentTaskRunId: RunId.fromFriendlyId(absentParentFriendlyId) }, + }); + expect(orphan).toBeNull(); + + await engine.quit(); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts b/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts new file mode 100644 index 00000000000..38b7c072abf --- /dev/null +++ b/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts @@ -0,0 +1,50 @@ +// Shared harness for the triggerFailedTask.*.test.ts family, split from the +// original single test file so CI's duration-based sharding can balance the +// container-heavy tests. Not a test file (vitest's include only matches *.test.ts). +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import { TriggerFailedTaskService } from "../../app/runEngine/services/triggerFailedTask.server"; +import { EventRepository } from "../../app/v3/eventRepository/eventRepository.server"; + +// Bind the service's trace-event writes to the testcontainer DB. Without this, +// call() resolves the repository via getEventRepository โ†’ global prisma, which +// points at a database that doesn't exist in CI. +export function makeService(prisma: any, engine: RunEngine) { + return new TriggerFailedTaskService({ + prisma, + engine, + // Read the parent through the same store the engine wrote it to. + runStore: engine.runStore, + eventRepository: { + repository: new EventRepository(prisma, prisma, { + batchSize: 100, + batchInterval: 1000, + retentionInDays: 30, + partitioningEnabled: false, + }), + store: "taskEvent", + }, + }); +} + +export function makeEngine(prisma: any, redisOptions: any) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { redis: redisOptions }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} From d0975058846203c10fd72bb94dd56cdb3b2c9289 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 13 Jul 2026 15:28:17 +0100 Subject: [PATCH 7/7] test timings --- test-timings.json | 522 +++++++++++++++++++++++----------------------- 1 file changed, 258 insertions(+), 264 deletions(-) diff --git a/test-timings.json b/test-timings.json index 85588499aa6..33fa9c5f32b 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,275 +1,269 @@ { "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, - "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 1945, - "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 42000, - "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 126000, - "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 126000, - "apps/webapp/app/utils/friendlyId.test.ts": 26, - "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 22, - "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3199, - "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 25, - "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 486, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2091, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49515, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132479, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132404, + "apps/webapp/app/utils/friendlyId.test.ts": 59, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 25, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3372, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 36, + "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 477, "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 27, - "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 10361, - "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 22, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 498, - "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 485, - "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 10337, - "apps/webapp/app/v3/runStore.server.test.ts": 8121, - "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 9867, - "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 116, - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 3847, - "apps/webapp/test/GCRARateLimiter.test.ts": 4735, - "apps/webapp/test/SpanPresenter.readthrough.test.ts": 10078, - "apps/webapp/test/activitySeries.server.test.ts": 22, - "apps/webapp/test/aiTitleRateLimiter.test.ts": 663, - "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 92382, - "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 174393, - "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5273, - "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 10127, - "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11165, - "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3001, - "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8581, - "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12372, - "apps/webapp/test/apiRunListPresenter.test.ts": 142529, - "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 7138, - "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3184, - "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 13383, - "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, - "apps/webapp/test/batchListPresenter.readroute.test.ts": 11236, - "apps/webapp/test/batchPresenter.test.ts": 12536, - "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 2021, - "apps/webapp/test/batchRunAccess.test.ts": 8380, - "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6380, - "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1472, - "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5515, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 6305, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 24, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 546, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 511, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 6073, + "apps/webapp/app/v3/runStore.server.test.ts": 8480, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5922, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 83, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 4096, + "apps/webapp/test/GCRARateLimiter.test.ts": 4787, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9668, + "apps/webapp/test/activitySeries.server.test.ts": 25, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 744, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 97711, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 179054, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5727, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5378, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11836, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3476, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8669, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12490, + "apps/webapp/test/apiRunListPresenter.test.ts": 148237, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 6843, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3321, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 9115, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 11138, + "apps/webapp/test/batchPresenter.test.ts": 16534, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 1959, + "apps/webapp/test/batchRunAccess.test.ts": 8353, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6556, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1037, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5624, "apps/webapp/test/billingAlertsFormat.test.ts": 34, - "apps/webapp/test/billingLimit.schemas.test.ts": 29, - "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 15295, - "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3409, - "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 20, - "apps/webapp/test/billingLimitConvergeResolve.test.ts": 211, - "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 565, - "apps/webapp/test/billingLimitHit.test.ts": 29, - "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 37, - "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12481, - "apps/webapp/test/billingLimitReconcileTick.test.ts": 231, - "apps/webapp/test/billingLimitReconciliation.test.ts": 544, - "apps/webapp/test/billingLimitResolve.test.ts": 20, - "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 20, - "apps/webapp/test/billingLimitsRoute.test.ts": 1013, - "apps/webapp/test/branchableEnvironment.test.ts": 23, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 22, - "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6047, + "apps/webapp/test/billingLimit.schemas.test.ts": 40, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16155, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3424, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 21, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 200, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 609, + "apps/webapp/test/billingLimitHit.test.ts": 25, + "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 35, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12281, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 213, + "apps/webapp/test/billingLimitReconciliation.test.ts": 630, + "apps/webapp/test/billingLimitResolve.test.ts": 17, + "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 26, + "apps/webapp/test/billingLimitsRoute.test.ts": 1102, + "apps/webapp/test/branchableEnvironment.test.ts": 20, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 25, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6183, "apps/webapp/test/calculateNextSchedule.test.ts": 208, - "apps/webapp/test/chartActivityTimeAxis.test.ts": 35, - "apps/webapp/test/chartXAxisTicks.test.ts": 51, - "apps/webapp/test/chartZoomRange.test.ts": 27, - "apps/webapp/test/chat-snapshot-integration.test.ts": 1654, - "apps/webapp/test/checkPermissions.test.ts": 20, - "apps/webapp/test/clickhouseFactory.test.ts": 3652, - "apps/webapp/test/components/DateTime.test.ts": 801, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 161, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 268, - "apps/webapp/test/components/runs/v3/RunTag.test.ts": 146, - "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 373, - "apps/webapp/test/computeBucket.test.ts": 106, - "apps/webapp/test/computeMigration.test.ts": 25, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 976, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8757, - "apps/webapp/test/crossSeamGuard.proof.test.ts": 5319, - "apps/webapp/test/dependentAttemptScope.test.ts": 16, - "apps/webapp/test/detectQueryTables.test.ts": 233, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 29, + "apps/webapp/test/chartXAxisTicks.test.ts": 29, + "apps/webapp/test/chartZoomRange.test.ts": 22, + "apps/webapp/test/chat-snapshot-integration.test.ts": 1896, + "apps/webapp/test/checkPermissions.test.ts": 22, + "apps/webapp/test/clickhouseFactory.test.ts": 3978, + "apps/webapp/test/components/DateTime.test.ts": 784, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 136, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 273, + "apps/webapp/test/components/runs/v3/RunTag.test.ts": 190, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 339, + "apps/webapp/test/computeBucket.test.ts": 123, + "apps/webapp/test/computeMigration.test.ts": 22, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 716, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8603, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 5538, + "apps/webapp/test/dependentAttemptScope.test.ts": 17, + "apps/webapp/test/detectQueryTables.test.ts": 259, "apps/webapp/test/detectbadJsonStrings.test.ts": 73, - "apps/webapp/test/devBranchServices.test.ts": 4024, - "apps/webapp/test/devPresenceRecency.test.ts": 1168, - "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2737, - "apps/webapp/test/duplicateTaskIds.test.ts": 19, - "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1598, - "apps/webapp/test/emailPattern.test.ts": 20, - "apps/webapp/test/engine/batchPayloads.test.ts": 5358, - "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10952, - "apps/webapp/test/engine/streamBatchItems.test.ts": 20294, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 3590, - "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 129000, - "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 86000, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 133586, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133409, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 171472, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 172402, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 172353, - "apps/webapp/test/engine/triggerTask.test.ts": 91815, - "apps/webapp/test/environmentSort.test.ts": 25, - "apps/webapp/test/environmentVariableDeduplication.test.ts": 35, - "apps/webapp/test/environmentVariableRules.test.ts": 25, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3239, - "apps/webapp/test/environmentVariablesRepository.test.ts": 3696, - "apps/webapp/test/errorFingerprinting.test.ts": 31, - "apps/webapp/test/errorGroupWebhook.test.ts": 58, - "apps/webapp/test/fairDequeuingStrategy.test.ts": 5705, - "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3820, - "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5478, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8693, - "apps/webapp/test/getDeploymentImageRef.test.ts": 324, - "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6427, - "apps/webapp/test/googleEmailVerification.test.ts": 19, - "apps/webapp/test/httpErrors.test.ts": 52, - "apps/webapp/test/idempotencyDedupResidency.test.ts": 9844, - "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 10956, - "apps/webapp/test/inviteRoleLadder.test.ts": 17, - "apps/webapp/test/marqsKeyProducer.test.ts": 7, - "apps/webapp/test/member.server.test.ts": 5081, - "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 246, - "apps/webapp/test/mfaRateLimiter.test.ts": 602, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 748, - "apps/webapp/test/mollifierClaimResolution.test.ts": 435, - "apps/webapp/test/mollifierDecisionLabels.test.ts": 43, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 470, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 2073, - "apps/webapp/test/mollifierDrainingGauge.test.ts": 700, - "apps/webapp/test/mollifierGate.test.ts": 444, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 442, - "apps/webapp/test/mollifierMollify.test.ts": 243, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 489, - "apps/webapp/test/mollifierReadFallback.test.ts": 533, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 260, - "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 498, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 614, - "apps/webapp/test/mollifierStaleSweep.test.ts": 975, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 609, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 34, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 753, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 19, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 23, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 234, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 342, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 684, - "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 24812, - "apps/webapp/test/objectStore.test.ts": 4979, - "apps/webapp/test/orgBanner.test.ts": 22, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 4856, - "apps/webapp/test/otlpExporter.test.ts": 174, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 4773, - "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 332, - "apps/webapp/test/pauseEnvironment.server.test.ts": 9290, - "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6662, - "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 9059, - "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 5369, - "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36728, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 20, - "apps/webapp/test/prismaErrors.test.ts": 230, - "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3465, + "apps/webapp/test/devBranchServices.test.ts": 4029, + "apps/webapp/test/devPresenceRecency.test.ts": 1179, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2678, + "apps/webapp/test/duplicateTaskIds.test.ts": 20, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1730, + "apps/webapp/test/emailPattern.test.ts": 18, + "apps/webapp/test/engine/batchPayloads.test.ts": 5365, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10574, + "apps/webapp/test/engine/streamBatchItems.test.ts": 19344, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4014, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 133741, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 91796, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 134153, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133523, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 173067, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 173296, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 172996, + "apps/webapp/test/engine/triggerTask.test.ts": 92296, + "apps/webapp/test/environmentSort.test.ts": 29, + "apps/webapp/test/environmentVariableDeduplication.test.ts": 24, + "apps/webapp/test/environmentVariableRules.test.ts": 19, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3690, + "apps/webapp/test/environmentVariablesRepository.test.ts": 4145, + "apps/webapp/test/errorFingerprinting.test.ts": 28, + "apps/webapp/test/errorGroupWebhook.test.ts": 59, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3764, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5555, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8848, + "apps/webapp/test/getDeploymentImageRef.test.ts": 306, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6408, + "apps/webapp/test/googleEmailVerification.test.ts": 27, + "apps/webapp/test/httpErrors.test.ts": 39, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 9623, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5778, + "apps/webapp/test/inviteRoleLadder.test.ts": 18, + "apps/webapp/test/member.server.test.ts": 5194, + "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 274, + "apps/webapp/test/mfaRateLimiter.test.ts": 632, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 850, + "apps/webapp/test/mollifierClaimResolution.test.ts": 469, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 47, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 450, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 2020, + "apps/webapp/test/mollifierDrainingGauge.test.ts": 716, + "apps/webapp/test/mollifierGate.test.ts": 492, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 423, + "apps/webapp/test/mollifierMollify.test.ts": 223, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 445, + "apps/webapp/test/mollifierReadFallback.test.ts": 405, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 249, + "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 522, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 448, + "apps/webapp/test/mollifierStaleSweep.test.ts": 1047, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 559, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 25, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 815, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 20, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 21, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 197, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 303, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 831, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 30311, + "apps/webapp/test/objectStore.test.ts": 5520, + "apps/webapp/test/orgBanner.test.ts": 18, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 5455, + "apps/webapp/test/otlpExporter.test.ts": 144, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5831, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 317, + "apps/webapp/test/pauseEnvironment.server.test.ts": 8847, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6886, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 16145, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 6231, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36865, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 21, + "apps/webapp/test/prismaErrors.test.ts": 219, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3592, "apps/webapp/test/promptOverrideSource.test.ts": 17, - "apps/webapp/test/queryResultsTimeTicks.test.ts": 732, - "apps/webapp/test/queueListPagination.test.ts": 17, - "apps/webapp/test/rbacFallbackBranch.test.ts": 3647, - "apps/webapp/test/realtime/boundedTtlCache.test.ts": 17, - "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 37321, - "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 61, - "apps/webapp/test/realtime/envChangeRouter.test.ts": 1180, - "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4133, - "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 317, - "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 601, - "apps/webapp/test/realtime/replayCursorStore.test.ts": 1600, - "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 577, - "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3546, - "apps/webapp/test/realtime/runReaderProjection.test.ts": 56, - "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7537, - "apps/webapp/test/realtime/shadowCompare.test.ts": 27, - "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5719, - "apps/webapp/test/realtimeClient.test.ts": 1, - "apps/webapp/test/redisRealtimeStreams.test.ts": 5733, - "apps/webapp/test/registryConfig.test.ts": 301, - "apps/webapp/test/reloadingRegistry.test.ts": 369, - "apps/webapp/test/removeTeamMember.test.ts": 9388, - "apps/webapp/test/replay-after-crash.test.ts": 1676, - "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4150, - "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5723, - "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6657, - "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5774, - "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 5620, - "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 586, - "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 6211, - "apps/webapp/test/runEngineHandlers.test.ts": 14676, - "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 370, - "apps/webapp/test/runOpsDbTopology.test.ts": 4175, - "apps/webapp/test/runOpsMintCutover.test.ts": 3733, - "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3409, - "apps/webapp/test/runOpsSplitMode.test.ts": 4141, - "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 503, - "apps/webapp/test/runOpsSplitReadGate.test.ts": 24, - "apps/webapp/test/runPresenterReadRoute.test.ts": 4068, - "apps/webapp/test/runsBackfiller.test.ts": 8145, - "apps/webapp/test/runsReplicationBenchmark.test.ts": 0, - "apps/webapp/test/runsReplicationInstance.test.ts": 23928, - "apps/webapp/test/runsReplicationService.part1.test.ts": 26040, - "apps/webapp/test/runsReplicationService.part2.test.ts": 22148, - "apps/webapp/test/runsReplicationService.part3.test.ts": 12273, - "apps/webapp/test/runsReplicationService.part4.test.ts": 27353, - "apps/webapp/test/runsReplicationService.part5.test.ts": 10249, - "apps/webapp/test/runsReplicationService.part6.test.ts": 12512, - "apps/webapp/test/runsReplicationService.part7.test.ts": 69936, - "apps/webapp/test/runsReplicationService.part8.test.ts": 25504, - "apps/webapp/test/runsReplicationService.part9.test.ts": 12873, - "apps/webapp/test/runsRepository.part1.test.ts": 23966, - "apps/webapp/test/runsRepository.part2.test.ts": 25074, - "apps/webapp/test/runsRepository.part3.test.ts": 21260, - "apps/webapp/test/runsRepository.part4.test.ts": 23884, - "apps/webapp/test/runsRepository.readthrough.test.ts": 36385, - "apps/webapp/test/runsRepositoryCpres.test.ts": 8383, - "apps/webapp/test/runsRepositoryCursor.test.ts": 29311, - "apps/webapp/test/safeEnvironmentLog.test.ts": 29, - "apps/webapp/test/safeIntegrationLog.test.ts": 20, - "apps/webapp/test/safeRequestLogContext.test.ts": 25, - "apps/webapp/test/safeWebhookFetch.test.ts": 230, - "apps/webapp/test/safeWebhookUrl.test.ts": 25, - "apps/webapp/test/sameOriginNavigation.test.ts": 34, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 22, - "apps/webapp/test/sanitizeUrl.test.ts": 30, - "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 283, - "apps/webapp/test/sentryTenantContext.test.ts": 24, - "apps/webapp/test/sentryTraceContext.server.test.ts": 92, - "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5811, - "apps/webapp/test/services/organizationAccessToken.test.ts": 255, - "apps/webapp/test/services/personalAccessToken.test.ts": 229, - "apps/webapp/test/sessionDuration.test.ts": 9116, - "apps/webapp/test/sessions.readthrough.test.ts": 6117, - "apps/webapp/test/sessionsReplicationService.test.ts": 16991, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 23, - "apps/webapp/test/slackErrorAlerts.test.ts": 0, - "apps/webapp/test/slackOAuthResultLog.test.ts": 20, - "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5397, - "apps/webapp/test/streamLoader.controlPlane.test.ts": 5239, - "apps/webapp/test/tenantContext.test.ts": 44, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 17, - "apps/webapp/test/tenantContextResolver.test.ts": 39, - "apps/webapp/test/timeGranularity.test.ts": 25, - "apps/webapp/test/timelineSpanEvents.test.ts": 28, - "apps/webapp/test/traceExport.test.ts": 32, - "apps/webapp/test/updateMetadata.test.ts": 18934, - "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 8027, - "apps/webapp/test/utils/timezones.test.ts": 29, - "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 6456, - "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 7447, - "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 6225, - "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4283, - "apps/webapp/test/validateGitBranchName.test.ts": 20, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 650, + "apps/webapp/test/queueListPagination.test.ts": 31, + "apps/webapp/test/rbacFallbackBranch.test.ts": 3600, + "apps/webapp/test/realtime/boundedTtlCache.test.ts": 22, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 41562, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 68, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 1176, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4496, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 342, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 607, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1529, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 584, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3540, + "apps/webapp/test/realtime/runReaderProjection.test.ts": 53, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7242, + "apps/webapp/test/realtime/shadowCompare.test.ts": 26, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5726, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5745, + "apps/webapp/test/registryConfig.test.ts": 343, + "apps/webapp/test/reloadingRegistry.test.ts": 362, + "apps/webapp/test/removeTeamMember.test.ts": 9473, + "apps/webapp/test/replay-after-crash.test.ts": 2002, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4349, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5698, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6634, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5232, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 6025, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 550, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 5799, + "apps/webapp/test/runEngineHandlers.test.ts": 14088, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 348, + "apps/webapp/test/runOpsDbTopology.test.ts": 4214, + "apps/webapp/test/runOpsMintCutover.test.ts": 3808, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3407, + "apps/webapp/test/runOpsSplitMode.test.ts": 4202, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 473, + "apps/webapp/test/runOpsSplitReadGate.test.ts": 18, + "apps/webapp/test/runPresenterReadRoute.test.ts": 3909, + "apps/webapp/test/runsBackfiller.test.ts": 8095, + "apps/webapp/test/runsReplicationInstance.test.ts": 24640, + "apps/webapp/test/runsReplicationService.part1.test.ts": 25612, + "apps/webapp/test/runsReplicationService.part2.test.ts": 22323, + "apps/webapp/test/runsReplicationService.part3.test.ts": 12303, + "apps/webapp/test/runsReplicationService.part4.test.ts": 27040, + "apps/webapp/test/runsReplicationService.part5.test.ts": 9175, + "apps/webapp/test/runsReplicationService.part6.test.ts": 12682, + "apps/webapp/test/runsReplicationService.part7.test.ts": 69508, + "apps/webapp/test/runsReplicationService.part8.test.ts": 24479, + "apps/webapp/test/runsReplicationService.part9.test.ts": 12463, + "apps/webapp/test/runsRepository.part1.test.ts": 22987, + "apps/webapp/test/runsRepository.part2.test.ts": 23786, + "apps/webapp/test/runsRepository.part3.test.ts": 18531, + "apps/webapp/test/runsRepository.part4.test.ts": 24073, + "apps/webapp/test/runsRepository.readthrough.test.ts": 39432, + "apps/webapp/test/runsRepositoryCpres.test.ts": 8672, + "apps/webapp/test/runsRepositoryCursor.test.ts": 28582, + "apps/webapp/test/safeEnvironmentLog.test.ts": 17, + "apps/webapp/test/safeIntegrationLog.test.ts": 15, + "apps/webapp/test/safeRequestLogContext.test.ts": 26, + "apps/webapp/test/safeWebhookFetch.test.ts": 199, + "apps/webapp/test/safeWebhookUrl.test.ts": 30, + "apps/webapp/test/sameOriginNavigation.test.ts": 53, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 30, + "apps/webapp/test/sanitizeUrl.test.ts": 22, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 280, + "apps/webapp/test/sentryTenantContext.test.ts": 23, + "apps/webapp/test/sentryTraceContext.server.test.ts": 70, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5401, + "apps/webapp/test/services/organizationAccessToken.test.ts": 241, + "apps/webapp/test/services/personalAccessToken.test.ts": 232, + "apps/webapp/test/sessionDuration.test.ts": 10233, + "apps/webapp/test/sessions.readthrough.test.ts": 5859, + "apps/webapp/test/sessionsReplicationService.test.ts": 17051, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 19, + "apps/webapp/test/slackOAuthResultLog.test.ts": 23, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5373, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 5162, + "apps/webapp/test/tenantContext.test.ts": 43, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 38, + "apps/webapp/test/tenantContextResolver.test.ts": 37, + "apps/webapp/test/timeGranularity.test.ts": 27, + "apps/webapp/test/timelineSpanEvents.test.ts": 25, + "apps/webapp/test/traceExport.test.ts": 29, + "apps/webapp/test/updateMetadata.test.ts": 19006, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7309, + "apps/webapp/test/utils/timezones.test.ts": 31, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 5949, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 6678, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 5388, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4587, + "apps/webapp/test/validateGitBranchName.test.ts": 23, "apps/webapp/test/vercelUrls.test.ts": 18, - "apps/webapp/test/verifyDeploymentImage.test.ts": 928, - "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 6413, - "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8929, - "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 4918, - "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8786, - "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 5354, - "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 6103, - "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 32457, - "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5498, - "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5732, - "apps/webapp/test/webhookErrorAlerts.test.ts": 86, - "apps/webapp/test/workerGroupAccess.test.ts": 34, - "apps/webapp/test/workerQueueSplit.server.test.ts": 38, - "apps/webapp/test/workerQueueSplit.test.ts": 26, - "apps/webapp/test/workerRegions.test.ts": 508, + "apps/webapp/test/verifyDeploymentImage.test.ts": 922, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7670, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8556, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5386, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8112, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4994, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5687, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 35556, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5062, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5653, + "apps/webapp/test/webhookErrorAlerts.test.ts": 61, + "apps/webapp/test/workerGroupAccess.test.ts": 33, + "apps/webapp/test/workerQueueSplit.server.test.ts": 23, + "apps/webapp/test/workerQueueSplit.test.ts": 27, + "apps/webapp/test/workerRegions.test.ts": 487, "internal-packages/cache/src/stores/lruMemory.test.ts": 65, "internal-packages/clickhouse/src/client/client.test.ts": 7547, "internal-packages/clickhouse/src/taskRuns.test.ts": 6768,