diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml index 4a995a08ed9..c795b3e642a 100644 --- a/.github/workflows/e2e-webapp.yml +++ b/.github/workflows/e2e-webapp.yml @@ -14,7 +14,7 @@ on: jobs: e2eTests: name: "๐Ÿงช E2E Tests: Webapp" - runs-on: warp-ubuntu-latest-x64-8x + runs-on: warp-ubuntu-latest-x64-16x timeout-minutes: 20 env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f2ae36243e2..f2015e0d39d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [warp-ubuntu-latest-x64-4x, warp-windows-latest-x64-4x] + os: [warp-ubuntu-latest-x64-4x, warp-windows-latest-x64-8x] package-manager: ["npm", "pnpm"] steps: - name: โฌ‡๏ธ Checkout repo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6bef7ac5c0..b7f8b14c991 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: release: name: ๐Ÿš€ Release npm packages - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: write @@ -281,7 +281,7 @@ jobs: # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims. prerelease: name: ๐Ÿงช Prerelease - runs-on: ubuntu-latest + runs-on: ubuntu-latest # this cannot run on non-GH runner environment: npm-publish permissions: contents: read diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 05d3a1496e5..d4c6cd2a5eb 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -8,7 +8,7 @@ permissions: jobs: typecheck: - runs-on: warp-ubuntu-latest-x64-8x + runs-on: warp-ubuntu-latest-x64-16x steps: - name: โฌ‡๏ธ Checkout repo diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml index 6869fe8a5f5..068d9e47a92 100644 --- a/.github/workflows/unit-tests-internal.yml +++ b/.github/workflows/unit-tests-internal.yml @@ -14,17 +14,14 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Internal" - runs-on: warp-ubuntu-latest-x64-8x - strategy: - # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard - fail-fast: false - matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - shardTotal: [12] + # Single big machine instead of a 12-job matrix: the internal suites are serial + # (fileParallelism: false) and container-wait-bound, so 12 in-machine shard processes + # fit comfortably in 32 vCPUs while paying the setup cost (install, prisma generate, + # image pulls) once instead of 12 times. + runs-on: warp-ubuntu-latest-x64-32x env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - SHARD_INDEX: ${{ matrix.shardIndex }} - SHARD_TOTAL: ${{ matrix.shardTotal }} + SHARD_TOTAL: 12 steps: - name: ๐Ÿ”ง Disable IPv6 run: | @@ -108,8 +105,34 @@ jobs: - name: ๐Ÿ“€ Generate Prisma Client run: pnpm run generate - - name: ๐Ÿงช Run Internal Unit Tests - run: pnpm run test:internal --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --passWithNoTests + - name: ๐Ÿ—๏ธ Build test dependencies + # Build once up-front so the parallel shard runs below (turbo --only) never race + # to build or cache-restore the same outputs concurrently. + run: pnpm exec turbo run build --filter "@internal/*..." + + - name: ๐Ÿงช Run Internal Unit Tests (${{ env.SHARD_TOTAL }} in-machine shards) + run: | + # Same shard partitioning as the old 12-job matrix (DurationShardingSequencer + # keys off --shard=i/N), but as parallel local processes. --only skips the + # ^build dependency handled by the step above. + status=0 + declare -a pids + for i in $(seq 1 "$SHARD_TOTAL"); do + pnpm exec turbo run test --only --concurrency=1 --filter "@internal/*" -- \ + --run --reporter=default --reporter=blob --shard="$i/$SHARD_TOTAL" --passWithNoTests \ + > "/tmp/internal-shard-$i.log" 2>&1 & + pids[i]=$! + done + for i in $(seq 1 "$SHARD_TOTAL"); do + if ! wait "${pids[i]}"; then + status=1 + echo "::error::internal unit test shard $i/$SHARD_TOTAL failed" + fi + echo "::group::๐Ÿงช shard $i/$SHARD_TOTAL" + cat "/tmp/internal-shard-$i.log" + echo "::endgroup::" + done + exit "$status" - name: Gather all reports if: ${{ !cancelled() }} @@ -118,44 +141,6 @@ jobs: find . -type f -path '*/.vitest-reports/blob-*.json' \ -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \; - - name: Upload blob reports to GitHub Actions Artifacts + - name: ๐Ÿ“Š Merge reports if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: internal-blob-report-${{ matrix.shardIndex }} - path: .vitest-reports/* - include-hidden-files: true - retention-days: 1 - - merge-reports: - name: "๐Ÿ“Š Merge Reports" - if: ${{ !cancelled() }} - needs: [unitTests] - runs-on: warp-ubuntu-latest-x64-2x - steps: - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 1 - persist-credentials: false - - - name: โŽ” Setup pnpm - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - with: - version: 10.33.2 - - - name: โŽ” Setup node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 22.23.1 - # no cache enabled, we're not installing deps - - - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: .vitest-reports - pattern: internal-blob-report-* - merge-multiple: true - - - name: Merge reports run: pnpm dlx vitest@4.1.7 run --merge-reports --pass-with-no-tests diff --git a/.github/workflows/unit-tests-webapp.yml b/.github/workflows/unit-tests-webapp.yml index eb1cd424c4b..0ba248ca7db 100644 --- a/.github/workflows/unit-tests-webapp.yml +++ b/.github/workflows/unit-tests-webapp.yml @@ -14,13 +14,18 @@ on: jobs: unitTests: name: "๐Ÿงช Unit Tests: Webapp" - runs-on: warp-ubuntu-latest-x64-8x + # 10 shards on 16x machines: webapp test throughput is limited per-machine (one + # docker daemon + disk absorbing all the per-file Postgres/ClickHouse container + # spin-up), so many machines beats few big ones - fewer/bigger (3x32) measured + # SLOWER than 10x8. The 16x (vs 8x) gives the fork pool the CPU headroom the 8x + # runners lacked. Setup overhead per machine is ~1 min on warm runners. + runs-on: warp-ubuntu-latest-x64-16x strategy: # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard fail-fast: false matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - shardTotal: [10] + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + shardTotal: [12] env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SHARD_INDEX: ${{ matrix.shardIndex }} diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts new file mode 100644 index 00000000000..a200d2f9e66 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts @@ -0,0 +1,191 @@ +// Split from triggerTask.server.test.ts (combined parent + locked-worker read) +// so CI's duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService combined parent + locked-worker reads", () => { + containerTest( + "issues two independent single-table reads when one call supplies both parentRunId and lockToVersion", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); + + const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: worker.id }, + }); + + // Count BOTH reads issued by the service on the control-plane client: + // the parent read (runStore.findRun โ†’ taskRun.findFirst) and the + // locked-worker read (backgroundWorker.findFirst). Capture every + // findFirst arg so we can assert no read carries a cross-seam include. + let taskRunFindFirstCalls = 0; + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + if (prop === "taskRun") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(trTarget, trProp, trReceiver) { + if (trProp === "findFirst") { + return async (args: any) => { + taskRunFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(trTarget, trProp, trReceiver); + return typeof value === "function" ? value.bind(trTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + // queueConcern/idempotency get the real unproxied prisma so the + // counting proxy only observes reads issued by the service itself. + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // ROOT parent first (uses the unproxied prisma via a separate service so + // its internal reads don't pollute the child's counts). + const parentService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + const parentResult = await parentService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // CHILD supplying BOTH parentRunId AND lockToVersion in one call. + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "child" }, + options: { + parentRunId: parentResult.run.friendlyId, + lockToVersion: workerRow.version, + }, + }, + }); + assertNonNullable(childResult); + + const parentRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: parentResult.run.id }, + }); + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + + // Child resolved the parent (single-table parent read). + expect(childRow.parentTaskRunId).toBe(parentRow.id); + expect(childRow.depth).toBe(parentRow.depth + 1); + + // Child locked to the worker (single-table worker read). + expect(childRow.lockedToVersionId).toBe(workerRow.id); + expect(childRow.taskVersion).toBe(workerRow.version); + + // Exactly one backgroundWorker.findFirst fired for the locked-worker read, + // and at least one taskRun.findFirst fired for the parent read. + expect(backgroundWorkerFindFirstCalls).toBe(1); + expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1); + + // NO-JOIN proof: no captured read carried an `include` joining + // taskRun <-> backgroundWorker. Every findFirst arg has include undefined. + for (const args of findFirstArgs) { + expect(args?.include).toBeUndefined(); + } + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts new file mode 100644 index 00000000000..af0b335bfae --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts @@ -0,0 +1,324 @@ +// Split from triggerTask.server.test.ts (locked-worker read concerns) so CI's +// duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService locked-worker reads", () => { + containerTest( + "resolves the locked background worker on the control-plane client with no cross-DB join", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); + + // Read the seeded worker row to get its real version/id. + const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: worker.id }, + }); + + // Counting proxy over the control-plane client. `this.prisma` is ALWAYS + // the control-plane client; the locked-worker lookup is a DIRECT + // backgroundWorker.findFirst on it. The parent read uses a DIFFERENT + // call (runStore.findRun โ†’ taskRun), so a single call() issues two + // separate single-table reads โ€” never one cross-seam join. Here we count + // the findFirst calls and capture their args to assert no include/join. + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + // The queue manager gets the real (unproxied) prisma so the counting + // proxy only observes reads issued by the service itself. + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: workerRow.version }, + }, + }); + assertNonNullable(result); + + // Observable proof the locked worker was resolved on the control-plane + // client: the created run records the worker id in lockedToVersionId. + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + expect(runRow.lockedToVersionId).toBe(workerRow.id); + expect(runRow.taskVersion).toBe(workerRow.version); + + // Exactly one backgroundWorker.findFirst fired for the locked-worker read. + expect(backgroundWorkerFindFirstCalls).toBe(1); + + // NO-JOIN assertion: the read referenced ONLY the backgroundWorker table. + // No `include` (which would join into another table); the `select` lists + // only backgroundWorker scalar columns. + const args = findFirstArgs[0]; + assertNonNullable(args); + expect(args.include).toBeUndefined(); + expect(Object.keys(args.select ?? {}).sort()).toEqual([ + "cliVersion", + "id", + "sdkVersion", + "version", + ]); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "lockToVersion matching no worker rejects the trigger after a single scalar-only worker read", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + let backgroundWorkerFindFirstCalls = 0; + const findFirstArgs: any[] = []; + const countingPrisma = new Proxy(prisma, { + get(target, prop, receiver) { + if (prop === "backgroundWorker") { + const delegate = Reflect.get(target, prop, receiver); + return new Proxy(delegate, { + get(bwTarget, bwProp, bwReceiver) { + if (bwProp === "findFirst") { + return async (args: any) => { + backgroundWorkerFindFirstCalls += 1; + findFirstArgs.push(args); + return (delegate as any).findFirst(args); + }; + } + const value = Reflect.get(bwTarget, bwProp, bwReceiver); + return typeof value === "function" ? value.bind(bwTarget) : value; + }, + }); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as typeof prisma; + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma: countingPrisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + const bogusVersion = "v-does-not-exist-0000"; + // The no-match worker read returns null; the queue concern then rejects + // the trigger rather than silently locking the run to a phantom version. + await expect( + triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: bogusVersion }, + }, + }) + ).rejects.toThrow(/no worker found with that version/); + + // No run was locked to the bogus version (none was created). + const lockedRuns = await prisma.taskRun.findMany({ + where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion }, + }); + expect(lockedRuns).toEqual([]); + + // The lone worker read fired exactly once with the scalar-only select and + // no cross-seam include. + expect(backgroundWorkerFindFirstCalls).toBe(1); + const args = findFirstArgs[0]; + assertNonNullable(args); + expect(args.include).toBeUndefined(); + expect(Object.keys(args.select ?? {}).sort()).toEqual([ + "cliVersion", + "id", + "sdkVersion", + "version", + ]); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "does not resolve a locked worker from a different environment", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + // Two independent authenticated environments. Rename envA's globally-unique + // fields before the second setup call to avoid unique-constraint collisions. + const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await prisma.organization.update({ + where: { id: envA.organizationId }, + data: { slug: `${envA.organization.slug}-a` }, + }); + await prisma.project.update({ + where: { id: envA.projectId }, + data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, + }); + await prisma.runtimeEnvironment.update({ + where: { id: envA.id }, + data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, + }); + await prisma.workerGroupToken.updateMany({ + where: { tokenHash: "token_hash" }, + data: { tokenHash: "token_hash_a" }, + }); + await prisma.workerInstanceGroup.updateMany({ + where: { masterQueue: "default" }, + data: { masterQueue: "default_a" }, + }); + const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + expect(envA.id).not.toBe(envB.id); + expect(envA.organizationId).not.toBe(envB.organizationId); + + const taskIdentifier = "test-task"; + const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier); + const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier); + + const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: workerA.id }, + }); + const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({ + where: { id: workerB.id }, + }); + // Both seeded workers share the same version string. + expect(workerARow.version).toBe(workerBRow.version); + expect(workerARow.id).not.toBe(workerBRow.id); + + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger in envB locking to the shared version string. + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envB, + body: { + payload: { kind: "locked" }, + options: { lockToVersion: workerBRow.version }, + }, + }); + assertNonNullable(result); + + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + // The projectId + runtimeEnvironmentId guard in the single-table worker + // read resolves envB's worker, never envA's same-version worker. + expect(runRow.lockedToVersionId).toBe(workerBRow.id); + expect(runRow.lockedToVersionId).not.toBe(workerARow.id); + expect(runRow.taskVersion).toBe(workerBRow.version); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts new file mode 100644 index 00000000000..7e8b3cd5310 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts @@ -0,0 +1,259 @@ +// Split from triggerTask.server.test.ts (parent-read concerns) so CI's +// duration-based sharding can balance the container-heavy tests. +import { describe, expect, vi } from "vitest"; + +// Mock the db prisma client. The service is constructed against a real +// testcontainer prisma instead โ€” these empty singletons only satisfy the +// module-level imports of the production wiring (infrastructure boundary). +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +// Inherited harness boilerplate. The parent read under test takes the +// findRun(where, client) overload with this.prisma, so it does not consult this +// flag; the mock only satisfies other wiring imported transitively. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); + +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout + +describe("RunEngineTriggerTaskService parent reads", () => { + containerTest( + "resolves the parent run through the run-ops store by minted run id", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger a ROOT run first to create a real parent TaskRun. + const parentResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // Trigger a CHILD pointing at the parent's friendlyId. The service must + // resolve the parent via runStore.findRun (minted RunId, env-scoped). + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { + payload: { kind: "child" }, + options: { parentRunId: parentResult.run.friendlyId }, + }, + }); + assertNonNullable(childResult); + + // The capturing validator observed the resolved parent โ€” proving the + // read ran (against the container DB) and returned the right row. + expect(validator.capturedParentRun).not.toBe("unset"); + const capturedParent = validator.capturedParentRun; + assertNonNullable(capturedParent); + expect(capturedParent.id).toBe(parentResult.run.id); + expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId); + + // depth and root carry through โ€” proving parentRun.depth and the parent + // id were read off the resolved row and threaded into the child. + const parentRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: parentResult.run.id }, + }); + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + + expect(childRow.depth).toBe(parentRow.depth + 1); + expect(childRow.parentTaskRunId).toBe(parentRow.id); + expect(childRow.rootTaskRunId).toBe(parentRow.id); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "scopes the parent lookup to the run's environment (cross-env parent is not resolved)", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + // Two independent authenticated environments. The setup helper hardcodes + // several globally-unique fields (org/project slug, env apiKey/pkApiKey, + // worker-group token hash), so rename envA's before the second call to + // avoid unique-constraint collisions. + const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + await prisma.organization.update({ + where: { id: envA.organizationId }, + data: { slug: `${envA.organization.slug}-a` }, + }); + await prisma.project.update({ + where: { id: envA.projectId }, + data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, + }); + await prisma.runtimeEnvironment.update({ + where: { id: envA.id }, + data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, + }); + await prisma.workerGroupToken.updateMany({ + where: { tokenHash: "token_hash" }, + data: { tokenHash: "token_hash_a" }, + }); + await prisma.workerInstanceGroup.updateMany({ + where: { masterQueue: "default" }, + data: { masterQueue: "default_a" }, + }); + const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + expect(envA.id).not.toBe(envB.id); + expect(envA.organizationId).not.toBe(envB.organizationId); + + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, envA, taskIdentifier); + await setupBackgroundWorker(engine, envB, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // A real parent run in envA. + const parentResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envA, + body: { payload: { kind: "parent" } }, + }); + assertNonNullable(parentResult); + + // Trigger a child in envB pointing at the envA parent's friendlyId. The + // env guard in runStore.findRun's `where` rejects the cross-env parent + // in a single query, so the resolved parentRun is null. + const childResult = await triggerTaskService.call({ + taskId: taskIdentifier, + environment: envB, + body: { + payload: { kind: "child" }, + options: { parentRunId: parentResult.run.friendlyId }, + }, + }); + assertNonNullable(childResult); + + // validateParentRun was called with no resolved parent. + expect(validator.capturedParentRun).not.toBe("unset"); + expect(validator.capturedParentRun ?? null).toBeNull(); + + // The child still triggered, at the root depth with no parent linkage โ€” + // confirming the cross-env parent was dropped, not silently joined. + const childRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: childResult.run.id }, + }); + expect(childRow.depth).toBe(0); + expect(childRow.parentTaskRunId).toBeNull(); + } finally { + await engine.quit(); + } + } + ); + + containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const validator = new CapturingParentRunValidator(); + const triggerTaskService = new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern( + prisma, + engine, + new MockTraceEventConcern() + ), + validator, + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); + + // Trigger with NO parentRunId. + const result = await triggerTaskService.call({ + taskId: taskIdentifier, + environment, + body: { payload: { kind: "root" } }, + }); + assertNonNullable(result); + + // The validator ran but received no resolved parent: the parent read was + // skipped because no parentRunId was supplied. + expect(validator.capturedParentRun).not.toBe("unset"); + expect(validator.capturedParentRun).toBeUndefined(); + + const runRow = await prisma.taskRun.findUniqueOrThrow({ + where: { id: result.run.id }, + }); + expect(runRow.depth).toBe(0); + expect(runRow.parentTaskRunId).toBeNull(); + } finally { + await engine.quit(); + } + }); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts b/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts new file mode 100644 index 00000000000..fed87481c06 --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.test.helpers.ts @@ -0,0 +1,156 @@ +// Shared harness for the triggerTask.server.*.test.ts family. The suite is split +// across per-concern files (parentReads / lockedWorker / combinedReads) so CI's +// duration-based sharding can spread the container-heavy tests over shards; these +// helpers hold the pure mocks and engine wiring they all share. +// +// NOTE: this is not a test file (vitest's include only matches *.test.ts) and it +// must not import from "vitest" โ€” vi.mock declarations live in each test file +// because vitest hoists them per test module. +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import type { IOPacket } from "@trigger.dev/core/v3"; +import type { TaskRun } from "@trigger.dev/database"; +import type { + EntitlementValidationParams, + MaxAttemptsValidationParams, + ParentRunValidationParams, + PayloadProcessor, + TagValidationParams, + TracedEventSpan, + TraceEventConcern, + TriggerTaskRequest, + TriggerTaskValidator, + ValidationResult, +} from "~/runEngine/types"; + +export class MockPayloadProcessor implements PayloadProcessor { + async process(request: TriggerTaskRequest): Promise { + return { + data: JSON.stringify(request.body.payload), + dataType: "application/json", + }; + } +} + +// Captures the `parentRun` the service resolved (via runStore.findRun) and +// passed into validation, so a test can assert on the resolved parent without +// mocking the read itself. Returns ok so the child triggers regardless. +export class CapturingParentRunValidator implements TriggerTaskValidator { + public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset"; + + validateTags(_params: TagValidationParams): ValidationResult { + return { ok: true }; + } + validateEntitlement(_params: EntitlementValidationParams): Promise { + return Promise.resolve({ ok: true }); + } + validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult { + return { ok: true }; + } + validateParentRun(params: ParentRunValidationParams): ValidationResult { + this.capturedParentRun = params.parentRun; + return { ok: true }; + } +} + +export class MockTraceEventConcern implements TraceEventConcern { + async traceRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } + + async traceIdempotentRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + _options: { + existingRun: TaskRun; + idempotencyKey: string; + incomplete: boolean; + isError: boolean; + }, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } + + async traceDebouncedRun( + _request: TriggerTaskRequest, + _parentStore: string | undefined, + _options: { + existingRun: TaskRun; + debounceKey: string; + incomplete: boolean; + isError: boolean; + }, + callback: (span: TracedEventSpan, store: string) => Promise + ): Promise { + return await callback( + { + traceId: "test", + spanId: "test", + traceContext: {}, + traceparent: undefined, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + }, + "test" + ); + } +} + +export function buildEngine(prisma: any, redisOptions: any) { + return new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.test.ts deleted file mode 100644 index 31c624a3864..00000000000 --- a/apps/webapp/app/runEngine/services/triggerTask.server.test.ts +++ /dev/null @@ -1,825 +0,0 @@ -import { describe, expect, vi } from "vitest"; - -// Mock the db prisma client. The service is constructed against a real -// testcontainer prisma instead โ€” these empty singletons only satisfy the -// module-level imports of the production wiring (infrastructure boundary). -vi.mock("~/db.server", () => ({ - prisma: {}, - $replica: {}, - runOpsNewPrisma: {}, - runOpsLegacyPrisma: {}, - runOpsNewReplica: {}, - runOpsLegacyReplica: {}, -})); -// Inherited harness boilerplate. The parent read under test takes the -// findRun(where, client) overload with this.prisma, so it does not consult this -// flag; the mock only satisfies other wiring imported transitively. -vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); - -vi.mock("~/services/platform.v3.server", async (importOriginal) => { - const actual = (await importOriginal()) as Record; - return { - ...actual, - getEntitlement: vi.fn(), - }; -}); - -import { RunEngine } from "@internal/run-engine"; -import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; -import { assertNonNullable, containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; -import type { IOPacket } from "@trigger.dev/core/v3"; -import type { TaskRun } from "@trigger.dev/database"; -import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; -import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; -import type { - EntitlementValidationParams, - MaxAttemptsValidationParams, - ParentRunValidationParams, - PayloadProcessor, - TagValidationParams, - TracedEventSpan, - TraceEventConcern, - TriggerTaskRequest, - TriggerTaskValidator, - ValidationResult, -} from "~/runEngine/types"; -import { RunEngineTriggerTaskService } from "./triggerTask.server"; - -vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout - -class MockPayloadProcessor implements PayloadProcessor { - async process(request: TriggerTaskRequest): Promise { - return { - data: JSON.stringify(request.body.payload), - dataType: "application/json", - }; - } -} - -// Captures the `parentRun` the service resolved (via runStore.findRun) and -// passed into validation, so a test can assert on the resolved parent without -// mocking the read itself. Returns ok so the child triggers regardless. -class CapturingParentRunValidator implements TriggerTaskValidator { - public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset"; - - validateTags(_params: TagValidationParams): ValidationResult { - return { ok: true }; - } - validateEntitlement(_params: EntitlementValidationParams): Promise { - return Promise.resolve({ ok: true }); - } - validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult { - return { ok: true }; - } - validateParentRun(params: ParentRunValidationParams): ValidationResult { - this.capturedParentRun = params.parentRun; - return { ok: true }; - } -} - -class MockTraceEventConcern implements TraceEventConcern { - async traceRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } - - async traceIdempotentRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - _options: { - existingRun: TaskRun; - idempotencyKey: string; - incomplete: boolean; - isError: boolean; - }, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } - - async traceDebouncedRun( - _request: TriggerTaskRequest, - _parentStore: string | undefined, - _options: { - existingRun: TaskRun; - debounceKey: string; - incomplete: boolean; - isError: boolean; - }, - callback: (span: TracedEventSpan, store: string) => Promise - ): Promise { - return await callback( - { - traceId: "test", - spanId: "test", - traceContext: {}, - traceparent: undefined, - setAttribute: () => {}, - failWithError: () => {}, - stop: () => {}, - }, - "test" - ); - } -} - -function buildEngine(prisma: any, redisOptions: any) { - return new RunEngine({ - prisma, - worker: { - redis: redisOptions, - workers: 1, - tasksPerWorker: 10, - pollIntervalMs: 100, - }, - queue: { - redis: redisOptions, - }, - runLock: { - redis: redisOptions, - }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { - name: "small-1x" as const, - cpu: 0.5, - memory: 0.5, - centsPerMs: 0.0001, - }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); -} - -describe("RunEngineTriggerTaskService parent + locked-worker reads", () => { - containerTest( - "resolves the parent run through the run-ops store by minted run id", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger a ROOT run first to create a real parent TaskRun. - const parentResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // Trigger a CHILD pointing at the parent's friendlyId. The service must - // resolve the parent via runStore.findRun (minted RunId, env-scoped). - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "child" }, - options: { parentRunId: parentResult.run.friendlyId }, - }, - }); - assertNonNullable(childResult); - - // The capturing validator observed the resolved parent โ€” proving the - // read ran (against the container DB) and returned the right row. - expect(validator.capturedParentRun).not.toBe("unset"); - const capturedParent = validator.capturedParentRun; - assertNonNullable(capturedParent); - expect(capturedParent.id).toBe(parentResult.run.id); - expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId); - - // depth and root carry through โ€” proving parentRun.depth and the parent - // id were read off the resolved row and threaded into the child. - const parentRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: parentResult.run.id }, - }); - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - - expect(childRow.depth).toBe(parentRow.depth + 1); - expect(childRow.parentTaskRunId).toBe(parentRow.id); - expect(childRow.rootTaskRunId).toBe(parentRow.id); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "scopes the parent lookup to the run's environment (cross-env parent is not resolved)", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - // Two independent authenticated environments. The setup helper hardcodes - // several globally-unique fields (org/project slug, env apiKey/pkApiKey, - // worker-group token hash), so rename envA's before the second call to - // avoid unique-constraint collisions. - const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await prisma.organization.update({ - where: { id: envA.organizationId }, - data: { slug: `${envA.organization.slug}-a` }, - }); - await prisma.project.update({ - where: { id: envA.projectId }, - data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, - }); - await prisma.runtimeEnvironment.update({ - where: { id: envA.id }, - data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, - }); - await prisma.workerGroupToken.updateMany({ - where: { tokenHash: "token_hash" }, - data: { tokenHash: "token_hash_a" }, - }); - await prisma.workerInstanceGroup.updateMany({ - where: { masterQueue: "default" }, - data: { masterQueue: "default_a" }, - }); - const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - expect(envA.id).not.toBe(envB.id); - expect(envA.organizationId).not.toBe(envB.organizationId); - - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, envA, taskIdentifier); - await setupBackgroundWorker(engine, envB, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // A real parent run in envA. - const parentResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envA, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // Trigger a child in envB pointing at the envA parent's friendlyId. The - // env guard in runStore.findRun's `where` rejects the cross-env parent - // in a single query, so the resolved parentRun is null. - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envB, - body: { - payload: { kind: "child" }, - options: { parentRunId: parentResult.run.friendlyId }, - }, - }); - assertNonNullable(childResult); - - // validateParentRun was called with no resolved parent. - expect(validator.capturedParentRun).not.toBe("unset"); - expect(validator.capturedParentRun ?? null).toBeNull(); - - // The child still triggered, at the root depth with no parent linkage โ€” - // confirming the cross-env parent was dropped, not silently joined. - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - expect(childRow.depth).toBe(0); - expect(childRow.parentTaskRunId).toBeNull(); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "resolves the locked background worker on the control-plane client with no cross-DB join", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); - - // Read the seeded worker row to get its real version/id. - const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: worker.id }, - }); - - // Counting proxy over the control-plane client. `this.prisma` is ALWAYS - // the control-plane client; the locked-worker lookup is a DIRECT - // backgroundWorker.findFirst on it. The parent read uses a DIFFERENT - // call (runStore.findRun โ†’ taskRun), so a single call() issues two - // separate single-table reads โ€” never one cross-seam join. Here we count - // the findFirst calls and capture their args to assert no include/join. - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - // The queue manager gets the real (unproxied) prisma so the counting - // proxy only observes reads issued by the service itself. - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: workerRow.version }, - }, - }); - assertNonNullable(result); - - // Observable proof the locked worker was resolved on the control-plane - // client: the created run records the worker id in lockedToVersionId. - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - expect(runRow.lockedToVersionId).toBe(workerRow.id); - expect(runRow.taskVersion).toBe(workerRow.version); - - // Exactly one backgroundWorker.findFirst fired for the locked-worker read. - expect(backgroundWorkerFindFirstCalls).toBe(1); - - // NO-JOIN assertion: the read referenced ONLY the backgroundWorker table. - // No `include` (which would join into another table); the `select` lists - // only backgroundWorker scalar columns. - const args = findFirstArgs[0]; - assertNonNullable(args); - expect(args.include).toBeUndefined(); - expect(Object.keys(args.select ?? {}).sort()).toEqual([ - "cliVersion", - "id", - "sdkVersion", - "version", - ]); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "issues two independent single-table reads when one call supplies both parentRunId and lockToVersion", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier); - - const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: worker.id }, - }); - - // Count BOTH reads issued by the service on the control-plane client: - // the parent read (runStore.findRun โ†’ taskRun.findFirst) and the - // locked-worker read (backgroundWorker.findFirst). Capture every - // findFirst arg so we can assert no read carries a cross-seam include. - let taskRunFindFirstCalls = 0; - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - if (prop === "taskRun") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(trTarget, trProp, trReceiver) { - if (trProp === "findFirst") { - return async (args: any) => { - taskRunFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(trTarget, trProp, trReceiver); - return typeof value === "function" ? value.bind(trTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - // queueConcern/idempotency get the real unproxied prisma so the - // counting proxy only observes reads issued by the service itself. - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // ROOT parent first (uses the unproxied prisma via a separate service so - // its internal reads don't pollute the child's counts). - const parentService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - const parentResult = await parentService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "parent" } }, - }); - assertNonNullable(parentResult); - - // CHILD supplying BOTH parentRunId AND lockToVersion in one call. - const childResult = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "child" }, - options: { - parentRunId: parentResult.run.friendlyId, - lockToVersion: workerRow.version, - }, - }, - }); - assertNonNullable(childResult); - - const parentRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: parentResult.run.id }, - }); - const childRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: childResult.run.id }, - }); - - // Child resolved the parent (single-table parent read). - expect(childRow.parentTaskRunId).toBe(parentRow.id); - expect(childRow.depth).toBe(parentRow.depth + 1); - - // Child locked to the worker (single-table worker read). - expect(childRow.lockedToVersionId).toBe(workerRow.id); - expect(childRow.taskVersion).toBe(workerRow.version); - - // Exactly one backgroundWorker.findFirst fired for the locked-worker read, - // and at least one taskRun.findFirst fired for the parent read. - expect(backgroundWorkerFindFirstCalls).toBe(1); - expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1); - - // NO-JOIN proof: no captured read carried an `include` joining - // taskRun <-> backgroundWorker. Every findFirst arg has include undefined. - for (const args of findFirstArgs) { - expect(args?.include).toBeUndefined(); - } - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "lockToVersion matching no worker rejects the trigger after a single scalar-only worker read", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - let backgroundWorkerFindFirstCalls = 0; - const findFirstArgs: any[] = []; - const countingPrisma = new Proxy(prisma, { - get(target, prop, receiver) { - if (prop === "backgroundWorker") { - const delegate = Reflect.get(target, prop, receiver); - return new Proxy(delegate, { - get(bwTarget, bwProp, bwReceiver) { - if (bwProp === "findFirst") { - return async (args: any) => { - backgroundWorkerFindFirstCalls += 1; - findFirstArgs.push(args); - return (delegate as any).findFirst(args); - }; - } - const value = Reflect.get(bwTarget, bwProp, bwReceiver); - return typeof value === "function" ? value.bind(bwTarget) : value; - }, - }); - } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; - }, - }) as typeof prisma; - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma: countingPrisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - const bogusVersion = "v-does-not-exist-0000"; - // The no-match worker read returns null; the queue concern then rejects - // the trigger rather than silently locking the run to a phantom version. - await expect( - triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: bogusVersion }, - }, - }) - ).rejects.toThrow(/no worker found with that version/); - - // No run was locked to the bogus version (none was created). - const lockedRuns = await prisma.taskRun.findMany({ - where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion }, - }); - expect(lockedRuns).toEqual([]); - - // The lone worker read fired exactly once with the scalar-only select and - // no cross-seam include. - expect(backgroundWorkerFindFirstCalls).toBe(1); - const args = findFirstArgs[0]; - assertNonNullable(args); - expect(args.include).toBeUndefined(); - expect(Object.keys(args.select ?? {}).sort()).toEqual([ - "cliVersion", - "id", - "sdkVersion", - "version", - ]); - } finally { - await engine.quit(); - } - } - ); - - containerTest( - "does not resolve a locked worker from a different environment", - async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - // Two independent authenticated environments. Rename envA's globally-unique - // fields before the second setup call to avoid unique-constraint collisions. - const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - await prisma.organization.update({ - where: { id: envA.organizationId }, - data: { slug: `${envA.organization.slug}-a` }, - }); - await prisma.project.update({ - where: { id: envA.projectId }, - data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` }, - }); - await prisma.runtimeEnvironment.update({ - where: { id: envA.id }, - data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` }, - }); - await prisma.workerGroupToken.updateMany({ - where: { tokenHash: "token_hash" }, - data: { tokenHash: "token_hash_a" }, - }); - await prisma.workerInstanceGroup.updateMany({ - where: { masterQueue: "default" }, - data: { masterQueue: "default_a" }, - }); - const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - expect(envA.id).not.toBe(envB.id); - expect(envA.organizationId).not.toBe(envB.organizationId); - - const taskIdentifier = "test-task"; - const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier); - const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier); - - const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: workerA.id }, - }); - const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({ - where: { id: workerB.id }, - }); - // Both seeded workers share the same version string. - expect(workerARow.version).toBe(workerBRow.version); - expect(workerARow.id).not.toBe(workerBRow.id); - - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator: new CapturingParentRunValidator(), - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger in envB locking to the shared version string. - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment: envB, - body: { - payload: { kind: "locked" }, - options: { lockToVersion: workerBRow.version }, - }, - }); - assertNonNullable(result); - - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - // The projectId + runtimeEnvironmentId guard in the single-table worker - // read resolves envB's worker, never envA's same-version worker. - expect(runRow.lockedToVersionId).toBe(workerBRow.id); - expect(runRow.lockedToVersionId).not.toBe(workerARow.id); - expect(runRow.taskVersion).toBe(workerBRow.version); - } finally { - await engine.quit(); - } - } - ); - - containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => { - const engine = buildEngine(prisma, redisOptions); - - try { - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "test-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const validator = new CapturingParentRunValidator(); - const triggerTaskService = new RunEngineTriggerTaskService({ - engine, - prisma, - payloadProcessor: new MockPayloadProcessor(), - queueConcern: new DefaultQueueManager(prisma, engine), - idempotencyKeyConcern: new IdempotencyKeyConcern( - prisma, - engine, - new MockTraceEventConcern() - ), - validator, - traceEventConcern: new MockTraceEventConcern(), - tracer: trace.getTracer("test", "0.0.0"), - metadataMaximumSize: 1024 * 1024 * 1, - }); - - // Trigger with NO parentRunId. - const result = await triggerTaskService.call({ - taskId: taskIdentifier, - environment, - body: { payload: { kind: "root" } }, - }); - assertNonNullable(result); - - // The validator ran but received no resolved parent: the parent read was - // skipped because no parentRunId was supplied. - expect(validator.capturedParentRun).not.toBe("unset"); - expect(validator.capturedParentRun).toBeUndefined(); - - const runRow = await prisma.taskRun.findUniqueOrThrow({ - where: { id: result.run.id }, - }); - expect(runRow.depth).toBe(0); - expect(runRow.parentTaskRunId).toBeNull(); - } finally { - await engine.quit(); - } - }); -}); diff --git a/apps/webapp/test/engine/triggerFailedTask.test.ts b/apps/webapp/test/engine/triggerFailedTask.call.test.ts similarity index 50% rename from apps/webapp/test/engine/triggerFailedTask.test.ts rename to apps/webapp/test/engine/triggerFailedTask.call.test.ts index 498312956aa..94b95d18786 100644 --- a/apps/webapp/test/engine/triggerFailedTask.test.ts +++ b/apps/webapp/test/engine/triggerFailedTask.call.test.ts @@ -1,59 +1,15 @@ +// Split from triggerFailedTask.test.ts (the call() path) so CI's +// duration-based sharding can balance the container-heavy tests. import { describe, expect } from "vitest"; -import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; import { containerTest } from "@internal/testcontainers"; -import { trace } from "@opentelemetry/api"; import { RunId, classifyKind, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; -import { TriggerFailedTaskService } from "../../app/runEngine/services/triggerFailedTask.server"; -import { EventRepository } from "../../app/v3/eventRepository/eventRepository.server"; +import { makeEngine, makeService } from "./triggerFailedTaskTestHelpers"; vi.setConfig?.({ testTimeout: 60_000 }); -// Bind the service's trace-event writes to the testcontainer DB. Without this, -// call() resolves the repository via getEventRepository โ†’ global prisma, which -// points at a database that doesn't exist in CI. -function makeService(prisma: any, engine: RunEngine) { - return new TriggerFailedTaskService({ - prisma, - engine, - // Read the parent through the same store the engine wrote it to. - runStore: engine.runStore, - eventRepository: { - repository: new EventRepository(prisma, prisma, { - batchSize: 100, - batchInterval: 1000, - retentionInDays: 30, - partitioningEnabled: false, - }), - store: "taskEvent", - }, - }); -} - -function makeEngine(prisma: any, redisOptions: any) { - return new RunEngine({ - prisma, - worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, - queue: { redis: redisOptions }, - runLock: { redis: redisOptions }, - machines: { - defaultMachine: "small-1x", - machines: { - "small-1x": { - name: "small-1x" as const, - cpu: 0.5, - memory: 0.5, - centsPerMs: 0.0001, - }, - }, - baseCostInCents: 0.0005, - }, - tracer: trace.getTracer("test", "0.0.0"), - }); -} - -describe("TriggerFailedTaskService โ€” failed run residency", () => { +describe("TriggerFailedTaskService โ€” failed run residency (call)", () => { containerTest( "root failed run mints cuid when split is off (call)", async ({ prisma, redisOptions }) => { @@ -180,85 +136,4 @@ describe("TriggerFailedTaskService โ€” failed run residency", () => { await engine.quit(); } ); - - containerTest( - "failed child of a NEW parent mints run-ops id (callWithoutTraceEvents)", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); - await engine.trigger( - { - friendlyId: parentFriendlyId, - environment, - taskIdentifier, - payload: "{}", - payloadType: "application/json", - traceId: "00000000000000000000000000000000", - spanId: "0000000000000000", - workerQueue: "main", - queue: `task/${taskIdentifier}`, - isTest: false, - tags: [], - } as any, - prisma - ); - - const service = makeService(prisma, engine); - - const friendlyId = await service.callWithoutTraceEvents({ - environmentId: environment.id, - environmentType: environment.type, - projectId: environment.projectId, - organizationId: environment.organizationId, - taskId: taskIdentifier, - payload: { test: "child" }, - errorMessage: "boom", - parentRunId: parentFriendlyId, - }); - - expect(classifyKind(friendlyId!)).toBe("runOpsId"); - - await engine.quit(); - } - ); - - containerTest( - "callWithoutTraceEvents returns null (best-effort) when the derived parent row is absent", - async ({ prisma, redisOptions }) => { - const engine = makeEngine(prisma, redisOptions); - const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); - const taskIdentifier = "failed-residency-task"; - await setupBackgroundWorker(engine, environment, taskIdentifier); - - const service = makeService(prisma, engine); - - // A well-formed run-ops parent friendlyId that was NEVER triggered โ†’ no row. - // Exercises the missing-parent fallback in callWithoutTraceEvents. - const absentParentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); - - const friendlyId = await service.callWithoutTraceEvents({ - environmentId: environment.id, - environmentType: environment.type, - projectId: environment.projectId, - organizationId: environment.organizationId, - taskId: taskIdentifier, - payload: { test: "absent-parent" }, - errorMessage: "boom", - parentRunId: absentParentFriendlyId, - }); - - // Fallback derives parentTaskRunId from an id with no row; the parentTaskRunId FK rejects the create, so the method returns null instead of throwing. - expect(friendlyId).toBeNull(); - const orphan = await prisma.taskRun.findFirst({ - where: { parentTaskRunId: RunId.fromFriendlyId(absentParentFriendlyId) }, - }); - expect(orphan).toBeNull(); - - await engine.quit(); - } - ); }); diff --git a/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts new file mode 100644 index 00000000000..a0be900fb82 --- /dev/null +++ b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts @@ -0,0 +1,93 @@ +// Split from triggerFailedTask.test.ts (the callWithoutTraceEvents() path) so +// CI's duration-based sharding can balance the container-heavy tests. +import { describe, expect } from "vitest"; + +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; +import { containerTest } from "@internal/testcontainers"; +import { RunId, classifyKind, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { makeEngine, makeService } from "./triggerFailedTaskTestHelpers"; + +vi.setConfig?.({ testTimeout: 60_000 }); + +describe("TriggerFailedTaskService โ€” failed run residency (callWithoutTraceEvents)", () => { + containerTest( + "failed child of a NEW parent mints run-ops id (callWithoutTraceEvents)", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + await engine.trigger( + { + friendlyId: parentFriendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], + } as any, + prisma + ); + + const service = makeService(prisma, engine); + + const friendlyId = await service.callWithoutTraceEvents({ + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.projectId, + organizationId: environment.organizationId, + taskId: taskIdentifier, + payload: { test: "child" }, + errorMessage: "boom", + parentRunId: parentFriendlyId, + }); + + expect(classifyKind(friendlyId!)).toBe("runOpsId"); + + await engine.quit(); + } + ); + + containerTest( + "callWithoutTraceEvents returns null (best-effort) when the derived parent row is absent", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const service = makeService(prisma, engine); + + // A well-formed run-ops parent friendlyId that was NEVER triggered โ†’ no row. + // Exercises the missing-parent fallback in callWithoutTraceEvents. + const absentParentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + + const friendlyId = await service.callWithoutTraceEvents({ + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.projectId, + organizationId: environment.organizationId, + taskId: taskIdentifier, + payload: { test: "absent-parent" }, + errorMessage: "boom", + parentRunId: absentParentFriendlyId, + }); + + // Fallback derives parentTaskRunId from an id with no row; the parentTaskRunId FK rejects the create, so the method returns null instead of throwing. + expect(friendlyId).toBeNull(); + const orphan = await prisma.taskRun.findFirst({ + where: { parentTaskRunId: RunId.fromFriendlyId(absentParentFriendlyId) }, + }); + expect(orphan).toBeNull(); + + await engine.quit(); + } + ); +}); diff --git a/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts b/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts new file mode 100644 index 00000000000..38b7c072abf --- /dev/null +++ b/apps/webapp/test/engine/triggerFailedTaskTestHelpers.ts @@ -0,0 +1,50 @@ +// Shared harness for the triggerFailedTask.*.test.ts family, split from the +// original single test file so CI's duration-based sharding can balance the +// container-heavy tests. Not a test file (vitest's include only matches *.test.ts). +import { RunEngine } from "@internal/run-engine"; +import { trace } from "@opentelemetry/api"; +import { TriggerFailedTaskService } from "../../app/runEngine/services/triggerFailedTask.server"; +import { EventRepository } from "../../app/v3/eventRepository/eventRepository.server"; + +// Bind the service's trace-event writes to the testcontainer DB. Without this, +// call() resolves the repository via getEventRepository โ†’ global prisma, which +// points at a database that doesn't exist in CI. +export function makeService(prisma: any, engine: RunEngine) { + return new TriggerFailedTaskService({ + prisma, + engine, + // Read the parent through the same store the engine wrote it to. + runStore: engine.runStore, + eventRepository: { + repository: new EventRepository(prisma, prisma, { + batchSize: 100, + batchInterval: 1000, + retentionInDays: 30, + partitioningEnabled: false, + }), + store: "taskEvent", + }, + }); +} + +export function makeEngine(prisma: any, redisOptions: any) { + return new RunEngine({ + prisma, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { redis: redisOptions }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} diff --git a/test-timings.json b/test-timings.json index 9f6cdfbbe77..33fa9c5f32b 100644 --- a/test-timings.json +++ b/test-timings.json @@ -1,227 +1,483 @@ { - "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 10249, - "apps/webapp/test/GCRARateLimiter.test.ts": 4984, - "apps/webapp/test/authorizationRateLimitMiddleware.test.ts": 1, - "apps/webapp/test/bufferedTriggerPayload.test.ts": 3, - "apps/webapp/test/calculateNextSchedule.test.ts": 345, - "apps/webapp/test/chat-snapshot-integration.test.ts": 2326, - "apps/webapp/test/clickhouseFactory.test.ts": 13885, - "apps/webapp/test/concurrentFlushScheduler.test.ts": 361, - "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 17889, - "apps/webapp/test/detectbadJsonStrings.test.ts": 98, - "apps/webapp/test/environmentVariableDeduplication.test.ts": 3, - "apps/webapp/test/environmentVariableRules.test.ts": 3, - "apps/webapp/test/environmentVariablesEnvironments.test.ts": 10355, - "apps/webapp/test/environmentVariablesRepository.test.ts": 18320, - "apps/webapp/test/errorFingerprinting.test.ts": 16, - "apps/webapp/test/errorGroupWebhook.test.ts": 7, - "apps/webapp/test/fairDequeuingStrategy.test.ts": 5705, - "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 17442, - "apps/webapp/test/getDeploymentImageRef.test.ts": 7, - "apps/webapp/test/httpErrors.test.ts": 25, - "apps/webapp/test/marqsKeyProducer.test.ts": 7, - "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 6, - "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 497, - "apps/webapp/test/mollifierClaimResolution.test.ts": 6, - "apps/webapp/test/mollifierDrainerHandler.test.ts": 21, - "apps/webapp/test/mollifierDrainerWorker.test.ts": 5, - "apps/webapp/test/mollifierDrainingGauge.test.ts": 449, - "apps/webapp/test/mollifierGate.test.ts": 16, - "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 13, - "apps/webapp/test/mollifierMollify.test.ts": 5, - "apps/webapp/test/mollifierMutateWithFallback.test.ts": 18, - "apps/webapp/test/mollifierReadFallback.test.ts": 15, - "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 2, - "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 9, - "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 7, - "apps/webapp/test/mollifierStaleSweep.test.ts": 969, - "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 5, - "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 5, - "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 805, - "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 2, - "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 4, - "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 8, - "apps/webapp/test/mollifierSyntheticTrace.test.ts": 6, - "apps/webapp/test/mollifierTripEvaluator.test.ts": 621, - "apps/webapp/test/objectStore.test.ts": 15979, - "apps/webapp/test/organizationDataStoresRegistry.test.ts": 16732, - "apps/webapp/test/otlpExporter.test.ts": 13, - "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 6540, - "apps/webapp/test/realtimeClient.test.ts": 1, - "apps/webapp/test/redisRealtimeStreams.test.ts": 6214, - "apps/webapp/test/registryConfig.test.ts": 652, - "apps/webapp/test/replay-after-crash.test.ts": 2233, - "apps/webapp/test/runsBackfiller.test.ts": 15478, - "apps/webapp/test/runsReplicationBenchmark.test.ts": 0, - "apps/webapp/test/runsRepository.part1.test.ts": 53000, - "apps/webapp/test/runsRepository.part2.test.ts": 57000, - "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 8, - "apps/webapp/test/sentryTenantContext.test.ts": 5, - "apps/webapp/test/sentryTraceContext.server.test.ts": 12, - "apps/webapp/test/sessionDuration.test.ts": 18416, - "apps/webapp/test/sessionsReplicationService.test.ts": 30000, - "apps/webapp/test/shouldRevalidateRunsList.test.ts": 5, - "apps/webapp/test/slackErrorAlerts.test.ts": 0, - "apps/webapp/test/tenantContext.test.ts": 26, - "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 2, - "apps/webapp/test/tenantContextResolver.test.ts": 19, - "apps/webapp/test/timeGranularity.test.ts": 3, - "apps/webapp/test/timelineSpanEvents.test.ts": 6, - "apps/webapp/test/updateMetadata.test.ts": 26380, - "apps/webapp/test/validateGitBranchName.test.ts": 7, - "apps/webapp/test/vercelUrls.test.ts": 3, - "apps/webapp/test/webhookErrorAlerts.test.ts": 5, - "apps/webapp/test/workerQueueSplit.test.ts": 3, - "apps/webapp/test/components/DateTime.test.ts": 24, - "apps/webapp/test/engine/batchPayloads.test.ts": 5018, - "apps/webapp/test/engine/streamBatchItems.test.ts": 45000, - "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 13152, - "apps/webapp/test/engine/triggerTask.debounce.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.residency.test.ts": 5501, - "apps/webapp/test/engine/triggerTask.test.ts": 4125, - "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 3, - "apps/webapp/test/services/organizationAccessToken.test.ts": 9, - "apps/webapp/test/services/personalAccessToken.test.ts": 8, - "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 10, - "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 237, - "apps/webapp/test/components/runs/v3/RunTag.test.ts": 5, - "packages/trigger-sdk/test/chat-snapshot.test.ts": 22, - "packages/trigger-sdk/test/chatHandover.test.ts": 1658, - "packages/trigger-sdk/test/merge-by-id.test.ts": 9, - "packages/trigger-sdk/test/mockChatAgent.test.ts": 2254, - "packages/trigger-sdk/test/recovery-boot.test.ts": 671, - "packages/trigger-sdk/test/replay-session-in.test.ts": 12, - "packages/trigger-sdk/test/replay-session-out.test.ts": 40, - "packages/trigger-sdk/test/skill.test.ts": 16, - "packages/trigger-sdk/test/skillsRuntime.test.ts": 131, - "packages/trigger-sdk/test/wire-shape.test.ts": 15, - "packages/trigger-sdk/src/v3/chat-server.test.ts": 185, - "packages/trigger-sdk/src/v3/chat-tab-coordinator.test.ts": 14, - "packages/trigger-sdk/src/v3/chat.test.ts": 77, - "packages/trigger-sdk/src/v3/createStartSessionAction.test.ts": 5, - "packages/trigger-sdk/src/v3/sessions.test.ts": 31, - "packages/trigger-sdk/src/v3/shared.test.ts": 68, - "packages/trigger-sdk/src/v3/streams.test.ts": 6, - "packages/trigger-sdk/src/v3/triggerClient.test.ts": 66, - "packages/trigger-sdk/src/v3/triggerClient.types.test.ts": 11, - "packages/redis-worker/src/cron.test.ts": 27371, - "packages/redis-worker/src/queue.test.ts": 3435, - "packages/redis-worker/src/worker.test.ts": 32870, - "packages/redis-worker/src/mollifier/buffer.test.ts": 3091, - "packages/redis-worker/src/mollifier/drainer.test.ts": 8403, - "packages/redis-worker/src/fair-queue/tests/concurrency.test.ts": 1341, - "packages/redis-worker/src/fair-queue/tests/drr.test.ts": 1203, - "packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts": 7728, - "packages/redis-worker/src/fair-queue/tests/raceConditions.test.ts": 14961, - "packages/redis-worker/src/fair-queue/tests/retry.test.ts": 10, - "packages/redis-worker/src/fair-queue/tests/tenantDispatch.test.ts": 5769, - "packages/redis-worker/src/fair-queue/tests/visibility.test.ts": 3783, - "packages/redis-worker/src/fair-queue/tests/workerQueue.test.ts": 1116, - "packages/core/test/duration.test.ts": 42, - "packages/core/test/errors.test.ts": 51, - "packages/core/test/eventFilterMatches.test.ts": 38, - "packages/core/test/externalSpanExporterWrapper.test.ts": 13, - "packages/core/test/flattenAttributes.test.ts": 59, - "packages/core/test/ioSerialization.test.ts": 306, - "packages/core/test/jumpHash.test.ts": 385, - "packages/core/test/mockTaskContext.test.ts": 61, - "packages/core/test/recordSpanException.test.ts": 65, - "packages/core/test/resourceCatalog.test.ts": 71, - "packages/core/test/runStream.test.ts": 245, - "packages/core/test/skillCatalog.test.ts": 17, - "packages/core/test/standardMetadataManager.test.ts": 456, - "packages/core/test/streamsWriterV1.test.ts": 84112, - "packages/core/test/taskExecutor.test.ts": 364, - "packages/core/test/utils.test.ts": 15, - "packages/core/src/v3/apiClient/runStream.test.ts": 1893, - "packages/core/src/v3/apiClient/streamBatchItems.test.ts": 278, - "packages/core/src/v3/build/flags.test.ts": 12, + "apps/webapp/app/runEngine/concerns/idempotencyResidency.server.test.ts": 26, + "apps/webapp/app/runEngine/services/triggerFailedTask.server.test.ts": 2091, + "apps/webapp/app/runEngine/services/triggerTask.server.combinedReads.test.ts": 49515, + "apps/webapp/app/runEngine/services/triggerTask.server.lockedWorker.test.ts": 132479, + "apps/webapp/app/runEngine/services/triggerTask.server.parentReads.test.ts": 132404, + "apps/webapp/app/utils/friendlyId.test.ts": 59, + "apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts": 25, + "apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 3372, + "apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts": 36, + "apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts": 477, + "apps/webapp/app/v3/runOpsMigration/mintFlipGrace.test.ts": 27, + "apps/webapp/app/v3/runOpsMigration/readThrough.server.test.ts": 6305, + "apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts": 24, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.flipLatency.test.ts": 546, + "apps/webapp/app/v3/runOpsMigration/runOpsMintKind.server.test.ts": 511, + "apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts": 6073, + "apps/webapp/app/v3/runStore.server.test.ts": 8480, + "apps/webapp/app/v3/services/bulk/BulkActionV2.batchReadThrough.server.test.ts": 5922, + "apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts": 83, + "apps/webapp/test/EnvironmentVariablesPresenter.test.ts": 4096, + "apps/webapp/test/GCRARateLimiter.test.ts": 4787, + "apps/webapp/test/SpanPresenter.readthrough.test.ts": 9668, + "apps/webapp/test/activitySeries.server.test.ts": 25, + "apps/webapp/test/aiTitleRateLimiter.test.ts": 744, + "apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts": 97711, + "apps/webapp/test/api.v1.waitpoints.tokens.test.ts": 179054, + "apps/webapp/test/apiBatchResultsPresenter.dedicatedSeam.test.ts": 5727, + "apps/webapp/test/apiBatchResultsPresenter.readroute.test.ts": 5378, + "apps/webapp/test/apiBatchResultsPresenter.readthrough.test.ts": 11836, + "apps/webapp/test/apiBatchResultsPresenter.splitNPlus1.test.ts": 3476, + "apps/webapp/test/apiRetrieveRunPresenter.groupedLockedWorker.test.ts": 8669, + "apps/webapp/test/apiRetrieveRunPresenter.readroute.test.ts": 12490, + "apps/webapp/test/apiRunListPresenter.test.ts": 148237, + "apps/webapp/test/apiRunResultPresenter.readthrough.test.ts": 6843, + "apps/webapp/test/apiWaitpointListPresenter.readroute.test.ts": 3321, + "apps/webapp/test/apiWaitpointPresenter.readthrough.test.ts": 9115, + "apps/webapp/test/batchListPresenter.readroute.test.ts": 11138, + "apps/webapp/test/batchPresenter.test.ts": 16534, + "apps/webapp/test/batchQueueItemResidencyAnchoring.test.ts": 1959, + "apps/webapp/test/batchRunAccess.test.ts": 8353, + "apps/webapp/test/batchTaskRunEnvironmentFkDrop.test.ts": 6556, + "apps/webapp/test/batchTriggerV3ResidencyInheritance.test.ts": 1037, + "apps/webapp/test/batchTriggerV3StoreRouting.test.ts": 5624, + "apps/webapp/test/billingAlertsFormat.test.ts": 34, + "apps/webapp/test/billingLimit.schemas.test.ts": 40, + "apps/webapp/test/billingLimitBulkCancelInProgress.test.ts": 16155, + "apps/webapp/test/billingLimitConvergeEnvironments.test.ts": 3424, + "apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts": 21, + "apps/webapp/test/billingLimitConvergeResolve.test.ts": 200, + "apps/webapp/test/billingLimitEnvCreatePause.test.ts": 609, + "apps/webapp/test/billingLimitHit.test.ts": 25, + "apps/webapp/test/billingLimitPauseEnvironment.test.ts": 35, + "apps/webapp/test/billingLimitQueuedRuns.test.ts": 12281, + "apps/webapp/test/billingLimitReconcileTick.test.ts": 213, + "apps/webapp/test/billingLimitReconciliation.test.ts": 630, + "apps/webapp/test/billingLimitResolve.test.ts": 17, + "apps/webapp/test/billingLimitTriggerEntitlement.test.ts": 26, + "apps/webapp/test/billingLimitsRoute.test.ts": 1102, + "apps/webapp/test/branchableEnvironment.test.ts": 20, + "apps/webapp/test/bufferedTriggerPayload.test.ts": 25, + "apps/webapp/test/bulkActionV2ReadRouting.test.ts": 6183, + "apps/webapp/test/calculateNextSchedule.test.ts": 208, + "apps/webapp/test/chartActivityTimeAxis.test.ts": 29, + "apps/webapp/test/chartXAxisTicks.test.ts": 29, + "apps/webapp/test/chartZoomRange.test.ts": 22, + "apps/webapp/test/chat-snapshot-integration.test.ts": 1896, + "apps/webapp/test/checkPermissions.test.ts": 22, + "apps/webapp/test/clickhouseFactory.test.ts": 3978, + "apps/webapp/test/components/DateTime.test.ts": 784, + "apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts": 136, + "apps/webapp/test/components/code/tsql/tsqlLinter.test.ts": 273, + "apps/webapp/test/components/runs/v3/RunTag.test.ts": 190, + "apps/webapp/test/components/runs/v3/agent/AgentMessageView.test.ts": 339, + "apps/webapp/test/computeBucket.test.ts": 123, + "apps/webapp/test/computeMigration.test.ts": 22, + "apps/webapp/test/concurrentFlushScheduler.test.ts": 716, + "apps/webapp/test/createDeploymentWithNextVersion.test.ts": 8603, + "apps/webapp/test/crossSeamGuard.proof.test.ts": 5538, + "apps/webapp/test/dependentAttemptScope.test.ts": 17, + "apps/webapp/test/detectQueryTables.test.ts": 259, + "apps/webapp/test/detectbadJsonStrings.test.ts": 73, + "apps/webapp/test/devBranchServices.test.ts": 4029, + "apps/webapp/test/devPresenceRecency.test.ts": 1179, + "apps/webapp/test/dropTaskRunToTaskRunTagJoin.test.ts": 2678, + "apps/webapp/test/duplicateTaskIds.test.ts": 20, + "apps/webapp/test/dynamicFlushSchedulerMetrics.test.ts": 1730, + "apps/webapp/test/emailPattern.test.ts": 18, + "apps/webapp/test/engine/batchPayloads.test.ts": 5365, + "apps/webapp/test/engine/idempotencyParentRunScope.test.ts": 10574, + "apps/webapp/test/engine/streamBatchItems.test.ts": 19344, + "apps/webapp/test/engine/taskIdentifierRegistry.test.ts": 4014, + "apps/webapp/test/engine/triggerFailedTask.call.test.ts": 133741, + "apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts": 91796, + "apps/webapp/test/engine/triggerTask.debounce.test.ts": 134153, + "apps/webapp/test/engine/triggerTask.idempotency.test.ts": 133523, + "apps/webapp/test/engine/triggerTask.metadataCache.test.ts": 173067, + "apps/webapp/test/engine/triggerTask.mollifier.test.ts": 173296, + "apps/webapp/test/engine/triggerTask.residency.test.ts": 172996, + "apps/webapp/test/engine/triggerTask.test.ts": 92296, + "apps/webapp/test/environmentSort.test.ts": 29, + "apps/webapp/test/environmentVariableDeduplication.test.ts": 24, + "apps/webapp/test/environmentVariableRules.test.ts": 19, + "apps/webapp/test/environmentVariablesEnvironments.test.ts": 3690, + "apps/webapp/test/environmentVariablesRepository.test.ts": 4145, + "apps/webapp/test/errorFingerprinting.test.ts": 28, + "apps/webapp/test/errorGroupWebhook.test.ts": 59, + "apps/webapp/test/findEnvironmentByApiKey.test.ts": 3764, + "apps/webapp/test/findEnvironmentFromRun.readthrough.test.ts": 5555, + "apps/webapp/test/findOrCreateBackgroundWorker.test.ts": 8848, + "apps/webapp/test/getDeploymentImageRef.test.ts": 306, + "apps/webapp/test/getTraceDetailedSubtreeSummary.integration.test.ts": 6408, + "apps/webapp/test/googleEmailVerification.test.ts": 27, + "apps/webapp/test/httpErrors.test.ts": 39, + "apps/webapp/test/idempotencyDedupResidency.test.ts": 9623, + "apps/webapp/test/idempotencyKeyConcernLegacyAuthority.test.ts": 5778, + "apps/webapp/test/inviteRoleLadder.test.ts": 18, + "apps/webapp/test/member.server.test.ts": 5194, + "apps/webapp/test/metadataRouteOperationsLogging.test.ts": 274, + "apps/webapp/test/mfaRateLimiter.test.ts": 632, + "apps/webapp/test/mollifierApplyMetadataMutation.test.ts": 850, + "apps/webapp/test/mollifierClaimResolution.test.ts": 469, + "apps/webapp/test/mollifierDecisionLabels.test.ts": 47, + "apps/webapp/test/mollifierDrainerHandler.test.ts": 450, + "apps/webapp/test/mollifierDrainerWorker.test.ts": 2020, + "apps/webapp/test/mollifierDrainingGauge.test.ts": 716, + "apps/webapp/test/mollifierGate.test.ts": 492, + "apps/webapp/test/mollifierIdempotencyClaim.test.ts": 423, + "apps/webapp/test/mollifierMollify.test.ts": 223, + "apps/webapp/test/mollifierMutateWithFallback.test.ts": 445, + "apps/webapp/test/mollifierReadFallback.test.ts": 405, + "apps/webapp/test/mollifierReplayPayloadShape.test.ts": 249, + "apps/webapp/test/mollifierResetIdempotencyKey.test.ts": 522, + "apps/webapp/test/mollifierResolveRunForMutation.test.ts": 448, + "apps/webapp/test/mollifierStaleSweep.test.ts": 1047, + "apps/webapp/test/mollifierSynthesiseFoundRun.test.ts": 559, + "apps/webapp/test/mollifierSyntheticApiResponses.test.ts": 25, + "apps/webapp/test/mollifierSyntheticRedirectInfo.test.ts": 815, + "apps/webapp/test/mollifierSyntheticReplayTaskRun.test.ts": 20, + "apps/webapp/test/mollifierSyntheticRunHeader.test.ts": 21, + "apps/webapp/test/mollifierSyntheticSpanRun.test.ts": 197, + "apps/webapp/test/mollifierSyntheticTrace.test.ts": 303, + "apps/webapp/test/mollifierTripEvaluator.test.ts": 831, + "apps/webapp/test/nextRunListPresenter.readthrough.test.ts": 30311, + "apps/webapp/test/objectStore.test.ts": 5520, + "apps/webapp/test/orgBanner.test.ts": 18, + "apps/webapp/test/organizationDataStoresRegistry.test.ts": 5455, + "apps/webapp/test/otlpExporter.test.ts": 144, + "apps/webapp/test/otlpUtf16Sanitization.integration.test.ts": 5831, + "apps/webapp/test/otlpWorkerPoolMetrics.test.ts": 317, + "apps/webapp/test/pauseEnvironment.server.test.ts": 8847, + "apps/webapp/test/performTaskRunAlertsStoreRouting.test.ts": 6886, + "apps/webapp/test/presenters/ApiBatchResultsPresenter.test.ts": 16145, + "apps/webapp/test/presenters/TaskDetailPresenter.getActivity.test.ts": 6231, + "apps/webapp/test/presenters/TestTaskPresenter.readthrough.test.ts": 36865, + "apps/webapp/test/presenters/mapRunToLiveFields.test.ts": 21, + "apps/webapp/test/prismaErrors.test.ts": 219, + "apps/webapp/test/prismaInfrastructureErrorCapture.test.ts": 3592, + "apps/webapp/test/promptOverrideSource.test.ts": 17, + "apps/webapp/test/queryResultsTimeTicks.test.ts": 650, + "apps/webapp/test/queueListPagination.test.ts": 31, + "apps/webapp/test/rbacFallbackBranch.test.ts": 3600, + "apps/webapp/test/realtime/boundedTtlCache.test.ts": 22, + "apps/webapp/test/realtime/clickHouseRunListResolver.test.ts": 41562, + "apps/webapp/test/realtime/electricStreamProtocol.test.ts": 68, + "apps/webapp/test/realtime/envChangeRouter.test.ts": 1176, + "apps/webapp/test/realtime/nativeHoldOnEmpty.test.ts": 4496, + "apps/webapp/test/realtime/nativeRealtimeClient.test.ts": 342, + "apps/webapp/test/realtime/nativeRunSetCache.test.ts": 607, + "apps/webapp/test/realtime/replayCursorStore.test.ts": 1529, + "apps/webapp/test/realtime/replicaLagEstimator.test.ts": 584, + "apps/webapp/test/realtime/runChangeNotifier.test.ts": 3540, + "apps/webapp/test/realtime/runReaderProjection.test.ts": 53, + "apps/webapp/test/realtime/runReaderReadThrough.test.ts": 7242, + "apps/webapp/test/realtime/shadowCompare.test.ts": 26, + "apps/webapp/test/realtime/streamRegistrationRouting.test.ts": 5726, + "apps/webapp/test/redisRealtimeStreams.test.ts": 5745, + "apps/webapp/test/registryConfig.test.ts": 343, + "apps/webapp/test/reloadingRegistry.test.ts": 362, + "apps/webapp/test/removeTeamMember.test.ts": 9473, + "apps/webapp/test/replay-after-crash.test.ts": 2002, + "apps/webapp/test/replayTaskRunEnvironmentScoping.test.ts": 4349, + "apps/webapp/test/resetIdempotencyKeyLegacyAuthority.test.ts": 5698, + "apps/webapp/test/resolveWaitpointThroughReadThrough.readthrough.test.ts": 6634, + "apps/webapp/test/routeLoaders.controlPlane.readthrough.test.ts": 5232, + "apps/webapp/test/runDetailLoaders.controlPlane.readthrough.test.ts": 6025, + "apps/webapp/test/runEngineBatchTriggerResidencyAnchoring.test.ts": 550, + "apps/webapp/test/runEngineBatchTriggerStoreRouting.test.ts": 5799, + "apps/webapp/test/runEngineHandlers.test.ts": 14088, + "apps/webapp/test/runOpsCrossSeamGuard.test.ts": 348, + "apps/webapp/test/runOpsDbTopology.test.ts": 4214, + "apps/webapp/test/runOpsMintCutover.test.ts": 3808, + "apps/webapp/test/runOpsMintGlobalFlipLock.test.ts": 3407, + "apps/webapp/test/runOpsSplitMode.test.ts": 4202, + "apps/webapp/test/runOpsSplitReadGate.glue.test.ts": 473, + "apps/webapp/test/runOpsSplitReadGate.test.ts": 18, + "apps/webapp/test/runPresenterReadRoute.test.ts": 3909, + "apps/webapp/test/runsBackfiller.test.ts": 8095, + "apps/webapp/test/runsReplicationInstance.test.ts": 24640, + "apps/webapp/test/runsReplicationService.part1.test.ts": 25612, + "apps/webapp/test/runsReplicationService.part2.test.ts": 22323, + "apps/webapp/test/runsReplicationService.part3.test.ts": 12303, + "apps/webapp/test/runsReplicationService.part4.test.ts": 27040, + "apps/webapp/test/runsReplicationService.part5.test.ts": 9175, + "apps/webapp/test/runsReplicationService.part6.test.ts": 12682, + "apps/webapp/test/runsReplicationService.part7.test.ts": 69508, + "apps/webapp/test/runsReplicationService.part8.test.ts": 24479, + "apps/webapp/test/runsReplicationService.part9.test.ts": 12463, + "apps/webapp/test/runsRepository.part1.test.ts": 22987, + "apps/webapp/test/runsRepository.part2.test.ts": 23786, + "apps/webapp/test/runsRepository.part3.test.ts": 18531, + "apps/webapp/test/runsRepository.part4.test.ts": 24073, + "apps/webapp/test/runsRepository.readthrough.test.ts": 39432, + "apps/webapp/test/runsRepositoryCpres.test.ts": 8672, + "apps/webapp/test/runsRepositoryCursor.test.ts": 28582, + "apps/webapp/test/safeEnvironmentLog.test.ts": 17, + "apps/webapp/test/safeIntegrationLog.test.ts": 15, + "apps/webapp/test/safeRequestLogContext.test.ts": 26, + "apps/webapp/test/safeWebhookFetch.test.ts": 199, + "apps/webapp/test/safeWebhookUrl.test.ts": 30, + "apps/webapp/test/sameOriginNavigation.test.ts": 53, + "apps/webapp/test/sanitizeRowsOnParseError.test.ts": 30, + "apps/webapp/test/sanitizeUrl.test.ts": 22, + "apps/webapp/test/sanitizeWorkerHeaders.test.ts": 280, + "apps/webapp/test/sentryTenantContext.test.ts": 23, + "apps/webapp/test/sentryTraceContext.server.test.ts": 70, + "apps/webapp/test/services.controlPlane.readthrough.test.ts": 5401, + "apps/webapp/test/services/organizationAccessToken.test.ts": 241, + "apps/webapp/test/services/personalAccessToken.test.ts": 232, + "apps/webapp/test/sessionDuration.test.ts": 10233, + "apps/webapp/test/sessions.readthrough.test.ts": 5859, + "apps/webapp/test/sessionsReplicationService.test.ts": 17051, + "apps/webapp/test/shouldRevalidateRunsList.test.ts": 19, + "apps/webapp/test/slackOAuthResultLog.test.ts": 23, + "apps/webapp/test/spanPresenterReadthroughDecompose.test.ts": 5373, + "apps/webapp/test/streamLoader.controlPlane.test.ts": 5162, + "apps/webapp/test/tenantContext.test.ts": 43, + "apps/webapp/test/tenantContextFromAuthEnvironment.test.ts": 38, + "apps/webapp/test/tenantContextResolver.test.ts": 37, + "apps/webapp/test/timeGranularity.test.ts": 27, + "apps/webapp/test/timelineSpanEvents.test.ts": 25, + "apps/webapp/test/traceExport.test.ts": 29, + "apps/webapp/test/updateMetadata.test.ts": 19006, + "apps/webapp/test/updateMetadataStoreRoutingHetero.test.ts": 7309, + "apps/webapp/test/utils/timezones.test.ts": 31, + "apps/webapp/test/v3/runOpsMigration/controlPlaneRepoint.server.test.ts": 5949, + "apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts": 6678, + "apps/webapp/test/v3/runOpsMigration/distinctDbSentinel.server.test.ts": 5388, + "apps/webapp/test/v3/runOpsMigration/runEngineControlPlaneResolver.server.test.ts": 4587, + "apps/webapp/test/validateGitBranchName.test.ts": 23, + "apps/webapp/test/vercelUrls.test.ts": 18, + "apps/webapp/test/verifyDeploymentImage.test.ts": 922, + "apps/webapp/test/waitpointCallback.controlPlane.test.ts": 7670, + "apps/webapp/test/waitpointListPresenter.readroute.test.ts": 8556, + "apps/webapp/test/waitpointPresenter.connectedRunsBounded.test.ts": 5386, + "apps/webapp/test/waitpointPresenter.controlPlane.test.ts": 8112, + "apps/webapp/test/waitpointPresenter.danglingConnectedRuns.test.ts": 4994, + "apps/webapp/test/waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts": 5687, + "apps/webapp/test/waitpointPresenter.readthrough.test.ts": 35556, + "apps/webapp/test/waitpointPresenter.splitConnectedRuns.test.ts": 5062, + "apps/webapp/test/waitpointTagListPresenter.readroute.test.ts": 5653, + "apps/webapp/test/webhookErrorAlerts.test.ts": 61, + "apps/webapp/test/workerGroupAccess.test.ts": 33, + "apps/webapp/test/workerQueueSplit.server.test.ts": 23, + "apps/webapp/test/workerQueueSplit.test.ts": 27, + "apps/webapp/test/workerRegions.test.ts": 487, + "internal-packages/cache/src/stores/lruMemory.test.ts": 65, + "internal-packages/clickhouse/src/client/client.test.ts": 7547, + "internal-packages/clickhouse/src/taskRuns.test.ts": 6768, + "internal-packages/clickhouse/src/tsql.test.ts": 8254, + "internal-packages/clickhouse/src/tsqlFunctions.test.ts": 9360, + "internal-packages/dashboard-agent/src/dashboard-agent.test.ts": 1429, + "internal-packages/dashboard-agent/src/repo-tools.test.ts": 116, + "internal-packages/llm-model-catalog/src/registry.test.ts": 84, + "internal-packages/llm-model-catalog/src/sync.test.ts": 3585, + "internal-packages/rbac/src/ability.test.ts": 55, + "internal-packages/rbac/src/loader.test.ts": 77, + "internal-packages/rbac/src/require-plugins.test.ts": 84, + "internal-packages/replication/src/client.test.ts": 33067, + "internal-packages/run-engine/src/batch-queue/tests/index.test.ts": 4370, + "internal-packages/run-engine/src/engine/systems/batchSystem.test.ts": 35042, + "internal-packages/run-engine/src/engine/systems/debounceSystem.test.ts": 14129, + "internal-packages/run-engine/src/engine/systems/delayedRunSystem.test.ts": 12805, + "internal-packages/run-engine/src/engine/systems/enqueueSystem.test.ts": 9585, + "internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.test.ts": 12815, + "internal-packages/run-engine/src/engine/systems/pendingVersionSystem.test.ts": 12791, + "internal-packages/run-engine/src/engine/systems/runAttemptSystem.test.ts": 17101, + "internal-packages/run-engine/src/engine/systems/ttlSystem.test.ts": 12916, + "internal-packages/run-engine/src/engine/systems/waitpointSystem.test.ts": 31249, + "internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts": 14624, + "internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts": 17248, + "internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts": 19939, + "internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts": 13594, + "internal-packages/run-engine/src/engine/tests/blockEdgeResidency.test.ts": 6962, + "internal-packages/run-engine/src/engine/tests/cancelling.test.ts": 11125, + "internal-packages/run-engine/src/engine/tests/checkpointSystem.controlPlaneResolver.test.ts": 12189, + "internal-packages/run-engine/src/engine/tests/checkpointSystemStore.test.ts": 15100, + "internal-packages/run-engine/src/engine/tests/checkpoints.test.ts": 19226, + "internal-packages/run-engine/src/engine/tests/clearBlockingWaitpointsResidency.test.ts": 6030, + "internal-packages/run-engine/src/engine/tests/completeWaitpointCrossSeamGuard.test.ts": 9881, + "internal-packages/run-engine/src/engine/tests/completeWaitpointReadResidency.test.ts": 7270, + "internal-packages/run-engine/src/engine/tests/controlPlaneResolverInjectability.test.ts": 9790, + "internal-packages/run-engine/src/engine/tests/createCancelledRun.test.ts": 10516, + "internal-packages/run-engine/src/engine/tests/createFailedTaskRun.test.ts": 8961, + "internal-packages/run-engine/src/engine/tests/crossVersionCompat.test.ts": 6563, + "internal-packages/run-engine/src/engine/tests/datetimeWaitpointColocation.test.ts": 10600, + "internal-packages/run-engine/src/engine/tests/debounce.test.ts": 31044, + "internal-packages/run-engine/src/engine/tests/delayedRunSystem.controlPlaneResolver.test.ts": 12846, + "internal-packages/run-engine/src/engine/tests/delays.test.ts": 32798, + "internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts": 13496, + "internal-packages/run-engine/src/engine/tests/dequeueSystem.recovery.controlPlaneResolver.test.ts": 4805, + "internal-packages/run-engine/src/engine/tests/dequeuing.test.ts": 15045, + "internal-packages/run-engine/src/engine/tests/engineResidualInversions.controlPlaneResolver.test.ts": 5341, + "internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts": 222657, + "internal-packages/run-engine/src/engine/tests/getSnapshotsSinceResidency.test.ts": 5582, + "internal-packages/run-engine/src/engine/tests/heartbeats.test.ts": 29022, + "internal-packages/run-engine/src/engine/tests/heteroPostgresFixture.test.ts": 5790, + "internal-packages/run-engine/src/engine/tests/lazyWaitpoint.test.ts": 20398, + "internal-packages/run-engine/src/engine/tests/legacyRunNewTokenResumeFlow.test.ts": 6757, + "internal-packages/run-engine/src/engine/tests/lifecycleRouter.test.ts": 16360, + "internal-packages/run-engine/src/engine/tests/locking.test.ts": 50934, + "internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts": 13218, + "internal-packages/run-engine/src/engine/tests/priority.test.ts": 11463, + "internal-packages/run-engine/src/engine/tests/runAttemptSystem.controlPlaneResolver.test.ts": 15028, + "internal-packages/run-engine/src/engine/tests/runStoreInjectability.test.ts": 9726, + "internal-packages/run-engine/src/engine/tests/startRunAttemptReadResidency.test.ts": 6902, + "internal-packages/run-engine/src/engine/tests/trigger.test.ts": 10838, + "internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts": 14414, + "internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts": 13971, + "internal-packages/run-engine/src/engine/tests/ttl.test.ts": 51947, + "internal-packages/run-engine/src/engine/tests/waitpointPublicRouter.test.ts": 14940, + "internal-packages/run-engine/src/engine/tests/waitpointRace.test.ts": 10633, + "internal-packages/run-engine/src/engine/tests/waitpointReplicaLag.test.ts": 5858, + "internal-packages/run-engine/src/engine/tests/waitpointSystem.controlPlaneResolver.test.ts": 11913, + "internal-packages/run-engine/src/engine/tests/waitpoints.test.ts": 29161, + "internal-packages/run-engine/src/engine/tests/workerQueueObservation.test.ts": 8582, + "internal-packages/run-engine/src/run-queue/index.test.ts": 82145, + "internal-packages/run-engine/src/run-queue/tests/ack.test.ts": 2823, + "internal-packages/run-engine/src/run-queue/tests/ckCounters.test.ts": 15673, + "internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts": 8611, + "internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts": 6162, + "internal-packages/run-engine/src/run-queue/tests/dequeueMessageFromWorkerQueue.test.ts": 67581, + "internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts": 8763, + "internal-packages/run-engine/src/run-queue/tests/fairQueueSelectionStrategy.test.ts": 3513, + "internal-packages/run-engine/src/run-queue/tests/keyProducer.test.ts": 80, + "internal-packages/run-engine/src/run-queue/tests/migrateLegacyMasterQueue.test.ts": 1609, + "internal-packages/run-engine/src/run-queue/tests/nack.test.ts": 5927, + "internal-packages/run-engine/src/run-queue/tests/releaseConcurrency.test.ts": 2649, + "internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts": 239, + "internal-packages/run-ops-database/prisma/schema.parity.test.ts": 43, + "internal-packages/run-store/src/PostgresRunStore.batchProbeReadClient.test.ts": 6303, + "internal-packages/run-store/src/PostgresRunStore.connectedRunsBounded.test.ts": 5104, + "internal-packages/run-store/src/PostgresRunStore.connectedRunsIncludeBounded.test.ts": 2362, + "internal-packages/run-store/src/PostgresRunStore.controlPlaneAlertFk.test.ts": 4777, + "internal-packages/run-store/src/PostgresRunStore.crossGenerationError.test.ts": 2646, + "internal-packages/run-store/src/PostgresRunStore.dedicatedRepro.test.ts": 12323, + "internal-packages/run-store/src/PostgresRunStore.dedicatedSelect.test.ts": 5884, + "internal-packages/run-store/src/PostgresRunStore.dualSchema.test.ts": 5443, + "internal-packages/run-store/src/PostgresRunStore.findRunsByIds.test.ts": 2707, + "internal-packages/run-store/src/PostgresRunStore.groupedRelations.test.ts": 2561, + "internal-packages/run-store/src/PostgresRunStore.hydratorSelectPushdown.test.ts": 2428, + "internal-packages/run-store/src/PostgresRunStore.sharedHydratedTarget.test.ts": 2289, + "internal-packages/run-store/src/PostgresRunStore.test.ts": 11219, + "internal-packages/run-store/src/PostgresRunStore.writeAtomicity.test.ts": 7791, + "internal-packages/run-store/src/batchCompletionResidency.test.ts": 6379, + "internal-packages/run-store/src/checkpointBatchReplicaLag.test.ts": 2667, + "internal-packages/run-store/src/clientCompat.test.ts": 32, + "internal-packages/run-store/src/runOpsStore.batchItemMisroute.test.ts": 5285, + "internal-packages/run-store/src/runOpsStore.crossDbCompletedWaitpoint.test.ts": 4837, + "internal-packages/run-store/src/runOpsStore.crossDbTokenBlock.test.ts": 4787, + "internal-packages/run-store/src/runOpsStore.flipWindowDuplicate.test.ts": 4844, + "internal-packages/run-store/src/runOpsStore.forWaitpointCompletion.test.ts": 96, + "internal-packages/run-store/src/runOpsStore.groupedReads.test.ts": 5767, + "internal-packages/run-store/src/runOpsStore.idempotencyDedup.test.ts": 6729, + "internal-packages/run-store/src/runOpsStore.mixedResidency.test.ts": 9524, + "internal-packages/run-store/src/runOpsStore.readAfterWrite.test.ts": 5519, + "internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts": 5564, + "internal-packages/run-store/src/runOpsStore.routedReadPrimary.test.ts": 6475, + "internal-packages/run-store/src/runOpsStore.snapshotCompletedWaitpoints.test.ts": 4889, + "internal-packages/run-store/src/runOpsStore.snapshots.test.ts": 5938, + "internal-packages/run-store/src/runOpsStore.test.ts": 22578, + "internal-packages/run-store/src/runOpsStore.waitpoints.test.ts": 6504, + "internal-packages/run-store/src/scheduleSeam.test.ts": 5180, + "internal-packages/schedule-engine/test/scheduleEngine.test.ts": 125260, + "internal-packages/schedule-engine/test/scheduleEngine2.test.ts": 9011, + "internal-packages/schedule-engine/test/scheduleRecovery.test.ts": 10259, + "internal-packages/sdk-compat-tests/src/tests/bundler.test.ts": 170, + "internal-packages/sdk-compat-tests/src/tests/import.test.ts": 2455, + "internal-packages/sso/src/loader.test.ts": 61, + "internal-packages/tsql/src/grammar/parser.test.ts": 423, + "internal-packages/tsql/src/index.test.ts": 561, + "internal-packages/tsql/src/query/escape.test.ts": 50, + "internal-packages/tsql/src/query/parser.test.ts": 518, + "internal-packages/tsql/src/query/printer.test.ts": 872, + "internal-packages/tsql/src/query/results.test.ts": 59, + "internal-packages/tsql/src/query/schema.test.ts": 78, + "internal-packages/tsql/src/query/security.test.ts": 627, + "internal-packages/tsql/src/query/time_buckets.test.ts": 39, + "internal-packages/tsql/src/query/validator.test.ts": 557, + "packages/build/src/extensions/core/syncEnvVars.test.ts": 33, + "packages/core/src/v3/apiClient/bulkActions.test.ts": 214, + "packages/core/src/v3/apiClient/runStream.test.ts": 2319, + "packages/core/src/v3/apiClient/streamBatchItems.test.ts": 328, + "packages/core/src/v3/apps/exec.test.ts": 24, + "packages/core/src/v3/build/flags.test.ts": 19, + "packages/core/src/v3/errors.test.ts": 51, + "packages/core/src/v3/idempotency-key-catalog/inMemoryIdempotencyKeyCatalog.test.ts": 19, "packages/core/src/v3/idempotency-key-catalog/lruIdempotencyKeyCatalog.test.ts": 21, - "packages/core/src/v3/machines/max-old-space.test.ts": 18, - "packages/core/src/v3/realtimeStreams/manager.test.ts": 28, - "packages/core/src/v3/realtimeStreams/streamsWriterV2.test.ts": 91, - "packages/core/src/v3/schemas/api-type.test.ts": 33, - "packages/core/src/v3/schemas/batchItemNDJSON.test.ts": 15, - "packages/core/src/v3/schemas/idempotencyKey.test.ts": 68, - "packages/core/src/v3/sessionStreams/manager.test.ts": 126, - "packages/core/src/v3/serverOnly/shutdownManager.test.ts": 57, - "packages/core/src/v3/taskContext/index.test.ts": 23, - "packages/core/src/v3/utils/reconnectBackoff.test.ts": 45, - "packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts": 123, - "packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts": 98, - "packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts": 92, - "packages/schema-to-json/tests/index.test.ts": 17, - "internal-packages/run-engine/src/run-queue/index.test.ts": 82296, - "internal-packages/run-engine/src/batch-queue/tests/index.test.ts": 5462, - "internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts": 35471, - "internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts": 33127, - "internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts": 33681, - "internal-packages/run-engine/src/engine/tests/batchTwoPhase.test.ts": 28159, - "internal-packages/run-engine/src/engine/tests/cancelling.test.ts": 26240, - "internal-packages/run-engine/src/engine/tests/checkpoints.test.ts": 29420, - "internal-packages/run-engine/src/engine/tests/createCancelledRun.test.ts": 31807, - "internal-packages/run-engine/src/engine/tests/createFailedTaskRun.test.ts": 23573, - "internal-packages/run-engine/src/engine/tests/debounce.test.ts": 58554, - "internal-packages/run-engine/src/engine/tests/delays.test.ts": 42748, - "internal-packages/run-engine/src/engine/tests/dequeuing.test.ts": 25542, - "internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts": 32158, - "internal-packages/run-engine/src/engine/tests/heartbeats.test.ts": 39634, - "internal-packages/run-engine/src/engine/tests/lazyWaitpoint.test.ts": 34757, - "internal-packages/run-engine/src/engine/tests/locking.test.ts": 51090, - "internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts": 28762, - "internal-packages/run-engine/src/engine/tests/priority.test.ts": 28808, - "internal-packages/run-engine/src/engine/tests/trigger.test.ts": 30601, - "internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts": 28893, - "internal-packages/run-engine/src/engine/tests/ttl.test.ts": 61981, - "internal-packages/run-engine/src/engine/tests/waitpointRace.test.ts": 22855, - "internal-packages/run-engine/src/engine/tests/waitpoints.test.ts": 39521, - "internal-packages/run-engine/src/run-queue/tests/ack.test.ts": 3156, - "internal-packages/run-engine/src/run-queue/tests/ckCounters.test.ts": 16715, - "internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts": 8916, - "internal-packages/run-engine/src/run-queue/tests/concurrencySweeper.test.ts": 6379, - "internal-packages/run-engine/src/run-queue/tests/dequeueMessageFromWorkerQueue.test.ts": 66701, - "internal-packages/run-engine/src/run-queue/tests/enqueueMessage.test.ts": 9108, - "internal-packages/run-engine/src/run-queue/tests/fairQueueSelectionStrategy.test.ts": 7997, - "internal-packages/run-engine/src/run-queue/tests/keyProducer.test.ts": 19, - "internal-packages/run-engine/src/run-queue/tests/migrateLegacyMasterQueue.test.ts": 1728, - "internal-packages/run-engine/src/run-queue/tests/nack.test.ts": 6184, - "internal-packages/run-engine/src/run-queue/tests/releaseConcurrency.test.ts": 3018, - "internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts": 38, - "internal-packages/cache/src/stores/lruMemory.test.ts": 44, - "internal-packages/schedule-engine/test/scheduleEngine.test.ts": 43000, - "internal-packages/schedule-engine/test/scheduleRecovery.test.ts": 17396, - "internal-packages/replication/src/client.test.ts": 31306, - "internal-packages/tsql/src/index.test.ts": 246, - "internal-packages/tsql/src/grammar/parser.test.ts": 150, - "internal-packages/tsql/src/query/escape.test.ts": 9, - "internal-packages/tsql/src/query/parser.test.ts": 368, - "internal-packages/tsql/src/query/printer.test.ts": 942, - "internal-packages/tsql/src/query/results.test.ts": 4, - "internal-packages/tsql/src/query/schema.test.ts": 18, - "internal-packages/tsql/src/query/security.test.ts": 487, - "internal-packages/tsql/src/query/time_buckets.test.ts": 5, - "internal-packages/tsql/src/query/validator.test.ts": 250, - "internal-packages/rbac/src/ability.test.ts": 6, - "internal-packages/rbac/src/loader.test.ts": 3, - "internal-packages/llm-model-catalog/src/registry.test.ts": 15, - "internal-packages/llm-model-catalog/src/sync.test.ts": 15852, - "internal-packages/clickhouse/src/taskRuns.test.ts": 6813, - "internal-packages/clickhouse/src/tsql.test.ts": 9021, - "internal-packages/clickhouse/src/tsqlFunctions.test.ts": 12971, - "internal-packages/clickhouse/src/client/client.test.ts": 9138, - "internal-packages/sdk-compat-tests/src/tests/bundler.test.ts": 348, - "internal-packages/sdk-compat-tests/src/tests/import.test.ts": 4742, - "apps/webapp/test/runsReplicationService.part1.test.ts": 74000, - "apps/webapp/test/runsReplicationService.part2.test.ts": 64000, - "apps/webapp/test/runsReplicationService.part3.test.ts": 30000, - "apps/webapp/test/runsReplicationService.part4.test.ts": 70000, - "apps/webapp/test/runsReplicationService.part5.test.ts": 43000, - "apps/webapp/test/runsReplicationService.part6.test.ts": 32000, - "apps/webapp/test/runsRepository.part3.test.ts": 43000, - "apps/webapp/test/runsRepository.part4.test.ts": 57000, - "apps/webapp/test/runsReplicationService.part7.test.ts": 43000, - "internal-packages/schedule-engine/test/scheduleEngine2.test.ts": 43000 + "packages/core/src/v3/idempotencyKeys.test.ts": 409, + "packages/core/src/v3/isomorphic/friendlyId.test.ts": 371, + "packages/core/src/v3/isomorphic/runOpsResidency.test.ts": 877, + "packages/core/src/v3/machines/max-old-space.test.ts": 20, + "packages/core/src/v3/realtimeStreams/manager.test.ts": 119, + "packages/core/src/v3/realtimeStreams/streamsWriterV2.test.ts": 109, + "packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts": 226, + "packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.test.ts": 65, + "packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts": 125, + "packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts": 57, + "packages/core/src/v3/schemas/api-type.test.ts": 55, + "packages/core/src/v3/schemas/batchItemNDJSON.test.ts": 91, + "packages/core/src/v3/schemas/idempotencyKey.test.ts": 64, + "packages/core/src/v3/serverOnly/shutdownManager.test.ts": 53, + "packages/core/src/v3/sessionStreams/manager.test.ts": 130, + "packages/core/src/v3/taskContext/index.test.ts": 37, + "packages/core/src/v3/telnetLogServer.test.ts": 233, + "packages/core/src/v3/utils/reconnectBackoff.test.ts": 17, + "packages/core/test/duration.test.ts": 28, + "packages/core/test/errors.test.ts": 60, + "packages/core/test/eventFilterMatches.test.ts": 63, + "packages/core/test/externalSpanExporterWrapper.test.ts": 285, + "packages/core/test/flattenAttributes.test.ts": 166, + "packages/core/test/ioSerialization.test.ts": 559, + "packages/core/test/jumpHash.test.ts": 297, + "packages/core/test/mockTaskContext.test.ts": 607, + "packages/core/test/recordSpanException.test.ts": 75, + "packages/core/test/resourceCatalog.test.ts": 48, + "packages/core/test/runStream.test.ts": 304, + "packages/core/test/skillCatalog.test.ts": 24, + "packages/core/test/standardMetadataManager.test.ts": 719, + "packages/core/test/streamsWriterV1.test.ts": 86772, + "packages/core/test/taskExecutor.test.ts": 1120, + "packages/core/test/utils.test.ts": 17, + "packages/redis-worker/src/cron.test.ts": 27511, + "packages/redis-worker/src/fair-queue/tests/concurrency.test.ts": 460, + "packages/redis-worker/src/fair-queue/tests/drr.test.ts": 437, + "packages/redis-worker/src/fair-queue/tests/fairQueue.test.ts": 6821, + "packages/redis-worker/src/fair-queue/tests/raceConditions.test.ts": 13720, + "packages/redis-worker/src/fair-queue/tests/retry.test.ts": 215, + "packages/redis-worker/src/fair-queue/tests/tenantDispatch.test.ts": 4820, + "packages/redis-worker/src/fair-queue/tests/visibility.test.ts": 1428, + "packages/redis-worker/src/fair-queue/tests/workerQueue.test.ts": 432, + "packages/redis-worker/src/mollifier/buffer.test.ts": 1354, + "packages/redis-worker/src/mollifier/drainer.test.ts": 7797, + "packages/redis-worker/src/queue.test.ts": 2866, + "packages/redis-worker/src/worker.test.ts": 16743, + "packages/schema-to-json/tests/index.test.ts": 395, + "packages/trigger-sdk/src/v3/chat-server.test.ts": 858, + "packages/trigger-sdk/src/v3/chat-tab-coordinator.test.ts": 30, + "packages/trigger-sdk/src/v3/chat.test.ts": 848, + "packages/trigger-sdk/src/v3/createStartSessionAction.test.ts": 276, + "packages/trigger-sdk/src/v3/runs-bulk.test.ts": 224, + "packages/trigger-sdk/src/v3/sessions.test.ts": 218, + "packages/trigger-sdk/src/v3/shared.test.ts": 266, + "packages/trigger-sdk/src/v3/streams.test.ts": 193, + "packages/trigger-sdk/src/v3/triggerClient.test.ts": 841, + "packages/trigger-sdk/src/v3/triggerClient.types.test.ts": 218, + "packages/trigger-sdk/test/chat-snapshot.test.ts": 1300, + "packages/trigger-sdk/test/chat-transport-events.test.ts": 292, + "packages/trigger-sdk/test/chat-turn-correlation.test.ts": 385, + "packages/trigger-sdk/test/chatHandover.test.ts": 2984, + "packages/trigger-sdk/test/chatHandoverBackends.test.ts": 1455, + "packages/trigger-sdk/test/declaration-emit.test.ts": 1699, + "packages/trigger-sdk/test/merge-by-id.test.ts": 364, + "packages/trigger-sdk/test/mockChatAgent.test.ts": 4622, + "packages/trigger-sdk/test/pending-message-drain.test.ts": 4563, + "packages/trigger-sdk/test/promptCaching.test.ts": 513, + "packages/trigger-sdk/test/recovery-boot.test.ts": 1893, + "packages/trigger-sdk/test/replay-session-in.test.ts": 514, + "packages/trigger-sdk/test/replay-session-out.test.ts": 526, + "packages/trigger-sdk/test/skill.test.ts": 194, + "packages/trigger-sdk/test/skillsRuntime.test.ts": 532, + "packages/trigger-sdk/test/wire-shape.test.ts": 1262 }