From 9e7b11998cb2727b05337f9133eae4647c979644 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 10 Jul 2026 09:47:22 -0700 Subject: [PATCH] test(producer): gate source tests by execution lane --- .github/workflows/ci.yml | 30 ++++++++ bun.lock | 27 +++---- package.json | 9 ++- packages/producer/package.json | 15 +++- .../scripts/check-test-classification.mjs | 9 +++ packages/producer/scripts/run-test-lane.mjs | 34 +++++++++ .../producer/scripts/test-classification.mjs | 74 +++++++++++++++++++ .../scripts/test-classification.test.mjs | 48 ++++++++++++ packages/producer/src/regression-harness.ts | 21 +++--- .../deterministicFonts-failClosed.test.ts | 3 + .../crossWorkerIdempotency.test.ts | 3 +- .../services/distributed/renderChunk.test.ts | 7 +- .../stages/captureStreamingStage.test.ts | 25 ++++++- .../src/services/renderOrchestrator.test.ts | 9 ++- .../tests/hdr-hlg-regression/README.md | 4 +- .../producer/tests/hdr-regression/README.md | 4 +- 16 files changed, 285 insertions(+), 37 deletions(-) create mode 100644 packages/producer/scripts/check-test-classification.mjs create mode 100644 packages/producer/scripts/run-test-lane.mjs create mode 100644 packages/producer/scripts/test-classification.mjs create mode 100644 packages/producer/scripts/test-classification.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ed4b23197..cc38804287 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,6 +235,36 @@ jobs: - run: bun run --cwd packages/core build:hyperframes-runtime - run: bun run --filter '!@hyperframes/producer' test + producer-source-tests: + name: "Producer: ${{ matrix.lane }} tests" + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + lane: [unit, integration] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + lfs: true + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - name: Install FFmpeg for integration tests + if: matrix.lane == 'integration' + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends ffmpeg + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bun install --frozen-lockfile + - run: bun run --filter '@hyperframes/{parsers,lint,studio-server}' build + - run: bun run --cwd packages/core build + - run: bun run --filter @hyperframes/engine build + - run: bun run producer:test:${{ matrix.lane }} + # Tests under skills/**/*.test.mjs are bare `node --test` files with only # `node:` built-in imports. They aren't part of any workspace package, and # the main `Test` job's `code` path filter excludes `skills/**`, so without diff --git a/bun.lock b/bun.lock index c6eb53154d..4339fb118a 100644 --- a/bun.lock +++ b/bun.lock @@ -22,7 +22,7 @@ }, "packages/aws-lambda": { "name": "@hyperframes/aws-lambda", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@aws-sdk/client-s3": "^3.700.0", "@aws-sdk/client-sfn": "^3.700.0", @@ -54,7 +54,7 @@ }, "packages/cli": { "name": "@hyperframes/cli", - "version": "0.7.45", + "version": "0.7.48", "bin": { "hyperframes": "./dist/cli.js", }, @@ -103,7 +103,7 @@ }, "packages/core": { "name": "@hyperframes/core", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@chenglou/pretext": "^0.0.5", "@hyperframes/lint": "workspace:*", @@ -128,7 +128,7 @@ }, "packages/engine": { "name": "@hyperframes/engine", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@hono/node-server": "^1.13.0", "@hyperframes/core": "workspace:^", @@ -146,7 +146,7 @@ }, "packages/gcp-cloud-run": { "name": "@hyperframes/gcp-cloud-run", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@google-cloud/storage": "^7.14.0", "@google-cloud/workflows": "^4.2.0", @@ -166,7 +166,7 @@ }, "packages/lint": { "name": "@hyperframes/lint", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@hyperframes/parsers": "workspace:*", "htmlparser2": "^10.1.0", @@ -183,7 +183,7 @@ }, "packages/parsers": { "name": "@hyperframes/parsers", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@babel/parser": "^7.27.0", "acorn": "^8.17.0", @@ -203,7 +203,7 @@ }, "packages/player": { "name": "@hyperframes/player", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@hyperframes/core": "workspace:*", }, @@ -218,7 +218,7 @@ }, "packages/producer": { "name": "@hyperframes/producer", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@fontsource/archivo-black": "^5.2.8", "@fontsource/eb-garamond": "^5.2.7", @@ -258,11 +258,12 @@ "esbuild": "^0.25.12", "tsx": "^4.21.0", "typescript": "^5.7.2", + "vitest": "^3.2.4", }, }, "packages/sdk": { "name": "@hyperframes/sdk", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@hyperframes/core": "workspace:*", "@hyperframes/parsers": "workspace:*", @@ -289,7 +290,7 @@ }, "packages/shader-transitions": { "name": "@hyperframes/shader-transitions", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "html2canvas": "^1.4.1", }, @@ -301,7 +302,7 @@ }, "packages/studio": { "name": "@hyperframes/studio", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@codemirror/autocomplete": "^6.20.1", "@codemirror/commands": "^6.10.3", @@ -349,7 +350,7 @@ }, "packages/studio-server": { "name": "@hyperframes/studio-server", - "version": "0.7.45", + "version": "0.7.48", "dependencies": { "@hyperframes/core": "workspace:*", "@hyperframes/parsers": "workspace:*", diff --git a/package.json b/package.json index 5535bb035e..12d17ffce3 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,14 @@ "lint:fix": "oxlint --fix .", "check:tracked-artifacts": "node scripts/check-tracked-artifacts.mjs", "format": "oxfmt .", - "test": "bun run --filter '*' test", + "test": "bun run test:unit", + "test:unit": "bun run --filter '*' test", + "producer:test:classification": "bun run --cwd packages/producer test:classification", + "producer:test:unit": "bun run --cwd packages/producer test:unit", + "producer:test:unit:bun": "bun run --cwd packages/producer test:unit:bun", + "producer:test:unit:vitest": "bun run --cwd packages/producer test:unit:vitest", + "producer:test:integration": "bun run --cwd packages/producer test:integration", + "test:regression": "bun run --cwd packages/producer test:regression", "player:perf": "bun run --filter @hyperframes/player perf", "format:check": "oxfmt --check .", "knip": "knip", diff --git a/packages/producer/package.json b/packages/producer/package.json index 5d76636ea1..aa417d3912 100644 --- a/packages/producer/package.json +++ b/packages/producer/package.json @@ -43,8 +43,16 @@ "check:runtime-conformance": "tsx src/runtime-conformance.ts", "benchmark": "tsx src/benchmark.ts", "bench:hdr": "tsx src/benchmark.ts --tags hdr", - "test": "tsx src/regression-harness.ts --exclude-tags transparency", - "test:update": "tsx src/regression-harness.ts --update --exclude-tags transparency", + "test": "bun run test:unit", + "test:classification": "node --test scripts/test-classification.test.mjs && node scripts/check-test-classification.mjs", + "test:unit": "bun run test:classification && node scripts/run-test-lane.mjs unit", + "test:unit:bun": "node scripts/run-test-lane.mjs unit bun", + "test:unit:vitest": "node scripts/run-test-lane.mjs unit vitest", + "test:integration": "bun run test:classification && node scripts/run-test-lane.mjs integration", + "test:integration:bun": "node scripts/run-test-lane.mjs integration bun", + "test:integration:vitest": "node scripts/run-test-lane.mjs integration vitest", + "test:regression": "tsx src/regression-harness.ts --exclude-tags transparency", + "test:regression:update": "tsx src/regression-harness.ts --update --exclude-tags transparency", "test:distributed": "tsx src/regression-harness.ts --exclude-tags transparency --mode=distributed-simulated", "test:transparency": "tsx src/transparency-test.ts", "docker:build:test": "docker build -f ../../Dockerfile.test -t hyperframes-producer:test ../..", @@ -93,7 +101,8 @@ "@webgpu/types": "^0.1.69", "esbuild": "^0.25.12", "tsx": "^4.21.0", - "typescript": "^5.7.2" + "typescript": "^5.7.2", + "vitest": "^3.2.4" }, "engines": { "node": ">=22" diff --git a/packages/producer/scripts/check-test-classification.mjs b/packages/producer/scripts/check-test-classification.mjs new file mode 100644 index 0000000000..338c9b6333 --- /dev/null +++ b/packages/producer/scripts/check-test-classification.mjs @@ -0,0 +1,9 @@ +import { discoverProducerTests, summarizeTests } from "./test-classification.mjs"; + +try { + const tests = discoverProducerTests(); + console.log(JSON.stringify({ event: "producer_tests_classified", ...summarizeTests(tests) })); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/packages/producer/scripts/run-test-lane.mjs b/packages/producer/scripts/run-test-lane.mjs new file mode 100644 index 0000000000..8c22ca09ec --- /dev/null +++ b/packages/producer/scripts/run-test-lane.mjs @@ -0,0 +1,34 @@ +import { spawnSync } from "node:child_process"; +import { discoverProducerTests, PRODUCER_ROOT } from "./test-classification.mjs"; + +const lane = process.argv[2]; +const requestedRunner = process.argv[3]; +if (lane !== "unit" && lane !== "integration") { + throw new Error("Usage: node scripts/run-test-lane.mjs [bun|vitest]"); +} +if (requestedRunner && requestedRunner !== "bun" && requestedRunner !== "vitest") { + throw new Error(`Unknown test runner: ${requestedRunner}`); +} + +const tests = discoverProducerTests().filter( + (test) => test.lane === lane && (!requestedRunner || test.runner === requestedRunner), +); + +function run(args) { + const result = spawnSync("bun", args, { + cwd: PRODUCER_ROOT, + env: { ...process.env, HYPERFRAMES_TEST_LANE: lane }, + stdio: "inherit", + }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); +} + +const vitestFiles = tests.filter((test) => test.runner === "vitest").map((test) => test.file); +if (vitestFiles.length > 0) run(["x", "vitest", "run", ...vitestFiles]); + +// Bun's mock.module registry is process-global. Run each file in a fresh +// process so mocks from one source test cannot mutate another test's imports. +for (const test of tests.filter((entry) => entry.runner === "bun")) { + run(["test", test.file]); +} diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjs new file mode 100644 index 0000000000..aa6fc43365 --- /dev/null +++ b/packages/producer/scripts/test-classification.mjs @@ -0,0 +1,74 @@ +// fallow-ignore-file complexity +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const PRODUCER_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +// These tests need host capabilities such as Chrome, ffmpeg, worker threads, +// or local sockets. Keep the list explicit so a filename-only rename does not +// make Git/fallow re-audit thousands of unchanged test lines as new code. +const INTEGRATION_TEST_FILES = new Set([ + "src/services/deterministicFonts-systemCapture.test.ts", + "src/services/distributed/assemble.test.ts", + "src/services/distributed/chunkBoundary.test.ts", + "src/services/distributed/crossWorkerIdempotency.test.ts", + "src/services/distributed/plan.test.ts", + "src/services/distributed/planSizeCap.test.ts", + "src/services/distributed/renderChunk.test.ts", + "src/services/fileServer.test.ts", + "src/services/healthWorker.test.ts", + "src/utils/audioRegression.test.ts", + "src/utils/streamDurationParity.test.ts", +]); + +function collectTestFiles(directory, files = []) { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = resolve(directory, entry.name); + if (entry.isDirectory()) collectTestFiles(entryPath, files); + else if (entry.isFile() && entry.name.endsWith(".test.ts")) files.push(entryPath); + } + return files; +} + +export function classifyTestSource(filePath, source, integrationFiles = INTEGRATION_TEST_FILES) { + const importsBun = /\bfrom\s+["']bun:test["']/.test(source); + const importsVitest = /\bfrom\s+["']vitest["']/.test(source); + if (importsBun === importsVitest) { + const detail = importsBun ? "imports both bun:test and vitest" : "imports neither runner"; + throw new Error(`${filePath}: ${detail}`); + } + return { + file: filePath, + runner: importsBun ? "bun" : "vitest", + lane: integrationFiles.has(filePath) ? "integration" : "unit", + }; +} + +export function discoverProducerTests(producerRoot = PRODUCER_ROOT) { + const srcDir = resolve(producerRoot, "src"); + const files = collectTestFiles(srcDir).map((absolutePath) => ({ + absolutePath, + filePath: relative(producerRoot, absolutePath).replaceAll("\\", "/"), + })); + const discoveredFiles = new Set(files.map((test) => test.filePath)); + const staleEntries = [...INTEGRATION_TEST_FILES].filter((file) => !discoveredFiles.has(file)); + if (staleEntries.length > 0) { + throw new Error(`Integration test manifest contains missing files: ${staleEntries.join(", ")}`); + } + return files + .map(({ absolutePath, filePath }) => { + return classifyTestSource(filePath, readFileSync(absolutePath, "utf8")); + }) + .sort((left, right) => left.file.localeCompare(right.file)); +} + +export function summarizeTests(tests) { + const summary = { + total: tests.length, + unit: { bun: 0, vitest: 0 }, + integration: { bun: 0, vitest: 0 }, + }; + for (const test of tests) summary[test.lane][test.runner] += 1; + return summary; +} diff --git a/packages/producer/scripts/test-classification.test.mjs b/packages/producer/scripts/test-classification.test.mjs new file mode 100644 index 0000000000..75652df5ef --- /dev/null +++ b/packages/producer/scripts/test-classification.test.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + classifyTestSource, + discoverProducerTests, + summarizeTests, +} from "./test-classification.mjs"; + +describe("producer test classification", () => { + it("classifies each supported runner into one lane", () => { + assert.deepEqual(classifyTestSource("src/a.test.ts", 'import { it } from "bun:test";'), { + file: "src/a.test.ts", + runner: "bun", + lane: "unit", + }); + assert.deepEqual( + classifyTestSource( + "src/a.test.ts", + 'import { it } from "vitest";', + new Set(["src/a.test.ts"]), + ), + { file: "src/a.test.ts", runner: "vitest", lane: "integration" }, + ); + }); + + it("rejects missing and ambiguous runner imports", () => { + assert.throws(() => classifyTestSource("src/a.test.ts", "export {};"), /neither runner/); + assert.throws( + () => + classifyTestSource( + "src/a.test.ts", + 'import { it } from "bun:test"; import { expect } from "vitest";', + ), + /both bun:test and vitest/, + ); + }); + + it("classifies every current source test exactly once", () => { + const tests = discoverProducerTests(); + assert.equal(new Set(tests.map((test) => test.file)).size, tests.length); + const summary = summarizeTests(tests); + assert.ok(summary.total > 0); + assert.equal( + summary.unit.bun + summary.unit.vitest + summary.integration.bun + summary.integration.vitest, + summary.total, + ); + }); +}); diff --git a/packages/producer/src/regression-harness.ts b/packages/producer/src/regression-harness.ts index 3f1101c619..d1c6dde2ef 100644 --- a/packages/producer/src/regression-harness.ts +++ b/packages/producer/src/regression-harness.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file complexity import { existsSync, mkdirSync, @@ -1510,12 +1511,14 @@ async function run(): Promise { } } -void run().catch((error) => { - console.error( - JSON.stringify({ - event: "test_suite_fatal", - message: error instanceof Error ? error.message : String(error), - }), - ); - process.exitCode = 1; -}); +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + void run().catch((error) => { + console.error( + JSON.stringify({ + event: "test_suite_fatal", + message: error instanceof Error ? error.message : String(error), + }), + ); + process.exitCode = 1; + }); +} diff --git a/packages/producer/src/services/deterministicFonts-failClosed.test.ts b/packages/producer/src/services/deterministicFonts-failClosed.test.ts index e64158da8a..d3860ed6f1 100644 --- a/packages/producer/src/services/deterministicFonts-failClosed.test.ts +++ b/packages/producer/src/services/deterministicFonts-failClosed.test.ts @@ -59,6 +59,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)" it("swallows a network failure and returns the original HTML (no throw)", async () => { const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: false, + allowSystemFontCapture: false, fetchImpl: makeFailingFetch(), }); // No @font-face was injected because the fetch failed — but the call @@ -69,6 +70,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)" it("swallows a 404 response and returns the original HTML (no throw)", async () => { const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: false, + allowSystemFontCapture: false, fetchImpl: makeHttp404Fetch(), }); expect(result.includes("data-hyperframes-deterministic-fonts")).toBe(false); @@ -77,6 +79,7 @@ describe("injectDeterministicFontFaces — failClosedFontFetch: false (default)" it("swallows a 5xx response and returns the original HTML (no throw)", async () => { const result = await injectDeterministicFontFaces(HTML_REQUESTING_UNRESOLVED_FONT, { failClosedFontFetch: false, + allowSystemFontCapture: false, fetchImpl: makeHttp503Fetch(), }); expect(result.includes("data-hyperframes-deterministic-fonts")).toBe(false); diff --git a/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts b/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts index 4d3e7ba84a..219e76c563 100644 --- a/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts +++ b/packages/producer/src/services/distributed/crossWorkerIdempotency.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication complexity /** * Cross-worker idempotency: rendering the same `(planDir, chunkIndex)` on two * different workers MUST produce byte-identical output. This is what makes a @@ -41,7 +42,7 @@ const FIXTURE_HTML = ` cross-worker idempotency fixture -
+

chunk fixture

diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.ts index e88230a705..d325aef137 100644 --- a/packages/producer/src/services/distributed/renderChunk.test.ts +++ b/packages/producer/src/services/distributed/renderChunk.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication complexity /** * Unit tests for `services/distributed/renderChunk.ts`. * @@ -41,7 +42,7 @@ const FIXTURE_HTML = ` renderChunk fixture -
+

chunk fixture

@@ -134,7 +135,7 @@ beforeAll(async () => { planDir = join(runRoot, "plan"); mkdirSync(planDir, { recursive: true }); await plan(projectDir, { fps: 30, width: 160, height: 120, format: "png-sequence" }, planDir); -}); +}, 30_000); afterAll(() => { rmSync(runRoot, { recursive: true, force: true }); @@ -331,7 +332,7 @@ describe("renderChunk() — variables threading", () => { renderChunk variables fixture -
+