From 7f8a126e2ec5f89d9247dc20d79a47fb46cfe6a5 Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Tue, 30 Jun 2026 03:47:45 +0000 Subject: [PATCH 1/6] feat(engine): fleet capacity & scale-pressure signal (lib + GET /api/v1/me/fleet/capacity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose a claim-faithful, autoscaler-pollable signal of engine queue depth vs idle container-instance capacity so an operator (or external autoscaler) can decide whether to spin up more daemon instances. Read-only/derivation only — no machine-local lease coordination, no in-daemon concurrency state (server surfaces depth + declares policy; daemon coordinates claims). - src/lib/engine/fleet-capacity.ts: pure deriveFleetCapacity(queueDepth, machines) → { queueDepth, online, busy, available, shouldScale, recommendedInstances }, plus an async getFleetCapacity that sources depth from listFleetGoalsOrdered (the SAME fleetOwnerId-scoped, occupied-excluded order the claim path consumes — counts only 'queued' rows) and busy from live leased runs (isRunLive). shouldScale = queueDepth > available is the server-declared policy, kept in the pure module. - Extract the daemon working/idle/stuck/error/offline decision tree from the fleet-dots route into classifyDaemonMachine (fleet.ts), so the capacity busy/idle counts and the fleet-dots strip cannot drift (faithful-scope). - GET /api/v1/me/fleet/capacity via authorizeDaemonOrSession (autoscaler token or browser session), optional ?projectIds=a,b — mirrors the fleet-queue route. - Unit tests for the pure derivation (zero machines, all busy, depth 0, depth>available, capacity>=depth, offline excluded); validated via node simulation since tsc/vitest can't run in this env. Agent-config: additive read-only HTTP endpoint, not an agent tool — no new domain entity/status/workflow/specialist. No PM-agent config change needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/api/v1/me/fleet/capacity/route.ts | 26 ++++ src/app/api/v1/me/machines/fleet/route.ts | 51 +++----- src/lib/engine/fleet-capacity.test.ts | 118 +++++++++++++++++ src/lib/engine/fleet-capacity.ts | 149 ++++++++++++++++++++++ src/lib/engine/fleet.ts | 50 ++++++++ src/lib/engine/index.ts | 1 + 6 files changed, 364 insertions(+), 31 deletions(-) create mode 100644 src/app/api/v1/me/fleet/capacity/route.ts create mode 100644 src/lib/engine/fleet-capacity.test.ts create mode 100644 src/lib/engine/fleet-capacity.ts diff --git a/src/app/api/v1/me/fleet/capacity/route.ts b/src/app/api/v1/me/fleet/capacity/route.ts new file mode 100644 index 000000000..04b12faa2 --- /dev/null +++ b/src/app/api/v1/me/fleet/capacity/route.ts @@ -0,0 +1,26 @@ +// Copyright (c) 2026 Interactor, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later +import { NextRequest, NextResponse } from "next/server"; +import { authorizeDaemonOrSession } from "@/lib/auth/with-daemon-token-auth"; +import { getFleetCapacity } from "@/lib/engine"; + +// Read-only fleet capacity & scale-pressure signal for the operator's cross-project fleet: +// engine queue depth vs idle container-instance capacity. Pollable by an EXTERNAL autoscaler +// holding a daemon token (and by the browser session) — auth mirrors the fleet-queue route. +// Optional `?projectIds=a,b` narrows to a subset of the operator's projects, mirroring the +// claim filter. Derivation only — no machine-local coordination, no in-daemon concurrency +// state (the server surfaces depth + declares the scale policy; the daemon coordinates claims). +export async function GET(req: NextRequest) { + const authed = await authorizeDaemonOrSession(req); + if (!authed) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const raw = new URL(req.url).searchParams.get("projectIds"); + const projectIds = raw + ? raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0) + : undefined; + + const capacity = await getFleetCapacity(authed.userId, { projectIds }); + return NextResponse.json({ data: capacity }); +} diff --git a/src/app/api/v1/me/machines/fleet/route.ts b/src/app/api/v1/me/machines/fleet/route.ts index bdf9150bf..775108cfb 100644 --- a/src/app/api/v1/me/machines/fleet/route.ts +++ b/src/app/api/v1/me/machines/fleet/route.ts @@ -4,10 +4,7 @@ import { NextResponse } from "next/server"; import { auth } from "@/lib/auth"; import { db } from "@/lib/db"; import { DedicatedMachineStatus } from "@prisma/client"; - -const OFFLINE_THRESHOLD_MS = 90 * 1000; -// A failed run within this window marks the machine as "error". -const ERROR_WINDOW_MS = 2 * 60 * 60 * 1000; +import { classifyDaemonMachine, MACHINE_ERROR_WINDOW_MS } from "@/lib/engine/fleet"; export interface FleetMachine { id: string; @@ -55,7 +52,7 @@ export async function GET() { : []; const now = Date.now(); - const errorWindowCutoff = new Date(now - ERROR_WINDOW_MS); + const errorWindowCutoff = new Date(now - MACHINE_ERROR_WINDOW_MS); // For daemon machines: check for a recent failed EngineRun to surface the "error" state, // and for an active EngineRun (non-expired lease) to surface the "working" state. @@ -95,32 +92,24 @@ export async function GET() { ); const fleet: FleetMachine[] = [ - ...daemonMachines.map((m) => { - const isOffline = now - m.lastSeenAt.getTime() > OFFLINE_THRESHOLD_MS; - const hasActiveSession = m.activeSessionId != null; - const hasActiveRun = activeRunMachineIds.has(m.id); - const hasRecentError = failedMachineIds.has(m.id); - - let status: FleetMachine["status"]; - if (isOffline && hasActiveSession) { - // Heartbeat went stale while a session was still active — agent is stuck. - status = "stuck"; - } else if (isOffline) { - status = hasRecentError ? "error" : "offline"; - } else if (hasActiveSession || hasActiveRun) { - // Working: either a user session is attached OR the daemon is executing an EngineRun. - status = "working"; - } else { - status = hasRecentError ? "error" : "idle"; - } - - return { - id: m.id, - type: "daemon" as const, - name: m.customName ?? m.name, - status, - } satisfies FleetMachine; - }), + ...daemonMachines.map((m) => ({ + id: m.id, + type: "daemon" as const, + name: m.customName ?? m.name, + // Shared with the capacity signal so the two surfaces can't drift (see + // classifyDaemonMachine). activeSessionId tracks user-session attachment only — it is + // NOT set during autonomous daemon execution, so the live EngineRun lease is what marks + // an autonomous machine "working". + status: classifyDaemonMachine( + { + lastSeenAt: m.lastSeenAt, + activeSessionId: m.activeSessionId, + hasActiveRun: activeRunMachineIds.has(m.id), + hasRecentError: failedMachineIds.has(m.id), + }, + new Date(now), + ), + } satisfies FleetMachine)), ...dedicatedMachines.map((m) => ({ id: m.id, type: "dedicated" as const, diff --git a/src/lib/engine/fleet-capacity.test.ts b/src/lib/engine/fleet-capacity.test.ts new file mode 100644 index 000000000..efe9dfe70 --- /dev/null +++ b/src/lib/engine/fleet-capacity.test.ts @@ -0,0 +1,118 @@ +// Copyright (c) 2026 Interactor, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later +import { describe, expect, it, vi } from "vitest"; + +// fleet-capacity.ts imports `@/lib/db` at module load; stub it so the PURE derivation can +// be unit-tested without a real Prisma client (the async `getFleetCapacity` is not exercised +// here — only `deriveFleetCapacity`). +vi.mock("@/lib/db", () => ({ db: { machine: { findMany: vi.fn() }, engineRun: { findMany: vi.fn() } } })); + +const { deriveFleetCapacity } = await import("./fleet-capacity"); +const { MACHINE_ONLINE_THRESHOLD_MS } = await import("./fleet"); + +const NOW = new Date("2026-06-30T12:00:00.000Z"); +const past = (ms: number) => new Date(NOW.getTime() - ms); + +/** An ONLINE machine that is WORKING (a live leased run). */ +const busyMachine = () => ({ + lastSeenAt: past(0), + activeSessionId: null, + hasActiveRun: true, + hasRecentError: false, +}); +/** An ONLINE machine that is IDLE (claimable now). */ +const idleMachine = () => ({ + lastSeenAt: past(0), + activeSessionId: null, + hasActiveRun: false, + hasRecentError: false, +}); +/** An OFFLINE machine (heartbeat stale) — not counted as online/available. */ +const offlineMachine = () => ({ + lastSeenAt: past(MACHINE_ONLINE_THRESHOLD_MS + 1), + activeSessionId: null, + hasActiveRun: false, + hasRecentError: false, +}); + +describe("deriveFleetCapacity — scale-pressure derivation", () => { + it("zero machines: nothing online; any depth wants exactly that many instances", () => { + expect(deriveFleetCapacity(0, [], NOW)).toEqual({ + queueDepth: 0, + online: 0, + busy: 0, + available: 0, + shouldScale: false, + recommendedInstances: 0, + }); + expect(deriveFleetCapacity(3, [], NOW)).toEqual({ + queueDepth: 3, + online: 0, + busy: 0, + available: 0, + shouldScale: true, + recommendedInstances: 3, + }); + }); + + it("all machines busy: available is 0, so a queued backlog wants one instance per goal", () => { + const machines = [busyMachine(), busyMachine(), busyMachine()]; + expect(deriveFleetCapacity(2, machines, NOW)).toEqual({ + queueDepth: 2, + online: 3, + busy: 3, + available: 0, + shouldScale: true, + recommendedInstances: 2, + }); + }); + + it("queueDepth = 0: never scales, regardless of idle capacity", () => { + const machines = [idleMachine(), busyMachine()]; + expect(deriveFleetCapacity(0, machines, NOW)).toMatchObject({ + queueDepth: 0, + online: 2, + busy: 1, + available: 1, + shouldScale: false, + recommendedInstances: 0, + }); + }); + + it("queueDepth > available: shouldScale true, recommends the shortfall", () => { + // 4 online (1 busy, 3 idle) ⇒ available 3; depth 5 ⇒ short by 2. + const machines = [busyMachine(), idleMachine(), idleMachine(), idleMachine()]; + expect(deriveFleetCapacity(5, machines, NOW)).toEqual({ + queueDepth: 5, + online: 4, + busy: 1, + available: 3, + shouldScale: true, + recommendedInstances: 2, + }); + }); + + it("capacity >= depth: shouldScale false, recommends 0 (no negative instances)", () => { + // 3 idle online ⇒ available 3; depth 2 ⇒ slack, no scale. + const machines = [idleMachine(), idleMachine(), idleMachine()]; + expect(deriveFleetCapacity(2, machines, NOW)).toEqual({ + queueDepth: 2, + online: 3, + busy: 0, + available: 3, + shouldScale: false, + recommendedInstances: 0, + }); + }); + + it("offline machines count toward neither online nor available", () => { + const machines = [idleMachine(), offlineMachine(), offlineMachine()]; + expect(deriveFleetCapacity(1, machines, NOW)).toMatchObject({ + online: 1, + busy: 0, + available: 1, + shouldScale: false, + recommendedInstances: 0, + }); + }); +}); diff --git a/src/lib/engine/fleet-capacity.ts b/src/lib/engine/fleet-capacity.ts new file mode 100644 index 000000000..87bd8b499 --- /dev/null +++ b/src/lib/engine/fleet-capacity.ts @@ -0,0 +1,149 @@ +// Copyright (c) 2026 Interactor, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later +import { db } from "@/lib/db"; +import { + classifyDaemonMachine, + isRunLive, + MACHINE_ERROR_WINDOW_MS, + MACHINE_ONLINE_THRESHOLD_MS, + type DaemonMachineSignals, +} from "./fleet"; +import { listFleetGoalsOrdered } from "./fleet-queue"; + +// ─── Fleet capacity & scale-pressure signal ─────────────────────────────────── +// +// A claim-faithful, autoscaler-pollable view of engine queue depth vs idle +// container-instance capacity, so an operator (or an external autoscaler with a daemon +// token) can decide whether to spin up more instances. +// +// Read-only / derivation only. This surfaces depth + declares a scale POLICY; it does NOT +// coordinate machine-local leases or hold any in-daemon concurrency state (Jason's +// boundary: the server surfaces depth + declares policy, the daemon coordinates claims). +// +// The fleet is instance-per-task: a daemon container drives exactly ONE goal at a time, so +// an idle (online, non-working) machine can take exactly one queued goal right now. That +// makes `available` the true "can take a queued goal now" count, and the scale threshold a +// simple depth-vs-available comparison. + +/** The scale-pressure signal. `shouldScale` / `recommendedInstances` are the POLICY the + * server declares (kept here, in the pure module — not in the daemon). */ +export interface FleetCapacity { + /** Queued-but-unclaimed fleet-claimable goals for this operator. */ + queueDepth: number; + /** Daemon machines heartbeating within the 90s window (matches `machineCounts`). */ + online: number; + /** Online machines the shared classifier rules "working" (a live leased run / session). */ + busy: number; + /** online − busy: idle online machines, each able to take one queued goal now. */ + available: number; + /** queueDepth > available — more queued goals than idle machines to take them. */ + shouldScale: boolean; + /** max(0, queueDepth − available): how many more instances would clear the backlog. */ + recommendedInstances: number; +} + +/** + * Pure capacity derivation. Given the operator's queued-claimable goal count and their + * daemon machines' signals, compute the scale-pressure signal. + * + * - `online` is derived from `lastSeenAt` directly (the same 90s window `machineCounts` + * uses) — NOT from the status enum, whose `error` value is ambiguous between an offline + * machine and an online-but-recently-failed one. + * - `busy` is the count of ONLINE machines the SHARED `classifyDaemonMachine` rules + * "working", so busy/idle can never drift from the fleet-dots strip (faithful-scope). + * - `available = online − busy`; `shouldScale` / `recommendedInstances` are the policy. + */ +export function deriveFleetCapacity( + queueDepth: number, + machines: DaemonMachineSignals[], + now: Date = new Date(), +): FleetCapacity { + let online = 0; + let busy = 0; + for (const m of machines) { + const seen = m.lastSeenAt instanceof Date ? m.lastSeenAt : new Date(m.lastSeenAt); + if (now.getTime() - seen.getTime() > MACHINE_ONLINE_THRESHOLD_MS) continue; // offline + online++; + if (classifyDaemonMachine(m, now) === "working") busy++; + } + const available = online - busy; + const shouldScale = queueDepth > available; + const recommendedInstances = Math.max(0, queueDepth - available); + return { queueDepth, online, busy, available, shouldScale, recommendedInstances }; +} + +/** + * Assemble the live capacity signal for an operator's fleet (optionally narrowed to a + * subset of their projects). Two derivations, both reusing existing definitions so this + * never becomes a second source of truth: + * - queueDepth: the QUEUED rows of `listFleetGoalsOrdered` — the SAME faithful, fleetOwnerId- + * scoped, occupied-goal-excluded order the daemon claim path consumes. A goal already + * running / waiting-on-you / waiting-on-deps is not queued-and-unclaimed, so only the + * `queued` rows count toward depth. + * - machines: the operator's daemon machines, each tagged with a LIVE leased run + * (`isRunLive` — keyed by EngineRun.machineId) and a recent-failure flag, fed through the + * shared classifier by `deriveFleetCapacity`. + */ +export async function getFleetCapacity( + userId: string, + opts: { projectIds?: string[]; now?: Date } = {}, +): Promise { + const now = opts.now ?? new Date(); + + const [goals, machineRows] = await Promise.all([ + listFleetGoalsOrdered(userId, { projectIds: opts.projectIds, now }), + db.machine.findMany({ + where: { userId, connectionKind: "daemon" }, + select: { id: true, lastSeenAt: true, activeSessionId: true }, + }), + ]); + + const queueDepth = goals.filter((g) => g.status === "queued").length; + + const machineIds = machineRows.map((m) => m.id); + const errorWindowCutoff = new Date(now.getTime() - MACHINE_ERROR_WINDOW_MS); + const [activeRuns, failedRuns] = + machineIds.length > 0 + ? await Promise.all([ + db.engineRun.findMany({ + where: { + machineId: { in: machineIds }, + status: "running", + endedAt: null, + leaseExpiresAt: { gt: now }, + }, + select: { machineId: true, status: true, leaseExpiresAt: true }, + }), + db.engineRun.findMany({ + where: { + machineId: { in: machineIds }, + status: "failed", + endedAt: { gte: errorWindowCutoff }, + }, + select: { machineId: true }, + distinct: ["machineId"], + }), + ]) + : [[], []]; + + // The lease predicate is already in the WHERE; the `isRunLive` re-filter is + // belt-and-suspenders against clock skew (same pattern as `getProjectFleet`). + const liveRunMachineIds = new Set( + activeRuns + .filter((r) => isRunLive(r, now)) + .map((r) => r.machineId) + .filter((id): id is string => id != null), + ); + const failedMachineIds = new Set( + failedRuns.map((r) => r.machineId).filter((id): id is string => id != null), + ); + + const signals: DaemonMachineSignals[] = machineRows.map((m) => ({ + lastSeenAt: m.lastSeenAt, + activeSessionId: m.activeSessionId, + hasActiveRun: liveRunMachineIds.has(m.id), + hasRecentError: failedMachineIds.has(m.id), + })); + + return deriveFleetCapacity(queueDepth, signals, now); +} diff --git a/src/lib/engine/fleet.ts b/src/lib/engine/fleet.ts index 9a246dbad..af8445ceb 100644 --- a/src/lib/engine/fleet.ts +++ b/src/lib/engine/fleet.ts @@ -80,6 +80,56 @@ export function machineCounts( return { online, total: machines.length }; } +// ─── Shared per-machine status classification (fleet-dots + capacity) ───────── +// +// The machines page renders a coloured "fleet dot" per daemon machine +// (working/idle/stuck/error/offline) and the capacity signal (./fleet-capacity.ts) +// counts busy vs idle machines. Both MUST classify a machine identically or the two +// surfaces drift (an operator seeing "3 idle dots" while capacity reports 0 available). +// This decision tree, extracted verbatim from the fleet-dots route, is the single +// source of truth they both consume. + +/** A daemon machine's coarse fleet status. */ +export type FleetMachineStatus = "working" | "idle" | "stuck" | "error" | "offline"; + +/** A failed run within this window marks an otherwise-idle online machine "error". */ +export const MACHINE_ERROR_WINDOW_MS = 2 * 60 * 60 * 1000; + +/** The per-machine signals the status decision tree reads: + * - `lastSeenAt` — last heartbeat (drives the 90s online window). + * - `activeSessionId` — user-session attachment; NULL during autonomous daemon execution, + * so for an autonomous fleet "working" is driven by `hasActiveRun`. + * - `hasActiveRun` — a LIVE leased run on this machine (the `isRunLive` predicate). + * - `hasRecentError` — a failed run within `MACHINE_ERROR_WINDOW_MS`. */ +export interface DaemonMachineSignals { + lastSeenAt: Date | string; + activeSessionId: string | null; + hasActiveRun: boolean; + hasRecentError: boolean; +} + +/** + * Classify one daemon machine. Decision tree (unchanged from the original fleet-dots + * route): heartbeat stale + a session still attached → `stuck` (went dark mid-work); + * heartbeat stale → `error` if a run failed recently else `offline`; online + + * (session attached OR a live run) → `working`; online + idle → `error` if a run + * failed recently else `idle`. Pure, so both surfaces and the unit tests share it. + */ +export function classifyDaemonMachine( + m: DaemonMachineSignals, + now: Date = new Date(), +): FleetMachineStatus { + const seen = m.lastSeenAt instanceof Date ? m.lastSeenAt : new Date(m.lastSeenAt); + // Offline complements `machineCounts`' online rule exactly (online ⇔ seen within the + // window): offline ⇔ seen strictly OUTSIDE it. + const isOffline = now.getTime() - seen.getTime() > MACHINE_ONLINE_THRESHOLD_MS; + const hasActiveSession = m.activeSessionId != null; + if (isOffline && hasActiveSession) return "stuck"; + if (isOffline) return m.hasRecentError ? "error" : "offline"; + if (hasActiveSession || m.hasActiveRun) return "working"; + return m.hasRecentError ? "error" : "idle"; +} + /** * Assemble the live `FleetActivityData` for a project + the requesting user's daemon * machines. Two reads: diff --git a/src/lib/engine/index.ts b/src/lib/engine/index.ts index cadcfc612..e319e58e6 100644 --- a/src/lib/engine/index.ts +++ b/src/lib/engine/index.ts @@ -8,6 +8,7 @@ export * from "./service"; export * from "./http-errors"; export * from "./queue"; export * from "./fleet-queue"; +export * from "./fleet-capacity"; export * from "./phase-job"; export * from "./risk"; export * from "./design-decisions"; From 8d5d257d8104e5f88c09dd6dc9a8a11c220bd85a Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Tue, 30 Jun 2026 06:49:17 +0000 Subject: [PATCH 2/6] test(engine): cover getFleetCapacity + capacity route; harden coverage:patch heap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review rework on the fleet-capacity goal. Blocker (system:patch-coverage-below-threshold) — the patch-coverage gate runs `npm run coverage:patch` and then checks only that coverage/lcov.info EXISTS (container-loop ignores the command's exit code). The report was missing because the full v8-instrumented suite (16080 tests) OOM'd at V8's old-space HEAP limit before vitest could write the report — a heap ceiling, not a physical-RAM limit (per the env-typecheck-incomplete finding), so the review runner has the RAM, the run just needed a higher ceiling. Fixes: - package.json: prefix coverage:patch with NODE_OPTIONS=--max-old-space-size=6144 so the instrumented run reaches the report-writing step. Raising the ceiling cannot increase a run's actual memory or break a passing run — it only lets a heap-bound run finish (monotonically safe); matches the repo's inline-env script style (build/dev:ee). - Add the missing unit tests so that once a report IS produced, PATCH coverage on the changed lines clears the 80% floor. Verified via a scoped lcov run: 43/43 changed-instrumented lines covered = 100%. * fleet-capacity.test.ts: exercise the async getFleetCapacity assembler — QUEUED-only depth counting, a live-leased machine as busy (with the isRunLive expired-lease + null-machineId re-filters), the zero-machines run-query short-circuit, and a recently-failed-but-online machine counted as available. Mocks @/lib/db + ./fleet-queue; keeps ./fleet REAL so the shared classifyDaemonMachine/isRunLive predicates are exercised. * me/fleet/capacity/route.test.ts: 401 path, default all-projects call, and the comma-separated projectIds parse — mirrors the green fleet-queue route test (mocks authorizeDaemonOrSession + getFleetCapacity). The refactored classifyDaemonMachine branches + the fleet-dots route map are already covered by the existing machines/fleet route test (all five statuses). Agent-config: still no PM-agent change — these are tests + a build-script tweak, no agent-visible tool/data/workflow change. Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 2 +- .../api/v1/me/fleet/capacity/route.test.ts | 51 ++++++++ src/lib/engine/fleet-capacity.test.ts | 110 +++++++++++++++++- 3 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 src/app/api/v1/me/fleet/capacity/route.test.ts diff --git a/package.json b/package.json index a16c7c673..1eaf5c7b8 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", - "coverage:patch": "vitest run --coverage --maxWorkers=25% --coverage.reporter=lcovonly --coverage.reporter=text-summary --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0", + "coverage:patch": "NODE_OPTIONS=--max-old-space-size=6144 vitest run --coverage --maxWorkers=25% --coverage.reporter=lcovonly --coverage.reporter=text-summary --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0", "test:unit": "vitest run tests/unit/", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", diff --git a/src/app/api/v1/me/fleet/capacity/route.test.ts b/src/app/api/v1/me/fleet/capacity/route.test.ts new file mode 100644 index 000000000..669144422 --- /dev/null +++ b/src/app/api/v1/me/fleet/capacity/route.test.ts @@ -0,0 +1,51 @@ +// Copyright (c) 2026 Interactor, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/auth/with-daemon-token-auth", () => ({ authorizeDaemonOrSession: vi.fn() })); +vi.mock("@/lib/engine", async (orig) => { + const actual = await orig(); + return { ...actual, getFleetCapacity: vi.fn() }; +}); + +const { authorizeDaemonOrSession } = await import("@/lib/auth/with-daemon-token-auth"); +const { getFleetCapacity } = await import("@/lib/engine"); +const { GET } = await import("./route"); + +const row = (v: T) => v as never; +const CAP = { + queueDepth: 2, + online: 3, + busy: 2, + available: 1, + shouldScale: true, + recommendedInstances: 1, +}; +const reqWith = (qs = "") => + new Request(`http://localhost/api/v1/me/fleet/capacity${qs}`) as never; + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(authorizeDaemonOrSession).mockResolvedValue(row({ userId: "u" })); + vi.mocked(getFleetCapacity).mockResolvedValue(row(CAP)); +}); + +describe("GET /api/v1/me/fleet/capacity", () => { + it("401s when unauthorized", async () => { + vi.mocked(authorizeDaemonOrSession).mockResolvedValue(row(null)); + const res = await GET(reqWith()); + expect(res.status).toBe(401); + expect(getFleetCapacity).not.toHaveBeenCalled(); + }); + + it("returns the operator's capacity signal for all projects by default", async () => { + const res = await GET(reqWith()); + expect((await res.json()).data).toEqual(CAP); + expect(getFleetCapacity).toHaveBeenCalledWith("u", { projectIds: undefined }); + }); + + it("parses a comma-separated projectIds filter", async () => { + await GET(reqWith("?projectIds=p1,%20p2")); + expect(getFleetCapacity).toHaveBeenCalledWith("u", { projectIds: ["p1", "p2"] }); + }); +}); diff --git a/src/lib/engine/fleet-capacity.test.ts b/src/lib/engine/fleet-capacity.test.ts index efe9dfe70..822039b7f 100644 --- a/src/lib/engine/fleet-capacity.test.ts +++ b/src/lib/engine/fleet-capacity.test.ts @@ -1,14 +1,22 @@ // Copyright (c) 2026 Interactor, Inc. // SPDX-License-Identifier: AGPL-3.0-or-later -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -// fleet-capacity.ts imports `@/lib/db` at module load; stub it so the PURE derivation can -// be unit-tested without a real Prisma client (the async `getFleetCapacity` is not exercised -// here — only `deriveFleetCapacity`). +// fleet-capacity.ts imports `@/lib/db` and `./fleet-queue` at module load. Stub both so the +// PURE derivation AND the async `getFleetCapacity` assembler are unit-testable without a real +// Prisma client or the full queue pipeline. `./fleet` stays REAL so the shared +// `classifyDaemonMachine` / `isRunLive` predicates the assembler relies on are exercised +// end-to-end (busy/idle counts can't silently drift from the fleet-dots classifier). vi.mock("@/lib/db", () => ({ db: { machine: { findMany: vi.fn() }, engineRun: { findMany: vi.fn() } } })); +vi.mock("./fleet-queue", async (orig) => ({ + ...(await orig()), + listFleetGoalsOrdered: vi.fn(), +})); -const { deriveFleetCapacity } = await import("./fleet-capacity"); +const { deriveFleetCapacity, getFleetCapacity } = await import("./fleet-capacity"); const { MACHINE_ONLINE_THRESHOLD_MS } = await import("./fleet"); +const { db } = await import("@/lib/db"); +const { listFleetGoalsOrdered } = await import("./fleet-queue"); const NOW = new Date("2026-06-30T12:00:00.000Z"); const past = (ms: number) => new Date(NOW.getTime() - ms); @@ -116,3 +124,95 @@ describe("deriveFleetCapacity — scale-pressure derivation", () => { }); }); }); + +describe("getFleetCapacity — live assembly from queued goals + leased machines", () => { + // Only the fields each layer reads are supplied; the casts keep the stubs minimal. + const goal = (status: string) => ({ status }) as never; + const future = new Date(NOW.getTime() + 5 * 60_000); + const expired = new Date(NOW.getTime() - 1_000); + const offlineSeen = new Date(NOW.getTime() - MACHINE_ONLINE_THRESHOLD_MS - 1); + + beforeEach(() => { + vi.mocked(db.machine.findMany).mockReset(); + vi.mocked(db.engineRun.findMany).mockReset(); + vi.mocked(listFleetGoalsOrdered).mockReset(); + }); + + it("counts only QUEUED goals as depth and a live-leased machine as busy", async () => { + // Depth = the 2 queued rows; running / waiting_on_you / waiting_on_deps are already claimed. + vi.mocked(listFleetGoalsOrdered).mockResolvedValue([ + goal("queued"), + goal("queued"), + goal("running"), + goal("waiting_on_you"), + goal("waiting_on_deps"), + ]); + vi.mocked(db.machine.findMany).mockResolvedValue([ + { id: "m1", lastSeenAt: NOW, activeSessionId: null }, // online + live run ⇒ busy + { id: "m2", lastSeenAt: NOW, activeSessionId: null }, // online + idle ⇒ available + { id: "m3", lastSeenAt: offlineSeen, activeSessionId: null }, // offline ⇒ neither + ] as never); + vi.mocked(db.engineRun.findMany) + .mockResolvedValueOnce([ + { machineId: "m1", status: "running", leaseExpiresAt: future }, + // belt-and-suspenders: an expired lease is dropped by the isRunLive re-filter… + { machineId: "m2", status: "running", leaseExpiresAt: expired }, + // …and a null machineId is dropped by the id-narrowing filter. + { machineId: null, status: "running", leaseExpiresAt: future }, + ] as never) + .mockResolvedValueOnce([] as never); // no recent failures + + const cap = await getFleetCapacity("u", { now: NOW }); + expect(cap).toEqual({ + queueDepth: 2, + online: 2, + busy: 1, + available: 1, + shouldScale: true, + recommendedInstances: 1, + }); + expect(listFleetGoalsOrdered).toHaveBeenCalledWith("u", { projectIds: undefined, now: NOW }); + }); + + it("short-circuits the run queries when the operator has no daemon machines", async () => { + vi.mocked(listFleetGoalsOrdered).mockResolvedValue([goal("queued")]); + vi.mocked(db.machine.findMany).mockResolvedValue([] as never); + + const cap = await getFleetCapacity("u", { projectIds: ["p1", "p2"] }); + expect(cap).toMatchObject({ + queueDepth: 1, + online: 0, + busy: 0, + available: 0, + shouldScale: true, + recommendedInstances: 1, + }); + // No machines ⇒ the run lookups are skipped entirely (the [[],[]] branch). + expect(db.engineRun.findMany).not.toHaveBeenCalled(); + expect(listFleetGoalsOrdered).toHaveBeenCalledWith("u", { + projectIds: ["p1", "p2"], + now: expect.any(Date), + }); + }); + + it("treats a recently-failed but online machine as available (online, not busy)", async () => { + vi.mocked(listFleetGoalsOrdered).mockResolvedValue([]); + vi.mocked(db.machine.findMany).mockResolvedValue([ + { id: "m1", lastSeenAt: NOW, activeSessionId: null }, + ] as never); + vi.mocked(db.engineRun.findMany) + .mockResolvedValueOnce([] as never) // no live runs + .mockResolvedValueOnce([{ machineId: "m1" }] as never); // a recent failure + + const cap = await getFleetCapacity("u", { now: NOW }); + // Depth 0 ⇒ never scales; a failed-but-online machine is still idle capacity. + expect(cap).toEqual({ + queueDepth: 0, + online: 1, + busy: 0, + available: 1, + shouldScale: false, + recommendedInstances: 0, + }); + }); +}); From 53fa6550043b2df5b7a11a045366b40566ba2e1f Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Tue, 30 Jun 2026 17:59:02 +0000 Subject: [PATCH 3/6] test: raise vitest test/hook timeouts to 30s to de-flake full-suite + coverage gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review rework on the fleet-capacity goal — the two system blockers (system:test-suite-failed and system:patch-coverage-below-threshold) share a single root cause: timing-sensitive suites timing out under the full ~16k-test run, not anything in the fleet-capacity change. The 3 intermittent failures (agent-tools-route beforeEach dynamic-import hook, authorization-fuzzing route import, engine-goals-client jsdom render) all pass in isolation in well under a second each, but under the constrained review container the 50%-worker pool oversubscribes the box and they crossed the old 15s test / default 10s hook ceilings. A timed-out test fails `npm run test` (exit 1) AND aborts the v8-instrumented coverage run before coverage/lcov.info is written — which is exactly the pair of gates that came back red. Fix: raise testTimeout 15s→30s and set hookTimeout 30s (was the 10s default). Raising a timeout only lets a slow run finish; it cannot turn a passing test red, so this is monotonically safe. Verified locally: `npm run coverage:patch` now exits 0, writes coverage/lcov.info (1.1MB), and the full suite reports 16086 passed / 0 failed (87.53% lines, above the 80% floor). No fleet-capacity source change; no PM-agent config change (test-config only). Co-Authored-By: Claude Opus 4.8 (1M context) --- vitest.config.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/vitest.config.ts b/vitest.config.ts index 3f4a080b6..9076384bf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,7 +7,16 @@ export default defineConfig({ test: { globals: true, environment: "node", - testTimeout: 15000, + // Timing headroom for the constrained CI/review containers. Under the full + // ~16k-test run the worker pool oversubscribes the box, so timing-sensitive + // suites (dynamic-import beforeEach hooks, jsdom render+act) intermittently + // crossed the old 15s test / default 10s hook ceilings and timed out — flaky + // failures in unrelated suites that also aborted the instrumented coverage run + // before coverage/lcov.info was written. They pass comfortably in isolation + // (well under a second each); raising the ceilings only lets a slow run finish + // and cannot turn a passing test red. + testTimeout: 30000, + hookTimeout: 30000, // Cap worker concurrency to half the available cores. The default (one worker per // core) oversubscribes CPU/memory on the constrained CI/review containers, which // starves timing-sensitive tests past `testTimeout` and OOMs the v8-instrumented From fe42126e848ee0ea74a0efb737f51ae15142ff4e Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Tue, 30 Jun 2026 18:51:12 +0000 Subject: [PATCH 4/6] feat(fleet-sidebar): surface queue-depth vs capacity + scale-up prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Poll GET /api/v1/me/fleet/capacity (30s cadence + visibility/realtime handling, alongside the existing fleet-dots/machines/queue polls) and consume the shared FleetCapacity signal. - Add an explicit capacity summary line: "N queued · M idle / K online". - When shouldScale, render an under-provisioned prompt card (styled like the existing setup cards) wired to the /settings/machines Add-machine path, using recommendedInstances in the copy when > 0. - Mirror the pressure as a small amber indicator in the collapsed strip. - Read-only; no claim/lease changes. Gated so an idle fleet or a zero-machine state never shows the scale prompt (the existing "No machines connected" card owns that case). No agent config update needed: read-only operator UI surface, adds no tools and changes no agent-visible domain data. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/components/layout/fleet-sidebar.tsx | 116 ++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 7 deletions(-) diff --git a/src/components/layout/fleet-sidebar.tsx b/src/components/layout/fleet-sidebar.tsx index 7f26d3486..1b59f1dd3 100644 --- a/src/components/layout/fleet-sidebar.tsx +++ b/src/components/layout/fleet-sidebar.tsx @@ -4,10 +4,11 @@ import { useCallback, useEffect, useState } from "react"; import Link from "next/link"; -import { Monitor, ChevronRight, ChevronLeft, Loader2, Target, Hand, GitBranch, ListOrdered } from "lucide-react"; +import { Monitor, ChevronRight, ChevronLeft, Loader2, Target, Hand, GitBranch, ListOrdered, Layers } from "lucide-react"; import { cn } from "@/lib/utils"; import { useChannel } from "@/lib/realtime/use-channel"; import type { FleetMachine } from "@/app/api/v1/me/machines/fleet/route"; +import type { FleetCapacity } from "@/lib/engine/fleet-capacity"; import { useCurrentProject } from "./project-context"; const LS_KEY = "fleet-sidebar-collapsed"; @@ -82,6 +83,7 @@ export function FleetSidebar({ userId }: { userId: string }) { const [machines, setMachines] = useState([]); const [machinesLoaded, setMachinesLoaded] = useState(false); const [goals, setGoals] = useState([]); + const [capacity, setCapacity] = useState(null); const [collapsed, setCollapsed] = useState(() => { if (typeof window === "undefined") return true; const stored = localStorage.getItem(LS_KEY); @@ -117,30 +119,56 @@ export function FleetSidebar({ userId }: { userId: string }) { } catch { /* transient */ } }, []); - // Poll fleet dots (60s) + machines list (60s) + goals queue (30s) + const refreshCapacity = useCallback(async () => { + if (typeof document !== "undefined" && document.visibilityState === "hidden") return; + try { + const res = await fetch("/api/v1/me/fleet/capacity"); + if (!res.ok) return; + const json = (await res.json()) as { data: FleetCapacity }; + setCapacity(json.data ?? null); + } catch { /* transient */ } + }, []); + + // Poll fleet dots (60s) + machines list (60s) + goals queue (30s) + capacity (30s) useEffect(() => { void refreshFleet(); void refreshMachines(); void refreshGoals(); + void refreshCapacity(); const fi = setInterval(() => void refreshFleet(), 60_000); const mi = setInterval(() => void refreshMachines(), 60_000); const gi = setInterval(() => void refreshGoals(), 30_000); + const ci = setInterval(() => void refreshCapacity(), 30_000); const onVisible = () => { - if (document.visibilityState === "visible") void refreshGoals(); + if (document.visibilityState === "visible") { + void refreshGoals(); + void refreshCapacity(); + } }; document.addEventListener("visibilitychange", onVisible); return () => { clearInterval(fi); clearInterval(mi); clearInterval(gi); + clearInterval(ci); document.removeEventListener("visibilitychange", onVisible); }; - }, [refreshFleet, refreshMachines, refreshGoals]); + }, [refreshFleet, refreshMachines, refreshGoals, refreshCapacity]); // Real-time updates on machine notifications - useChannel(`notifications:${userId}`, () => void refreshFleet(), { - onOpen: () => void refreshFleet(), - }); + useChannel( + `notifications:${userId}`, + () => { + void refreshFleet(); + void refreshCapacity(); + }, + { + onOpen: () => { + void refreshFleet(); + void refreshCapacity(); + }, + }, + ); // Auto-expand on first paint when setup is incomplete useEffect(() => { @@ -172,6 +200,16 @@ export function FleetSidebar({ userId }: { userId: string }) { const workingCount = fleet.filter((m) => m.status === "working").length; + // Capacity summary is meaningful only once there is a fleet or queued work to compare. + const showCapacityLine = + capacity !== null && (capacity.online > 0 || capacity.queueDepth > 0); + // Under-provisioned prompt: more queued goals than idle machines to take them. Gated on + // there being at least one connected machine — when there are none, the "No machines + // connected" setup card already owns that call to action, so we don't double up. And + // `shouldScale` already implies `queueDepth > 0`, so an idle fleet never triggers it. + const showScalePrompt = + capacity !== null && capacity.shouldScale && capacity.queueDepth > 0 && !noMachine; + // Collapsed: thin strip with icons + dots + toggle if (collapsed) { return ( @@ -206,6 +244,14 @@ export function FleetSidebar({ userId }: { userId: string }) { )} ))} + {showScalePrompt && ( + + + + )} ); @@ -252,7 +298,63 @@ export function FleetSidebar({ userId }: { userId: string }) { + {/* Capacity summary: queued work vs idle capacity (read-only; mirrors the autoscaler signal) */} + {showCapacityLine && ( +
+ + 0 ? "text-[#1C1C1E]" : "text-[#8E8E93]", + )} + > + {capacity!.queueDepth} queued + + + + 0 ? "text-[#2F855A]" : "text-[#1C1C1E]", + )} + > + {capacity!.available} idle + + {" / "} + {capacity!.online} online + +
+ )} +
+ {/* Under-provisioned prompt: more queued goals than idle machines to take them. */} + {showScalePrompt && ( +
+
+
+ + Fleet under-provisioned +
+

+ {capacity!.queueDepth} goal{capacity!.queueDepth === 1 ? "" : "s"} queued,{" "} + {capacity!.available > 0 + ? `only ${capacity!.available} idle machine${capacity!.available === 1 ? "" : "s"}` + : "no idle machine"}{" "} + — add capacity + {capacity!.recommendedInstances > 0 && + ` (≈${capacity!.recommendedInstances} more instance${capacity!.recommendedInstances === 1 ? "" : "s"})`} + . +

+ + Add machine + +
+
+ )} + {/* Setup guidance cards */} {showSetup && (
From 06aa6202260419ab0fc7172265243d79c5c548bb Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Tue, 30 Jun 2026 19:43:46 +0000 Subject: [PATCH 5/6] feat(fleet-activity): surface queue depth + idle capacity on the project widget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the per-project Fleet Activity glance beyond live runs to also show queue depth and idle fleet capacity, reusing the fleet-capacity signal's shared definitions so the two surfaces can never drift. - fleet.ts: extend FleetActivityData with `queueDepth` (this project's runnable-but-unclaimed goals via `listRunnableGoalsWithKeys` — fleetOwnerId scope + occupied-goal exclusion, NOT a new query, bounded by FLEET_QUEUE_DEPTH_CAP) and `machines.busy`/`available`. getProjectFleet now derives capacity through new shared helpers `countMachineCapacity` + `loadDaemonMachineSignals`, kept as the single serializer (route + server render stay byte-identical). - fleet-capacity.ts: refactor deriveFleetCapacity/getFleetCapacity onto the same shared helpers so busy/idle has one definition (faithful-scope). - GET .../engine/fleet: forwards the extended payload (no logic change). - fleet-activity-widget.tsx: header now reads "N running · M queued · online/total (K idle)" with a subtle "under capacity" link to /settings/machines when queueDepth > available. The primary scale prompt stays in the Fleet sidebar. Read-only; no claim/lease changes. Adds widget + helper tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[projectId]/engine/fleet/route.ts | 7 +- .../dashboards/fleet-activity-widget.test.tsx | 60 ++++++++ .../dashboards/fleet-activity-widget.tsx | 22 ++- src/lib/engine/fleet-capacity.ts | 78 ++-------- src/lib/engine/fleet.test.ts | 136 ++++++++++++++-- src/lib/engine/fleet.ts | 145 ++++++++++++++++-- 6 files changed, 352 insertions(+), 96 deletions(-) create mode 100644 src/components/dashboards/fleet-activity-widget.test.tsx diff --git a/src/app/api/v2/projects/[projectId]/engine/fleet/route.ts b/src/app/api/v2/projects/[projectId]/engine/fleet/route.ts index e18f75adb..35fce4468 100644 --- a/src/app/api/v2/projects/[projectId]/engine/fleet/route.ts +++ b/src/app/api/v2/projects/[projectId]/engine/fleet/route.ts @@ -8,9 +8,10 @@ type Params = { params: Promise<{ projectId: string }> }; // Fleet Activity (project dashboard widget): the project's LIVE fleet runs (read from // the durable run-liveness lease, NOT ClaudeSession) + the requesting user's daemon -// machine online/total. Read-only. Returns the SAME `FleetActivityData` the dashboard -// server-renders (one shared serializer in @/lib/engine/fleet), so a live client -// refetch off this route re-renders byte-identical to the first paint. +// machine online/total/busy/available and this project's queue depth. Read-only. Returns +// the SAME `FleetActivityData` the dashboard server-renders (one shared serializer in +// @/lib/engine/fleet), so a live client refetch off this route re-renders byte-identical +// to the first paint. export async function GET(_req: NextRequest, { params }: Params) { const { projectId } = await params; const authed = await authorizeProject(projectId); diff --git a/src/components/dashboards/fleet-activity-widget.test.tsx b/src/components/dashboards/fleet-activity-widget.test.tsx new file mode 100644 index 000000000..820f13b8d --- /dev/null +++ b/src/components/dashboards/fleet-activity-widget.test.tsx @@ -0,0 +1,60 @@ +// @vitest-environment jsdom +// Copyright (c) 2026 Interactor, Inc. +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import React from "react"; + +vi.mock("next/link", () => ({ + default: ({ href, children, ...rest }: { href: string; children: React.ReactNode }) => ( + + {children} + + ), +})); +// The widget subscribes to a realtime channel + ticks a clock; neither matters for the +// header glance under test, so stub the hook to a no-op (it would otherwise open an SSE). +vi.mock("@/lib/realtime/use-channel", () => ({ useChannel: vi.fn() })); + +import { FleetActivityWidget } from "./fleet-activity-widget"; +import type { FleetActivityData } from "@/lib/engine/fleet"; + +function data(overrides: Partial = {}): FleetActivityData { + return { + runs: [], + machines: { online: 2, total: 3, busy: 1, available: 1 }, + queueDepth: 0, + ...overrides, + }; +} + +describe("FleetActivityWidget — header glance", () => { + it("shows running · queued · online/total (idle) in one line", () => { + render(); + expect(screen.getByText("0 running · 4 queued · 2/3 (1 idle)")).toBeInTheDocument(); + }); + + it("surfaces an 'under capacity' link to /settings/machines when depth > idle", () => { + render(); + const link = screen.getByRole("link", { name: /under capacity/i }); + expect(link).toHaveAttribute("href", "/settings/machines"); + }); + + it("omits the hint when idle capacity covers the queue", () => { + // 1 queued vs 1 idle — covered, so no nudge. + render(); + expect(screen.queryByRole("link", { name: /under capacity/i })).not.toBeInTheDocument(); + }); + + it("treats any queue with zero idle machines as under capacity", () => { + render( + , + ); + expect(screen.getByRole("link", { name: /under capacity/i })).toBeInTheDocument(); + }); +}); diff --git a/src/components/dashboards/fleet-activity-widget.tsx b/src/components/dashboards/fleet-activity-widget.tsx index 34627f1b6..a45c454cb 100644 --- a/src/components/dashboards/fleet-activity-widget.tsx +++ b/src/components/dashboards/fleet-activity-widget.tsx @@ -3,6 +3,7 @@ "use client"; import { useCallback, useEffect, useRef, useState } from "react"; +import Link from "next/link"; import { Activity, Cpu } from "lucide-react"; import { useChannel } from "@/lib/realtime/use-channel"; import { cn } from "@/lib/utils"; @@ -17,6 +18,11 @@ import type { FleetActivityData } from "@/lib/engine/fleet"; // run going live / ending reflects within ~1s. The shared serializer // (@/lib/engine/fleet) backs both the page render and the route, so a refetch // re-renders byte-identical to first paint. +// +// The header line gives a queue-depth + idle-capacity glance alongside the live runs: +// "N running · M queued · online/total (K idle)", plus a subtle "under capacity" link to +// /settings/machines when queued depth exceeds idle machines. The primary scale-up prompt +// lives in the Fleet sidebar; this stays a lightweight operator glance. interface Props { projectId: string; @@ -90,7 +96,10 @@ export function FleetActivityWidget({ projectId, data: initialData }: Props) { }, ); - const { runs, machines } = data; + const { runs, machines, queueDepth } = data; + // More queued goals than idle machines to take them — a subtle, lightweight nudge. The + // primary "scale up" prompt lives in the Fleet sidebar; here it's just a glance + link. + const underCapacity = queueDepth > machines.available; return (
@@ -98,8 +107,17 @@ export function FleetActivityWidget({ projectId, data: initialData }: Props) {

Fleet Activity

- {runs.length} running · {machines.online}/{machines.total} machines online + {runs.length} running · {queueDepth} queued · {machines.online}/{machines.total} ({machines.available} idle) + {underCapacity && ( + + under capacity + + )}
{runs.length === 0 ? ( diff --git a/src/lib/engine/fleet-capacity.ts b/src/lib/engine/fleet-capacity.ts index 87bd8b499..0d37d792e 100644 --- a/src/lib/engine/fleet-capacity.ts +++ b/src/lib/engine/fleet-capacity.ts @@ -1,13 +1,6 @@ // Copyright (c) 2026 Interactor, Inc. // SPDX-License-Identifier: AGPL-3.0-or-later -import { db } from "@/lib/db"; -import { - classifyDaemonMachine, - isRunLive, - MACHINE_ERROR_WINDOW_MS, - MACHINE_ONLINE_THRESHOLD_MS, - type DaemonMachineSignals, -} from "./fleet"; +import { countMachineCapacity, loadDaemonMachineSignals, type DaemonMachineSignals } from "./fleet"; import { listFleetGoalsOrdered } from "./fleet-queue"; // ─── Fleet capacity & scale-pressure signal ─────────────────────────────────── @@ -58,15 +51,9 @@ export function deriveFleetCapacity( machines: DaemonMachineSignals[], now: Date = new Date(), ): FleetCapacity { - let online = 0; - let busy = 0; - for (const m of machines) { - const seen = m.lastSeenAt instanceof Date ? m.lastSeenAt : new Date(m.lastSeenAt); - if (now.getTime() - seen.getTime() > MACHINE_ONLINE_THRESHOLD_MS) continue; // offline - online++; - if (classifyDaemonMachine(m, now) === "working") busy++; - } - const available = online - busy; + // online/busy/available come from the SHARED counter (the same one the project idle glance + // uses) so the two surfaces can never disagree; this module only layers the policy on top. + const { online, busy, available } = countMachineCapacity(machines, now); const shouldScale = queueDepth > available; const recommendedInstances = Math.max(0, queueDepth - available); return { queueDepth, online, busy, available, shouldScale, recommendedInstances }; @@ -90,60 +77,15 @@ export async function getFleetCapacity( ): Promise { const now = opts.now ?? new Date(); - const [goals, machineRows] = await Promise.all([ + // Both reads reuse existing definitions so this never becomes a second source of truth: + // queueDepth from the faithful claim-order list, machine signals from the SHARED loader + // (the same one the project idle glance uses). + const [goals, machineSignals] = await Promise.all([ listFleetGoalsOrdered(userId, { projectIds: opts.projectIds, now }), - db.machine.findMany({ - where: { userId, connectionKind: "daemon" }, - select: { id: true, lastSeenAt: true, activeSessionId: true }, - }), + loadDaemonMachineSignals(userId, now), ]); const queueDepth = goals.filter((g) => g.status === "queued").length; - const machineIds = machineRows.map((m) => m.id); - const errorWindowCutoff = new Date(now.getTime() - MACHINE_ERROR_WINDOW_MS); - const [activeRuns, failedRuns] = - machineIds.length > 0 - ? await Promise.all([ - db.engineRun.findMany({ - where: { - machineId: { in: machineIds }, - status: "running", - endedAt: null, - leaseExpiresAt: { gt: now }, - }, - select: { machineId: true, status: true, leaseExpiresAt: true }, - }), - db.engineRun.findMany({ - where: { - machineId: { in: machineIds }, - status: "failed", - endedAt: { gte: errorWindowCutoff }, - }, - select: { machineId: true }, - distinct: ["machineId"], - }), - ]) - : [[], []]; - - // The lease predicate is already in the WHERE; the `isRunLive` re-filter is - // belt-and-suspenders against clock skew (same pattern as `getProjectFleet`). - const liveRunMachineIds = new Set( - activeRuns - .filter((r) => isRunLive(r, now)) - .map((r) => r.machineId) - .filter((id): id is string => id != null), - ); - const failedMachineIds = new Set( - failedRuns.map((r) => r.machineId).filter((id): id is string => id != null), - ); - - const signals: DaemonMachineSignals[] = machineRows.map((m) => ({ - lastSeenAt: m.lastSeenAt, - activeSessionId: m.activeSessionId, - hasActiveRun: liveRunMachineIds.has(m.id), - hasRecentError: failedMachineIds.has(m.id), - })); - - return deriveFleetCapacity(queueDepth, signals, now); + return deriveFleetCapacity(queueDepth, machineSignals.signals, now); } diff --git a/src/lib/engine/fleet.test.ts b/src/lib/engine/fleet.test.ts index ad1626c27..0b9e6fa73 100644 --- a/src/lib/engine/fleet.test.ts +++ b/src/lib/engine/fleet.test.ts @@ -9,9 +9,24 @@ vi.mock("@/lib/db", () => ({ }, })); +// getProjectFleet now derives queue depth from the SAME goal-grain enumeration the daemon +// claim path consumes. Stub it so this file unit-tests the serializer without the full +// queue pipeline; queue.ts is exercised end-to-end by its own suite. +vi.mock("./queue", () => ({ listRunnableGoalsWithKeys: vi.fn() })); + const { db } = await import("@/lib/db"); -const { getProjectFleet, isRunLive, machineCounts, MACHINE_ONLINE_THRESHOLD_MS } = - await import("./fleet"); +const { listRunnableGoalsWithKeys } = await import("./queue"); +const { + getProjectFleet, + isRunLive, + machineCounts, + countMachineCapacity, + loadDaemonMachineSignals, + MACHINE_ONLINE_THRESHOLD_MS, +} = await import("./fleet"); + +/** A goal-grain runnable candidate (only `.length` is read for depth). */ +const goalCandidate = () => ({}) as never; const row = (v: T) => v as never; @@ -21,6 +36,10 @@ const past = (ms: number) => new Date(NOW.getTime() - ms); beforeEach(() => { vi.resetAllMocks(); + // Safe defaults so each test overrides only what it asserts on. + vi.mocked(db.engineRun.findMany).mockResolvedValue(row([])); + vi.mocked(db.machine.findMany).mockResolvedValue(row([])); + vi.mocked(listRunnableGoalsWithKeys).mockResolvedValue(row([])); }); describe("isRunLive — the lease-unexpired live-run filter", () => { @@ -69,11 +88,84 @@ describe("machineCounts — online (within 90s) / total", () => { }); }); -describe("getProjectFleet", () => { - it("queries live runs with the indexed lease predicate, scoped to the project", async () => { - vi.mocked(db.engineRun.findMany).mockResolvedValue(row([])); +describe("countMachineCapacity — online / busy / available (idle) from signals", () => { + const onlineBusy = { lastSeenAt: NOW, activeSessionId: null, hasActiveRun: true, hasRecentError: false }; + const onlineIdle = { lastSeenAt: NOW, activeSessionId: null, hasActiveRun: false, hasRecentError: false }; + const offline = { + lastSeenAt: past(MACHINE_ONLINE_THRESHOLD_MS + 1), + activeSessionId: null, + hasActiveRun: false, + hasRecentError: false, + }; + + it("available = online − busy; offline machines count toward neither", () => { + expect(countMachineCapacity([onlineBusy, onlineIdle, onlineIdle, offline], NOW)).toEqual({ + online: 3, + busy: 1, + available: 2, + }); + }); + + it("is all-zero for an empty fleet", () => { + expect(countMachineCapacity([], NOW)).toEqual({ online: 0, busy: 0, available: 0 }); + }); + + it("an attached session marks an online machine busy even without a live run", () => { + const sessionAttached = { lastSeenAt: NOW, activeSessionId: "s1", hasActiveRun: false, hasRecentError: false }; + expect(countMachineCapacity([sessionAttached], NOW)).toEqual({ online: 1, busy: 1, available: 0 }); + }); +}); + +describe("loadDaemonMachineSignals — operator fleet → shared busy/idle signals", () => { + it("scopes to the user's daemon machines and flags live-run / recent-failure machines", async () => { + vi.mocked(db.machine.findMany).mockResolvedValue( + row([ + { id: "m1", lastSeenAt: NOW, activeSessionId: null }, // a live leased run ⇒ busy + { id: "m2", lastSeenAt: NOW, activeSessionId: null }, // a recent failure, but online ⇒ idle + { id: "m3", lastSeenAt: NOW, activeSessionId: null }, // plain idle + ]), + ); + vi.mocked(db.engineRun.findMany) + .mockResolvedValueOnce(row([{ machineId: "m1", status: "running", leaseExpiresAt: future(60_000) }])) + .mockResolvedValueOnce(row([{ machineId: "m2" }])); + + const { total, signals } = await loadDaemonMachineSignals("user-1", NOW); + + expect(vi.mocked(db.machine.findMany).mock.calls[0][0]?.where).toEqual({ + userId: "user-1", + connectionKind: "daemon", + }); + expect(total).toBe(3); + expect(signals).toEqual([ + { lastSeenAt: NOW, activeSessionId: null, hasActiveRun: true, hasRecentError: false }, + { lastSeenAt: NOW, activeSessionId: null, hasActiveRun: false, hasRecentError: true }, + { lastSeenAt: NOW, activeSessionId: null, hasActiveRun: false, hasRecentError: false }, + ]); + }); + + it("skips the run lookups entirely when the operator has no daemon machines", async () => { vi.mocked(db.machine.findMany).mockResolvedValue(row([])); + const { total, signals } = await loadDaemonMachineSignals("user-1", NOW); + + expect(total).toBe(0); + expect(signals).toEqual([]); + expect(db.engineRun.findMany).not.toHaveBeenCalled(); + }); + + it("drops an expired-lease run via the isRunLive re-filter (clock-skew defense)", async () => { + vi.mocked(db.machine.findMany).mockResolvedValue(row([{ id: "m1", lastSeenAt: NOW, activeSessionId: null }])); + vi.mocked(db.engineRun.findMany) + .mockResolvedValueOnce(row([{ machineId: "m1", status: "running", leaseExpiresAt: past(1) }])) + .mockResolvedValueOnce(row([])); + + const { signals } = await loadDaemonMachineSignals("user-1", NOW); + expect(signals[0].hasActiveRun).toBe(false); + }); +}); + +describe("getProjectFleet", () => { + it("queries live runs (project-scoped) + machines (user-scoped) + the queue-depth enumeration", async () => { await getProjectFleet("proj-1", "user-1", NOW); const where = vi.mocked(db.engineRun.findMany).mock.calls[0][0]?.where; @@ -87,10 +179,12 @@ describe("getProjectFleet", () => { userId: "user-1", connectionKind: "daemon", }); + // Queue depth reuses the goal-grain claim enumeration, fleetOwnerId-scoped to the caller. + expect(listRunnableGoalsWithKeys).toHaveBeenCalledWith("proj-1", expect.any(Number), NOW, "user-1"); }); - it("serializes a live run with its task + goal, and counts machines", async () => { - vi.mocked(db.engineRun.findMany).mockResolvedValue( + it("serializes a live run with its task + goal, counts machine capacity, and reports queue depth", async () => { + vi.mocked(db.engineRun.findMany).mockResolvedValueOnce( row([ { id: "run-1", @@ -109,13 +203,20 @@ describe("getProjectFleet", () => { }, ]), ); + // 3 daemon machines: 2 online (1 busy, 1 idle), 1 offline ⇒ online 2, busy 1, available 1, total 3. vi.mocked(db.machine.findMany).mockResolvedValue( - row([{ lastSeenAt: past(1000) }, { lastSeenAt: past(10 * 60_000) }]), + row([ + { id: "m1", lastSeenAt: past(1000), activeSessionId: "sess-1" }, // online + session ⇒ busy + { id: "m2", lastSeenAt: past(1000), activeSessionId: null }, // online idle + { id: "m3", lastSeenAt: past(10 * 60_000), activeSessionId: null }, // offline + ]), ); + vi.mocked(listRunnableGoalsWithKeys).mockResolvedValue(row([goalCandidate(), goalCandidate(), goalCandidate()])); const data = await getProjectFleet("proj-1", "user-1", NOW); - expect(data.machines).toEqual({ online: 1, total: 2 }); + expect(data.machines).toEqual({ online: 2, total: 3, busy: 1, available: 1 }); + expect(data.queueDepth).toBe(3); expect(data.runs).toHaveLength(1); expect(data.runs[0]).toEqual({ runId: "run-1", @@ -129,7 +230,7 @@ describe("getProjectFleet", () => { }); it("emits a null goal for a goalless task", async () => { - vi.mocked(db.engineRun.findMany).mockResolvedValue( + vi.mocked(db.engineRun.findMany).mockResolvedValueOnce( row([ { id: "run-2", @@ -142,17 +243,16 @@ describe("getProjectFleet", () => { }, ]), ); - vi.mocked(db.machine.findMany).mockResolvedValue(row([])); const data = await getProjectFleet("proj-1", "user-1", NOW); expect(data.runs[0].goal).toBeNull(); expect(data.runs[0].machineId).toBeNull(); }); - it("defensively drops a row whose lease slipped past now between query and map", async () => { + it("defensively drops a run whose lease slipped past now between query and map", async () => { // Belt-and-suspenders: the DB predicate filters, but isRunLive re-checks against // a single `now` so a row that expired mid-request never leaks into the payload. - vi.mocked(db.engineRun.findMany).mockResolvedValue( + vi.mocked(db.engineRun.findMany).mockResolvedValueOnce( row([ { id: "stale", @@ -165,9 +265,17 @@ describe("getProjectFleet", () => { }, ]), ); - vi.mocked(db.machine.findMany).mockResolvedValue(row([])); const data = await getProjectFleet("proj-1", "user-1", NOW); expect(data.runs).toHaveLength(0); }); + + it("reports zero depth and an all-zero fleet when nothing is queued or connected", async () => { + const data = await getProjectFleet("proj-1", "user-1", NOW); + expect(data).toEqual({ + runs: [], + machines: { online: 0, total: 0, busy: 0, available: 0 }, + queueDepth: 0, + }); + }); }); diff --git a/src/lib/engine/fleet.ts b/src/lib/engine/fleet.ts index af8445ceb..4b30c77a5 100644 --- a/src/lib/engine/fleet.ts +++ b/src/lib/engine/fleet.ts @@ -6,6 +6,10 @@ import { engine_run_mode } from "@prisma/client"; // Readiness panel can import it without dragging `@/lib/db` into the browser // bundle. Re-exported here so existing fleet importers/tests are unchanged. import { MACHINE_ONLINE_THRESHOLD_MS } from "@/lib/agent/repo-readiness"; +// Queue-depth reuses the EXACT goal-grain candidate enumeration the daemon claim path +// consumes (fleetOwnerId scope + occupied-goal exclusion live there) — NOT a second +// query — so the widget's "M queued" can never drift from what the fleet would claim. +import { listRunnableGoalsWithKeys } from "./queue"; // Fleet Activity (project dashboard widget): a LIVE window into what the daemon fleet // is doing right now, read from the durable run-liveness lease (PR #1106, ./lease.ts) @@ -24,6 +28,12 @@ export { MACHINE_ONLINE_THRESHOLD_MS }; /** How many live runs the widget shows. A sane cap — a fleet rarely drives more. */ export const FLEET_RUNS_CAP = 50; +/** Upper bound on the queue-depth count the project glance reports. The widget shows a + * glance ("M queued"), not an exact backlog census — bounding the enumeration keeps the + * read cheap. Depth ≥ this many runnable goals still trips the under-capacity hint, which + * is all the lightweight glance needs (the precise scale prompt lives in the Fleet sidebar). */ +export const FLEET_QUEUE_DEPTH_CAP = 50; + /** One live run on the fleet. Lease-derived liveness; `startedAt`/`leaseExpiresAt` ISO. */ export interface FleetRun { runId: string; @@ -43,7 +53,14 @@ export interface FleetRun { /** The serializable payload the Fleet Activity widget renders against. */ export interface FleetActivityData { runs: FleetRun[]; - machines: { online: number; total: number }; + /** Operator's daemon fleet: `online`/`total` for the existing "machines online" glance, + * plus `busy`/`available` (available = idle online machines, each able to take one queued + * goal now) so the widget can show an idle count alongside it. */ + machines: { online: number; total: number; busy: number; available: number }; + /** Count of this project's runnable-but-unclaimed goals (fleetOwnerId-scoped, occupied + * goals excluded) — the same goal-grain depth the daemon claim path would pop, bounded + * by `FLEET_QUEUE_DEPTH_CAP`. */ + queueDepth: number; } /** @@ -130,13 +147,113 @@ export function classifyDaemonMachine( return m.hasRecentError ? "error" : "idle"; } +/** online / busy / available counts for a daemon fleet. `online` uses the same 90s window as + * `machineCounts`; `busy` = online machines the shared `classifyDaemonMachine` rules + * "working"; `available` = online − busy (idle online machines, each able to take one queued + * goal now). The instance-per-task fleet model makes `available` the true "idle capacity". */ +export interface MachineCapacityCounts { + online: number; + busy: number; + available: number; +} + +/** + * Derive online/busy/available from daemon machine signals. Pure, and the SINGLE definition + * of "idle capacity" — shared by the project glance (`getProjectFleet`) and the scale-pressure + * policy (`deriveFleetCapacity`, which layers shouldScale/recommendedInstances on top), so the + * two surfaces can never report a different idle count for the same fleet (faithful-scope). + */ +export function countMachineCapacity( + machines: DaemonMachineSignals[], + now: Date = new Date(), +): MachineCapacityCounts { + let online = 0; + let busy = 0; + for (const m of machines) { + const seen = m.lastSeenAt instanceof Date ? m.lastSeenAt : new Date(m.lastSeenAt); + if (now.getTime() - seen.getTime() > MACHINE_ONLINE_THRESHOLD_MS) continue; // offline + online++; + if (classifyDaemonMachine(m, now) === "working") busy++; + } + return { online, busy, available: online - busy }; +} + +/** + * Load an operator's daemon machines and resolve each to the shared `DaemonMachineSignals` + * the capacity/idle derivation consumes. ONE definition of "busy": a machine is working iff + * it has a LIVE leased run (`isRunLive`, keyed by `EngineRun.machineId`) or an attached + * session; a run that failed within `MACHINE_ERROR_WINDOW_MS` flags it (but a failed-yet- + * online machine is still idle capacity). Reused by `getProjectFleet` (the idle glance) and + * `getFleetCapacity` (the scale signal) so the two can never drift. Returns `total` (every + * daemon machine, incl. offline — the widget's "/total") alongside the per-machine signals. + */ +export async function loadDaemonMachineSignals( + userId: string, + now: Date = new Date(), +): Promise<{ total: number; signals: DaemonMachineSignals[] }> { + const machineRows = await db.machine.findMany({ + where: { userId, connectionKind: "daemon" }, + select: { id: true, lastSeenAt: true, activeSessionId: true }, + }); + + const machineIds = machineRows.map((m) => m.id); + const errorWindowCutoff = new Date(now.getTime() - MACHINE_ERROR_WINDOW_MS); + const [activeRuns, failedRuns] = + machineIds.length > 0 + ? await Promise.all([ + db.engineRun.findMany({ + where: { + machineId: { in: machineIds }, + status: "running", + endedAt: null, + leaseExpiresAt: { gt: now }, + }, + select: { machineId: true, status: true, leaseExpiresAt: true }, + }), + db.engineRun.findMany({ + where: { + machineId: { in: machineIds }, + status: "failed", + endedAt: { gte: errorWindowCutoff }, + }, + select: { machineId: true }, + distinct: ["machineId"], + }), + ]) + : [[], []]; + + // The lease predicate is already in the WHERE; the `isRunLive` re-filter is + // belt-and-suspenders against clock skew (same pattern as the live-runs read). + const liveRunMachineIds = new Set( + activeRuns + .filter((r) => isRunLive(r, now)) + .map((r) => r.machineId) + .filter((id): id is string => id != null), + ); + const failedMachineIds = new Set( + failedRuns.map((r) => r.machineId).filter((id): id is string => id != null), + ); + + const signals: DaemonMachineSignals[] = machineRows.map((m) => ({ + lastSeenAt: m.lastSeenAt, + activeSessionId: m.activeSessionId, + hasActiveRun: liveRunMachineIds.has(m.id), + hasRecentError: failedMachineIds.has(m.id), + })); + return { total: machineRows.length, signals }; +} + /** * Assemble the live `FleetActivityData` for a project + the requesting user's daemon - * machines. Two reads: + * machines. Three reads, run concurrently: * - live runs: every `running` EngineRun on one of THIS project's tasks whose lease is * unexpired (the indexed lease predicate), oldest-first, capped. The `isRunLive` * re-filter is belt-and-suspenders against clock skew between the query and the map. - * - machines: the USER's daemon machines (the fleet they own), counted online/total. + * - machines: the USER's daemon machines (the fleet they own), resolved to the shared + * busy/idle signals and counted online/total + busy/available (idle capacity). + * - queueDepth: this project's runnable-but-unclaimed GOALS via the same + * `listRunnableGoalsWithKeys` the daemon claim path consumes (fleetOwnerId-scoped to + * the requesting operator, occupied goals excluded), bounded by `FLEET_QUEUE_DEPTH_CAP`. * * Project-scoped: callers gate on project membership before calling. The machine counts * are user-scoped (a daemon belongs to a user, not a project) — they reflect the @@ -147,7 +264,7 @@ export async function getProjectFleet( userId: string, now: Date = new Date(), ): Promise { - const [runRows, machineRows] = await Promise.all([ + const [runRows, machineSignals, runnableGoals] = await Promise.all([ db.engineRun.findMany({ where: { status: "running", @@ -174,10 +291,10 @@ export async function getProjectFleet( orderBy: { startedAt: "asc" }, take: FLEET_RUNS_CAP, }), - db.machine.findMany({ - where: { userId, connectionKind: "daemon" }, - select: { lastSeenAt: true }, - }), + loadDaemonMachineSignals(userId, now), + // Pass the caller so the enumeration honors fleetOwnerId scope exactly as the claim + // path does; the returned candidates are already occupied-goal-excluded. + listRunnableGoalsWithKeys(projectId, FLEET_QUEUE_DEPTH_CAP, now, userId), ]); const runs: FleetRun[] = runRows @@ -199,5 +316,15 @@ export async function getProjectFleet( leaseExpiresAt: r.leaseExpiresAt ? r.leaseExpiresAt.toISOString() : null, })); - return { runs, machines: machineCounts(machineRows, now) }; + const cap = countMachineCapacity(machineSignals.signals, now); + return { + runs, + machines: { + online: cap.online, + total: machineSignals.total, + busy: cap.busy, + available: cap.available, + }, + queueDepth: runnableGoals.length, + }; } From d1bbacdb797c268dbf3d8ee98819b71f586723cb Mon Sep 17 00:00:00 2001 From: pulzzejaehoon Date: Wed, 1 Jul 2026 00:02:47 +0000 Subject: [PATCH 6/6] =?UTF-8?q?test:=20trim=20vitest=20fork=20concurrency?= =?UTF-8?q?=2050%=E2=86=9240%=20to=20de-flake=20full-suite=20+=20coverage?= =?UTF-8?q?=20gates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full-suite gate (npm run test) and the patch-coverage gate both aborted on the constrained 8-core review box: 16055/16057 tests passed, but 2 test files died with 'Failed to start forks worker' / 'Timeout waiting for worker to respond' — vitest's hardcoded 60s worker START_TIMEOUT exceeded because a freshly forked worker could not get scheduled through its heavy module-import phase while the other forks saturated the box. Any such abort fails the run and leaves no coverage/lcov.info, blocking the coverage gate too. None of the failing files are touched by this goal. Even at 50% (4 forks here) the spawn window starved. Trimming to 40% (3 forks on an 8-core box) widens the margin so a fresh fork can initialize in time, while a proportional cap still scales up on larger CI hosts. Scheduling only — it cannot turn a passing test red, only let a slow run finish cleanly and emit lcov.info. Co-Authored-By: Claude Opus 4.8 (1M context) --- vitest.config.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/vitest.config.ts b/vitest.config.ts index 9076384bf..6f4066892 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -17,13 +17,18 @@ export default defineConfig({ // and cannot turn a passing test red. testTimeout: 30000, hookTimeout: 30000, - // Cap worker concurrency to half the available cores. The default (one worker per + // Cap worker concurrency below half the available cores. The default (one worker per // core) oversubscribes CPU/memory on the constrained CI/review containers, which // starves timing-sensitive tests past `testTimeout` and OOMs the v8-instrumented // coverage run — producing non-deterministic failures in unrelated suites and, on - // any failure, no coverage/lcov.info at all. A proportional cap keeps the run - // deterministic on small boxes while still scaling on larger CI hosts. - maxWorkers: "50%", + // any failure, no coverage/lcov.info at all. Even at 50% the small (8-core) review box + // could not schedule a freshly forked worker's module-import phase within vitest's + // hardcoded 60s START_TIMEOUT, so spawns intermittently aborted with "Failed to start + // forks worker" / "Timeout waiting for worker to respond" — failing the whole run and + // leaving no lcov.info behind. Trimming to 40% (3 forks on an 8-core box) widens that + // spawn margin while a proportional cap still scales up on larger CI hosts. Scheduling + // only — it can never turn a passing test red, only let a slow run finish cleanly. + maxWorkers: "40%", // Node 26 exposes an experimental globalThis.localStorage that returns undefined // when accessed without --localstorage-file, which shadows jsdom's implementation. // Setting a URL in the jsdom environment options forces jsdom to register its own