Fleet scaling: surface queue depth + capacity so more container instances spin up when work is queued

package.json

@@ -30,7 +30,7 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
-    "coverage:patch": "vitest run --coverage --maxWorkers=25% --coverage.reporter=lcovonly --coverage.reporter=text-summary --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0",
+    "coverage:patch": "NODE_OPTIONS=--max-old-space-size=6144 vitest run --coverage --maxWorkers=25% --coverage.reporter=lcovonly --coverage.reporter=text-summary --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0",
     "test:unit": "vitest run tests/unit/",
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",

src/app/api/v1/me/fleet/capacity/route.test.ts

@@ -0,0 +1,51 @@
+// Copyright (c) 2026 Interactor, Inc.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/lib/auth/with-daemon-token-auth", () => ({ authorizeDaemonOrSession: vi.fn() }));
+vi.mock("@/lib/engine", async (orig) => {
+  const actual = await orig<typeof import("@/lib/engine")>();
+  return { ...actual, getFleetCapacity: vi.fn() };
+});
+
+const { authorizeDaemonOrSession } = await import("@/lib/auth/with-daemon-token-auth");
+const { getFleetCapacity } = await import("@/lib/engine");
+const { GET } = await import("./route");
+
+const row = <T>(v: T) => v as never;
+const CAP = {
+  queueDepth: 2,
+  online: 3,
+  busy: 2,
+  available: 1,
+  shouldScale: true,
+  recommendedInstances: 1,
+};
+const reqWith = (qs = "") =>
+  new Request(`http://localhost/api/v1/me/fleet/capacity${qs}`) as never;
+
+beforeEach(() => {
+  vi.resetAllMocks();
+  vi.mocked(authorizeDaemonOrSession).mockResolvedValue(row({ userId: "u" }));
+  vi.mocked(getFleetCapacity).mockResolvedValue(row(CAP));
+});
+
+describe("GET /api/v1/me/fleet/capacity", () => {
+  it("401s when unauthorized", async () => {
+    vi.mocked(authorizeDaemonOrSession).mockResolvedValue(row(null));
+    const res = await GET(reqWith());
+    expect(res.status).toBe(401);
+    expect(getFleetCapacity).not.toHaveBeenCalled();
+  });
+
+  it("returns the operator's capacity signal for all projects by default", async () => {
+    const res = await GET(reqWith());
+    expect((await res.json()).data).toEqual(CAP);
+    expect(getFleetCapacity).toHaveBeenCalledWith("u", { projectIds: undefined });
+  });
+
+  it("parses a comma-separated projectIds filter", async () => {
+    await GET(reqWith("?projectIds=p1,%20p2"));
+    expect(getFleetCapacity).toHaveBeenCalledWith("u", { projectIds: ["p1", "p2"] });
+  });
+});

src/app/api/v1/me/fleet/capacity/route.ts

@@ -0,0 +1,26 @@
+// Copyright (c) 2026 Interactor, Inc.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { NextRequest, NextResponse } from "next/server";
+import { authorizeDaemonOrSession } from "@/lib/auth/with-daemon-token-auth";
+import { getFleetCapacity } from "@/lib/engine";
+
+// Read-only fleet capacity & scale-pressure signal for the operator's cross-project fleet:
+// engine queue depth vs idle container-instance capacity. Pollable by an EXTERNAL autoscaler
+// holding a daemon token (and by the browser session) — auth mirrors the fleet-queue route.
+// Optional `?projectIds=a,b` narrows to a subset of the operator's projects, mirroring the
+// claim filter. Derivation only — no machine-local coordination, no in-daemon concurrency
+// state (the server surfaces depth + declares the scale policy; the daemon coordinates claims).
+export async function GET(req: NextRequest) {
+  const authed = await authorizeDaemonOrSession(req);
+  if (!authed) {
+    return NextResponse.json({ error: "unauthorized" }, { status: 401 });
+  }
+
+  const raw = new URL(req.url).searchParams.get("projectIds");
+  const projectIds = raw
+    ? raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
+    : undefined;
+
+  const capacity = await getFleetCapacity(authed.userId, { projectIds });
+  return NextResponse.json({ data: capacity });
+}

src/app/api/v1/me/machines/fleet/route.ts

@@ -4,10 +4,7 @@ import { NextResponse } from "next/server";
 import { auth } from "@/lib/auth";
 import { db } from "@/lib/db";
 import { DedicatedMachineStatus } from "@prisma/client";
-
-const OFFLINE_THRESHOLD_MS = 90 * 1000;
-// A failed run within this window marks the machine as "error".
-const ERROR_WINDOW_MS = 2 * 60 * 60 * 1000;
+import { classifyDaemonMachine, MACHINE_ERROR_WINDOW_MS } from "@/lib/engine/fleet";
 
 export interface FleetMachine {
   id: string;
@@ -55,7 +52,7 @@ export async function GET() {
       : [];
 
   const now = Date.now();
-  const errorWindowCutoff = new Date(now - ERROR_WINDOW_MS);
+  const errorWindowCutoff = new Date(now - MACHINE_ERROR_WINDOW_MS);
 
   // For daemon machines: check for a recent failed EngineRun to surface the "error" state,
   // and for an active EngineRun (non-expired lease) to surface the "working" state.
@@ -95,32 +92,24 @@ export async function GET() {
   );
 
   const fleet: FleetMachine[] = [
-    ...daemonMachines.map((m) => {
-      const isOffline = now - m.lastSeenAt.getTime() > OFFLINE_THRESHOLD_MS;
-      const hasActiveSession = m.activeSessionId != null;
-      const hasActiveRun = activeRunMachineIds.has(m.id);
-      const hasRecentError = failedMachineIds.has(m.id);
-
-      let status: FleetMachine["status"];
-      if (isOffline && hasActiveSession) {
-        // Heartbeat went stale while a session was still active — agent is stuck.
-        status = "stuck";
-      } else if (isOffline) {
-        status = hasRecentError ? "error" : "offline";
-      } else if (hasActiveSession || hasActiveRun) {
-        // Working: either a user session is attached OR the daemon is executing an EngineRun.
-        status = "working";
-      } else {
-        status = hasRecentError ? "error" : "idle";
-      }
-
-      return {
-        id: m.id,
-        type: "daemon" as const,
-        name: m.customName ?? m.name,
-        status,
-      } satisfies FleetMachine;
-    }),
+    ...daemonMachines.map((m) => ({
+      id: m.id,
+      type: "daemon" as const,
+      name: m.customName ?? m.name,
+      // Shared with the capacity signal so the two surfaces can't drift (see
+      // classifyDaemonMachine). activeSessionId tracks user-session attachment only — it is
+      // NOT set during autonomous daemon execution, so the live EngineRun lease is what marks
+      // an autonomous machine "working".
+      status: classifyDaemonMachine(
+        {
+          lastSeenAt: m.lastSeenAt,
+          activeSessionId: m.activeSessionId,
+          hasActiveRun: activeRunMachineIds.has(m.id),
+          hasRecentError: failedMachineIds.has(m.id),
+        },
+        new Date(now),
+      ),
+    } satisfies FleetMachine)),
     ...dedicatedMachines.map((m) => ({
       id: m.id,
       type: "dedicated" as const,

src/app/api/v2/projects/[projectId]/engine/fleet/route.ts

@@ -8,9 +8,10 @@ type Params = { params: Promise<{ projectId: string }> };
 
 // Fleet Activity (project dashboard widget): the project's LIVE fleet runs (read from
 // the durable run-liveness lease, NOT ClaudeSession) + the requesting user's daemon
-// machine online/total. Read-only. Returns the SAME `FleetActivityData` the dashboard
-// server-renders (one shared serializer in @/lib/engine/fleet), so a live client
-// refetch off this route re-renders byte-identical to the first paint.
+// machine online/total/busy/available and this project's queue depth. Read-only. Returns
+// the SAME `FleetActivityData` the dashboard server-renders (one shared serializer in
+// @/lib/engine/fleet), so a live client refetch off this route re-renders byte-identical
+// to the first paint.
 export async function GET(_req: NextRequest, { params }: Params) {
   const { projectId } = await params;
   const authed = await authorizeProject(projectId);

src/components/dashboards/fleet-activity-widget.test.tsx

@@ -0,0 +1,60 @@
+// @vitest-environment jsdom
+// Copyright (c) 2026 Interactor, Inc.
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import { describe, it, expect, vi } from "vitest";
+import { render, screen } from "@testing-library/react";
+import "@testing-library/jest-dom";
+import React from "react";
+
+vi.mock("next/link", () => ({
+  default: ({ href, children, ...rest }: { href: string; children: React.ReactNode }) => (
+    <a href={href} {...rest}>
+      {children}
+    </a>
+  ),
+}));
+// The widget subscribes to a realtime channel + ticks a clock; neither matters for the
+// header glance under test, so stub the hook to a no-op (it would otherwise open an SSE).
+vi.mock("@/lib/realtime/use-channel", () => ({ useChannel: vi.fn() }));
+
+import { FleetActivityWidget } from "./fleet-activity-widget";
+import type { FleetActivityData } from "@/lib/engine/fleet";
+
+function data(overrides: Partial<FleetActivityData> = {}): FleetActivityData {
+  return {
+    runs: [],
+    machines: { online: 2, total: 3, busy: 1, available: 1 },
+    queueDepth: 0,
+    ...overrides,
+  };
+}
+
+describe("FleetActivityWidget — header glance", () => {
+  it("shows running · queued · online/total (idle) in one line", () => {
+    render(<FleetActivityWidget projectId="p1" data={data({ queueDepth: 4 })} />);
+    expect(screen.getByText("0 running · 4 queued · 2/3 (1 idle)")).toBeInTheDocument();
+  });
+
+  it("surfaces an 'under capacity' link to /settings/machines when depth > idle", () => {
+    render(<FleetActivityWidget projectId="p1" data={data({ queueDepth: 5 })} />);
+    const link = screen.getByRole("link", { name: /under capacity/i });
+    expect(link).toHaveAttribute("href", "/settings/machines");
+  });
+
+  it("omits the hint when idle capacity covers the queue", () => {
+    // 1 queued vs 1 idle — covered, so no nudge.
+    render(<FleetActivityWidget projectId="p1" data={data({ queueDepth: 1 })} />);
+    expect(screen.queryByRole("link", { name: /under capacity/i })).not.toBeInTheDocument();
+  });
+
+  it("treats any queue with zero idle machines as under capacity", () => {
+    render(
+      <FleetActivityWidget
+        projectId="p1"
+        data={data({ queueDepth: 1, machines: { online: 1, total: 1, busy: 1, available: 0 } })}
+      />,
+    );
+    expect(screen.getByRole("link", { name: /under capacity/i })).toBeInTheDocument();
+  });
+});

src/components/dashboards/fleet-activity-widget.tsx

@@ -3,6 +3,7 @@
 "use client";
 
 import { useCallback, useEffect, useRef, useState } from "react";
+import Link from "next/link";
 import { Activity, Cpu } from "lucide-react";
 import { useChannel } from "@/lib/realtime/use-channel";
 import { cn } from "@/lib/utils";
@@ -17,6 +18,11 @@ import type { FleetActivityData } from "@/lib/engine/fleet";
 // run going live / ending reflects within ~1s. The shared serializer
 // (@/lib/engine/fleet) backs both the page render and the route, so a refetch
 // re-renders byte-identical to first paint.
+//
+// The header line gives a queue-depth + idle-capacity glance alongside the live runs:
+// "N running · M queued · online/total (K idle)", plus a subtle "under capacity" link to
+// /settings/machines when queued depth exceeds idle machines. The primary scale-up prompt
+// lives in the Fleet sidebar; this stays a lightweight operator glance.
 
 interface Props {
   projectId: string;
@@ -90,16 +96,28 @@ export function FleetActivityWidget({ projectId, data: initialData }: Props) {
     },
   );
 
-  const { runs, machines } = data;
+  const { runs, machines, queueDepth } = data;
+  // More queued goals than idle machines to take them — a subtle, lightweight nudge. The
+  // primary "scale up" prompt lives in the Fleet sidebar; here it's just a glance + link.
+  const underCapacity = queueDepth > machines.available;
 
   return (
     <div className="rounded-2xl border border-gray-100 bg-white p-5 shadow-sm">
       <div className="mb-3 flex items-center gap-2">
         <Activity className="h-4 w-4 text-[#4CD964]" strokeWidth={1.5} />
         <h3 className="text-sm font-semibold text-[#1C1C1E]">Fleet Activity</h3>
         <span className="ml-auto text-xs font-medium text-[var(--ds-text-secondary)]">
-          {runs.length} running · {machines.online}/{machines.total} machines online
+          {runs.length} running · {queueDepth} queued · {machines.online}/{machines.total} ({machines.available} idle)
         </span>
+        {underCapacity && (
+          <Link
+            href="/settings/machines"
+            className="text-xs font-medium text-amber-600 underline-offset-2 hover:underline"
+            title={`${queueDepth} queued vs ${machines.available} idle — add fleet capacity`}
+          >
+            under capacity
+          </Link>
+        )}
       </div>
 
       {runs.length === 0 ? (