From aea338c1333e57a238f9799b1991a66b2b3b1a7d Mon Sep 17 00:00:00 2001 From: Dan Shapiro Date: Tue, 5 May 2026 18:43:33 -0700 Subject: [PATCH 01/25] Keep sidebar reopen button outside tab scroller --- src/components/TabBar.tsx | 25 +++++++++++-------- .../client/components/TabBar.mobile.test.tsx | 5 +++- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/components/TabBar.tsx b/src/components/TabBar.tsx index 2d170f67b..36904e9cf 100644 --- a/src/components/TabBar.tsx +++ b/src/components/TabBar.tsx @@ -400,6 +400,21 @@ export default function TabBar({ sidebarCollapsed, onToggleSidebar }: TabBarProp className="pointer-events-none absolute inset-x-0 bottom-0 h-px bg-muted-foreground/45" aria-hidden="true" /> + {sidebarCollapsed && onToggleSidebar && ( +
+ +
+ )} - {sidebarCollapsed && onToggleSidebar && ( - - )} {tabs.map(renderSortableTab)} diff --git a/test/unit/client/components/TabBar.mobile.test.tsx b/test/unit/client/components/TabBar.mobile.test.tsx index a7b3a6f6b..bc317009d 100644 --- a/test/unit/client/components/TabBar.mobile.test.tsx +++ b/test/unit/client/components/TabBar.mobile.test.tsx @@ -137,7 +137,7 @@ describe('TabBar sidebar toggle integration', () => { ;(globalThis as any).setMobileForTest?.(false) }) - it('renders show-sidebar button before tabs on desktop when sidebar is collapsed', () => { + it('renders show-sidebar button in a fixed desktop slot when sidebar is collapsed', () => { const onToggleSidebar = vi.fn() const store = createStore(defaultTabsState, defaultPanesState) render( @@ -147,7 +147,10 @@ describe('TabBar sidebar toggle integration', () => { ) const showButton = screen.getByTitle('Show sidebar') + const fixedSlot = screen.getByTestId('desktop-sidebar-reopen-slot') expect(showButton).toBeInTheDocument() + expect(fixedSlot).toContainElement(showButton) + expect(showButton.closest('.overflow-x-auto')).toBeNull() fireEvent.click(showButton) expect(onToggleSidebar).toHaveBeenCalled() }) From 85660396e6ec28a127b5231f67021ef0c403c7be Mon Sep 17 00:00:00 2001 From: Dan Shapiro Date: Tue, 5 May 2026 18:55:11 -0700 Subject: [PATCH 02/25] Handle Codex app-server startup disconnect race --- server/coding-cli/codex-app-server/runtime.ts | 116 ++++++++++++++---- .../codex-app-server/runtime.test.ts | 67 +++++++--- 2 files changed, 146 insertions(+), 37 deletions(-) diff --git a/server/coding-cli/codex-app-server/runtime.ts b/server/coding-cli/codex-app-server/runtime.ts index 96471533c..e36f40eec 100644 --- a/server/coding-cli/codex-app-server/runtime.ts +++ b/server/coding-cli/codex-app-server/runtime.ts @@ -7,6 +7,7 @@ import { allocateLocalhostPort, type LoopbackServerEndpoint } from '../../local- import { logger } from '../../logger.js' import { CodexAppServerClient, + type CodexAppServerDisconnectEvent, type CodexThreadLifecycleEvent, type CodexThreadLifecycleLossEvent, } from './client.js' @@ -500,6 +501,7 @@ export class CodexAppServerRuntime { private ownership: ActiveOwnership | null = null private ownershipTeardownPromise: Promise | null = null private ownershipTeardownFailure: Error | null = null + private startupAbortError: Error | null = null private shutdownRequested = false private lifecycleLossHandlers = new Set<(event: CodexThreadLifecycleLossEvent) => void>() private readonly exitHandlers = new Set<(error?: Error, source?: CodexAppServerRuntimeFailureSource) => void>() @@ -551,9 +553,17 @@ export class CodexAppServerRuntime { this.ensureReadyPromise = null }) - this.ready = await this.ensureReadyPromise + return this.publishReady(await this.ensureReadyPromise) + } + + private publishReady(ready: ReadyState): ReadyState { + if (this.startupAbortError) { + throw this.startupAbortError + } + this.startupAbortError = null + this.ready = ready this.statusValue = 'running' - return this.ready + return ready } async startThread( @@ -597,7 +607,7 @@ export class CodexAppServerRuntime { ...(input.codexHome !== undefined ? { codexHome: input.codexHome } : {}), updatedAt: new Date().toISOString(), } - await atomicWriteJson(this.ownership.metadataPath, this.ownership.metadata) + await this.writeOwnershipRecord(this.ownership) } onThreadLifecycleLoss(handler: (event: CodexThreadLifecycleLossEvent) => void): () => void { @@ -682,6 +692,7 @@ export class CodexAppServerRuntime { if (this.shutdownRequested) { throw new Error('Codex app-server startup was cancelled because the sidecar is shutting down.') } + this.startupAbortError = null const endpoint = await this.portAllocator() if (this.shutdownRequested) { throw new Error('Codex app-server startup was cancelled because the sidecar is shutting down.') @@ -732,7 +743,7 @@ export class CodexAppServerRuntime { this.ownership = ownership attemptOwnership = ownership await this.writeOwnershipRecord(ownership) - await this.readWrapperIdentityInto(ownership) + await this.readWrapperIdentityInto(ownership, child, childErrorPromise) this.ownership = ownership await this.writeOwnershipRecord(ownership) @@ -761,16 +772,14 @@ export class CodexAppServerRuntime { } }) client.onDisconnect((event) => { - if (this.shutdownRequested) return - for (const handler of this.exitHandlers) { - handler(event.error, 'app_server_client_disconnect') - } + this.handleClientDisconnect(client, event) }) this.client = client const initialized = await this.waitForInitialize(client, child, childErrorPromise) + this.assertStartupRuntimeStillActive(client, child) await this.updateOwnershipMetadata({ codexHome: initialized.codexHome }) - this.statusValue = 'running' + this.assertStartupRuntimeStillActive(client, child) return { wsUrl, processPid: child.pid, @@ -841,18 +850,41 @@ export class CodexAppServerRuntime { } } - private async readWrapperIdentityInto(ownership: ActiveOwnership): Promise { - const wrapperIdentity = await this.processIdentityReader(ownership.metadata.wrapperPid) - if (!isCompleteWrapperIdentity(wrapperIdentity)) { - throw new Error( - `Codex app-server wrapper identity could not be completely read for PID ${ownership.metadata.wrapperPid}.`, - ) - } - ownership.metadata = { - ...ownership.metadata, - wrapperIdentity, - updatedAt: new Date().toISOString(), + private async readWrapperIdentityInto( + ownership: ActiveOwnership, + child: ChildProcessHandle, + childErrorPromise: Promise, + ): Promise { + const deadline = Date.now() + this.startupAttemptTimeoutMs + + while (true) { + const [wrapperIdentity, hasOwnershipEnv] = await Promise.race([ + Promise.all([ + this.processIdentityReader(ownership.metadata.wrapperPid), + processHasOwnershipEnv(ownership.metadata.wrapperPid, ownership.metadata.ownershipId), + ]), + childErrorPromise.then((error) => { + throw error + }), + ]) + if (hasOwnershipEnv && isCompleteWrapperIdentity(wrapperIdentity)) { + ownership.metadata = { + ...ownership.metadata, + wrapperIdentity, + updatedAt: new Date().toISOString(), + } + return + } + + if (child.exitCode !== null || child.signalCode !== null || Date.now() >= deadline) { + break + } + await sleep(STARTUP_POLL_MS) } + + throw new Error( + `Codex app-server wrapper identity could not be completely read for PID ${ownership.metadata.wrapperPid}.`, + ) } private async writeOwnershipRecord(ownership: ActiveOwnership): Promise { @@ -894,6 +926,18 @@ export class CodexAppServerRuntime { throw lastError ?? new Error('Codex app-server exited before it finished initializing.') } + private assertStartupRuntimeStillActive(client: CodexAppServerClient, child: ChildProcessHandle): void { + if (this.startupAbortError) { + throw this.startupAbortError + } + if (this.child !== child) { + throw new Error('Codex app-server child exited before startup completed.') + } + if (this.client !== client) { + throw new Error('Codex app-server client disconnected before startup completed.') + } + } + private watchChildError(child: ChildProcessHandle): Promise { return new Promise((resolve) => { child.once('error', (error) => { @@ -919,6 +963,7 @@ export class CodexAppServerRuntime { return } + const wasReady = this.ready !== null const ownership = this.ownership this.child = null this.ready = null @@ -948,7 +993,7 @@ export class CodexAppServerRuntime { void closeClient } - if (!this.shutdownRequested) { + if (wasReady && !this.shutdownRequested) { for (const handler of this.exitHandlers) { handler(undefined, 'app_server_exit') } @@ -956,6 +1001,35 @@ export class CodexAppServerRuntime { }) } + private handleClientDisconnect(client: CodexAppServerClient, event: CodexAppServerDisconnectEvent): void { + if (this.client !== client) { + return + } + + const wasReady = this.ready !== null + this.client = null + this.ready = null + this.statusValue = 'stopped' + + if (!wasReady || this.shutdownRequested) { + if (!this.shutdownRequested) { + this.startupAbortError = new Error(event.reason === 'error' + ? `Codex app-server client socket errored before startup completed: ${event.error?.message ?? 'unknown error'}` + : 'Codex app-server client disconnected before startup completed.') + } + void this.stopActiveChild().catch(() => undefined) + return + } + + const error = event.reason === 'error' + ? new Error(`Codex app-server client socket errored: ${event.error?.message ?? 'unknown error'}`) + : new Error('Codex app-server client socket closed unexpectedly.') + for (const handler of this.exitHandlers) { + handler(error, 'app_server_client_disconnect') + } + void this.stopActiveChild().catch(() => undefined) + } + private async stopActiveChild(): Promise { const ownership = this.ownership const child = this.child diff --git a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts index b2108bd4f..497d6bc8d 100644 --- a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts @@ -204,6 +204,24 @@ describe('CodexAppServerRuntime', () => { expect(runtime.status()).toBe('running') }) + it('waits for a complete wrapper identity proof during startup', async () => { + let identityReads = 0 + const runtime = createRuntime({ + startupAttemptLimit: 1, + startupAttemptTimeoutMs: 1_000, + processIdentityReader: async (pid) => { + identityReads += 1 + if (identityReads === 1) return null + return readWrapperIdentityForTest(pid) + }, + }) + + await expect(runtime.ensureReady()).resolves.toEqual(expect.objectContaining({ + wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), + })) + expect(identityReads).toBeGreaterThan(1) + }) + it('rejects before spawning on platforms without Linux /proc ownership support', async () => { const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') if (!originalPlatform?.configurable) { @@ -479,12 +497,11 @@ describe('CodexAppServerRuntime', () => { expect(Date.now() - start).toBeLessThan(1_500) }, 3_000) - it('tears down the owned process group before retry when wrapper identity cannot be read', async () => { + it('waits for a transiently unreadable wrapper identity without retrying startup', async () => { const tempDir = await makeTempDir() const metadataDir = path.join(tempDir, 'metadata') const processGroups: number[] = [] const seenProcessGroups = new Set() - let previousAttemptGoneBeforeRetry = false let identityReadAttempts = 0 const runtime = createRuntime({ metadataDir, @@ -499,9 +516,6 @@ describe('CodexAppServerRuntime', () => { }, metadataWriter: async (filePath, metadata) => { if (!seenProcessGroups.has(metadata.processGroupId)) { - if (processGroups.length > 0) { - previousAttemptGoneBeforeRetry = !(await isProcessGroupAlive(processGroups[0])) - } seenProcessGroups.add(metadata.processGroupId) processGroups.push(metadata.processGroupId) } @@ -513,19 +527,17 @@ describe('CodexAppServerRuntime', () => { const ready = await runtime.ensureReady() const record = JSON.parse(await fsp.readFile(ready.metadataPath, 'utf8')) - expect(processGroups).toHaveLength(2) + expect(processGroups).toHaveLength(1) expect(identityReadAttempts).toBe(2) - expect(previousAttemptGoneBeforeRetry).toBe(true) - expect(record.processGroupId).toBe(processGroups[1]) + expect(record.processGroupId).toBe(processGroups[0]) expect(record.wrapperIdentity.startTimeTicks).toEqual(expect.any(Number)) }, 3_000) - it('tears down the owned process group before retry when wrapper identity is incomplete', async () => { + it('waits for a transiently incomplete wrapper identity without retrying startup', async () => { const tempDir = await makeTempDir() const metadataDir = path.join(tempDir, 'metadata') const processGroups: number[] = [] const seenProcessGroups = new Set() - let previousAttemptGoneBeforeRetry = false let identityReadAttempts = 0 const runtime = createRuntime({ metadataDir, @@ -542,9 +554,6 @@ describe('CodexAppServerRuntime', () => { }, metadataWriter: async (filePath, metadata) => { if (!seenProcessGroups.has(metadata.processGroupId)) { - if (processGroups.length > 0) { - previousAttemptGoneBeforeRetry = !(await isProcessGroupAlive(processGroups[0])) - } seenProcessGroups.add(metadata.processGroupId) processGroups.push(metadata.processGroupId) } @@ -556,10 +565,9 @@ describe('CodexAppServerRuntime', () => { const ready = await runtime.ensureReady() const record = JSON.parse(await fsp.readFile(ready.metadataPath, 'utf8')) - expect(processGroups).toHaveLength(2) + expect(processGroups).toHaveLength(1) expect(identityReadAttempts).toBe(2) - expect(previousAttemptGoneBeforeRetry).toBe(true) - expect(record.processGroupId).toBe(processGroups[1]) + expect(record.processGroupId).toBe(processGroups[0]) expect(record.wrapperIdentity.commandLine.length).toBeGreaterThan(0) expect(record.wrapperIdentity.cwd).toEqual(expect.any(String)) expect(record.wrapperIdentity.startTimeTicks).toEqual(expect.any(Number)) @@ -909,6 +917,7 @@ describe('CodexAppServerRuntime', () => { metadataDir, serverInstanceId: 'srv-current', }) + runtimes.delete(runtime) expect(result.reapedOwnershipIds).toContain(ready.ownershipId) await waitForProcessExit(ready.processPid) @@ -1248,6 +1257,32 @@ describe('CodexAppServerRuntime', () => { await closeBlocker(blocker) }) + it('does not publish ready when the app-server client socket disconnects before startup completes', async () => { + const runtime = createRuntime({ + startupAttemptLimit: 1, + metadataWriter: async (filePath, metadata) => { + if (metadata.codexHome) { + await new Promise((resolve) => setTimeout(resolve, 500)) + } + await fsp.mkdir(path.dirname(filePath), { recursive: true }) + await fsp.writeFile(filePath, `${JSON.stringify(metadata, null, 2)}\n`, { mode: 0o600 }) + }, + env: { + FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ + closeSocketAfterMethodsOnce: ['initialize'], + }), + }, + }) + const onExit = vi.fn() + runtime.onExit(onExit) + + await expect(runtime.ensureReady()).rejects.toThrow( + /Codex app-server client disconnected before startup completed/, + ) + expect(runtime.status()).toBe('stopped') + expect(onExit).not.toHaveBeenCalled() + }) + it('keeps child stdio drained so large app-server logs do not stall thread/start replies', async () => { const runtime = createRuntime({ env: { From 41aab4a35eeb1dcde11ec0e4faf479b82a81be6a Mon Sep 17 00:00:00 2001 From: Dan Shapiro Date: Tue, 5 May 2026 19:03:53 -0700 Subject: [PATCH 03/25] Handle transient Codex sidecar identity reads --- docs/lab-notes/2026-04-20-coding-cli-session-contract.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md index 34123ed2b..2f98d54ea 100644 --- a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md +++ b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md @@ -81,7 +81,7 @@ The implementation plan file is dated `2026-04-19` because the design work was w "executable": "claude", "resolvedPath": "/home/user/bin/claude", "isolatedBinaryPath": "/home/user/.local/bin/claude", - "version": "2.1.126 (Claude Code)", + "version": "2.1.129 (Claude Code)", "exactIdCommandTemplate": "HOME= /home/user/.local/bin/claude --bare --dangerously-skip-permissions -p --session-id ", "namedResumeCommandTemplate": "HOME= /home/user/.local/bin/claude --bare --dangerously-skip-permissions -p --resume [--name ] <prompt>", "transcriptGlob": ".claude/projects/*/<uuid>.jsonl", @@ -238,7 +238,7 @@ command -v claude # /home/user/bin/claude claude --version -# 2.1.126 (Claude Code) +# 2.1.129 (Claude Code) ``` The wrapper at `/home/user/bin/claude` shells out to `/home/user/.local/bin/claude`. The isolated probes used the actual binary and overrode `HOME` to keep persistence inside the probe temp root. From 7eae9acf13d2ecf36de6ecade8354cb22b944f7b Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 00:07:02 -0700 Subject: [PATCH 04/25] Cover sidebar reopen overflow behavior --- .../2026-04-20-coding-cli-session-contract.md | 7 ++- test/e2e-browser/specs/sidebar.spec.ts | 55 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md index 2f98d54ea..7832e64d2 100644 --- a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md +++ b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md @@ -1,6 +1,6 @@ # Coding CLI Session Contract Lab Note -This note records the real-binary provider probes rerun on `2026-04-26` inside `/home/user/code/freshell/.worktrees/trycycle-codex-session-resilience`. Binary version facts were refreshed on `2026-05-03` inside `/home/user/code/freshell/.worktrees/land-local-main-codex-sidecar-lifecycle`. +This note records the real-binary provider probes rerun on `2026-04-26` inside `/home/user/code/freshell/.worktrees/trycycle-codex-session-resilience`. Binary version facts were refreshed on `2026-05-03` inside `/home/user/code/freshell/.worktrees/land-local-main-codex-sidecar-lifecycle`; the Claude Code binary version fact was refreshed again on `2026-05-05` inside `/home/user/code/freshell/.worktrees/codex-sidebar-reopen-corner-20260505` after the installed binary changed. The later version-only refreshes did not re-prove the behavior contract, so `capturedOn` remains `2026-04-26`. The implementation plan file is dated `2026-04-19` because the design work was written the day before. This note is dated `2026-04-26` because the real-provider contracts were re-proved on the implementation machine on that date, and that verification date is the one Freshell is allowed to build on. @@ -9,7 +9,8 @@ The implementation plan file is dated `2026-04-19` because the design work was w { "capturedOn": "2026-04-26", "planCreatedOn": "2026-04-19", - "dateReason": "The plan was drafted on 2026-04-19, but the checked-in note is dated 2026-04-26 because that is when the durable behavior contract was re-proved on the implementation machine and the earlier 2026-04-23 contract capture was superseded by the newer provider behavior. Binary version facts were refreshed on 2026-05-03 after the installed provider versions changed.", + "binaryVersionFactsRefreshedOn": "2026-05-05", + "dateReason": "The plan was drafted on 2026-04-19, but the checked-in note is dated 2026-04-26 because that is when the durable behavior contract was re-proved on the implementation machine and the earlier 2026-04-23 contract capture was superseded by the newer provider behavior. Binary version facts were refreshed on 2026-05-03 after installed provider versions changed, and the Claude Code binary version fact was refreshed on 2026-05-05 after the local installed binary changed to 2.1.129. These later version-only refreshes did not re-prove the behavior contract.", "cleanup": { "liveProcessAuditCommand": "ps -eo pid,ppid,stat,cmd --sort=pid | rg \"codex|claude|opencode\"", "ownershipReportFields": [ @@ -241,6 +242,8 @@ claude --version # 2.1.129 (Claude Code) ``` +This Claude Code version line was refreshed on `2026-05-05`; the behavior observations below remain from the `2026-04-26` real-provider proof. + The wrapper at `/home/user/bin/claude` shells out to `/home/user/.local/bin/claude`. The isolated probes used the actual binary and overrode `HOME` to keep persistence inside the probe temp root. Fresh exact-id durability was probed with: diff --git a/test/e2e-browser/specs/sidebar.spec.ts b/test/e2e-browser/specs/sidebar.spec.ts index d08b6f23e..3ebc6d05a 100644 --- a/test/e2e-browser/specs/sidebar.spec.ts +++ b/test/e2e-browser/specs/sidebar.spec.ts @@ -26,6 +26,61 @@ test.describe('Sidebar', () => { await expect(page.getByRole('button', { name: /hide sidebar/i })).toBeVisible() }) + test('sidebar reopen button stays fixed when overflowing tabs are scrolled', async ({ freshellPage, page }) => { + await page.setViewportSize({ width: 900, height: 700 }) + await page.evaluate(() => { + const harness = window.__FRESHELL_TEST_HARNESS__ + if (!harness) throw new Error('Freshell test harness is not installed') + for (let i = 0; i < 18; i += 1) { + harness.dispatch({ + type: 'tabs/addTab', + payload: { + id: `overflow-tab-${i}`, + createRequestId: `overflow-tab-${i}`, + title: `Overflow tab ${i}`, + mode: 'shell', + shell: 'system', + status: 'running', + }, + }) + } + }) + await page.waitForFunction(() => ( + window.__FRESHELL_TEST_HARNESS__?.getState()?.tabs?.tabs?.length ?? 0 + ) >= 18) + + const collapseButton = page.getByRole('button', { name: /hide sidebar/i }) + await collapseButton.click() + + const showButton = page.getByRole('button', { name: /show sidebar/i }) + await expect(showButton).toBeVisible({ timeout: 3_000 }) + + const tabBar = page.locator('[data-context="global"]').filter({ + has: page.getByRole('button', { name: /new shell tab/i }), + }).first() + const tabScroller = tabBar.locator('.overflow-x-auto').first() + await expect(tabScroller).toBeVisible() + await expect.poll(async () => tabScroller.evaluate((element) => element.scrollWidth > element.clientWidth)).toBe(true) + + const before = await showButton.boundingBox() + expect(before).not.toBeNull() + + const scrollLeft = await tabScroller.evaluate((element) => { + element.scrollLeft = element.scrollWidth + element.dispatchEvent(new Event('scroll', { bubbles: true })) + return element.scrollLeft + }) + expect(scrollLeft).toBeGreaterThan(0) + + const after = await showButton.boundingBox() + const scrollerBox = await tabScroller.boundingBox() + expect(after).not.toBeNull() + expect(scrollerBox).not.toBeNull() + expect(after!.x).toBeCloseTo(before!.x, 0) + expect(after!.x).toBeGreaterThanOrEqual(0) + expect(after!.x + after!.width).toBeLessThanOrEqual(scrollerBox!.x) + }) + test('sidebar shows navigation buttons', async ({ freshellPage, page }) => { // Nav buttons have title attributes like "Settings (Ctrl+B ,)", "Tabs (Ctrl+B A)", etc. // Playwright matches title as accessible name for buttons with no text/aria-label. From 913f8887b1459e22306fe656ef9385d5a0272ae0 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:08:18 -0700 Subject: [PATCH 05/25] docs: define main mirror and dev self-host model --- AGENTS.md | 19 +++++----- docs/development/branch-model.md | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 docs/development/branch-model.md diff --git a/AGENTS.md b/AGENTS.md index d26f34bf4..ea4825e27 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ Freshell is a self-hosted, browser-accessible terminal multiplexer and session o ## Repo Rules - Always work in a worktree (in \.worktrees\) +- New behavior changes start on a worktree branch from `origin/main` and are submitted as PRs to `origin/main`; local `dev` only consumes PR heads. - Many agents may be working in the worktree at the same time. If you see activity from other agents (for example test runs or file changes), respect it. - Specific user instructions override ALL other instructions, including the above, and including superpowers or skills - Server uses NodeNext/ESM; relative imports must include `.js` extensions @@ -26,16 +27,18 @@ Freshell is a self-hosted, browser-accessible terminal multiplexer and session o - Use `npm run test:vitest -- ...` for a repo-owned direct Vitest path. Raw `npx vitest` is not a coordinated workflow. - `test:unit` is the exact default-config `test/unit` workload, `test:integration` is the exact server-config `test/server` workload, and `test:server` stays watch-capable unless you pass an explicit broad `--run`. -## Merging to Main (CRITICAL - Read This) +## Branch Model And Self-Hosting (CRITICAL - Read This) -**You are running inside Freshell right now. This session, the terminal the user is typing in, is served by the main branch. If you break main, you kill yourself mid-operation and the user has to clean up your mess with a separate agent.** +**This checkout is self-hosted from local `dev`, not local `main`.** -- Never run `git merge` directly on main - merge conflicts write `<<<<<<< HEAD` markers into source files, which crashes the server instantly -- Always merge main INTO the feature branch in the worktree first, resolve any conflicts there -- Before fast-forwarding main, run `npm test` and confirm all tests pass (not just related tests) -- If you find failing tests, you must stop everything to understand them and make improvements, even if you do not think you were responsible for them. -- Then fast-forward main: `git merge --ff-only feature/branch` - this is atomic (pointer move, no intermediate states) -- If `--ff-only` fails, go back to the worktree and rebase/merge until it can fast-forward +- Local `main` is a mirror of `origin/main`; do not commit to it, merge into it, or self-host from it. +- Local `dev` is the self-hosted integration branch. It is rebuilt from `origin/main` plus pending PR heads. +- Do not edit production behavior directly on `dev`. +- If a change is needed on `dev`, create or update a PR against `origin/main`, then apply that PR head to `dev`. +- If applying a PR to `dev` needs semantic conflict resolution, stop and fix the PR branch or create a replacement PR. Do not hide behavior changes in a local-only `dev` merge commit. +- Never run `git merge` directly on `main`. +- Never reset, force-push, or fast-forward local `main` unless the user explicitly asks to realign it to `origin/main`, the running server is no longer using `main`, and the intentional OpenCode notification-argument removal has been preserved in a PR that is included in `dev`. +- We cannot approve our own PRs. `dev` may contain unapproved pending work, but `origin/main` changes still require independent review. ## Process Safety (CRITICAL) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md new file mode 100644 index 000000000..da8260bd6 --- /dev/null +++ b/docs/development/branch-model.md @@ -0,0 +1,61 @@ +# Branch Model + +Freshell development uses two local integration concepts: + +- `main`: exact mirror of `origin/main` +- `dev`: self-hosted local integration branch + +## Branch Responsibilities + +`main` is disposable. It should always be resettable to `origin/main` with no local work lost. + +`dev` is where the local Freshell instance runs. It is assembled from `origin/main` plus pending PR heads. It is not where new behavior is authored. + +## Pending PR Definition + +A PR is pending for `dev` only when all of these are true: + +- It is open. +- It targets `main`. +- It is not draft. +- It is not marked do-not-merge, superseded, or approval-artifact-only. +- The user wants it in the self-hosted integration queue. +- Its branch applies cleanly to `origin/main`, or its branch has been updated so it does. + +If a PR cannot be amended because it comes from an external fork, create a replacement PR before adding that behavior to `dev`. + +## Change Flow + +1. Start work from `origin/main` in a worktree. +2. Implement the change. +3. Push a PR against `origin/main`. +4. Add that PR head to local `dev`. +5. Wait for independent review before merging the PR to `origin/main`. + +Never put behavior changes only on `dev`. + +## Conflict Policy + +If a PR conflicts with `origin/main`, fix the PR branch. + +If two pending PRs conflict with each other, fix one or both PR branches. + +Do not resolve semantic conflicts only on `dev`. `dev` must remain reproducible from `origin/main` plus PR heads. + +## Excluded PRs + +Draft PRs, do-not-merge PRs, closed PRs, superseded PRs, and approval artifacts are excluded from `dev` unless the user explicitly says otherwise. + +## Local Main Realignment + +Only realign local `main` after Freshell is self-hosting from `dev`, the user has explicitly approved the reset, and the intentional OpenCode notification-argument removal has been preserved in an open PR that is included in `dev` or confirmed already present in a selected pending PR. + +The intended final state is: + +```bash +git switch main +git fetch origin +git reset --hard origin/main +``` + +Do not run that command during ordinary development. It belongs only to the migration task that realigns local `main` after self-hosting has moved to `dev`. From 1d541762cf968a5d9f1fec2b4fdf258a846539db Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:12:03 -0700 Subject: [PATCH 06/25] fix: guard self-host launch and source updates --- scripts/launch.sh | 45 +++++------- scripts/precheck.ts | 15 +++- scripts/selfhost-branch.ts | 51 +++++++++++++ server/updater/index.ts | 4 ++ shared/selfhost-branch-policy.ts | 58 +++++++++++++++ test/unit/scripts/selfhost-branch.test.ts | 46 ++++++++++++ test/unit/server/updater/index.test.ts | 52 ++++++++++++-- .../shared/selfhost-branch-policy.test.ts | 72 +++++++++++++++++++ 8 files changed, 309 insertions(+), 34 deletions(-) create mode 100644 scripts/selfhost-branch.ts create mode 100644 shared/selfhost-branch-policy.ts create mode 100644 test/unit/scripts/selfhost-branch.test.ts create mode 100644 test/unit/shared/selfhost-branch-policy.test.ts diff --git a/scripts/launch.sh b/scripts/launch.sh index 8b7529a3f..5fa7a99a8 100755 --- a/scripts/launch.sh +++ b/scripts/launch.sh @@ -1,20 +1,29 @@ #!/usr/bin/env bash -# Launch Freshell: pull upstream, build, start server in background. +# Launch Freshell: build, start server in background. set -euo pipefail FRESHELL_DIR="$(cd "$(dirname "$0")/.." && pwd)" -FRESHELL_HOME="$HOME/.freshell" +FRESHELL_HOME="${FRESHELL_HOME:-$HOME/.freshell}" LOG_FILE="$FRESHELL_HOME/logs/server.log" URL_FILE="$FRESHELL_HOME/url" PID_FILE="$FRESHELL_HOME/server.pid" cd "$FRESHELL_DIR" -# Check for already-running server (verify PID is actually a node process from this project) +is_freshell_pid() { + local pid="$1" + local cwd="" + local args="" + cwd="$(readlink "/proc/$pid/cwd" 2>/dev/null || true)" + args="$(ps -p "$pid" -o args= 2>/dev/null || true)" + [[ "$cwd" == "$FRESHELL_DIR" && ( "$args" == *"dist/server/index.js"* || "$args" == *"npm start"* ) ]] +} + +# Check for already-running server (verify PID belongs to this project) if [[ -f "$PID_FILE" ]]; then saved_pid="$(cat "$PID_FILE")" - if kill -0 "$saved_pid" 2>/dev/null && ps -p "$saved_pid" -o args= 2>/dev/null | grep -q "dist/server/index.js"; then + if kill -0 "$saved_pid" 2>/dev/null && is_freshell_pid "$saved_pid"; then echo "freshell is already running (pid $saved_pid)" if [[ -f "$URL_FILE" ]]; then echo " $(cat "$URL_FILE")" @@ -25,30 +34,8 @@ if [[ -f "$PID_FILE" ]]; then fi fi -# Pull latest from upstream (only on main) -current_branch="$(git branch --show-current)" -if [[ "$current_branch" == "main" ]]; then - echo "Pulling latest from upstream..." - if git remote get-url upstream >/dev/null 2>&1; then - git fetch upstream - if git merge-base --is-ancestor upstream/main HEAD 2>/dev/null; then - echo " Already up to date." - elif git merge --ff-only upstream/main 2>/dev/null; then - echo " Merged upstream changes." - else - echo " Local main has diverged from upstream. Resetting to upstream/main..." - git reset --hard upstream/main - echo " Done." - fi - else - echo " No upstream remote, skipping pull." - fi - - echo "Pushing to origin..." - git push origin main 2>/dev/null && echo " Done." || echo " Push failed (non-fatal)." -else - echo "Warning: on branch '$current_branch', skipping upstream pull. Switch to main for auto-update." -fi +echo "Checking self-host branch..." +npx tsx scripts/selfhost-branch.ts validate-launch # Build echo "Building..." @@ -59,7 +46,7 @@ echo "Starting server..." mkdir -p "$(dirname "$LOG_FILE")" rm -f "$URL_FILE" -NODE_ENV=production node dist/server/index.js > "$LOG_FILE" 2>&1 & +npm start > "$LOG_FILE" 2>&1 & SERVER_PID=$! echo "$SERVER_PID" > "$PID_FILE" diff --git a/scripts/precheck.ts b/scripts/precheck.ts index f0240fa77..ea1c07ed8 100644 --- a/scripts/precheck.ts +++ b/scripts/precheck.ts @@ -10,6 +10,7 @@ */ import { readFileSync } from 'fs' +import { execFileSync } from 'child_process' import { resolve, dirname } from 'path' import { fileURLToPath } from 'url' import { createRequire } from 'module' @@ -30,6 +31,18 @@ function getPackageVersion(): string { } } +function getCurrentBranch(): string | undefined { + try { + return execFileSync('git', ['branch', '--show-current'], { + cwd: rootDir, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim() || undefined + } catch { + return undefined + } +} + /** * Check if node_modules is missing required dependencies from package.json. * Returns list of missing packages. @@ -182,7 +195,7 @@ async function checkVitePort(): Promise<PortCheckResult> { async function main(): Promise<void> { // 1. Check for updates first (before anything else can fail) - if (!shouldSkipUpdateCheck()) { + if (!shouldSkipUpdateCheck({ branch: getCurrentBranch() })) { const currentVersion = getPackageVersion() const updateResult = await runUpdateCheck(currentVersion) diff --git a/scripts/selfhost-branch.ts b/scripts/selfhost-branch.ts new file mode 100644 index 000000000..199ae269e --- /dev/null +++ b/scripts/selfhost-branch.ts @@ -0,0 +1,51 @@ +#!/usr/bin/env tsx + +import { execFile } from 'child_process' +import { promisify } from 'util' +import { pathToFileURL } from 'url' +import { classifySelfHostBranch, type SelfHostPolicyEnv } from '../shared/selfhost-branch-policy.js' + +const execFileAsync = promisify(execFile) + +export type LaunchBranchValidation = + | { ok: true; branch: string } + | { ok: false; message: string } + +export async function getCurrentGitBranch(cwd: string = process.cwd()): Promise<string | undefined> { + try { + const { stdout } = await execFileAsync('git', ['branch', '--show-current'], { cwd }) + return stdout.trim() || undefined + } catch { + return undefined + } +} + +export async function validateLaunchBranch(input: { + env: SelfHostPolicyEnv + getBranch?: () => Promise<string | undefined> +}): Promise<LaunchBranchValidation> { + const branch = await (input.getBranch ?? (() => getCurrentGitBranch()))() + const result = classifySelfHostBranch({ branch, env: input.env }) + if (result.ok === true) return { ok: true, branch: branch ?? result.expectedBranch } + return { ok: false, message: result.message } +} + +async function main(argv: string[]): Promise<number> { + const command = argv[0] + if (command !== 'validate-launch') { + console.error('Usage: tsx scripts/selfhost-branch.ts validate-launch') + return 2 + } + + const result = await validateLaunchBranch({ env: process.env }) + if (result.ok === false) { + console.error(result.message) + return 1 + } + + return 0 +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then((code) => process.exit(code)) +} diff --git a/server/updater/index.ts b/server/updater/index.ts index af964e791..d4edee871 100644 --- a/server/updater/index.ts +++ b/server/updater/index.ts @@ -2,6 +2,7 @@ import { checkForUpdate } from './version-checker.js' import { promptForUpdate } from './prompt.js' import { executeUpdate, type UpdateProgress } from './executor.js' +import { shouldSkipSourceUpdateForBranch } from '../../shared/selfhost-branch-policy.js' export type UpdateAction = 'none' | 'updated' | 'skipped' | 'error' | 'check-failed' @@ -67,6 +68,7 @@ export async function runUpdateCheck(currentVersion: string): Promise<UpdateChec * - --skip-update-check CLI flag is present * - SKIP_UPDATE_CHECK env var is 'true' * - Running via 'npm run dev' (predev lifecycle event) + * - Current branch is not main or cannot be determined * * Does NOT skip based on NODE_ENV because that may be set persistently * in dev environments even when running 'npm run serve'. @@ -74,6 +76,7 @@ export async function runUpdateCheck(currentVersion: string): Promise<UpdateChec export interface SkipCheckOptions { argv?: string[] env?: NodeJS.ProcessEnv + branch?: string } export function shouldSkipUpdateCheck(options: SkipCheckOptions = {}): boolean { @@ -83,6 +86,7 @@ export function shouldSkipUpdateCheck(options: SkipCheckOptions = {}): boolean { if (argv.includes('--skip-update-check')) return true if (env.SKIP_UPDATE_CHECK === 'true') return true if (env.npm_lifecycle_event === 'predev') return true + if (shouldSkipSourceUpdateForBranch({ branch: options.branch, env })) return true return false } diff --git a/shared/selfhost-branch-policy.ts b/shared/selfhost-branch-policy.ts new file mode 100644 index 000000000..f64faea8e --- /dev/null +++ b/shared/selfhost-branch-policy.ts @@ -0,0 +1,58 @@ +export type SelfHostPolicyEnv = { + FRESHELL_SELFHOST_BRANCH?: string + SKIP_UPDATE_CHECK?: string +} + +export type SelfHostBranchResult = + | { ok: true; expectedBranch: string } + | { ok: false; code: 'mirror-branch' | 'unexpected-branch' | 'unknown-branch'; message: string } + +export function getExpectedSelfHostBranch(env: SelfHostPolicyEnv): string { + const configured = env.FRESHELL_SELFHOST_BRANCH?.trim() + if (!configured || configured === 'main') return 'dev' + return configured +} + +export function classifySelfHostBranch(input: { + branch: string | undefined + env: SelfHostPolicyEnv +}): SelfHostBranchResult { + const expectedBranch = getExpectedSelfHostBranch(input.env) + const branch = input.branch?.trim() + + if (!branch) { + return { + ok: false, + code: 'unknown-branch', + message: `Could not determine the current Git branch. Switch to '${expectedBranch}' before self-hosting.`, + } + } + + if (branch === 'main') { + return { + ok: false, + code: 'mirror-branch', + message: `Refusing to self-host from local 'main'. Local 'main' must mirror 'origin/main'. Switch to '${expectedBranch}' or set FRESHELL_SELFHOST_BRANCH.`, + } + } + + if (branch === expectedBranch) { + return { ok: true, expectedBranch } + } + + return { + ok: false, + code: 'unexpected-branch', + message: `Refusing to self-host from '${branch}'. Expected '${expectedBranch}'. Set FRESHELL_SELFHOST_BRANCH only if the user explicitly chose another integration branch.`, + } +} + +export function shouldSkipSourceUpdateForBranch(input: { + branch: string | undefined + env: SelfHostPolicyEnv +}): boolean { + if (input.env.SKIP_UPDATE_CHECK === 'true') return true + const branch = input.branch?.trim() + if (!branch) return true + return branch !== 'main' +} diff --git a/test/unit/scripts/selfhost-branch.test.ts b/test/unit/scripts/selfhost-branch.test.ts new file mode 100644 index 000000000..58e9d8511 --- /dev/null +++ b/test/unit/scripts/selfhost-branch.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from 'vitest' +import { validateLaunchBranch } from '../../../scripts/selfhost-branch.js' + +describe('selfhost branch CLI helper', () => { + it('returns success on dev', async () => { + const result = await validateLaunchBranch({ + env: {}, + getBranch: async () => 'dev', + }) + + expect(result).toEqual({ ok: true, branch: 'dev' }) + }) + + it('returns a clear error on main', async () => { + const result = await validateLaunchBranch({ + env: {}, + getBranch: async () => 'main', + }) + + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.message).toContain("Refusing to self-host from local 'main'") + } + }) + + it('rejects main even when FRESHELL_SELFHOST_BRANCH is main', async () => { + const result = await validateLaunchBranch({ + env: { FRESHELL_SELFHOST_BRANCH: 'main' }, + getBranch: async () => 'main', + }) + + expect(result.ok).toBe(false) + }) + + it('surfaces git branch lookup failures', async () => { + const result = await validateLaunchBranch({ + env: {}, + getBranch: vi.fn(async () => undefined), + }) + + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.message).toContain('Could not determine the current Git branch') + } + }) +}) diff --git a/test/unit/server/updater/index.test.ts b/test/unit/server/updater/index.test.ts index 2540cd1bb..92cbaa0dd 100644 --- a/test/unit/server/updater/index.test.ts +++ b/test/unit/server/updater/index.test.ts @@ -343,7 +343,8 @@ describe('update orchestrator', () => { it('returns false by default when no skip conditions are met', () => { const result = shouldSkipUpdateCheck({ argv: ['node', 'script.js'], - env: { npm_lifecycle_event: 'preserve' } + env: { npm_lifecycle_event: 'preserve' }, + branch: 'main', }) expect(result).toBe(false) }) @@ -367,7 +368,8 @@ describe('update orchestrator', () => { it('returns false when SKIP_UPDATE_CHECK env var is other value', () => { const result = shouldSkipUpdateCheck({ argv: ['node', 'script.js'], - env: { SKIP_UPDATE_CHECK: 'false' } + env: { SKIP_UPDATE_CHECK: 'false' }, + branch: 'main', }) expect(result).toBe(false) }) @@ -383,7 +385,8 @@ describe('update orchestrator', () => { it('returns false when npm_lifecycle_event is "preserve"', () => { const result = shouldSkipUpdateCheck({ argv: ['node', 'script.js'], - env: { npm_lifecycle_event: 'preserve' } + env: { npm_lifecycle_event: 'preserve' }, + branch: 'main', }) expect(result).toBe(false) }) @@ -392,11 +395,52 @@ describe('update orchestrator', () => { // This is the key behavior change - NODE_ENV should not affect the check const result = shouldSkipUpdateCheck({ argv: ['node', 'script.js'], - env: { NODE_ENV: 'development', npm_lifecycle_event: 'preserve' } + env: { NODE_ENV: 'development', npm_lifecycle_event: 'preserve' }, + branch: 'main', }) expect(result).toBe(false) }) + it('skips update checks on dev branch', () => { + const result = shouldSkipUpdateCheck({ + argv: ['node', 'script.js'], + env: {}, + branch: 'dev', + }) + + expect(result).toBe(true) + }) + + it('skips update checks on feature branches', () => { + const result = shouldSkipUpdateCheck({ + argv: ['node', 'script.js'], + env: {}, + branch: 'feature/x', + }) + + expect(result).toBe(true) + }) + + it('does not skip update checks on main branch by branch policy alone', () => { + const result = shouldSkipUpdateCheck({ + argv: ['node', 'script.js'], + env: {}, + branch: 'main', + }) + + expect(result).toBe(false) + }) + + it('skips update checks when branch detection fails', () => { + const result = shouldSkipUpdateCheck({ + argv: ['node', 'script.js'], + env: {}, + branch: undefined, + }) + + expect(result).toBe(true) + }) + it('uses process.argv and process.env by default', () => { // When called with no options, it should use the actual process values // Just verify it returns a boolean without throwing diff --git a/test/unit/shared/selfhost-branch-policy.test.ts b/test/unit/shared/selfhost-branch-policy.test.ts new file mode 100644 index 000000000..ec2d68cb8 --- /dev/null +++ b/test/unit/shared/selfhost-branch-policy.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' +import { + classifySelfHostBranch, + getExpectedSelfHostBranch, + shouldSkipSourceUpdateForBranch, +} from '../../../shared/selfhost-branch-policy.js' + +describe('selfhost branch policy', () => { + it('defaults the expected self-host branch to dev', () => { + expect(getExpectedSelfHostBranch({})).toBe('dev') + }) + + it('allows overriding the expected self-host branch', () => { + expect(getExpectedSelfHostBranch({ FRESHELL_SELFHOST_BRANCH: 'dev/pr-queue' })).toBe('dev/pr-queue') + }) + + it('rejects self-host launch from main', () => { + expect(classifySelfHostBranch({ branch: 'main', env: {} })).toEqual({ + ok: false, + code: 'mirror-branch', + message: "Refusing to self-host from local 'main'. Local 'main' must mirror 'origin/main'. Switch to 'dev' or set FRESHELL_SELFHOST_BRANCH.", + }) + }) + + it('rejects self-host launch from main even if configured by env', () => { + expect(classifySelfHostBranch({ + branch: 'main', + env: { FRESHELL_SELFHOST_BRANCH: 'main' }, + })).toMatchObject({ + ok: false, + code: 'mirror-branch', + }) + }) + + it('accepts self-host launch from dev by default', () => { + expect(classifySelfHostBranch({ branch: 'dev', env: {} })).toEqual({ ok: true, expectedBranch: 'dev' }) + }) + + it('rejects unexpected non-main branches unless they are configured', () => { + expect(classifySelfHostBranch({ branch: 'feature/x', env: {} })).toMatchObject({ + ok: false, + code: 'unexpected-branch', + }) + }) + + it('accepts a configured non-main self-host branch', () => { + expect(classifySelfHostBranch({ + branch: 'dev/pr-queue', + env: { FRESHELL_SELFHOST_BRANCH: 'dev/pr-queue' }, + })).toEqual({ ok: true, expectedBranch: 'dev/pr-queue' }) + }) + + it('skips source updates on dev and feature branches', () => { + expect(shouldSkipSourceUpdateForBranch({ branch: 'dev', env: {} })).toBe(true) + expect(shouldSkipSourceUpdateForBranch({ branch: 'feature/x', env: {} })).toBe(true) + }) + + it('does not skip source updates on main unless another skip condition applies', () => { + expect(shouldSkipSourceUpdateForBranch({ branch: 'main', env: {} })).toBe(false) + }) + + it('skips source updates when the explicit skip env var is set', () => { + expect(shouldSkipSourceUpdateForBranch({ + branch: 'main', + env: { SKIP_UPDATE_CHECK: 'true' }, + })).toBe(true) + }) + + it('skips source updates when branch detection fails', () => { + expect(shouldSkipSourceUpdateForBranch({ branch: undefined, env: {} })).toBe(true) + }) +}) From 4c1f0dc042642726f2dd4e9aec3604304dcdf591 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:12:34 -0700 Subject: [PATCH 07/25] chore: add dev PR queue tooling --- docs/development/branch-model.md | 12 ++ package.json | 2 + scripts/dev-pr-queue.ts | 283 +++++++++++++++++++++++++ test/unit/scripts/dev-pr-queue.test.ts | 271 +++++++++++++++++++++++ 4 files changed, 568 insertions(+) create mode 100644 scripts/dev-pr-queue.ts create mode 100644 test/unit/scripts/dev-pr-queue.test.ts diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index da8260bd6..9e0162fdd 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -46,6 +46,18 @@ Do not resolve semantic conflicts only on `dev`. `dev` must remain reproducible Draft PRs, do-not-merge PRs, closed PRs, superseded PRs, and approval artifacts are excluded from `dev` unless the user explicitly says otherwise. +## Building `dev` + +Use an explicit queue. Do not blindly apply every open PR. + +Example: + +```bash +npm run dev:queue -- plan --prs 321,309,319 +``` + +The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. + ## Local Main Realignment Only realign local `main` after Freshell is self-hosting from `dev`, the user has explicitly approved the reset, and the intentional OpenCode notification-argument removal has been preserved in an open PR that is included in `dev` or confirmed already present in a selected pending PR. diff --git a/package.json b/package.json index 1b2b1f3cd..c1e1b8172 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "dev": "cross-env PORT=3002 concurrently -n client,server -c blue,green \"vite\" \"tsx watch server/index.ts\"", "dev:client": "vite", "dev:server": "cross-env PORT=3002 tsx watch server/index.ts", + "dev:queue": "tsx scripts/dev-pr-queue.ts", + "selfhost:check-branch": "tsx scripts/selfhost-branch.ts validate-launch", "typecheck": "npm run typecheck:client && npm run typecheck:server", "typecheck:client": "tsc -p tsconfig.json --noEmit", "typecheck:server": "tsc -p tsconfig.server.json --noEmit", diff --git a/scripts/dev-pr-queue.ts b/scripts/dev-pr-queue.ts new file mode 100644 index 000000000..6766c7e70 --- /dev/null +++ b/scripts/dev-pr-queue.ts @@ -0,0 +1,283 @@ +#!/usr/bin/env tsx + +import { execFile } from 'child_process' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) + +const REPO = 'danshapiro/freshell' +const PR_JSON_FIELDS = 'number,state,isDraft,baseRefName,headRefOid,mergeStateStatus,title,labels' +const EXCLUDED_LABELS = new Set([ + 'do-not-merge', + 'superseded', + 'approval-artifact', + 'approval-artifact-only', +]) + +export type DevQueuePr = { + number: number + state: 'OPEN' | 'CLOSED' | string + isDraft: boolean + baseRefName: string + headRefOid: string + mergeStateStatus: string + title: string + labels: Array<{ name: string }> +} + +export type DevQueuePlanStep = { + label: string + command: string[] +} + +export type DevQueuePlan = { + originMain: string + steps: DevQueuePlanStep[] +} + +export type DevQueueCommandRunner = (command: string, args: string[]) => Promise<string> + +export function parsePrList(input: string): number[] { + const prs = input + .split(',') + .map((part) => part.trim()) + .filter(Boolean) + .map((part) => Number(part)) + + if (prs.length === 0 || prs.some((number) => !Number.isInteger(number) || number <= 0)) { + throw new Error('At least one PR number is required, formatted like 321,309,319.') + } + + return prs +} + +export function buildPrMetadataCommand(number: number): [string, string[]] { + return [ + 'gh', + [ + 'pr', + 'view', + String(number), + '--repo', + REPO, + '--json', + PR_JSON_FIELDS, + ], + ] +} + +export async function loadPrMetadata( + numbers: number[], + run: DevQueueCommandRunner, +): Promise<DevQueuePr[]> { + const prs: DevQueuePr[] = [] + for (const number of numbers) { + const [command, args] = buildPrMetadataCommand(number) + const stdout = await run(command, args) + try { + prs.push(JSON.parse(stdout) as DevQueuePr) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to parse gh metadata for PR #${number}: ${message}`) + } + } + return prs +} + +function validatePrForDev(pr: DevQueuePr): void { + if (pr.state !== 'OPEN') { + throw new Error(`PR #${pr.number} is ${pr.state}, expected OPEN.`) + } + if (pr.isDraft) { + throw new Error(`PR #${pr.number} is draft and is not pending for dev.`) + } + if (pr.baseRefName !== 'main') { + throw new Error(`PR #${pr.number} targets ${pr.baseRefName}, expected main.`) + } + + const excludedLabel = pr.labels.find((label) => EXCLUDED_LABELS.has(label.name.toLowerCase())) + if (excludedLabel) { + throw new Error(`PR #${pr.number} is labeled ${excludedLabel.name} and is not pending for dev.`) + } +} + +export function buildDevQueuePlan(input: { + originMain: string + requestedPrs: number[] + prs: DevQueuePr[] +}): DevQueuePlan { + const byNumber = new Map(input.prs.map((pr) => [pr.number, pr])) + + for (const number of input.requestedPrs) { + const pr = byNumber.get(number) + if (!pr) { + throw new Error(`PR #${number} was not found.`) + } + validatePrForDev(pr) + } + + const steps: DevQueuePlanStep[] = [ + { label: 'fetch-origin-main', command: ['git', 'fetch', 'origin', 'main'] }, + { label: 'reset-dev-to-origin-main', command: ['git', 'reset', '--hard', input.originMain] }, + ] + + for (const number of input.requestedPrs) { + steps.push({ + label: `fetch-pr-${number}`, + command: ['git', 'fetch', 'origin', `+refs/pull/${number}/head:refs/remotes/pr/${number}`], + }) + steps.push({ + label: `merge-pr-${number}`, + command: ['git', 'merge', '--no-ff', '--no-edit', `refs/remotes/pr/${number}`], + }) + } + + return { originMain: input.originMain, steps } +} + +export async function assertAssemblePreconditions(input: { + getBranch: () => Promise<string | undefined> + getStatus: () => Promise<string> +}): Promise<void> { + const branch = await input.getBranch() + if (branch !== 'dev') { + throw new Error(`Refusing to assemble dev from ${branch ?? 'an unknown branch'}. Switch to dev first.`) + } + + const status = await input.getStatus() + if (status.trim()) { + throw new Error('Refusing to reset dev with a dirty worktree. Commit, stash, or discard local changes first.') + } +} + +export async function executeDevQueuePlan( + plan: DevQueuePlan, + run: DevQueueCommandRunner, +): Promise<void> { + for (const step of plan.steps) { + try { + await run(step.command[0], step.command.slice(1)) + } catch (error) { + const mergePrMatch = step.label.match(/^merge-pr-(\d+)$/) + if (mergePrMatch) { + throw new Error(`PR #${mergePrMatch[1]} did not merge cleanly. Fix the PR branch, abort the merge, and rerun the dev queue.`) + } + + const cherryPickPrMatch = step.label.match(/^cherry-pick-pr-(\d+)$/) + if (cherryPickPrMatch) { + throw new Error(`PR #${cherryPickPrMatch[1]} did not cherry-pick cleanly. Fix the PR branch, abort the cherry-pick, and rerun the dev queue.`) + } + + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Step ${step.label} failed: ${message}`) + } + } +} + +export async function assembleDevQueue(input: { + requestedPrs: number[] + run: DevQueueCommandRunner + getBranch: () => Promise<string | undefined> + getStatus: () => Promise<string> +}): Promise<void> { + await assertAssemblePreconditions(input) + await input.run('git', ['fetch', 'origin', 'main']) + const originMain = await input.run('git', ['rev-parse', 'origin/main']) + const prs = await loadPrMetadata(input.requestedPrs, input.run) + const plan = buildDevQueuePlan({ originMain, requestedPrs: input.requestedPrs, prs }) + await executeDevQueuePlan(plan, input.run) +} + +const runCommand: DevQueueCommandRunner = async (command, args) => { + try { + const { stdout } = await execFileAsync(command, args, { encoding: 'utf8' }) + return stdout.trim() + } catch (error) { + if (error instanceof Error) { + const details = error as Error & { stderr?: string; stdout?: string } + const output = [details.stderr, details.stdout] + .filter((value): value is string => typeof value === 'string' && value.trim().length > 0) + .map((value) => value.trim()) + .join('\n') + if (output) { + throw new Error(output) + } + throw error + } + throw new Error(String(error)) + } +} + +function printUsage(): void { + console.log('Usage: tsx scripts/dev-pr-queue.ts plan --prs 321,309,319') + console.log(' tsx scripts/dev-pr-queue.ts assemble --prs 321,309,319') +} + +function parseCliPrList(argv: string[]): number[] { + const prsFlagIndex = argv.indexOf('--prs') + const prsValue = prsFlagIndex >= 0 ? argv[prsFlagIndex + 1] : '' + return parsePrList(prsValue) +} + +async function buildPlanForCli(requestedPrs: number[]): Promise<{ + plan: DevQueuePlan + prs: DevQueuePr[] +}> { + await runCommand('git', ['fetch', 'origin', 'main']) + const originMain = await runCommand('git', ['rev-parse', 'origin/main']) + const prs = await loadPrMetadata(requestedPrs, runCommand) + const plan = buildDevQueuePlan({ originMain, requestedPrs, prs }) + return { plan, prs } +} + +export async function main(argv: string[]): Promise<number> { + const command = argv[0] + if (!command || command === '--help' || command === '-h') { + printUsage() + return command ? 0 : 2 + } + + try { + const requestedPrs = parseCliPrList(argv) + + if (command === 'plan') { + const { plan, prs } = await buildPlanForCli(requestedPrs) + for (const step of plan.steps) { + console.log(`${step.label}: ${step.command.join(' ')}`) + } + console.log(`origin/main: ${plan.originMain}`) + for (const pr of prs) { + console.log(`PR #${pr.number}: ${pr.headRefOid}`) + } + return 0 + } + + if (command === 'assemble') { + console.log('This will reset local dev to origin/main before applying PRs.') + console.log('Refusing to continue unless current branch is dev and worktree is clean.') + await assembleDevQueue({ + requestedPrs, + run: runCommand, + getBranch: async () => { + const branch = await runCommand('git', ['branch', '--show-current']) + return branch || undefined + }, + getStatus: async () => runCommand('git', ['status', '--porcelain']), + }) + console.log('Local dev has been reset to origin/main and updated with the requested PR heads.') + return 0 + } + + console.error(`Unsupported command: ${command}`) + printUsage() + return 2 + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.error(message) + return 1 + } +} + +if (process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/'))) { + main(process.argv.slice(2)).then((code) => process.exit(code)) +} diff --git a/test/unit/scripts/dev-pr-queue.test.ts b/test/unit/scripts/dev-pr-queue.test.ts new file mode 100644 index 000000000..902e56f5c --- /dev/null +++ b/test/unit/scripts/dev-pr-queue.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it } from 'vitest' +import { + assembleDevQueue, + assertAssemblePreconditions, + buildDevQueuePlan, + buildPrMetadataCommand, + executeDevQueuePlan, + loadPrMetadata, + parsePrList, + type DevQueuePr, +} from '../../../scripts/dev-pr-queue.js' + +const pr = (input: Partial<DevQueuePr> & Pick<DevQueuePr, 'number'>): DevQueuePr => ({ + number: input.number, + state: input.state ?? 'OPEN', + isDraft: input.isDraft ?? false, + baseRefName: input.baseRefName ?? 'main', + headRefOid: input.headRefOid ?? `sha-${input.number}`, + mergeStateStatus: input.mergeStateStatus ?? 'CLEAN', + title: input.title ?? `PR ${input.number}`, + labels: input.labels ?? [], +}) + +describe('dev PR queue planner', () => { + it('parses explicit PR numbers', () => { + expect(parsePrList('321,309,319')).toEqual([321, 309, 319]) + }) + + it('rejects empty PR lists', () => { + expect(() => parsePrList('')).toThrow('At least one PR number is required') + }) + + it('rejects malformed PR lists', () => { + expect(() => parsePrList('321,nope')).toThrow('At least one PR number is required') + expect(() => parsePrList('0')).toThrow('At least one PR number is required') + }) + + it('plans origin/main plus PR heads in the requested order', () => { + const plan = buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [321, 309], + prs: [pr({ number: 321 }), pr({ number: 309 })], + }) + + expect(plan.steps.map((step) => step.label)).toEqual([ + 'fetch-origin-main', + 'reset-dev-to-origin-main', + 'fetch-pr-321', + 'merge-pr-321', + 'fetch-pr-309', + 'merge-pr-309', + ]) + }) + + it('uses no-ff merges so PR boundaries remain visible on dev', () => { + const plan = buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [321], + prs: [pr({ number: 321 })], + }) + + expect(plan.steps.find((step) => step.label === 'merge-pr-321')?.command).toEqual([ + 'git', + 'merge', + '--no-ff', + '--no-edit', + 'refs/remotes/pr/321', + ]) + }) + + it('rejects missing metadata for requested PRs', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [321], + prs: [], + })).toThrow('PR #321 was not found') + }) + + it('rejects draft PRs', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [289], + prs: [pr({ number: 289, isDraft: true })], + })).toThrow('PR #289 is draft') + }) + + it('rejects closed PRs', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [310], + prs: [pr({ number: 310, state: 'CLOSED' })], + })).toThrow('PR #310 is CLOSED, expected OPEN') + }) + + it('rejects PRs that do not target main', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [310], + prs: [pr({ number: 310, baseRefName: 'other' })], + })).toThrow('PR #310 targets other, expected main') + }) + + it('rejects do-not-merge labels', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [289], + prs: [pr({ number: 289, labels: [{ name: 'do-not-merge' }] })], + })).toThrow('PR #289 is labeled do-not-merge') + }) + + it('rejects superseded and approval-artifact labels', () => { + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [310], + prs: [pr({ number: 310, labels: [{ name: 'superseded' }] })], + })).toThrow('PR #310 is labeled superseded') + + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [311], + prs: [pr({ number: 311, labels: [{ name: 'approval-artifact-only' }] })], + })).toThrow('PR #311 is labeled approval-artifact-only') + + expect(() => buildDevQueuePlan({ + originMain: 'origin-sha', + requestedPrs: [312], + prs: [pr({ number: 312, labels: [{ name: 'approval-artifact' }] })], + })).toThrow('PR #312 is labeled approval-artifact') + }) + + it('builds gh commands for exact PR metadata', () => { + expect(buildPrMetadataCommand(321)).toEqual([ + 'gh', + [ + 'pr', + 'view', + '321', + '--repo', + 'danshapiro/freshell', + '--json', + 'number,state,isDraft,baseRefName,headRefOid,mergeStateStatus,title,labels', + ], + ]) + }) + + it('loads explicit PR metadata through the injected runner', async () => { + const calls: string[] = [] + const prs = await loadPrMetadata([321, 309], async (command, args) => { + calls.push([command, ...args].join(' ')) + const number = Number(args[2]) + return JSON.stringify(pr({ number })) + }) + + expect(prs.map((item) => item.number)).toEqual([321, 309]) + expect(calls).toEqual([ + 'gh pr view 321 --repo danshapiro/freshell --json number,state,isDraft,baseRefName,headRefOid,mergeStateStatus,title,labels', + 'gh pr view 309 --repo danshapiro/freshell --json number,state,isDraft,baseRefName,headRefOid,mergeStateStatus,title,labels', + ]) + }) + + it('reports invalid gh metadata clearly', async () => { + await expect(loadPrMetadata([321], async () => 'not json')).rejects.toThrow( + 'Failed to parse gh metadata for PR #321', + ) + }) + + it('refuses assemble outside dev', async () => { + await expect(assertAssemblePreconditions({ + getBranch: async () => 'feature/x', + getStatus: async () => '', + })).rejects.toThrow('Refusing to assemble dev from feature/x') + }) + + it('refuses assemble on an unknown branch', async () => { + await expect(assertAssemblePreconditions({ + getBranch: async () => undefined, + getStatus: async () => '', + })).rejects.toThrow('Refusing to assemble dev from an unknown branch') + }) + + it('refuses assemble with a dirty worktree', async () => { + await expect(assertAssemblePreconditions({ + getBranch: async () => 'dev', + getStatus: async () => ' M package.json', + })).rejects.toThrow('Refusing to reset dev with a dirty worktree') + }) + + it('stops on the first failed merge and reports the PR', async () => { + const executed: string[] = [] + await expect(executeDevQueuePlan({ + originMain: 'origin-sha', + steps: [ + { label: 'reset-dev-to-origin-main', command: ['git', 'reset', '--hard', 'origin-sha'] }, + { label: 'merge-pr-321', command: ['git', 'merge', '--no-ff', '--no-edit', 'refs/remotes/pr/321'] }, + { label: 'merge-pr-309', command: ['git', 'merge', '--no-ff', '--no-edit', 'refs/remotes/pr/309'] }, + ], + }, async (_command, args) => { + executed.push(args.join(' ')) + if (args.includes('refs/remotes/pr/321')) throw new Error('merge failed') + return '' + })).rejects.toThrow('PR #321 did not merge cleanly') + + expect(executed).toHaveLength(2) + }) + + it('stops on the first failed cherry-pick and reports the PR', async () => { + await expect(executeDevQueuePlan({ + originMain: 'origin-sha', + steps: [ + { label: 'cherry-pick-pr-321', command: ['git', 'cherry-pick', 'sha-321'] }, + ], + }, async () => { + throw new Error('conflict') + })).rejects.toThrow('PR #321 did not cherry-pick cleanly') + }) + + it('reports non-merge command failures clearly', async () => { + await expect(executeDevQueuePlan({ + originMain: 'origin-sha', + steps: [ + { label: 'fetch-origin-main', command: ['git', 'fetch', 'origin', 'main'] }, + ], + }, async () => { + throw new Error('network unavailable') + })).rejects.toThrow('Step fetch-origin-main failed: network unavailable') + }) + + it('validates metadata before resetting dev', async () => { + const events: string[] = [] + await expect(assembleDevQueue({ + requestedPrs: [289], + run: async (command, args) => { + events.push([command, ...args].join(' ')) + if (args.join(' ') === 'rev-parse origin/main') return 'origin-sha' + if (args.includes('view')) { + return JSON.stringify(pr({ number: 289, labels: [{ name: 'do-not-merge' }] })) + } + return '' + }, + getBranch: async () => 'dev', + getStatus: async () => '', + })).rejects.toThrow('PR #289 is labeled do-not-merge') + + expect(events.some((event) => event.includes('reset --hard'))).toBe(false) + }) + + it('assembles dev after preconditions and metadata validation', async () => { + const events: string[] = [] + await assembleDevQueue({ + requestedPrs: [321], + run: async (command, args) => { + events.push([command, ...args].join(' ')) + if (args.join(' ') === 'rev-parse origin/main') return 'origin-sha' + if (args.includes('view')) return JSON.stringify(pr({ number: 321 })) + return '' + }, + getBranch: async () => 'dev', + getStatus: async () => '', + }) + + expect(events).toEqual([ + 'git fetch origin main', + 'git rev-parse origin/main', + 'gh pr view 321 --repo danshapiro/freshell --json number,state,isDraft,baseRefName,headRefOid,mergeStateStatus,title,labels', + 'git fetch origin main', + 'git reset --hard origin-sha', + 'git fetch origin +refs/pull/321/head:refs/remotes/pr/321', + 'git merge --no-ff --no-edit refs/remotes/pr/321', + ]) + }) +}) From 2bc879012154a81ba706b6086b7d54c6e8aed3ae Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Tue, 5 May 2026 08:18:48 -0700 Subject: [PATCH 08/25] fix: disable scroll-to-arrowkey fallback for opencode provider OpenCode CLI's TUI enables mouse tracking for click-handling, which caused the fallbackToCursorKeysWhenAltScreenMouseCapture policy to intercept scroll events and send up/down arrow keys instead. The CLI doesn't interpret arrow keys as scroll commands, so scrolling broke. Change scrollInputPolicy to 'native' so xterm.js passes mouse scroll SGR sequences through to the CLI, letting it handle them natively. --- extensions/opencode/freshell.json | 2 +- test/unit/client/lib/terminal-behavior.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/opencode/freshell.json b/extensions/opencode/freshell.json index 1bfb4482e..f3b946e43 100644 --- a/extensions/opencode/freshell.json +++ b/extensions/opencode/freshell.json @@ -19,7 +19,7 @@ "supportsModel": true, "terminalBehavior": { "preferredRenderer": "canvas", - "scrollInputPolicy": "fallbackToCursorKeysWhenAltScreenMouseCapture" + "scrollInputPolicy": "native" } }, "picker": { diff --git a/test/unit/client/lib/terminal-behavior.test.ts b/test/unit/client/lib/terminal-behavior.test.ts index 74305251d..6067a7dcf 100644 --- a/test/unit/client/lib/terminal-behavior.test.ts +++ b/test/unit/client/lib/terminal-behavior.test.ts @@ -17,7 +17,7 @@ const extensions: ClientExtensionEntry[] = [{ cli: { terminalBehavior: { preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }, }, }] @@ -26,7 +26,7 @@ describe('terminal behavior', () => { it('returns provider terminal behavior from the extension registry', () => { expect(getProviderTerminalBehavior('opencode', extensions)).toEqual({ preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }) }) From ed440dce75b9b6b0323f2e7ecb93ad1a1230d0e6 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Tue, 5 May 2026 08:30:03 -0700 Subject: [PATCH 09/25] fix: update scroll policy tests and skip useless scrollLines in alt screen - Skip term.scrollLines() in alt screen for touch scrolling (it's a no-op since alternate buffer has no scrollback) - Update opencode scroll/touch test fixtures and assertions to match the new native policy: OpenCode no longer translates scroll events to cursor keys --- src/components/TerminalView.tsx | 4 ++- .../e2e/opencode-scroll-input-policy.test.tsx | 8 ++--- ...pencode-touch-scroll-input-policy.test.tsx | 12 +++---- .../TerminalView.scroll-input-policy.test.tsx | 32 +++++++------------ ...nalView.touch-scroll-input-policy.test.tsx | 8 ++--- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/components/TerminalView.tsx b/src/components/TerminalView.tsx index def853dc9..ac2217d56 100644 --- a/src/components/TerminalView.tsx +++ b/src/components/TerminalView.tsx @@ -706,7 +706,9 @@ function TerminalView({ tabId, paneId, paneContent, hidden }: TerminalViewProps) const lines = rawLines > 0 ? Math.floor(rawLines) : Math.ceil(rawLines) if (lines !== 0) { if (!translateScrollLinesToInput(term, lines)) { - term.scrollLines(lines) + if (term.buffer.active.type !== 'alternate') { + term.scrollLines(lines) + } } touchScrollAccumulatorRef.current -= lines * TOUCH_SCROLL_PIXELS_PER_LINE diff --git a/test/e2e/opencode-scroll-input-policy.test.tsx b/test/e2e/opencode-scroll-input-policy.test.tsx index 42cd883d0..fe3c23dc5 100644 --- a/test/e2e/opencode-scroll-input-policy.test.tsx +++ b/test/e2e/opencode-scroll-input-policy.test.tsx @@ -103,7 +103,7 @@ const opencodeExtensionWithBehaviorHint: ClientExtensionEntry = { cli: { terminalBehavior: { preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }, }, } @@ -178,7 +178,7 @@ describe('opencode scroll input policy (e2e)', () => { vi.unstubAllGlobals() }) - it('sends cursor-key input when an OpenCode pane receives wheel input in alt screen mouse mode', async () => { + it('does not translate wheel input for opencode providers when policy is native', async () => { const { store, tabId, paneId, paneContent } = createStore('opencode', [opencodeExtensionWithBehaviorHint]) const { getByTestId } = render( @@ -195,10 +195,8 @@ describe('opencode scroll input policy (e2e)', () => { fireEvent.wheel(getByTestId('terminal-xterm-container'), { deltaY: 24 }) - expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', })) }) diff --git a/test/e2e/opencode-touch-scroll-input-policy.test.tsx b/test/e2e/opencode-touch-scroll-input-policy.test.tsx index 5dab864ca..6f041d09a 100644 --- a/test/e2e/opencode-touch-scroll-input-policy.test.tsx +++ b/test/e2e/opencode-touch-scroll-input-policy.test.tsx @@ -101,7 +101,7 @@ const opencodeExtensionWithBehaviorHint: ClientExtensionEntry = { cli: { terminalBehavior: { preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }, }, } @@ -177,7 +177,7 @@ describe('opencode touch scroll input policy (e2e)', () => { ;(globalThis as any).setMobileForTest(false) }) - it('sends translated cursor-key input instead of local scrollback for OpenCode touch scrolling', async () => { + it('does not translate touch scroll for opencode providers when policy is native', async () => { const { store, tabId, paneId, paneContent } = createStore('opencode', [opencodeExtensionWithBehaviorHint]) const { getByTestId } = render( @@ -202,14 +202,12 @@ describe('opencode touch scroll input policy (e2e)', () => { }) expect(latestTerminal?.scrollLines).not.toHaveBeenCalled() - expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', })) }) - it('keeps native touch scrolling for non-opted-in providers', async () => { + it('skips scrollLines in alt screen for non-opted-in providers', async () => { const { store, tabId, paneId, paneContent } = createStore('shell') const { getByTestId } = render( @@ -233,7 +231,7 @@ describe('opencode touch scroll input policy (e2e)', () => { touches: [{ clientX: 20, clientY: 100 }], }) - expect(latestTerminal?.scrollLines).toHaveBeenCalledWith(1) + expect(latestTerminal?.scrollLines).not.toHaveBeenCalled() expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input', })) diff --git a/test/unit/client/components/TerminalView.scroll-input-policy.test.tsx b/test/unit/client/components/TerminalView.scroll-input-policy.test.tsx index 06a362dc8..5d763c363 100644 --- a/test/unit/client/components/TerminalView.scroll-input-policy.test.tsx +++ b/test/unit/client/components/TerminalView.scroll-input-policy.test.tsx @@ -111,7 +111,7 @@ const opencodeExtensionWithBehaviorHint: ClientExtensionEntry = { cli: { terminalBehavior: { preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }, }, } @@ -193,7 +193,7 @@ describe('TerminalView wheel scroll input policy', () => { vi.unstubAllGlobals() }) - it('loads the extension registry for coding-cli panes before applying wheel translation', async () => { + it('loads the extension registry for coding-cli panes and does not translate when policy is native', async () => { authMocks.getAuthToken.mockReturnValue('token-test') apiMocks.get.mockResolvedValue([opencodeExtensionWithBehaviorHint]) const { store, tabId, paneId, paneContent } = createStore('opencode', [], 'term-opencode') @@ -212,15 +212,11 @@ describe('TerminalView wheel scroll input policy', () => { wsMocks.send.mockClear() const event = new WheelEvent('wheel', { deltaY: 24, cancelable: true }) - expect(wheelHandler?.(event)).toBe(false) - expect(wsMocks.send).toHaveBeenCalledWith({ - type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', - }) + expect(wheelHandler?.(event)).toBe(true) + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input' })) }) - it('translates wheel scrolling into cursor-key input for opted-in providers', async () => { + it('does not translate wheel scrolling for opencode providers when policy is native', async () => { const { store, tabId, paneId, paneContent } = createStore('opencode', [opencodeExtensionWithBehaviorHint], 'term-opencode') render( @@ -242,14 +238,10 @@ describe('TerminalView wheel scroll input policy', () => { preventDefault, stopPropagation, } as unknown as WheelEvent - expect(wheelHandler?.(event)).toBe(false) - expect(preventDefault).toHaveBeenCalledOnce() - expect(stopPropagation).toHaveBeenCalledOnce() - expect(wsMocks.send).toHaveBeenCalledWith({ - type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', - }) + expect(wheelHandler?.(event)).toBe(true) + expect(preventDefault).not.toHaveBeenCalled() + expect(stopPropagation).not.toHaveBeenCalled() + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input' })) }) it('keeps non-opted-in providers on native wheel behavior', async () => { @@ -307,11 +299,9 @@ describe('TerminalView wheel scroll input policy', () => { }) wsMocks.send.mockClear() - expect(wheelHandler?.(event)).toBe(false) - expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ + expect(wheelHandler?.(event)).toBe(true) + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', })) }) }) diff --git a/test/unit/client/components/TerminalView.touch-scroll-input-policy.test.tsx b/test/unit/client/components/TerminalView.touch-scroll-input-policy.test.tsx index d1f92c682..1c1f1e923 100644 --- a/test/unit/client/components/TerminalView.touch-scroll-input-policy.test.tsx +++ b/test/unit/client/components/TerminalView.touch-scroll-input-policy.test.tsx @@ -101,7 +101,7 @@ const opencodeExtensionWithBehaviorHint: ClientExtensionEntry = { cli: { terminalBehavior: { preferredRenderer: 'canvas', - scrollInputPolicy: 'fallbackToCursorKeysWhenAltScreenMouseCapture', + scrollInputPolicy: 'native', }, }, } @@ -177,7 +177,7 @@ describe('TerminalView touch scroll input policy', () => { ;(globalThis as any).setMobileForTest(false) }) - it('translates touch scrolling into cursor-key input for opted-in providers', async () => { + it('does not translate touch scrolling for opencode providers when policy is native', async () => { const { store, tabId, paneId, paneContent } = createStore('opencode', [opencodeExtensionWithBehaviorHint]) const { getByTestId } = render( @@ -203,10 +203,8 @@ describe('TerminalView touch scroll input policy', () => { }) expect(latestTerminal?.scrollLines).not.toHaveBeenCalled() - expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ + expect(wsMocks.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.input', - terminalId: 'term-opencode', - data: '\u001b[B', })) }) }) From 93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 12:07:41 -0700 Subject: [PATCH 10/25] fix(codex): start app server in requested cwd --- .../coding-cli/codex-app-server/launch-planner.ts | 10 +++++++--- server/coding-cli/codex-app-server/runtime.ts | 15 +++++++++++++++ server/index.ts | 5 ++++- .../codex-app-server/launch-planner.test.ts | 5 ++++- .../coding-cli/codex-app-server/runtime.test.ts | 10 ++++++++++ 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/server/coding-cli/codex-app-server/launch-planner.ts b/server/coding-cli/codex-app-server/launch-planner.ts index 020bf3278..2b9a2346d 100644 --- a/server/coding-cli/codex-app-server/launch-planner.ts +++ b/server/coding-cli/codex-app-server/launch-planner.ts @@ -35,6 +35,10 @@ type PlanCreateInput = { approvalPolicy?: string } +type RuntimeCreateInput = { + cwd?: string +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error) } @@ -53,11 +57,11 @@ export function isCodexSidecarTeardownError(error: unknown): error is CodexSidec export class CodexLaunchPlanner { private readonly activeSidecars = new Set<CodexLaunchSidecar>() private readonly failedSidecarShutdowns = new Set<CodexLaunchSidecar>() - private readonly runtimeFactory: () => CodexRuntimeLike + private readonly runtimeFactory: (input: RuntimeCreateInput) => CodexRuntimeLike private shutdownStarted = false private shutdownPromise: Promise<void> | null = null - constructor(runtimeOrFactory: CodexRuntimeLike | (() => CodexRuntimeLike)) { + constructor(runtimeOrFactory: CodexRuntimeLike | ((input: RuntimeCreateInput) => CodexRuntimeLike)) { this.runtimeFactory = typeof runtimeOrFactory === 'function' ? runtimeOrFactory : () => runtimeOrFactory @@ -68,7 +72,7 @@ export class CodexLaunchPlanner { await this.retryFailedSidecarShutdownsBeforePlan() this.assertAcceptingPlans() - const runtime = this.runtimeFactory() + const runtime = this.runtimeFactory({ cwd: input.cwd }) const sidecar = this.createSidecar(runtime) this.activeSidecars.add(sidecar) diff --git a/server/coding-cli/codex-app-server/runtime.ts b/server/coding-cli/codex-app-server/runtime.ts index 96471533c..d422e6e61 100644 --- a/server/coding-cli/codex-app-server/runtime.ts +++ b/server/coding-cli/codex-app-server/runtime.ts @@ -5,6 +5,7 @@ import os from 'node:os' import path from 'node:path' import { allocateLocalhostPort, type LoopbackServerEndpoint } from '../../local-port.js' import { logger } from '../../logger.js' +import { convertWindowsPathToWslPath, isWslEnvironment, sanitizeUserPathInput } from '../../path-utils.js' import { CodexAppServerClient, type CodexThreadLifecycleEvent, @@ -65,6 +66,7 @@ type ChildProcessHandle = ReturnType<typeof spawn> type RuntimeOptions = { command?: string commandArgs?: string[] + cwd?: string env?: NodeJS.ProcessEnv requestTimeoutMs?: number startupAttemptLimit?: number @@ -100,6 +102,16 @@ function sleep(ms: number): Promise<void> { return new Promise((resolve) => setTimeout(resolve, ms)) } +function resolveAppServerCwd(cwd: string | undefined): string | undefined { + if (typeof cwd !== 'string') return undefined + const candidate = sanitizeUserPathInput(cwd) + if (!candidate) return undefined + if (isWslEnvironment()) { + return convertWindowsPathToWslPath(candidate) ?? candidate + } + return candidate +} + function defaultMetadataDir(): string { return process.env.FRESHELL_CODEX_SIDECAR_DIR || path.join(os.homedir(), '.freshell', 'codex-sidecars') @@ -509,6 +521,7 @@ export class CodexAppServerRuntime { private readonly command: string private readonly commandArgs: string[] + private readonly cwd?: string private readonly env?: NodeJS.ProcessEnv private readonly requestTimeoutMs?: number private readonly startupAttemptLimit: number @@ -523,6 +536,7 @@ export class CodexAppServerRuntime { constructor(options: RuntimeOptions = {}) { this.command = options.command ?? (process.env.CODEX_CMD || 'codex') this.commandArgs = options.commandArgs ?? [] + this.cwd = resolveAppServerCwd(options.cwd) this.env = options.env this.requestTimeoutMs = options.requestTimeoutMs this.startupAttemptLimit = options.startupAttemptLimit ?? DEFAULT_STARTUP_ATTEMPT_LIMIT @@ -695,6 +709,7 @@ export class CodexAppServerRuntime { wsUrl, ], { detached: true, + ...(this.cwd ? { cwd: this.cwd } : {}), env: { ...process.env, ...this.env, diff --git a/server/index.ts b/server/index.ts index d443fae17..14f51fcf8 100644 --- a/server/index.ts +++ b/server/index.ts @@ -298,7 +298,10 @@ async function main() { sdkBridge = new SdkBridge(agentHistorySource) const server = http.createServer(app) - const codexLaunchPlanner = new CodexLaunchPlanner(() => new CodexAppServerRuntime({ serverInstanceId })) + const codexLaunchPlanner = new CodexLaunchPlanner((input) => new CodexAppServerRuntime({ + serverInstanceId, + cwd: input.cwd, + })) const wsHandler = new WsHandler( server, registry, diff --git a/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts b/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts index 8e8061986..151513c21 100644 --- a/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts @@ -71,7 +71,9 @@ class FakeRuntime { describe('CodexLaunchPlanner', () => { it('creates a distinct owned sidecar for each launch plan', async () => { const runtimes: FakeRuntime[] = [] - const planner = new CodexLaunchPlanner(() => { + const runtimeInputs: Array<{ cwd?: string }> = [] + const planner = new CodexLaunchPlanner((input) => { + runtimeInputs.push(input) const index = runtimes.length + 1 const runtime = new FakeRuntime(`ws://127.0.0.1:${43000 + index}`, `thread-${index}`) runtimes.push(runtime) @@ -82,6 +84,7 @@ describe('CodexLaunchPlanner', () => { const second = await planner.planCreate({ cwd: '/repo/two' }) expect(runtimes).toHaveLength(2) + expect(runtimeInputs).toEqual([{ cwd: '/repo/one' }, { cwd: '/repo/two' }]) expect(first.remote.wsUrl).toBe('ws://127.0.0.1:43001') expect(second.remote.wsUrl).toBe('ws://127.0.0.1:43002') diff --git a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts index b2108bd4f..5c3bdb77b 100644 --- a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts @@ -263,6 +263,16 @@ describe('CodexAppServerRuntime', () => { } }) + it('starts the app-server process in the requested cwd', async () => { + if (process.platform !== 'linux') return + const runtimeCwd = await makeTempDir() + const runtime = createRuntime({ cwd: runtimeCwd }) + + const ready = await runtime.ensureReady() + + await expect(fsp.readlink(`/proc/${ready.processPid}/cwd`)).resolves.toBe(runtimeCwd) + }) + it('reuses the same process for repeated ensureReady calls on one runtime', async () => { const runtime = createRuntime() From 0a334be42553929aed033c3aa3920d0bf58a2a65 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:24:00 -0700 Subject: [PATCH 11/25] feat: support factory terminal orchestration --- package-lock.json | 3 + server/agent-api/layout-store.ts | 11 ++++ server/agent-api/router.ts | 64 +++++++++++++++++++ server/terminal-view/service.ts | 25 ++++++++ server/terminal-view/types.ts | 2 + server/ws-handler.ts | 78 +++++++++++++++++++++- src/lib/ui-commands.ts | 1 + test/server/agent-tabs-write.test.ts | 96 ++++++++++++++++++++++++++++ test/server/terminals-api.test.ts | 18 ++++++ test/unit/client/ui-commands.test.ts | 18 ++++++ 10 files changed, 314 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 75df5123d..e5166f884 100644 --- a/package-lock.json +++ b/package-lock.json @@ -92,6 +92,9 @@ "typescript": "^5.7.2", "vite": "^6.4.1", "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22.5.0" } }, "node_modules/@adobe/css-tools": { diff --git a/server/agent-api/layout-store.ts b/server/agent-api/layout-store.ts index 9803fb63a..a66570df0 100644 --- a/server/agent-api/layout-store.ts +++ b/server/agent-api/layout-store.ts @@ -311,6 +311,17 @@ export class LayoutStore { return undefined } + findPaneByTerminalId(terminalId: string): { tabId: string; paneId: string } | undefined { + if (!this.snapshot) return undefined + for (const tab of this.snapshot.tabs) { + const root = this.snapshot.layouts?.[tab.id] + const leaves = this.collectLeaves(root, []) + const match = leaves.find((leaf) => leaf.content?.terminalId === terminalId) + if (match) return { tabId: tab.id, paneId: match.id } + } + return undefined + } + getPaneSnapshot(paneId: string): PaneSnapshot | undefined { if (!this.snapshot) return undefined for (const tab of this.snapshot.tabs) { diff --git a/server/agent-api/router.ts b/server/agent-api/router.ts index b2fb02db4..eb6633675 100644 --- a/server/agent-api/router.ts +++ b/server/agent-api/router.ts @@ -242,6 +242,14 @@ export function createAgentApiRouter({ assertTerminalCreateAccepted?.() } + const broadcastReplayableUiCommand = (command: { command: string; payload?: any }) => { + if (typeof wsHandler?.broadcastUiCommandWithReplay === 'function') { + wsHandler.broadcastUiCommandWithReplay(command) + return + } + wsHandler?.broadcastUiCommand?.(command) + } + const resolvePaneTarget = (raw: string) => { if (layoutStore.resolveTarget) { const resolved = layoutStore.resolveTarget(raw) @@ -516,6 +524,62 @@ export function createAgentApiRouter({ res.json(ok({ tabs, activeTabId })) }) + router.post('/terminals/:id/open', (req, res) => { + const terminalId = typeof req.params.id === 'string' ? req.params.id.trim() : '' + if (!terminalId) return res.status(400).json(fail('terminal id required')) + const term = registry.get?.(terminalId) + if (!term) return res.status(404).json(fail('terminal not found')) + + const existing = layoutStore.findPaneByTerminalId?.(terminalId) + if (existing?.tabId && existing?.paneId) { + const result = layoutStore.selectPane?.(existing.tabId, existing.paneId) || existing + const tabId = result?.tabId || existing.tabId + const paneId = result?.paneId || existing.paneId + if (tabId && paneId) { + broadcastReplayableUiCommand({ + command: 'tab.select', + payload: { id: tabId }, + }) + broadcastReplayableUiCommand({ + command: 'pane.select', + payload: { tabId, paneId }, + }) + } + return res.json(ok({ tabId, paneId, terminalId, reused: true }, result?.message || 'terminal selected')) + } + + if (!layoutStore.createTab || !layoutStore.attachPaneContent) { + return res.status(503).json(fail('layout store does not support terminal attach')) + } + + const title = parseRequiredName(req.body?.name) || term.title || terminalId + const { tabId, paneId } = layoutStore.createTab({ title }) + const paneContent = { + kind: 'terminal', + terminalId, + status: term.status || 'running', + mode: term.mode || 'shell', + initialCwd: term.cwd, + resumeSessionId: term.resumeSessionId, + } + layoutStore.attachPaneContent(tabId, paneId, paneContent) + broadcastReplayableUiCommand({ + command: 'tab.create', + payload: { + id: tabId, + title, + mode: term.mode || 'shell', + terminalId, + initialCwd: term.cwd, + resumeSessionId: term.resumeSessionId, + paneId, + paneContent, + status: term.status || 'running', + }, + }) + res.json(ok({ tabId, paneId, terminalId, reused: false }, 'terminal opened')) + }) + router.get('/panes', (req, res) => { const tabId = req.query.tabId as string | undefined const panes = layoutStore.listPanes?.(tabId) || [] diff --git a/server/terminal-view/service.ts b/server/terminal-view/service.ts index 969a26a15..92d23ddfe 100644 --- a/server/terminal-view/service.ts +++ b/server/terminal-view/service.ts @@ -77,6 +77,28 @@ function buildRuntime(record: TerminalRecord): TerminalViewportRuntime { } } +const ANSI_ESCAPE_RE = /\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g +const MAX_LAST_LINE_CHARS = 500 + +function isShellPromptLine(line: string): boolean { + return /^[^\s@:]+@[^\s:]+:.+[#$%]\s*$/.test(line) +} + +function lastEmittedLine(snapshot: string): string | undefined { + const lastLine = snapshot + .replace(ANSI_ESCAPE_RE, '') + .replace(/\r/g, '\n') + .split('\n') + .map((line) => line.trim()) + .filter((line) => !isShellPromptLine(line)) + .filter(Boolean) + .at(-1) + + if (!lastLine) return undefined + if (lastLine.length <= MAX_LAST_LINE_CHARS) return lastLine + return `${lastLine.slice(0, MAX_LAST_LINE_CHARS - 3)}...` +} + function encodeCursor(payload: CursorPayload): string { return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url') } @@ -179,10 +201,13 @@ export function createTerminalViewService(deps: TerminalViewServiceDeps): Termin .filter((terminal) => !config.terminalOverrides?.[terminal.terminalId]?.deleted) .map((terminal) => { const override = config.terminalOverrides?.[terminal.terminalId] + const lastLine = lastEmittedLine(deps.registry.get(terminal.terminalId)?.buffer.snapshot() || '') return { ...buildDirectoryItem(terminal), title: override?.titleOverride || terminal.title, description: override?.descriptionOverride || terminal.description, + lastLine, + last_line: lastLine, } }) .sort(compareTerminals) diff --git a/server/terminal-view/types.ts b/server/terminal-view/types.ts index d5abf8666..f6450fd95 100644 --- a/server/terminal-view/types.ts +++ b/server/terminal-view/types.ts @@ -13,6 +13,8 @@ export type TerminalDirectoryItem = { status: 'running' | 'exited' hasClients: boolean cwd?: string + lastLine?: string + last_line?: string } export type TerminalDirectoryPage = { diff --git a/server/ws-handler.ts b/server/ws-handler.ts index e7cee3ca8..ed0245bf2 100644 --- a/server/ws-handler.ts +++ b/server/ws-handler.ts @@ -350,6 +350,10 @@ type PendingScreenshot = { } type ScreenshotErrorCode = 'NO_SCREENSHOT_CLIENT' | 'SCREENSHOT_TIMEOUT' | 'SCREENSHOT_CONNECTION_CLOSED' +type UiCommand = { command: string; payload?: any } +type PendingUiCommand = { command: UiCommand; expiresAt: number } +const UI_COMMAND_REPLAY_TTL_MS = 15_000 +const UI_COMMAND_RECENT_CONNECTION_MS = 3_000 function createScreenshotError(code: ScreenshotErrorCode, message: string): Error & { code: ScreenshotErrorCode } { const err = new Error(message) as Error & { code: ScreenshotErrorCode } @@ -391,6 +395,7 @@ export class WsHandler { private createdSdkSessionByRequestId = new Map<string, string>() private sdkSessionByCreateOwnerKey = new Map<string, string>() private screenshotRequests = new Map<string, PendingScreenshot>() + private pendingUiCommands: PendingUiCommand[] = [] private sessionsRevision = 0 private terminalsRevision = 0 @@ -1738,6 +1743,7 @@ export class WsHandler { bootId: this.bootId, }) this.scheduleHandshakeSnapshot(ws, state) + this.flushPendingUiCommands(ws) return } @@ -3156,8 +3162,76 @@ export class WsHandler { } } - broadcastUiCommand(command: { command: string; payload?: any }) { - this.broadcast({ type: 'ui.command', ...command }) + private authenticatedUiConnections(): LiveWebSocket[] { + return [...this.connections].filter((ws) => { + if (ws.readyState !== WebSocket.OPEN) return false + return !!this.clientStates.get(ws)?.authenticated + }) + } + + private uiCommandKey(command: UiCommand): string { + return JSON.stringify(command) + } + + private queueUiCommand(command: UiCommand, now = Date.now()): void { + const key = this.uiCommandKey(command) + this.pendingUiCommands = this.pendingUiCommands.filter((item) => ( + item.expiresAt > now && this.uiCommandKey(item.command) !== key + )) + this.pendingUiCommands.push({ command, expiresAt: now + UI_COMMAND_REPLAY_TTL_MS }) + } + + private flushPendingUiCommands(target?: LiveWebSocket): void { + const now = Date.now() + const pending = this.pendingUiCommands.filter((item) => item.expiresAt > now) + this.pendingUiCommands = [] + if (!pending.length) return + + const targets = target ? [target] : this.authenticatedUiConnections() + if (!targets.length) { + this.pendingUiCommands.push(...pending) + return + } + + for (const item of pending) { + for (const ws of targets) { + if (ws.readyState === WebSocket.OPEN) { + this.send(ws, { type: 'ui.command', ...item.command }) + } + } + } + } + + broadcastUiCommand(command: UiCommand) { + const targets = this.authenticatedUiConnections() + if (!targets.length) { + this.queueUiCommand(command) + return + } + + for (const ws of targets) { + this.send(ws, { type: 'ui.command', ...command }) + } + } + + broadcastUiCommandWithReplay(command: UiCommand) { + const now = Date.now() + const targets = this.authenticatedUiConnections() + if (!targets.length) { + this.queueUiCommand(command, now) + return + } + + const hasRecentTarget = targets.some((ws) => ( + typeof ws.connectedAt === 'number' && now - ws.connectedAt <= UI_COMMAND_RECENT_CONNECTION_MS + )) + if (!hasRecentTarget) { + this.queueUiCommand(command, now) + } + + for (const ws of targets) { + this.send(ws, { type: 'ui.command', ...command }) + } } broadcastSessionsChanged(revision: number): void { diff --git a/src/lib/ui-commands.ts b/src/lib/ui-commands.ts index 17f201d78..0cfd7048b 100644 --- a/src/lib/ui-commands.ts +++ b/src/lib/ui-commands.ts @@ -123,6 +123,7 @@ export function handleUiCommand(msg: any, runtimeOrDispatch: UiCommandRuntime | case 'pane.close': return dispatch(closePaneWithCleanup({ tabId: msg.payload.tabId, paneId: msg.payload.paneId })) case 'pane.select': + dispatch(setActiveTab(msg.payload.tabId)) return dispatch(setActivePane({ tabId: msg.payload.tabId, paneId: msg.payload.paneId })) case 'pane.rename': return dispatch(applyPaneRename({ diff --git a/test/server/agent-tabs-write.test.ts b/test/server/agent-tabs-write.test.ts index 68e455c3f..e224d15ef 100644 --- a/test/server/agent-tabs-write.test.ts +++ b/test/server/agent-tabs-write.test.ts @@ -9,6 +9,7 @@ const expectedFreshellUrl = process.env.FRESHELL_URL || 'http://localhost:3001' class FakeRegistry { create = vi.fn((opts?: { terminalId?: string }) => ({ terminalId: opts?.terminalId ?? 'term_1' })) + get = vi.fn() } describe('tab endpoints', () => { @@ -85,6 +86,101 @@ describe('tab endpoints', () => { })) }) + it('opens an existing terminal in a new tab when it is detached', async () => { + const app = express() + app.use(express.json()) + const registry = new FakeRegistry() + registry.get.mockReturnValue({ + terminalId: 'term_1', + title: 'Detached shell', + mode: 'shell', + status: 'running', + cwd: '/workspace', + }) + const createTab = vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })) + const attachPaneContent = vi.fn() + const broadcastUiCommandWithReplay = vi.fn() + const layoutStore = { + createTab, + attachPaneContent, + findPaneByTerminalId: vi.fn(() => undefined), + } + app.use('/api', createAgentApiRouter({ + layoutStore, + registry, + wsHandler: { broadcastUiCommandWithReplay }, + })) + + const res = await request(app) + .post('/api/terminals/term_1/open') + .send({ name: 'Work shell' }) + + expect(res.status).toBe(200) + expect(createTab).toHaveBeenCalledWith({ title: 'Work shell' }) + expect(attachPaneContent).toHaveBeenCalledWith('tab_1', 'pane_1', { + kind: 'terminal', + terminalId: 'term_1', + status: 'running', + mode: 'shell', + initialCwd: '/workspace', + resumeSessionId: undefined, + }) + expect(broadcastUiCommandWithReplay).toHaveBeenCalledWith({ + command: 'tab.create', + payload: expect.objectContaining({ + id: 'tab_1', + paneId: 'pane_1', + terminalId: 'term_1', + title: 'Work shell', + }), + }) + expect(res.body.data).toMatchObject({ + tabId: 'tab_1', + paneId: 'pane_1', + terminalId: 'term_1', + reused: false, + }) + }) + + it('selects the existing pane when opening an already-attached terminal', async () => { + const app = express() + app.use(express.json()) + const registry = new FakeRegistry() + registry.get.mockReturnValue({ terminalId: 'term_1', title: 'Shell', mode: 'shell', status: 'running' }) + const selectPane = vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })) + const broadcastUiCommand = vi.fn() + const broadcastUiCommandWithReplay = vi.fn() + const layoutStore = { + findPaneByTerminalId: vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })), + selectPane, + } + app.use('/api', createAgentApiRouter({ + layoutStore, + registry, + wsHandler: { broadcastUiCommand, broadcastUiCommandWithReplay }, + })) + + const res = await request(app).post('/api/terminals/term_1/open').send({}) + + expect(res.status).toBe(200) + expect(selectPane).toHaveBeenCalledWith('tab_1', 'pane_1') + expect(broadcastUiCommandWithReplay).toHaveBeenCalledWith({ + command: 'tab.select', + payload: { id: 'tab_1' }, + }) + expect(broadcastUiCommandWithReplay).toHaveBeenCalledWith({ + command: 'pane.select', + payload: { tabId: 'tab_1', paneId: 'pane_1' }, + }) + expect(broadcastUiCommand).not.toHaveBeenCalled() + expect(res.body.data).toMatchObject({ + tabId: 'tab_1', + paneId: 'pane_1', + terminalId: 'term_1', + reused: true, + }) + }) + it('creates terminal tabs from canonical sessionRef without mirroring legacy resumeSessionId payloads', async () => { const app = express() app.use(express.json()) diff --git a/test/server/terminals-api.test.ts b/test/server/terminals-api.test.ts index 7fbf07b43..998621318 100644 --- a/test/server/terminals-api.test.ts +++ b/test/server/terminals-api.test.ts @@ -261,6 +261,24 @@ describe('Terminals API', () => { expect(secondTerminal.mode).toBe('claude') }) + it('includes the last emitted terminal line', async () => { + const terminal = registry.addTerminal({ + terminalId: 'term_last_line', + title: 'Repo push', + mode: 'shell', + }) + terminal.buffer.append('first line\nsecond line\nvagrant@gf-software-factory-vm:/workspace/project$ ') + + const response = await request(app) + .get('/api/terminals') + .set('x-auth-token', AUTH_TOKEN) + .expect(200) + + const item = response.body.find((t: any) => t.terminalId === 'term_last_line') + expect(item.lastLine).toBe('second line') + expect(item.last_line).toBe('second line') + }) + it('applies title override from config', async () => { registry.addTerminal({ terminalId: 'term_with_override', diff --git a/test/unit/client/ui-commands.test.ts b/test/unit/client/ui-commands.test.ts index 6fd5cae6c..95ccebd49 100644 --- a/test/unit/client/ui-commands.test.ts +++ b/test/unit/client/ui-commands.test.ts @@ -80,6 +80,24 @@ describe('handleUiCommand', () => { expect(actions[1].type).toBe('panes/swapPanes') }) + it('selects the tab before selecting a pane', () => { + const actions: any[] = [] + const dispatch = (action: any) => { + actions.push(action) + return action + } + + handleUiCommand({ + type: 'ui.command', + command: 'pane.select', + payload: { tabId: 't1', paneId: 'p1' }, + }, dispatch) + + expect(actions.map((a) => a.type)).toEqual(['tabs/setActiveTab', 'panes/setActivePane']) + expect(actions[0].payload).toBe('t1') + expect(actions[1].payload).toEqual({ tabId: 't1', paneId: 'p1' }) + }) + it('handles pane.rename', () => { const actions: any[] = [] const dispatch = (action: any) => { From d6831adc5f56d75adec66daf24af90f2d87b386a Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:31:18 -0700 Subject: [PATCH 12/25] docs: complete dev self-host workflow documentation --- .claude/skills/release-freshell/SKILL.md | 24 ++++++++---------- .claude/skills/reviewing-prs/SKILL.md | 31 +++++++++--------------- README.md | 2 +- docs/development/branch-model.md | 9 +++++++ 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.claude/skills/release-freshell/SKILL.md b/.claude/skills/release-freshell/SKILL.md index b6a9bda30..46a9d739d 100644 --- a/.claude/skills/release-freshell/SKILL.md +++ b/.claude/skills/release-freshell/SKILL.md @@ -94,13 +94,14 @@ Present the proposed README changes to the user for approval before proceeding t ## Release Steps -All work happens on a release branch in a worktree — main is untouched until the final atomic fast-forward. This protects the running Freshell instance. +All release preparation happens on a release branch in a worktree from `origin/main`. Local `main` is a mirror of `origin/main`; do not commit to it, fast-forward it, or push it directly. ### 1. Create the release branch ```bash -# From the main repo -git worktree add .worktrees/release-vX.Y.Z -b release/vX.Y.Z main +# From the repo root +git fetch origin +git worktree add .worktrees/release-vX.Y.Z -b release/vX.Y.Z origin/main cd .worktrees/release-vX.Y.Z npm install ``` @@ -129,19 +130,14 @@ All of these are committed to the release branch: 2. **Update README:** Change `--branch vOLD` to `--branch vNEW` in the clone command, and apply the approved Features changes 3. **Commit** with message like `release: vX.Y.Z` -### 4. Fast-forward main +### 4. Open and merge a release PR -```bash -# Back in the main repo working directory -git merge --ff-only release/vX.Y.Z -``` - -If `--ff-only` fails, go back to the worktree and rebase onto main until it can fast-forward. +Push the release branch and open a PR against `main`. After user approval and required checks, merge it through GitHub. If conflicts appear, rebase the release branch onto `origin/main`, resolve in the worktree, retest, and force-push with `--force-with-lease`. ### 5. Tag and publish ```bash -git push origin main +git fetch origin git tag -a vX.Y.Z -m "vX.Y.Z" git push --tags gh release create vX.Y.Z --title "vX.Y.Z" --notes "..." # with the release notes @@ -156,8 +152,8 @@ git branch -d release/vX.Y.Z ## Safety -- Main can contain work-in-progress; users clone a specific release tag -- You are running inside Freshell — if you break main mid-release, you kill yourself -- All release prep happens on a branch in a worktree, so main is never modified until the atomic fast-forward +- Local `main` mirrors `origin/main`; release work happens on a PR branch. +- Users clone a specific release tag. +- The self-hosted integration branch is `dev`, not local `main`. - Commit the version bump before tagging so the tag points to the right commit - If any step fails, stop and assess, then make recommendations to the user, rather than pushing forward diff --git a/.claude/skills/reviewing-prs/SKILL.md b/.claude/skills/reviewing-prs/SKILL.md index 4d7b8ad4e..3f337a85f 100644 --- a/.claude/skills/reviewing-prs/SKILL.md +++ b/.claude/skills/reviewing-prs/SKILL.md @@ -17,7 +17,7 @@ We never push work back onto contributors. Our goal is to harvest **good ideas f 1. Create a detached worktree for all PR work: ```bash - git worktree add .worktrees/pr-review --detach HEAD + git worktree add .worktrees/pr-review --detach origin/main ``` 2. List open PRs with `gh pr list` 3. Process oldest-to-newest (reduces cascading merge conflicts) @@ -28,11 +28,11 @@ We never push work back onto contributors. Our goal is to harvest **good ideas f ```bash cd .worktrees/pr-review -git fetch origin && git checkout --detach origin/main +git fetch origin && git switch --detach origin/main gh pr checkout <N> ``` -Always reset the worktree to latest main before each PR. +Always reset the worktree to latest `origin/main` before each PR. ### 2. Review the Diff @@ -85,7 +85,7 @@ Present build, test, and fresheyes results to the user. Include: Address all fresheyes findings and test failures before merging: - Commit fixes on the PR branch with detailed messages -- **Show the diff and get approval before pushing** to the PR branch or main +- **Show the diff and get approval before pushing** to the PR branch - Re-build and re-test after fixes ### 7. Merge (only after user approves) @@ -96,14 +96,9 @@ gh pr merge <N> --merge ``` **If GitHub's merge state is stale** (common after rebase/force-push): -```bash -# In the main repo (not the worktree) -git fetch origin && git merge --ff-only origin/main -git merge --no-ff <branch> -m "Merge pull request #N from ..." -git push origin main -``` +refresh the PR branch on `origin/main`, force-push it with `--force-with-lease`, and use GitHub merge after the merge state updates. Do not merge locally into `main` or push `main` directly. -**If conflicts exist:** rebase the PR branch on main, resolve, rebuild, retest, force-push, then merge. +**If conflicts exist:** rebase the PR branch on `origin/main`, resolve, rebuild, retest, force-push with `--force-with-lease`, then merge through GitHub. ### 8. Comment and Close @@ -116,7 +111,7 @@ Leave an effusive comment summarizing: gh pr comment <N> --body "..." ``` -Then ensure the PR is closed. Local merges (the fallback path) don't trigger GitHub's auto-close: +Then ensure the PR is closed: ```bash # Check if still open; close if needed @@ -128,10 +123,6 @@ gh pr view <N> --json state -q '.state' | grep -q OPEN && gh pr close <N> Once all PRs are landed and there is no unfinished work, clean up: ```bash -# Switch back to main in the primary repo -cd <primary-repo> -git checkout main && git pull --ff-only origin main - # Remove the pr-review worktree git worktree remove .worktrees/pr-review ``` @@ -192,7 +183,7 @@ The user may respond with: ##### c. Fix (if approved) -- Create a branch: `git checkout -b fix/issue-<N> origin/main` +- Create a branch in a worktree from `origin/main`. - Implement the fix in the worktree - Build and run targeted tests - **Show the diff and get approval before pushing** @@ -219,7 +210,7 @@ Always sign the comment `— Codex CLI`. | Tests | `go test ./...` (targeted) | — | | Present results | Build/test/fresheyes summary | **User approval (Gate 2)** | | Fix | Commit + show diff + get approval before push | Build + tests green | -| Merge | `gh pr merge` or local merge | All fixes landed + user says merge | +| Merge | `gh pr merge` | All fixes landed + user says merge | | Comment + Close | `gh pr comment` + `gh pr close` if still open | Merge complete | | Reset | `git checkout --detach origin/main` | Before next PR | | Teardown | `git worktree remove .worktrees/pr-review` | Queue empty, all landed | @@ -248,10 +239,10 @@ If you catch yourself thinking any of these, you are about to violate a gate: | Proceeding without user approval | There are TWO hard gates. Always wait at both. | | Treating ambiguous signals as approval | Only explicit approval words count. When in doubt, ask. | | Merging after build+fresheyes without asking | Gate 2 requires presenting results and waiting for go/no-go. | -| Pushing fixes to main without showing diff | Show the diff first. Get approval before any push. | +| Pushing fixes without showing diff | Show the diff first. Get approval before any push to the PR branch. | | Merging with fresheyes FAILED | Fix all findings first. Never merge a failed review. | | Forgetting to update worktree between PRs | Stale base causes unnecessary conflicts. Always reset to origin/main. | -| Forgetting to close after local merge | Local merges don't trigger GitHub auto-close. Always check and `gh pr close` if still open. | +| Pushing directly to `main` | `main` mirrors `origin/main`. Use PR branches and GitHub merge. | | Signing comment as "Codex" or "Claude" | Always sign as `— Codex CLI`. | | Processing PRs newest-first | Oldest-first minimizes cascading conflicts since earlier PRs often touch files later ones depend on. | | Processing issues oldest-first | Issue triage is newest-first so the most recent reports and follow-ups are evaluated first. | diff --git a/README.md b/README.md index 593830b76..c9431ea8f 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Drop a directory with a `freshell.json` manifest into `~/.freshell/extensions/` ## Contributing -Contributions are welcome! Please feel free to submit a Pull Request. +Contributions are welcome. Start from `origin/main`, submit a Pull Request against `main`, and keep behavior changes on PR branches. Local development self-hosting uses the `dev` integration branch described in [docs/development/branch-model.md](docs/development/branch-model.md). ## Community Projects diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 9e0162fdd..3c03935bf 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -58,6 +58,15 @@ npm run dev:queue -- plan --prs 321,309,319 The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. +To rebuild local `dev`: + +```bash +git switch dev +npm run dev:queue -- assemble --prs 321,309,319 +``` + +Use replacement PR numbers instead of external or superseded PRs. If the script stops on a conflict, do not resolve the conflict on `dev`. Abort the merge, fix the PR branch, and rerun the queue. + ## Local Main Realignment Only realign local `main` after Freshell is self-hosting from `dev`, the user has explicitly approved the reset, and the intentional OpenCode notification-argument removal has been preserved in an open PR that is included in `dev` or confirmed already present in a selected pending PR. From 2080ba1cde29296cffa353c4cebaae59faa8c3df Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:32:31 -0700 Subject: [PATCH 13/25] docs: record initial dev PR queue --- docs/development/branch-model.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 3c03935bf..373e735c4 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -53,7 +53,7 @@ Use an explicit queue. Do not blindly apply every open PR. Example: ```bash -npm run dev:queue -- plan --prs 321,309,319 +npm run dev:queue -- plan --prs 323,321,309,319,322 ``` The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. @@ -62,11 +62,28 @@ To rebuild local `dev`: ```bash git switch dev -npm run dev:queue -- assemble --prs 321,309,319 +npm run dev:queue -- assemble --prs 323,321,309,319,322 ``` Use replacement PR numbers instead of external or superseded PRs. If the script stops on a conflict, do not resolve the conflict on `dev`. Abort the merge, fix the PR branch, and rerun the queue. +Initial migration queue: + +| PR | Head SHA | Purpose | +| --- | --- | --- | +| #323 | Current PR head | `dev` branch workflow, launch guardrails, and queue tooling | +| #321 | `7eae9acf13d2ecf36de6ecade8354cb22b944f7b` | Sidebar reopen corner behavior | +| #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | +| #319 | `a66568daf0dfd0dd447cc217d6f4c39d1cf22398` | OpenCode native scroll behavior | +| #322 | `0a334be42553929aed033c3aa3920d0bf58a2a65` | Replacement for externally-owned factory terminal orchestration PR | + +Initial migration exclusions: + +| PR | Head SHA | Reason | +| --- | --- | --- | +| #297 | `8cad328c158a6b33d9779ce1748bfe725ecd0d1c` | Externally-owned and superseded by #322 | +| #289 | `4e4782699adadc3e006b96143f6ead6bda8b136d` | Draft approval artifact | + ## Local Main Realignment Only realign local `main` after Freshell is self-hosting from `dev`, the user has explicitly approved the reset, and the intentional OpenCode notification-argument removal has been preserved in an open PR that is included in `dev` or confirmed already present in a selected pending PR. From 48927eef6b46a2232ebe31d1e1dea38d2203eb72 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Tue, 5 May 2026 10:29:55 -0700 Subject: [PATCH 14/25] fix: update opencode contract lab note version facts --- docs/lab-notes/2026-04-20-coding-cli-session-contract.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md index 34123ed2b..afccd65a2 100644 --- a/docs/lab-notes/2026-04-20-coding-cli-session-contract.md +++ b/docs/lab-notes/2026-04-20-coding-cli-session-contract.md @@ -94,7 +94,7 @@ The implementation plan file is dated `2026-04-19` because the design work was w "opencode": { "executable": "opencode", "resolvedPath": "/home/user/.opencode/bin/opencode", - "version": "1.14.33", + "version": "1.14.39", "runCommandTemplate": "opencode run <prompt> --format json --dangerously-skip-permissions", "serveCommandTemplate": "opencode serve --hostname 127.0.0.1 --port <port>", "globalHealthPath": "/global/health", @@ -287,7 +287,7 @@ command -v opencode # /home/user/.opencode/bin/opencode opencode --version -# 1.14.33 +# 1.14.39 ``` Fresh isolated runs were probed with: @@ -312,7 +312,7 @@ curl http://127.0.0.1:<port>/session/status Observed control behavior: -- `/global/health` returned a healthy payload with version `1.14.33`. +- `/global/health` returned a healthy payload with version `1.14.39`. - `/session/status` returned `{}` while idle. - During an attached `opencode run ... --attach http://127.0.0.1:<port>`, `/session/status` returned the same authoritative `sessionID` with `{ "type": "busy" }`. From faf1f828ad2561af12dc6478d87115caf24806a0 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:36:17 -0700 Subject: [PATCH 15/25] docs: update initial dev queue for amended PR 319 --- docs/development/branch-model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 373e735c4..36c60f47a 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -74,7 +74,7 @@ Initial migration queue: | #323 | Current PR head | `dev` branch workflow, launch guardrails, and queue tooling | | #321 | `7eae9acf13d2ecf36de6ecade8354cb22b944f7b` | Sidebar reopen corner behavior | | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | -| #319 | `a66568daf0dfd0dd447cc217d6f4c39d1cf22398` | OpenCode native scroll behavior | +| #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | | #322 | `0a334be42553929aed033c3aa3920d0bf58a2a65` | Replacement for externally-owned factory terminal orchestration PR | Initial migration exclusions: From e15a86db992ab5d70048dec9c45da8e8ff0b7652 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:02:16 -0700 Subject: [PATCH 16/25] Remove Codex terminal notification args --- server/terminal-registry.ts | 6 +----- test/integration/server/codex-session-flow.test.ts | 5 +++-- test/unit/server/terminal-registry.test.ts | 9 ++++----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/server/terminal-registry.ts b/server/terminal-registry.ts index 8e0395bda..80c2c1036 100644 --- a/server/terminal-registry.ts +++ b/server/terminal-registry.ts @@ -134,11 +134,7 @@ function providerNotificationArgs( if (mode === 'codex') { return { - args: [ - '-c', 'tui.notification_method=bel', - '-c', "tui.notifications=['agent-turn-complete']", - ...mcpInjection.args, - ], + args: mcpInjection.args, env: mcpInjection.env, } } diff --git a/test/integration/server/codex-session-flow.test.ts b/test/integration/server/codex-session-flow.test.ts index e18eeb89e..ff98fd2cb 100644 --- a/test/integration/server/codex-session-flow.test.ts +++ b/test/integration/server/codex-session-flow.test.ts @@ -30,7 +30,7 @@ vi.mock('../../../server/logger', () => { child: vi.fn(), } logger.child.mockReturnValue(logger) - return { logger } + return { logger, sessionLifecycleLogger: logger } }) process.env.AUTH_TOKEN = 'test-token' @@ -386,7 +386,8 @@ describe('Codex Session Flow Integration', () => { ]) expect(recordedArgs).toContain('resume') expect(recordedArgs).toContain('thread-new-1') - expect(recordedArgs).toContain('tui.notification_method=bel') + expect(recordedArgs).not.toContain('tui.notification_method=bel') + expect(recordedArgs).not.toContain("tui.notifications=['agent-turn-complete']") expect(recordedArgs).not.toContain('--model') expect(recordedArgs).not.toContain('--sandbox') } finally { diff --git a/test/unit/server/terminal-registry.test.ts b/test/unit/server/terminal-registry.test.ts index ba2b915df..7a41f0b3d 100644 --- a/test/unit/server/terminal-registry.test.ts +++ b/test/unit/server/terminal-registry.test.ts @@ -83,9 +83,8 @@ const OTHER_CLAUDE_SESSION_ID = '6f1c2b3a-4d5e-6f70-8a9b-0c1d2e3f4a5b' const TEST_OPENCODE_SERVER = { hostname: '127.0.0.1' as const, port: 4173 } function expectCodexMcpArgs(args: string[]) { - // Bell notification still present - expect(args).toContain('-c') - expect(args).toContain('tui.notification_method=bel') + expect(args).not.toContain('tui.notification_method=bel') + expect(args).not.toContain("tui.notifications=['agent-turn-complete']") // MCP server config instead of skills.config const mcpArg = args.find(a => a.includes('mcp_servers.freshell')) expect(mcpArg).toBeDefined() @@ -1722,8 +1721,8 @@ describe('buildSpawnSpec resume validation on Windows shells', () => { expect(spec.args).toContain('-NoExit') expect(spec.args[3]).toContain("& 'C:\\Program Files\\Codex\\codex.cmd'") expect(spec.args[3]).toContain("'-c'") - expect(spec.args[3]).toContain("'tui.notification_method=bel'") - expect(spec.args[3]).toContain("'tui.notifications=[''agent-turn-complete'']'") + expect(spec.args[3]).not.toContain("'tui.notification_method=bel'") + expect(spec.args[3]).not.toContain("'tui.notifications=[''agent-turn-complete'']'") expect(spec.args[3]).toContain("'resume'") expect(spec.args[3]).toContain("'session-123'") }) From 26601cec20434790936af3a3f9cc823c8c19f984 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:04:39 -0700 Subject: [PATCH 17/25] Use sessionRef in terminal orchestration payloads --- server/agent-api/router.ts | 50 ++++++++++++++++++++++------ test/server/agent-tabs-write.test.ts | 49 ++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/server/agent-api/router.ts b/server/agent-api/router.ts index eb6633675..c116acda1 100644 --- a/server/agent-api/router.ts +++ b/server/agent-api/router.ts @@ -16,10 +16,29 @@ import { ok, approx, fail } from './response.js' import { renderCapture } from './capture.js' import { waitForMatch } from './wait-for.js' import { resolveScreenshotOutputPath } from './screenshot-path.js' +import { sanitizeSessionRef, type SessionRef } from '../../shared/session-contract.js' const truthy = (value: unknown) => value === true || value === 'true' || value === '1' || value === 'yes' const SYNCABLE_TERMINAL_MODES = new Set(['claude', 'codex', 'opencode', 'gemini', 'kimi']) +function buildSessionRef(provider: unknown, sessionId: unknown): SessionRef | undefined { + if (typeof provider !== 'string' || provider === 'shell') return undefined + if (typeof sessionId !== 'string' || !sessionId) return undefined + return sanitizeSessionRef({ provider, sessionId }) +} + +function resolveTerminalSessionRef({ + sessionRef, + mode, + resumeSessionId, +}: { + sessionRef?: unknown + mode?: unknown + resumeSessionId?: unknown +}): SessionRef | undefined { + return sanitizeSessionRef(sessionRef) ?? buildSessionRef(mode, resumeSessionId) +} + function agentRouteErrorStatus(error: unknown): number { return error instanceof CodexLaunchConfigError ? 400 : 500 } @@ -324,9 +343,10 @@ export function createAgentApiRouter({ const meta = terminalId ? terminalMetadata?.list?.().find((entry) => entry.terminalId === terminalId) : undefined - const resumeSessionId = typeof paneContent?.resumeSessionId === 'string' + const paneSessionRef = sanitizeSessionRef(paneContent?.sessionRef) + const resumeSessionId = paneSessionRef?.sessionId ?? (typeof paneContent?.resumeSessionId === 'string' ? paneContent.resumeSessionId - : undefined + : undefined) const modeCandidates = [ typeof paneContent?.mode === 'string' ? paneContent.mode : undefined, terminalId ? registry.get?.(terminalId)?.mode : undefined, @@ -344,7 +364,7 @@ export function createAgentApiRouter({ await configStore.patchTerminalOverride?.(terminalId, { titleOverride: title }) registry.updateTitle?.(terminalId, title) - const sessionProvider = typeof meta?.provider === 'string' ? meta.provider : mode + const sessionProvider = paneSessionRef?.provider ?? (typeof meta?.provider === 'string' ? meta.provider : mode) const sessionId = typeof meta?.sessionId === 'string' ? meta.sessionId : resumeSessionId if (sessionProvider && sessionId) { try { @@ -365,7 +385,7 @@ export function createAgentApiRouter({ } router.post('/tabs', async (req, res) => { - const { name, mode, shell, cwd, browser, editor, resumeSessionId, permissionMode, model, sandbox } = req.body || {} + const { name, mode, shell, cwd, browser, editor, resumeSessionId, sessionRef, permissionMode, model, sandbox } = req.body || {} const wantsBrowser = !!browser const wantsEditor = !!editor let launch: ResolvedSpawnProviderSettings | undefined @@ -381,16 +401,18 @@ export function createAgentApiRouter({ paneContent = { kind: 'editor', filePath: editor, language: null, readOnly: false, content: '', viewMode: 'source' } } else { const effectiveMode = mode || 'shell' + const requestedSessionRef = resolveTerminalSessionRef({ sessionRef, mode: effectiveMode, resumeSessionId }) + const requestedResumeSessionId = requestedSessionRef?.sessionId ?? resumeSessionId assertTerminalAdmission() launch = await resolveSpawnProviderSettings( effectiveMode, configStore, { permissionMode, model, sandbox }, - { cwd, resumeSessionId, codexLaunchPlanner, assertTerminalCreateAccepted: assertTerminalAdmission }, + { cwd, resumeSessionId: requestedResumeSessionId, codexLaunchPlanner, assertTerminalCreateAccepted: assertTerminalAdmission }, ) assertTerminalAdmission() const { tabId, paneId } = layoutStore.createTab({ title: name, browser, editor }) - const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, resumeSessionId) + const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, requestedResumeSessionId) assertTerminalAdmission() const terminal = registry.create({ mode: effectiveMode, @@ -403,10 +425,11 @@ export function createAgentApiRouter({ }) createdTerminalId = terminal.terminalId const launchResumeSessionId = launch.resumeSessionId + const launchSessionRef = requestedSessionRef ?? buildSessionRef(effectiveMode, launchResumeSessionId) assertTerminalAdmission() await adoptCodexLaunch(launch, terminal.terminalId) assertTerminalAdmission() - await waitForCodexResumeReadiness(launch, resumeSessionId) + await waitForCodexResumeReadiness(launch, requestedResumeSessionId) assertCodexCreateTerminalRunning(terminal) assertTerminalAdmission() publishCodexLaunch(registry, launch, terminal.terminalId) @@ -418,7 +441,7 @@ export function createAgentApiRouter({ status: 'running', mode: mode || 'shell', shell: shell || 'system', - resumeSessionId: launchResumeSessionId, + ...(launchSessionRef ? { sessionRef: launchSessionRef } : {}), initialCwd: cwd, } @@ -433,7 +456,7 @@ export function createAgentApiRouter({ shell, terminalId, initialCwd: cwd, - resumeSessionId: paneContent?.resumeSessionId, + ...(launchSessionRef ? { sessionRef: launchSessionRef } : {}), paneId, paneContent, }, @@ -554,13 +577,18 @@ export function createAgentApiRouter({ const title = parseRequiredName(req.body?.name) || term.title || terminalId const { tabId, paneId } = layoutStore.createTab({ title }) + const sessionRef = resolveTerminalSessionRef({ + sessionRef: term.sessionRef, + mode: term.mode, + resumeSessionId: term.resumeSessionId, + }) const paneContent = { kind: 'terminal', terminalId, status: term.status || 'running', mode: term.mode || 'shell', initialCwd: term.cwd, - resumeSessionId: term.resumeSessionId, + ...(sessionRef ? { sessionRef } : {}), } layoutStore.attachPaneContent(tabId, paneId, paneContent) broadcastReplayableUiCommand({ @@ -571,7 +599,7 @@ export function createAgentApiRouter({ mode: term.mode || 'shell', terminalId, initialCwd: term.cwd, - resumeSessionId: term.resumeSessionId, + ...(sessionRef ? { sessionRef } : {}), paneId, paneContent, status: term.status || 'running', diff --git a/test/server/agent-tabs-write.test.ts b/test/server/agent-tabs-write.test.ts index e224d15ef..89f28c3da 100644 --- a/test/server/agent-tabs-write.test.ts +++ b/test/server/agent-tabs-write.test.ts @@ -123,8 +123,8 @@ describe('tab endpoints', () => { status: 'running', mode: 'shell', initialCwd: '/workspace', - resumeSessionId: undefined, }) + expect(attachPaneContent.mock.calls[0]?.[2]).not.toHaveProperty('resumeSessionId') expect(broadcastUiCommandWithReplay).toHaveBeenCalledWith({ command: 'tab.create', payload: expect.objectContaining({ @@ -142,6 +142,53 @@ describe('tab endpoints', () => { }) }) + it('opens detached coding terminals with canonical sessionRef payloads', async () => { + const app = express() + app.use(express.json()) + const registry = new FakeRegistry() + registry.get.mockReturnValue({ + terminalId: 'term_1', + title: 'Detached Claude', + mode: 'claude', + status: 'running', + cwd: '/workspace', + resumeSessionId: '550e8400-e29b-41d4-a716-446655440000', + }) + const createTab = vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })) + const attachPaneContent = vi.fn() + const broadcastUiCommandWithReplay = vi.fn() + const layoutStore = { + createTab, + attachPaneContent, + findPaneByTerminalId: vi.fn(() => undefined), + } + app.use('/api', createAgentApiRouter({ + layoutStore, + registry, + wsHandler: { broadcastUiCommandWithReplay }, + })) + + const res = await request(app) + .post('/api/terminals/term_1/open') + .send({}) + + const sessionRef = { provider: 'claude', sessionId: '550e8400-e29b-41d4-a716-446655440000' } + expect(res.status).toBe(200) + expect(attachPaneContent).toHaveBeenCalledWith('tab_1', 'pane_1', expect.objectContaining({ + kind: 'terminal', + terminalId: 'term_1', + sessionRef, + })) + expect(attachPaneContent.mock.calls[0]?.[2]).not.toHaveProperty('resumeSessionId') + expect(broadcastUiCommandWithReplay).toHaveBeenCalledWith({ + command: 'tab.create', + payload: expect.objectContaining({ + sessionRef, + }), + }) + expect(broadcastUiCommandWithReplay.mock.calls[0]?.[0]?.payload).not.toHaveProperty('resumeSessionId') + }) + it('selects the existing pane when opening an already-attached terminal', async () => { const app = express() app.use(express.json()) From fc8a953557cd19b0534a512b9364ab2bc4f34ae9 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 16:57:40 -0700 Subject: [PATCH 18/25] Port durable session restore identity --- server/ws-handler.ts | 107 ++++++++++++++---- src/components/agent-chat/AgentChatView.tsx | 74 ++++++++---- test/e2e/agent-chat-restore-flow.test.tsx | 24 ++-- .../server/codex-session-flow.test.ts | 11 +- test/server/ws-protocol.test.ts | 20 +++- .../agent-chat/AgentChatView.reload.test.tsx | 77 +++++++++---- 6 files changed, 223 insertions(+), 90 deletions(-) diff --git a/server/ws-handler.ts b/server/ws-handler.ts index e7cee3ca8..630ce9cf2 100644 --- a/server/ws-handler.ts +++ b/server/ws-handler.ts @@ -43,6 +43,7 @@ import { ErrorCode, ShellSchema, CodingCliProviderSchema, + SessionLocatorSchema, TerminalMetaUpdatedSchema, CodexActivityListResponseSchema, CodexActivityListSchema, @@ -74,6 +75,7 @@ import { } from '../shared/ws-protocol.js' import { UiLayoutSyncSchema } from './agent-api/layout-schema.js' import type { LayoutStore } from './agent-api/layout-store.js' +import { LiveTerminalHandleSchema } from '../shared/session-contract.js' type WsHandlerConfig = { maxConnections: number @@ -466,11 +468,12 @@ export class WsHandler { }), shell: ShellSchema.default('system'), cwd: z.string().optional(), - resumeSessionId: z.string().optional(), + sessionRef: SessionLocatorSchema.optional(), + liveTerminal: LiveTerminalHandleSchema.optional(), restore: z.boolean().optional(), tabId: z.string().min(1).optional(), paneId: z.string().min(1).optional(), - }) + }).strict() const dynamicProviderSchema = CodingCliProviderSchema.superRefine((val, ctx) => { if (!canEnumerateCliExtensions || extensionModes.includes(val)) return @@ -1627,7 +1630,20 @@ export class WsHandler { } // Send terminal inventory so the client knows what's alive - const terminals = this.registry.list() + const terminals = this.registry.list().map((terminal) => { + const { resumeSessionId, ...rest } = terminal + return { + ...rest, + ...(resumeSessionId + ? { + sessionRef: { + provider: terminal.mode, + sessionId: resumeSessionId, + }, + } + : {}), + } + }) const terminalMeta = this.terminalMetaListProvider?.() ?? [] this.safeSend(ws, { type: 'terminal.inventory', @@ -1806,12 +1822,23 @@ export class WsHandler { return } case 'terminal.create': { + const requestedSessionRef = m.sessionRef?.provider === m.mode + ? m.sessionRef + : undefined + const canonicalSessionId = requestedSessionRef?.sessionId + const restoreRequested = m.restore === true + const localLiveTerminalId = ( + m.liveTerminal?.serverInstanceId === this.serverInstanceId + && typeof m.liveTerminal?.terminalId === 'string' + ) + ? m.liveTerminal.terminalId + : undefined log.debug({ requestId: m.requestId, connectionId: ws.connectionId, mode: m.mode, - resumeSessionId: m.resumeSessionId, - }, '[TRACE resumeSessionId] terminal.create received') + sessionRef: requestedSessionRef, + }, '[TRACE sessionRef] terminal.create received') recordSessionLifecycleEvent({ kind: 'terminal_create_requested', requestId: m.requestId, @@ -1820,9 +1847,9 @@ export class WsHandler { ...(m.paneId ? { paneId: m.paneId } : {}), ...(m.cwd ? { cwd: m.cwd } : {}), mode: m.mode as TerminalMode, - restoreRequested: m.restore === true, - hasRequestedSessionRef: false, - ...(m.resumeSessionId ? { requestedSessionId: m.resumeSessionId } : {}), + restoreRequested, + hasRequestedSessionRef: !!requestedSessionRef, + ...(canonicalSessionId ? { requestedSessionId: canonicalSessionId } : {}), }) const endCreateTimer = startPerfTimer( 'terminal_create', @@ -1834,10 +1861,10 @@ export class WsHandler { let reused = false let error = false let rateLimited = false - let effectiveResumeSessionId = m.resumeSessionId + let effectiveResumeSessionId = canonicalSessionId try { await this.withTerminalCreateLock( - this.terminalCreateLockKey(m.mode as TerminalMode, m.requestId, effectiveResumeSessionId), + this.terminalCreateLockKey(m.mode as TerminalMode, m.requestId, canonicalSessionId), async () => { const resolveExistingRequestTerminalId = (requestId: string): string | undefined => { const local = state.createdByRequestId.get(requestId) @@ -1855,7 +1882,6 @@ export class WsHandler { requestId: string terminalId: string createdAt: number - effectiveResumeSessionId?: string }): Promise<boolean> => { if (opts.ws.readyState !== WebSocket.OPEN) { return false @@ -1866,7 +1892,6 @@ export class WsHandler { requestId: opts.requestId, terminalId: opts.terminalId, createdAt: opts.createdAt, - ...(opts.effectiveResumeSessionId ? { effectiveResumeSessionId: opts.effectiveResumeSessionId } : {}), }) return true } @@ -1881,7 +1906,6 @@ export class WsHandler { requestId: m.requestId, terminalId: reusedTerminalId, createdAt, - effectiveResumeSessionId: resumeSessionId, }) if (!sent) { return false @@ -1923,19 +1947,27 @@ export class WsHandler { this.forgetCreatedRequestId(m.requestId) } - if (modeSupportsResume(m.mode as TerminalMode) && effectiveResumeSessionId) { + if (localLiveTerminalId) { + const liveTerminal = this.registry.get(localLiveTerminalId) + if (liveTerminal?.status === 'running' && liveTerminal.mode === m.mode) { + await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt, liveTerminal.resumeSessionId) + return + } + } + + if (modeSupportsResume(m.mode as TerminalMode) && canonicalSessionId) { let existing = this.registry.getCanonicalRunningTerminalBySession( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) if (!existing) { this.registry.repairLegacySessionOwners( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) existing = this.registry.getCanonicalRunningTerminalBySession( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) } if (existing) { @@ -1967,7 +1999,7 @@ export class WsHandler { } // Rate limit: prevent runaway terminal creation (e.g., infinite respawn loops) - if (!m.restore) { + if (!restoreRequested) { const now = Date.now() state.terminalCreateTimestamps = state.terminalCreateTimestamps.filter( (t) => now - t < this.config.terminalCreateRateWindowMs @@ -1983,19 +2015,27 @@ export class WsHandler { // Re-check session ownership after async config loading in case another request // created or repaired a matching running session while we were waiting. - if (modeSupportsResume(m.mode as TerminalMode) && effectiveResumeSessionId) { + if (localLiveTerminalId) { + const liveTerminal = this.registry.get(localLiveTerminalId) + if (liveTerminal?.status === 'running' && liveTerminal.mode === m.mode) { + await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt, liveTerminal.resumeSessionId) + return + } + } + + if (modeSupportsResume(m.mode as TerminalMode) && canonicalSessionId) { let existing = this.registry.getCanonicalRunningTerminalBySession( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) if (!existing) { this.registry.repairLegacySessionOwners( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) existing = this.registry.getCanonicalRunningTerminalBySession( m.mode as TerminalMode, - effectiveResumeSessionId, + canonicalSessionId, ) } if (existing) { @@ -2035,6 +2075,24 @@ export class WsHandler { } } + if (m.mode === 'opencode' && restoreRequested && !canonicalSessionId) { + this.sendError(ws, { + code: 'RESTORE_UNAVAILABLE', + message: 'OpenCode restore requires a canonical durable session id', + requestId: m.requestId, + }) + return + } + + if (m.mode === 'claude' && restoreRequested && !isValidClaudeSessionId(effectiveResumeSessionId)) { + this.sendError(ws, { + code: 'RESTORE_UNAVAILABLE', + message: 'Claude restore requires a canonical durable session id', + requestId: m.requestId, + }) + return + } + // After async repair wait, check if the client disconnected if (ws.readyState !== WebSocket.OPEN) { log.debug({ connectionId: ws.connectionId, requestId: m.requestId }, @@ -2048,9 +2106,9 @@ export class WsHandler { log.debug({ requestId: m.requestId, connectionId: ws.connectionId, - originalResumeSessionId: m.resumeSessionId, + sessionRef: requestedSessionRef, effectiveResumeSessionId, - }, '[TRACE resumeSessionId] about to create terminal') + }, '[TRACE sessionRef] about to create terminal') const requestedCodexResumeSessionId = m.mode === 'codex' ? effectiveResumeSessionId @@ -2162,7 +2220,6 @@ export class WsHandler { requestId: m.requestId, terminalId: record.terminalId, createdAt: record.createdAt, - effectiveResumeSessionId, }) if (!sent) { // Terminal may still exist even if created delivery failed (for diff --git a/src/components/agent-chat/AgentChatView.tsx b/src/components/agent-chat/AgentChatView.tsx index 5112fea10..4e7ccc6ff 100644 --- a/src/components/agent-chat/AgentChatView.tsx +++ b/src/components/agent-chat/AgentChatView.tsx @@ -52,10 +52,10 @@ import { buildAgentChatPersistedIdentityUpdate, flushPersistedLayoutNow, getCanonicalDurableSessionId, - getPreferredResumeSessionId, } from '@/store/persistControl' import { useMobile } from '@/hooks/useMobile' import { useKeyboardInset } from '@/hooks/useKeyboardInset' +import { buildRestoreError } from '@shared/session-contract' /** Early lifecycle states that should not be re-entered once the session has advanced. */ const EARLY_STATES = new Set(['creating', 'starting']) @@ -136,6 +136,7 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag const currentTab = useAppSelector((s) => ( (s as { tabs?: { tabs?: Tab[] } }).tabs?.tabs?.find((entry) => entry.id === tabId) )) + const tabHasSinglePane = useAppSelector((s) => s.panes.layouts[tabId]?.type === 'leaf') const tabTitleSetByUser = currentTab?.titleSetByUser ?? false const providerCapabilitiesState = useAppSelector( (s) => s.agentChat.capabilitiesByProvider?.[paneContent.provider], @@ -177,18 +178,15 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag const surfaceVisibleMarkedRef = useRef(false) const sessionRef = useRef(session) sessionRef.current = session - const persistedTimelineSessionId = isValidClaudeSessionId(paneContent.resumeSessionId) - ? paneContent.resumeSessionId - : undefined + const persistedTimelineSessionId = ( + paneContent.sessionRef?.provider === 'claude' + && isValidClaudeSessionId(paneContent.sessionRef.sessionId) + ) + ? paneContent.sessionRef.sessionId + : (isValidClaudeSessionId(paneContent.resumeSessionId) ? paneContent.resumeSessionId : undefined) const canonicalDurableSessionId = getCanonicalDurableSessionId(session) ?? persistedTimelineSessionId - const timelineSessionId = getPreferredResumeSessionId(session) ?? persistedTimelineSessionId - const restoreHistoryQueryId = timelineSessionId ?? paneContent.sessionId - const attachResumeSessionId = getPreferredResumeSessionId(session) - ?? ( - typeof paneContent.resumeSessionId === 'string' && paneContent.resumeSessionId.trim().length > 0 - ? paneContent.resumeSessionId - : undefined - ) + const restoreHistoryQueryId = canonicalDurableSessionId ?? paneContent.sessionId + const attachResumeSessionId = canonicalDurableSessionId const attachPayload = useMemo(() => { if (!paneContent.sessionId) return null return { @@ -236,21 +234,47 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag ) const isRestoring = !!paneContent.sessionId && !session?.historyLoaded && !hasRestoreFailure - // Shared recovery logic: clears stale sessionId and resets to 'creating' so a new - // SDK session is spawned. Preserves resumeSessionId for CLI session continuity. + // Shared recovery logic: clears stale SDK sessionId and recreates through the + // canonical durable identity when one is available. const triggerRecovery = useCallback(() => { + const durableResumeSessionId = getCanonicalDurableSessionId(sessionRef.current) + ?? ( + paneContentRef.current.sessionRef?.provider === 'claude' + && isValidClaudeSessionId(paneContentRef.current.sessionRef.sessionId) + ? paneContentRef.current.sessionRef.sessionId + : (isValidClaudeSessionId(paneContentRef.current.resumeSessionId) ? paneContentRef.current.resumeSessionId : undefined) + ) + if (!durableResumeSessionId) { + dispatch(updatePaneContent({ + tabId, + paneId, + content: { + ...paneContentRef.current, + sessionId: undefined, + status: 'idle' as const, + restoreError: buildRestoreError('dead_live_handle'), + }, + })) + createSentRef.current = false + attachSentRef.current = false + return + } + const newRequestId = nanoid() - const resumeSessionId = getPreferredResumeSessionId(sessionRef.current) - ?? paneContentRef.current.resumeSessionId dispatch(updatePaneContent({ tabId, paneId, content: { ...paneContentRef.current, sessionId: undefined, - resumeSessionId, + sessionRef: { + provider: 'claude', + sessionId: durableResumeSessionId, + }, + resumeSessionId: undefined, createRequestId: newRequestId, status: 'creating' as const, + restoreError: undefined, }, })) createSentRef.current = false @@ -410,7 +434,7 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag const identityUpdate = buildAgentChatPersistedIdentityUpdate({ session, paneContent: paneContentRef.current, - currentTab, + currentTab: tabHasSinglePane ? currentTab : undefined, metadataProvider, }) if (!identityUpdate) return @@ -433,7 +457,7 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag if (identityUpdate.shouldFlush) { dispatch(flushPersistedLayoutNow()) } - }, [currentTab, dispatch, paneId, providerConfig?.codingCliProvider, session, tabId]) + }, [currentTab, dispatch, paneId, providerConfig?.codingCliProvider, session, tabHasSinglePane, tabId]) // Tag this Claude Code session as belonging to this agent-chat provider. // Fires once when cliSessionId first becomes available (including resumes). @@ -441,21 +465,20 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag const taggedSessionRef = useRef<string | null>(null) useEffect(() => { if (suppressNetworkEffects) return - const preferredResumeSessionId = getPreferredResumeSessionId(session) - if (!preferredResumeSessionId) return - if (taggedSessionRef.current === preferredResumeSessionId) return - taggedSessionRef.current = preferredResumeSessionId + if (!canonicalDurableSessionId) return + if (taggedSessionRef.current === canonicalDurableSessionId) return + taggedSessionRef.current = canonicalDurableSessionId if (providerConfig?.codingCliProvider) { setSessionMetadata( providerConfig.codingCliProvider, - preferredResumeSessionId, + canonicalDurableSessionId, paneContent.provider, ).catch((err) => { console.warn('Failed to tag session metadata:', err) }) } - }, [paneContent.provider, providerConfig?.codingCliProvider, session?.cliSessionId, session?.timelineSessionId, suppressNetworkEffects]) + }, [canonicalDurableSessionId, paneContent.provider, providerConfig?.codingCliProvider, suppressNetworkEffects]) // Reset createSentRef when createRequestId changes const prevCreateRequestIdRef = useRef(paneContent.createRequestId) @@ -469,6 +492,7 @@ export default function AgentChatView({ tabId, paneId, paneContent, hidden }: Ag if (suppressNetworkEffects) return if (paneContent.sessionId || createSentRef.current) return if (paneContent.status !== 'creating') return + if (paneContent.restoreError) return const requestId = paneContent.createRequestId createSentRef.current = true diff --git a/test/e2e/agent-chat-restore-flow.test.tsx b/test/e2e/agent-chat-restore-flow.test.tsx index 18fb21ea8..3be1c1ba6 100644 --- a/test/e2e/agent-chat-restore-flow.test.tsx +++ b/test/e2e/agent-chat-restore-flow.test.tsx @@ -136,6 +136,7 @@ describe('agent chat restore flow', () => { }) it('restores a reloaded pane from sdk.session.snapshot, persists the durable id into pane and tab state, and shows partial output without a blank running gap', async () => { + const canonicalSessionId = '00000000-0000-4000-8000-000000000211' const store = makeStore({ resumeSessionId: 'named-resume', sessionMetadataByKey: { @@ -160,11 +161,11 @@ describe('agent chat restore flow', () => { })) getAgentTimelinePage.mockResolvedValue({ - sessionId: 'cli-session-1', + sessionId: canonicalSessionId, items: [ { turnId: 'turn-2', - sessionId: 'cli-session-1', + sessionId: canonicalSessionId, role: 'assistant', summary: 'Recent summary', timestamp: '2026-03-10T10:01:00.000Z', @@ -174,7 +175,7 @@ describe('agent chat restore flow', () => { revision: 2, bodies: { 'turn-2': { - sessionId: 'cli-session-1', + sessionId: canonicalSessionId, turnId: 'turn-2', message: { role: 'assistant', @@ -197,7 +198,7 @@ describe('agent chat restore flow', () => { sessionId: 'sdk-sess-1', latestTurnId: 'turn-2', status: 'running', - timelineSessionId: 'cli-session-1', + timelineSessionId: canonicalSessionId, revision: 2, streamingActive: true, streamingText: 'partial reply', @@ -209,7 +210,7 @@ describe('agent chat restore flow', () => { await waitFor(() => { expect(getAgentTimelinePage).toHaveBeenCalledWith( - 'cli-session-1', + canonicalSessionId, expect.objectContaining({ priority: 'visible', includeBodies: true }), expect.anything(), ) @@ -222,11 +223,18 @@ describe('agent chat restore flow', () => { await waitFor(() => { const root = store.getState().panes.layouts.t1 const leaf = root && findLeaf(root, 'p1') - expect(leaf?.content.kind === 'agent-chat' ? leaf.content.resumeSessionId : undefined).toBe('cli-session-1') + expect(leaf?.content.kind === 'agent-chat' ? leaf.content.sessionRef : undefined).toEqual({ + provider: 'claude', + sessionId: canonicalSessionId, + }) const tab = store.getState().tabs.tabs.find((entry) => entry.id === 't1') - expect(tab?.resumeSessionId).toBe('cli-session-1') - expect(tab?.sessionMetadataByKey?.['claude:cli-session-1']).toEqual(expect.objectContaining({ + expect(tab?.resumeSessionId).toBeUndefined() + expect(tab?.sessionRef).toEqual({ + provider: 'claude', + sessionId: canonicalSessionId, + }) + expect(tab?.sessionMetadataByKey?.[`claude:${canonicalSessionId}`]).toEqual(expect.objectContaining({ sessionType: 'freshclaude', firstUserMessage: 'Continue from the old tab', })) diff --git a/test/integration/server/codex-session-flow.test.ts b/test/integration/server/codex-session-flow.test.ts index e18eeb89e..b4df4837c 100644 --- a/test/integration/server/codex-session-flow.test.ts +++ b/test/integration/server/codex-session-flow.test.ts @@ -30,7 +30,7 @@ vi.mock('../../../server/logger', () => { child: vi.fn(), } logger.child.mockReturnValue(logger) - return { logger } + return { logger, sessionLifecycleLogger: logger } }) process.env.AUTH_TOKEN = 'test-token' @@ -373,7 +373,7 @@ describe('Codex Session Flow Integration', () => { throw new Error(`terminal.create failed: ${created.message}`) } - expect(created.effectiveResumeSessionId).toBe('thread-new-1') + expect(created).not.toHaveProperty('effectiveResumeSessionId') const record = registry.get(created.terminalId) expect(record?.resumeSessionId).toBe('thread-new-1') @@ -415,7 +415,10 @@ describe('Codex Session Flow Integration', () => { requestId: 'test-req-codex-restore', mode: 'codex', cwd: tempDir, - resumeSessionId: 'thread-existing-1', + sessionRef: { + provider: 'codex', + sessionId: 'thread-existing-1', + }, })) const created = await waitForMessage( @@ -429,7 +432,7 @@ describe('Codex Session Flow Integration', () => { throw new Error(`terminal.create failed: ${created.message}`) } - expect(created.effectiveResumeSessionId).toBe('thread-existing-1') + expect(created).not.toHaveProperty('effectiveResumeSessionId') const record = registry.get(created.terminalId) expect(record?.resumeSessionId).toBe('thread-existing-1') diff --git a/test/server/ws-protocol.test.ts b/test/server/ws-protocol.test.ts index 7af2b1e1a..11e2c5a0f 100644 --- a/test/server/ws-protocol.test.ts +++ b/test/server/ws-protocol.test.ts @@ -819,7 +819,10 @@ describe('ws protocol', () => { type: 'terminal.create', requestId: 'shutdown-during-readiness', mode: 'codex', - resumeSessionId: 'thread-during-readiness', + sessionRef: { + provider: 'codex', + sessionId: 'thread-during-readiness', + }, })), ) await vi.waitFor(() => expect(sidecar.waitForLoadedThreadCalls).toHaveLength(1)) @@ -848,7 +851,10 @@ describe('ws protocol', () => { type: 'terminal.create', requestId, mode: 'codex', - resumeSessionId: 'thread-resume-1', + sessionRef: { + provider: 'codex', + sessionId: 'thread-resume-1', + }, })) const created = await waitForMessage( ws, @@ -880,7 +886,10 @@ describe('ws protocol', () => { type: 'terminal.create', requestId, mode: 'codex', - resumeSessionId: 'thread-missing', + sessionRef: { + provider: 'codex', + sessionId: 'thread-missing', + }, })) const error = await waitForMessage( ws, @@ -916,7 +925,10 @@ describe('ws protocol', () => { type: 'terminal.create', requestId, mode: 'codex', - resumeSessionId: 'thread-resume-exits', + sessionRef: { + provider: 'codex', + sessionId: 'thread-resume-exits', + }, })) const error = await waitForMessage( ws, diff --git a/test/unit/client/components/agent-chat/AgentChatView.reload.test.tsx b/test/unit/client/components/agent-chat/AgentChatView.reload.test.tsx index 7d7176938..6e7c06945 100644 --- a/test/unit/client/components/agent-chat/AgentChatView.reload.test.tsx +++ b/test/unit/client/components/agent-chat/AgentChatView.reload.test.tsx @@ -30,6 +30,10 @@ beforeAll(() => { Element.prototype.scrollIntoView = vi.fn() }) +const DURABLE_SESSION_ID = '00000000-0000-4000-8000-000000000201' +const DURABLE_SESSION_ID_ALT = '00000000-0000-4000-8000-000000000202' +const DURABLE_SHELL_SESSION_ID = '00000000-0000-4000-8000-000000000203' + const wsSend = vi.fn() const getAgentTimelinePage = vi.fn() const getAgentTurnBody = vi.fn() @@ -142,7 +146,10 @@ const RELOAD_PANE: AgentChatPaneContent = { const RELOAD_PANE_WITH_CANONICAL_RESUME: AgentChatPaneContent = { ...RELOAD_PANE, - resumeSessionId: '00000000-0000-4000-8000-000000000321', + sessionRef: { + provider: 'claude', + sessionId: '00000000-0000-4000-8000-000000000321', + }, } const RELOAD_PANE_WITH_NAMED_RESUME: AgentChatPaneContent = { @@ -201,7 +208,7 @@ describe('AgentChatView reload/restore behavior', () => { }) }) - it('includes the named resumeSessionId when attaching a persisted pane before the canonical durable id exists', () => { + it('does not attach through a named legacy resumeSessionId before the canonical durable id exists', () => { const store = makeStore() render( <Provider store={store}> @@ -216,7 +223,6 @@ describe('AgentChatView reload/restore behavior', () => { expect(wsSend).toHaveBeenCalledWith({ type: 'sdk.attach', sessionId: 'sess-reload-1', - resumeSessionId: 'named-resume-token', }) }) @@ -840,7 +846,7 @@ describe('AgentChatView reload/restore behavior', () => { sessionId: 'sess-reload-1', latestTurnId: 'turn-2', status: 'idle', - timelineSessionId: 'cli-sess-1', + timelineSessionId: DURABLE_SESSION_ID, revision: 12, })) @@ -856,7 +862,7 @@ describe('AgentChatView reload/restore behavior', () => { expect(attachCalls[1]?.[0]).toEqual({ type: 'sdk.attach', sessionId: 'sess-reload-1', - resumeSessionId: 'cli-sess-1', + resumeSessionId: DURABLE_SESSION_ID, }) }) }) @@ -1076,15 +1082,15 @@ describe('AgentChatView reload/restore behavior', () => { }) }) - it('uses timelineSessionId from sdk.session.snapshot for visible restore hydration', async () => { - getAgentTimelinePage.mockResolvedValue({ sessionId: 'cli-sess-1', items: [], nextCursor: null, revision: 1 }) + it('uses canonical timelineSessionId from sdk.session.snapshot for visible restore hydration', async () => { + getAgentTimelinePage.mockResolvedValue({ sessionId: DURABLE_SESSION_ID, items: [], nextCursor: null, revision: 1 }) const store = makeStore() store.dispatch(sessionSnapshotReceived({ sessionId: 'sess-reload-1', latestTurnId: 'turn-2', status: 'idle', - timelineSessionId: 'cli-sess-1', + timelineSessionId: DURABLE_SESSION_ID, revision: 2, })) @@ -1096,7 +1102,7 @@ describe('AgentChatView reload/restore behavior', () => { await waitFor(() => { expect(getAgentTimelinePage).toHaveBeenCalledWith( - 'cli-sess-1', + DURABLE_SESSION_ID, expect.objectContaining({ includeBodies: true, revision: 2 }), expect.anything(), ) @@ -1164,15 +1170,22 @@ describe('AgentChatView reload/restore behavior', () => { sessionId: 'sdk-sess-1', latestTurnId: 'turn-2', status: 'idle', - timelineSessionId: 'cli-session-abc-123', + timelineSessionId: DURABLE_SESSION_ID_ALT, revision: 2, })) }) - expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't1', 'p1')?.resumeSessionId).toBe('cli-session-abc-123') + expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't1', 'p1')?.sessionRef).toEqual({ + provider: 'claude', + sessionId: DURABLE_SESSION_ID_ALT, + }) const tab = store.getState().tabs.tabs.find((entry) => entry.id === 't1') - expect(tab?.resumeSessionId).toBe('cli-session-abc-123') - expect(tab?.sessionMetadataByKey?.['claude:cli-session-abc-123']).toEqual(expect.objectContaining({ + expect(tab?.resumeSessionId).toBeUndefined() + expect(tab?.sessionRef).toEqual({ + provider: 'claude', + sessionId: DURABLE_SESSION_ID_ALT, + }) + expect(tab?.sessionMetadataByKey?.[`claude:${DURABLE_SESSION_ID_ALT}`]).toEqual(expect.objectContaining({ sessionType: 'freshclaude', firstUserMessage: 'Continue from the old tab', })) @@ -1212,16 +1225,23 @@ describe('AgentChatView reload/restore behavior', () => { sessionId: 'sdk-shell-1', latestTurnId: 'turn-2', status: 'idle', - timelineSessionId: 'cli-shell-abc-123', + timelineSessionId: DURABLE_SHELL_SESSION_ID, revision: 2, })) }) - expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't-shell', 'p1')?.resumeSessionId).toBe('cli-shell-abc-123') + expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't-shell', 'p1')?.sessionRef).toEqual({ + provider: 'claude', + sessionId: DURABLE_SHELL_SESSION_ID, + }) const tab = store.getState().tabs.tabs.find((entry) => entry.id === 't-shell') - expect(tab?.resumeSessionId).toBe('cli-shell-abc-123') + expect(tab?.resumeSessionId).toBeUndefined() + expect(tab?.sessionRef).toEqual({ + provider: 'claude', + sessionId: DURABLE_SHELL_SESSION_ID, + }) expect(tab?.codingCliProvider).toBe('claude') - expect(tab?.sessionMetadataByKey?.['claude:cli-shell-abc-123']).toEqual(expect.objectContaining({ + expect(tab?.sessionMetadataByKey?.[`claude:${DURABLE_SHELL_SESSION_ID}`]).toEqual(expect.objectContaining({ sessionType: 'freshclaude', firstUserMessage: 'Continue from shell fallback', })) @@ -1362,14 +1382,13 @@ describe('AgentChatView reload/restore behavior', () => { sessionId: 'sdk-meta-upgrade-1', latestTurnId: 'turn-2', status: 'idle', - timelineSessionId: 'named-resume', revision: 1, })) }) await waitFor(() => { expect(getAgentTimelinePage).toHaveBeenCalledWith( - 'named-resume', + 'sdk-meta-upgrade-1', expect.objectContaining({ includeBodies: true, revision: 1 }), expect.anything(), ) @@ -1440,9 +1459,16 @@ describe('AgentChatView reload/restore behavior', () => { expect(screen.queryByText('Live-only full body')).not.toBeInTheDocument() expect(screen.getAllByText('Post-watermark live delta')).toHaveLength(1) - expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't-meta', 'p1')?.resumeSessionId).toBe(canonicalSessionId) + expect(getPaneContent(store as unknown as ReturnType<typeof makeStore>, 't-meta', 'p1')?.sessionRef).toEqual({ + provider: 'claude', + sessionId: canonicalSessionId, + }) const tab = store.getState().tabs.tabs.find((entry) => entry.id === 't-meta') - expect(tab?.resumeSessionId).toBe(canonicalSessionId) + expect(tab?.resumeSessionId).toBeUndefined() + expect(tab?.sessionRef).toEqual({ + provider: 'claude', + sessionId: canonicalSessionId, + }) expect(tab?.sessionMetadataByKey?.['claude:00000000-0000-4000-8000-000000000321']).toEqual(expect.objectContaining({ sessionType: 'freshclaude', firstUserMessage: 'Continue from metadata upgrade', @@ -1893,14 +1919,17 @@ describe('AgentChatView server-restart recovery', () => { store.dispatch(sessionCreated({ requestId: 'req-1', sessionId: 'sdk-sess-1' })) store.dispatch(sessionInit({ sessionId: 'sdk-sess-1', - cliSessionId: 'cli-session-abc-123', + cliSessionId: DURABLE_SESSION_ID_ALT, model: 'claude-opus-4-6', })) }) - // Pane content should now have resumeSessionId persisted + // Pane content should now have canonical sessionRef persisted const content = getPaneContent(store, 't1', 'p1') - expect(content?.resumeSessionId).toBe('cli-session-abc-123') + expect(content?.sessionRef).toEqual({ + provider: 'claude', + sessionId: DURABLE_SESSION_ID_ALT, + }) }) it('does not reset the pane or send sdk.create when restore remains pending past the legacy timeout window', () => { From b34c7ae9ac91c4773927fd768bdaa3de6696622d Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:37:21 -0700 Subject: [PATCH 19/25] fix: restore codex sidecar resilience --- server/agent-api/router.ts | 450 ++-- server/coding-cli/codex-app-server/client.ts | 121 +- .../durable-rollout-tracker.ts | 250 +++ .../codex-app-server/launch-planner.ts | 296 +-- .../coding-cli/codex-app-server/protocol.ts | 5 - server/coding-cli/codex-app-server/runtime.ts | 548 +++-- server/coding-cli/codex-app-server/sidecar.ts | 218 ++ server/index.ts | 10 +- server/terminal-registry.ts | 1813 +++++++++++------ server/ws-handler.ts | 539 ++--- .../codex-app-server/fake-app-server.mjs | 289 ++- .../coding-cli/fake-codex-launch-planner.ts | 73 +- .../server/codex-session-flow.test.ts | 1136 ++++++++--- test/server/agent-run.test.ts | 168 +- test/server/agent-tabs-write.test.ts | 292 --- test/server/session-association.test.ts | 82 +- test/server/ws-protocol.test.ts | 633 ++---- .../codex-app-server/client.test.ts | 247 ++- .../durable-rollout-tracker.test.ts | 154 ++ .../codex-app-server/launch-planner.test.ts | 388 +--- .../codex-app-server/runtime.test.ts | 1301 +++--------- .../codex-app-server/sidecar.test.ts | 423 ++++ .../terminal-registry.codex-recovery.test.ts | 2 +- .../terminal-registry.codex-sidecar.test.ts | 1065 ---------- test/unit/server/terminal-registry.test.ts | 2 +- 25 files changed, 4852 insertions(+), 5653 deletions(-) create mode 100644 server/coding-cli/codex-app-server/durable-rollout-tracker.ts create mode 100644 server/coding-cli/codex-app-server/sidecar.ts create mode 100644 test/unit/server/coding-cli/codex-app-server/durable-rollout-tracker.test.ts create mode 100644 test/unit/server/coding-cli/codex-app-server/sidecar.test.ts delete mode 100644 test/unit/server/terminal-registry.codex-sidecar.test.ts diff --git a/server/agent-api/router.ts b/server/agent-api/router.ts index b2fb02db4..f9246b8a4 100644 --- a/server/agent-api/router.ts +++ b/server/agent-api/router.ts @@ -3,15 +3,21 @@ import fs from 'node:fs/promises' import { randomUUID } from 'node:crypto' import { nanoid } from 'nanoid' import { allocateLocalhostPort } from '../local-port.js' -import type { CodexLaunchPlan, CodexLaunchPlanner } from '../coding-cli/codex-app-server/launch-planner.js' +import { + runCodexLaunchWithRetry, + type CodexLaunchFactory, + type CodexLaunchPlan, + type CodexLaunchPlanner, +} from '../coding-cli/codex-app-server/launch-planner.js' import { CodexLaunchConfigError, getCodexSessionBindingReason, normalizeCodexSandboxSetting, } from '../coding-cli/codex-launch-config.js' import { makeSessionKey } from '../coding-cli/types.js' -import { terminalIdFromCreateError, type ProviderSettings } from '../terminal-registry.js' +import { buildFreshellTerminalEnv, type ProviderSettings, type TerminalEnvContext } from '../terminal-registry.js' import { MAX_TERMINAL_TITLE_OVERRIDE_LENGTH } from '../terminals-router.js' +import { sanitizeSessionRef, type SessionRef } from '../../shared/session-contract.js' import { ok, approx, fail } from './response.js' import { renderCapture } from './capture.js' import { waitForMatch } from './wait-for.js' @@ -24,14 +30,9 @@ function agentRouteErrorStatus(error: unknown): number { return error instanceof CodexLaunchConfigError ? 400 : 500 } -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} - -function combineWithCleanupError(primary: unknown, cleanupError: unknown): Error { - const primaryMessage = errorMessage(primary) - const cleanupMessage = errorMessage(cleanupError) - return new Error(`${primaryMessage}; cleanup failed: ${cleanupMessage}`) +async function shutdownUnownedCodexSidecar(sidecar: CodexLaunchPlan['sidecar'] | undefined): Promise<void> { + if (!sidecar) return + await sidecar.shutdown().catch(() => undefined) } /** @@ -60,53 +61,78 @@ async function resolveSpawnProviderSettings( overrides: { permissionMode?: string; model?: string; sandbox?: string }, opts: { cwd?: string - resumeSessionId?: string + terminalId?: string + envContext?: TerminalEnvContext + sessionRef?: SessionRef codexLaunchPlanner?: CodexLaunchPlanner - assertTerminalCreateAccepted?: () => void } = {}, -): Promise<{ resumeSessionId?: string; providerSettings?: ProviderSettings; codexPlan?: CodexLaunchPlan }> { +): Promise<{ + sessionRef?: SessionRef + providerSettings?: ProviderSettings + codexLaunchBaseProviderSettings?: { + model?: string + sandbox?: string + permissionMode?: string + } + codexSidecar?: CodexLaunchPlan['sidecar'] + codexLaunchFactory?: CodexLaunchFactory +}> { const providerSettings = await resolveProviderSettings(mode, configStore, overrides) + const sessionRef = opts.sessionRef?.provider === mode ? opts.sessionRef : undefined if (mode === 'codex') { if (!opts.codexLaunchPlanner) { - throw new Error('Codex terminal launch requires the app-server launch planner.') + throw new Error('Codex terminal launch requires the shared app-server planner.') + } + if (!opts.terminalId) { + throw new Error('Codex terminal launch requires a preallocated terminal id.') } - opts.assertTerminalCreateAccepted?.() - const plan = await opts.codexLaunchPlanner.planCreate({ - cwd: opts.cwd, - resumeSessionId: opts.resumeSessionId, - model: providerSettings?.model, - sandbox: normalizeCodexSandboxSetting(providerSettings?.sandbox), - approvalPolicy: providerSettings?.permissionMode, + const terminalId = opts.terminalId + const codexLaunchFactory: CodexLaunchFactory = async (input) => opts.codexLaunchPlanner!.planCreate({ + cwd: input.cwd, + terminalId: input.terminalId, + env: buildFreshellTerminalEnv(input.terminalId, input.envContext), + resumeSessionId: input.resumeSessionId, + model: input.providerSettings?.model ?? providerSettings?.model, + sandbox: normalizeCodexSandboxSetting(input.providerSettings?.sandbox ?? providerSettings?.sandbox), + approvalPolicy: input.providerSettings?.permissionMode ?? providerSettings?.permissionMode, }) + const plan = await runCodexLaunchWithRetry( + () => opts.codexLaunchPlanner!.planCreate({ + cwd: opts.cwd, + terminalId, + env: buildFreshellTerminalEnv(terminalId, opts.envContext), + resumeSessionId: sessionRef?.sessionId, + model: providerSettings?.model, + sandbox: normalizeCodexSandboxSetting(providerSettings?.sandbox), + approvalPolicy: providerSettings?.permissionMode, + }), + { + shouldRetry: (error) => !(error instanceof CodexLaunchConfigError), + }, + ) return { - resumeSessionId: plan.sessionId, - providerSettings: { - codexAppServer: { - ...plan.remote, - sidecar: plan.sidecar, - deferLifecycleUntilPublished: true, - recovery: { - planCreate: (input) => opts.codexLaunchPlanner!.planCreate({ - cwd: input.cwd ?? opts.cwd, - resumeSessionId: input.resumeSessionId, - model: providerSettings?.model, - sandbox: normalizeCodexSandboxSetting(providerSettings?.sandbox), - approvalPolicy: providerSettings?.permissionMode, - }), - }, + ...(plan.sessionId ? { + sessionRef: { + provider: mode, + sessionId: plan.sessionId, }, + } : {}), + codexSidecar: plan.sidecar, + codexLaunchFactory, + codexLaunchBaseProviderSettings: providerSettings, + providerSettings: { + codexAppServer: plan.remote, }, - codexPlan: plan, } } if (mode !== 'opencode') { return { - resumeSessionId: opts.resumeSessionId, + ...(sessionRef ? { sessionRef } : {}), providerSettings, } } return { - resumeSessionId: opts.resumeSessionId, + ...(sessionRef ? { sessionRef } : {}), providerSettings: { ...(providerSettings ?? {}), opencodeServer: await allocateLocalhostPort(), @@ -114,64 +140,10 @@ async function resolveSpawnProviderSettings( } } -type ResolvedSpawnProviderSettings = Awaited<ReturnType<typeof resolveSpawnProviderSettings>> - -async function adoptCodexLaunch( - launch: ResolvedSpawnProviderSettings | undefined, - terminalId: string, -): Promise<void> { - await launch?.codexPlan?.sidecar.adopt({ terminalId, generation: 0 }) -} - -async function cleanupUnadoptedCodexLaunch(launch: ResolvedSpawnProviderSettings | undefined): Promise<void> { - await launch?.codexPlan?.sidecar.shutdown() -} - -async function waitForCodexResumeReadiness( - launch: ResolvedSpawnProviderSettings | undefined, - requestedResumeSessionId: string | undefined, -): Promise<void> { - if (!launch?.codexPlan || !requestedResumeSessionId) return - await launch.codexPlan.sidecar.waitForLoadedThread(requestedResumeSessionId) -} - -function publishCodexLaunch(registry: any, launch: ResolvedSpawnProviderSettings | undefined, terminalId: string): void { - if (!launch?.codexPlan) return - registry.publishCodexSidecar?.(terminalId) -} - -function assertCodexCreateTerminalRunning(terminal: { status?: unknown }): void { - if (terminal.status === 'exited') { - throw new Error('Codex terminal PTY exited before create completed.') - } -} - -async function cleanupCreatedTerminal(registry: any, terminalId: string | undefined): Promise<void> { - if (!terminalId) return - if (typeof registry?.killAndWait === 'function') { - await registry.killAndWait(terminalId) - return - } - if (typeof registry?.kill === 'function') { - registry.kill(terminalId) - } -} - -async function cleanupFailedCodexCreate( - registry: any, - terminalId: string | undefined, - launch: ResolvedSpawnProviderSettings | undefined, -): Promise<void> { - const cleanupErrors: string[] = [] - await cleanupCreatedTerminal(registry, terminalId).catch((error) => { - cleanupErrors.push(`created terminal cleanup failed: ${errorMessage(error)}`) - }) - await cleanupUnadoptedCodexLaunch(launch).catch((error) => { - cleanupErrors.push(`Codex sidecar cleanup failed: ${errorMessage(error)}`) - }) - if (cleanupErrors.length > 0) { - throw new Error(cleanupErrors.join('; ')) - } +function resolveRequestedSessionRef(mode: string, value: unknown): SessionRef | undefined { + const sessionRef = sanitizeSessionRef(value) + if (!sessionRef) return undefined + return sessionRef.provider === mode ? sessionRef : undefined } type ResizeLayoutStore = { @@ -238,9 +210,7 @@ export function createAgentApiRouter({ assertTerminalCreateAccepted?: () => void }) { const router = Router() - const assertTerminalAdmission = () => { - assertTerminalCreateAccepted?.() - } + const assertTerminalAdmission = assertTerminalCreateAccepted ?? (() => undefined) const resolvePaneTarget = (raw: string) => { if (layoutStore.resolveTarget) { @@ -316,9 +286,9 @@ export function createAgentApiRouter({ const meta = terminalId ? terminalMetadata?.list?.().find((entry) => entry.terminalId === terminalId) : undefined - const resumeSessionId = typeof paneContent?.resumeSessionId === 'string' - ? paneContent.resumeSessionId - : undefined + const paneSessionRef = sanitizeSessionRef(paneContent?.sessionRef) + const resumeSessionId = paneSessionRef?.sessionId + ?? (typeof paneContent?.resumeSessionId === 'string' ? paneContent.resumeSessionId : undefined) const modeCandidates = [ typeof paneContent?.mode === 'string' ? paneContent.mode : undefined, terminalId ? registry.get?.(terminalId)?.mode : undefined, @@ -357,11 +327,11 @@ export function createAgentApiRouter({ } router.post('/tabs', async (req, res) => { - const { name, mode, shell, cwd, browser, editor, resumeSessionId, permissionMode, model, sandbox } = req.body || {} + const { name, mode, shell, cwd, browser, editor, sessionRef, permissionMode, model, sandbox } = req.body || {} const wantsBrowser = !!browser const wantsEditor = !!editor - let launch: ResolvedSpawnProviderSettings | undefined - let createdTerminalId: string | undefined + let rollbackTabId: string | undefined + let unownedCodexSidecar: CodexLaunchPlan['sidecar'] | undefined try { let paneContent: any @@ -374,35 +344,71 @@ export function createAgentApiRouter({ } else { const effectiveMode = mode || 'shell' assertTerminalAdmission() - launch = await resolveSpawnProviderSettings( - effectiveMode, - configStore, - { permissionMode, model, sandbox }, - { cwd, resumeSessionId, codexLaunchPlanner, assertTerminalCreateAccepted: assertTerminalAdmission }, - ) - assertTerminalAdmission() - const { tabId, paneId } = layoutStore.createTab({ title: name, browser, editor }) - const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, resumeSessionId) + const requestedSessionRef = resolveRequestedSessionRef(effectiveMode, sessionRef) + const isCodexMode = effectiveMode === 'codex' + const preallocatedTerminalId = isCodexMode ? nanoid() : undefined + let tabId: string + let paneId: string + let launch: Awaited<ReturnType<typeof resolveSpawnProviderSettings>> + if (isCodexMode) { + tabId = nanoid() + paneId = nanoid() + launch = await resolveSpawnProviderSettings( + effectiveMode, + configStore, + { permissionMode, model, sandbox }, + { + cwd, + terminalId: preallocatedTerminalId, + envContext: { tabId, paneId }, + sessionRef: requestedSessionRef, + codexLaunchPlanner, + }, + ) + unownedCodexSidecar = launch.codexSidecar + layoutStore.createTab({ + title: name, + terminalId: preallocatedTerminalId, + tabId, + paneId, + }) + rollbackTabId = tabId + } else { + const created = layoutStore.createTab({ title: name }) + tabId = created.tabId + paneId = created.paneId + rollbackTabId = tabId + launch = await resolveSpawnProviderSettings( + effectiveMode, + configStore, + { permissionMode, model, sandbox }, + { + cwd, + envContext: { tabId, paneId }, + sessionRef: requestedSessionRef, + codexLaunchPlanner, + }, + ) + } + unownedCodexSidecar = launch.codexSidecar + const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, requestedSessionRef?.sessionId) assertTerminalAdmission() const terminal = registry.create({ + ...(preallocatedTerminalId ? { terminalId: preallocatedTerminalId } : {}), mode: effectiveMode, shell, cwd, - resumeSessionId: launch.resumeSessionId, + resumeSessionId: launch.sessionRef?.sessionId, ...(sessionBindingReason ? { sessionBindingReason } : {}), + ...(launch.codexSidecar ? { codexSidecar: launch.codexSidecar } : {}), + ...(launch.codexLaunchFactory ? { codexLaunchFactory: launch.codexLaunchFactory } : {}), + ...(launch.codexLaunchBaseProviderSettings + ? { codexLaunchBaseProviderSettings: launch.codexLaunchBaseProviderSettings } + : {}), providerSettings: launch.providerSettings, envContext: { tabId, paneId }, }) - createdTerminalId = terminal.terminalId - const launchResumeSessionId = launch.resumeSessionId - assertTerminalAdmission() - await adoptCodexLaunch(launch, terminal.terminalId) - assertTerminalAdmission() - await waitForCodexResumeReadiness(launch, resumeSessionId) - assertCodexCreateTerminalRunning(terminal) - assertTerminalAdmission() - publishCodexLaunch(registry, launch, terminal.terminalId) - launch = undefined + unownedCodexSidecar = undefined terminalId = terminal.terminalId paneContent = { kind: 'terminal', @@ -410,11 +416,12 @@ export function createAgentApiRouter({ status: 'running', mode: mode || 'shell', shell: shell || 'system', - resumeSessionId: launchResumeSessionId, + ...(launch.sessionRef ? { sessionRef: launch.sessionRef } : {}), initialCwd: cwd, } layoutStore.attachPaneContent(tabId, paneId, paneContent) + rollbackTabId = undefined wsHandler?.broadcastUiCommand({ command: 'tab.create', @@ -425,14 +432,13 @@ export function createAgentApiRouter({ shell, terminalId, initialCwd: cwd, - resumeSessionId: paneContent?.resumeSessionId, + sessionRef: paneContent?.sessionRef, paneId, paneContent, }, }) res.json(ok({ tabId, paneId, terminalId }, 'tab created')) - createdTerminalId = undefined return } @@ -448,7 +454,7 @@ export function createAgentApiRouter({ shell, terminalId, initialCwd: cwd, - resumeSessionId: paneContent?.resumeSessionId, + sessionRef: paneContent?.sessionRef, paneId, paneContent, }, @@ -456,12 +462,12 @@ export function createAgentApiRouter({ res.json(ok({ tabId, paneId, terminalId }, 'tab created')) } catch (err: any) { - let responseError = err - await cleanupFailedCodexCreate(registry, createdTerminalId ?? terminalIdFromCreateError(err), launch).catch((cleanupError) => { - responseError = combineWithCleanupError(err, cleanupError) - }) - const status = agentRouteErrorStatus(responseError) - res.status(status).json(fail(responseError?.message || 'Failed to create tab')) + await shutdownUnownedCodexSidecar(unownedCodexSidecar) + if (rollbackTabId) { + layoutStore.closeTab?.(rollbackTabId) + } + const status = agentRouteErrorStatus(err) + res.status(status).json(fail(err?.message || 'Failed to create tab')) } }) @@ -744,37 +750,59 @@ export function createAgentApiRouter({ const rawTimeout = payload.timeout || payload.T const timeoutSeconds = typeof rawTimeout === 'number' ? rawTimeout : Number(rawTimeout) const timeoutMs = Number.isFinite(timeoutSeconds) ? timeoutSeconds * 1000 : 30000 - let launch: ResolvedSpawnProviderSettings | undefined - let createdTerminalId: string | undefined + let rollbackTabId: string | undefined + let unownedCodexSidecar: CodexLaunchPlan['sidecar'] | undefined try { assertTerminalAdmission() - launch = await resolveSpawnProviderSettings(mode, configStore, {}, { - cwd, - codexLaunchPlanner, - assertTerminalCreateAccepted: assertTerminalAdmission, - }) - assertTerminalAdmission() - const created = layoutStore.createTab?.({ title }) - const tabId = created?.tabId || nanoid() - const paneId = created?.paneId || nanoid() - const sessionBindingReason = getCodexSessionBindingReason(mode) + const isCodexMode = mode === 'codex' + const preallocatedTerminalId = isCodexMode ? nanoid() : undefined + let tabId: string + let paneId: string + let launch: Awaited<ReturnType<typeof resolveSpawnProviderSettings>> + if (isCodexMode) { + tabId = nanoid() + paneId = nanoid() + launch = await resolveSpawnProviderSettings(mode, configStore, {}, { + cwd, + terminalId: preallocatedTerminalId, + envContext: { tabId, paneId }, + codexLaunchPlanner, + }) + unownedCodexSidecar = launch.codexSidecar + const created = layoutStore.createTab?.({ title, terminalId: preallocatedTerminalId, tabId, paneId }) + rollbackTabId = created?.tabId + } else { + const created = layoutStore.createTab?.({ title }) + tabId = created?.tabId || nanoid() + paneId = created?.paneId || nanoid() + rollbackTabId = created?.tabId + launch = await resolveSpawnProviderSettings(mode, configStore, {}, { + cwd, + envContext: { tabId, paneId }, + codexLaunchPlanner, + }) + } + unownedCodexSidecar = launch.codexSidecar + const sessionBindingReason = getCodexSessionBindingReason(mode, launch.sessionRef?.sessionId) assertTerminalAdmission() const terminal = registry.create({ + ...(preallocatedTerminalId ? { terminalId: preallocatedTerminalId } : {}), mode, shell, cwd, - resumeSessionId: launch.resumeSessionId, + resumeSessionId: launch.sessionRef?.sessionId, ...(sessionBindingReason ? { sessionBindingReason } : {}), + ...(launch.codexSidecar ? { codexSidecar: launch.codexSidecar } : {}), + ...(launch.codexLaunchFactory ? { codexLaunchFactory: launch.codexLaunchFactory } : {}), + ...(launch.codexLaunchBaseProviderSettings + ? { codexLaunchBaseProviderSettings: launch.codexLaunchBaseProviderSettings } + : {}), providerSettings: launch.providerSettings, envContext: { tabId, paneId }, }) - createdTerminalId = terminal.terminalId - assertTerminalAdmission() - await adoptCodexLaunch(launch, terminal.terminalId) - assertTerminalAdmission() - publishCodexLaunch(registry, launch, terminal.terminalId) - launch = undefined + unownedCodexSidecar = undefined layoutStore.attachPaneContent?.(tabId, paneId, { kind: 'terminal', terminalId: terminal.terminalId }) + rollbackTabId = undefined wsHandler?.broadcastUiCommand({ command: 'tab.create', payload: { id: tabId, title, mode, shell, terminalId: terminal.terminalId, initialCwd: cwd }, @@ -786,7 +814,6 @@ export function createAgentApiRouter({ if (!capture || detached) { const message = detached ? 'command started (detached)' : 'command sent' - createdTerminalId = undefined return res.json(ok({ terminalId: terminal.terminalId, tabId, paneId }, message)) } @@ -800,21 +827,20 @@ export function createAgentApiRouter({ const output = rawOutput.split(sentinel).join('').trim() const responder = result.matched ? ok : approx const message = result.matched ? 'run complete' : 'timeout waiting for command' - createdTerminalId = undefined return res.json(responder({ terminalId: terminal.terminalId, tabId, paneId, output }, message)) } catch (err: any) { - let responseError = err - await cleanupFailedCodexCreate(registry, createdTerminalId ?? terminalIdFromCreateError(err), launch).catch((cleanupError) => { - responseError = combineWithCleanupError(err, cleanupError) - }) - const status = agentRouteErrorStatus(responseError) - return res.status(status).json(fail(responseError?.message || 'Failed to run command')) + await shutdownUnownedCodexSidecar(unownedCodexSidecar) + if (rollbackTabId) { + layoutStore.closeTab?.(rollbackTabId) + } + const status = agentRouteErrorStatus(err) + return res.status(status).json(fail(err?.message || 'Failed to run command')) } }) router.post('/panes/:id/split', async (req, res) => { - let launch: ResolvedSpawnProviderSettings | undefined - let createdTerminalId: string | undefined + let unownedCodexSidecar: CodexLaunchPlan['sidecar'] | undefined + let rollbackPaneId: string | undefined try { const rawPaneId = req.params.id const resolved = resolvePaneTarget(rawPaneId) @@ -823,9 +849,6 @@ export function createAgentApiRouter({ const direction = req.body?.direction || 'vertical' const wantsBrowser = !!req.body?.browser const wantsEditor = !!req.body?.editor - if (!wantsBrowser && !wantsEditor) { - assertTerminalAdmission() - } const result = layoutStore.splitPane({ paneId, @@ -841,6 +864,7 @@ export function createAgentApiRouter({ const tabId = result.tabId const newPaneId = result.newPaneId + rollbackPaneId = newPaneId let content: any let terminalId: string | undefined @@ -849,40 +873,40 @@ export function createAgentApiRouter({ } else if (wantsEditor) { content = { kind: 'editor', filePath: req.body.editor, language: null, readOnly: false, content: '', viewMode: 'source' } } else { + assertTerminalAdmission() const splitMode = req.body?.mode || 'shell' - launch = await resolveSpawnProviderSettings( + const preallocatedTerminalId = nanoid() + const launch = await resolveSpawnProviderSettings( splitMode, configStore, {}, { cwd: req.body?.cwd, - resumeSessionId: req.body?.resumeSessionId, + terminalId: preallocatedTerminalId, + envContext: { tabId, paneId: newPaneId }, + sessionRef: resolveRequestedSessionRef(splitMode, req.body?.sessionRef), codexLaunchPlanner, - assertTerminalCreateAccepted: assertTerminalAdmission, }, ) - assertTerminalAdmission() - const sessionBindingReason = getCodexSessionBindingReason(splitMode, req.body?.resumeSessionId) + unownedCodexSidecar = launch.codexSidecar + const sessionBindingReason = getCodexSessionBindingReason(splitMode, launch.sessionRef?.sessionId) assertTerminalAdmission() const terminal = registry.create({ + terminalId: preallocatedTerminalId, mode: splitMode, shell: req.body?.shell, cwd: req.body?.cwd, - resumeSessionId: launch.resumeSessionId, + resumeSessionId: launch.sessionRef?.sessionId, ...(sessionBindingReason ? { sessionBindingReason } : {}), + ...(launch.codexSidecar ? { codexSidecar: launch.codexSidecar } : {}), + ...(launch.codexLaunchFactory ? { codexLaunchFactory: launch.codexLaunchFactory } : {}), + ...(launch.codexLaunchBaseProviderSettings + ? { codexLaunchBaseProviderSettings: launch.codexLaunchBaseProviderSettings } + : {}), providerSettings: launch.providerSettings, envContext: { tabId, paneId: newPaneId }, }) - createdTerminalId = terminal.terminalId - const launchResumeSessionId = launch.resumeSessionId - assertTerminalAdmission() - await adoptCodexLaunch(launch, terminal.terminalId) - assertTerminalAdmission() - await waitForCodexResumeReadiness(launch, req.body?.resumeSessionId) - assertCodexCreateTerminalRunning(terminal) - assertTerminalAdmission() - publishCodexLaunch(registry, launch, terminal.terminalId) - launch = undefined + unownedCodexSidecar = undefined terminalId = terminal.terminalId content = { kind: 'terminal', @@ -890,11 +914,12 @@ export function createAgentApiRouter({ status: 'running', mode: req.body?.mode || 'shell', shell: req.body?.shell || 'system', - ...(launchResumeSessionId ? { resumeSessionId: launchResumeSessionId } : {}), + ...(launch.sessionRef ? { sessionRef: launch.sessionRef } : {}), } } layoutStore.attachPaneContent(tabId, newPaneId, content) + rollbackPaneId = undefined wsHandler?.broadcastUiCommand({ command: 'pane.split', @@ -908,14 +933,13 @@ export function createAgentApiRouter({ }) const message = wantsBrowser || wantsEditor ? 'pane split (non-terminal)' : 'pane split' - createdTerminalId = undefined res.json(ok({ paneId: newPaneId, terminalId }, message)) } catch (err: any) { - let responseError = err - await cleanupFailedCodexCreate(registry, createdTerminalId ?? terminalIdFromCreateError(err), launch).catch((cleanupError) => { - responseError = combineWithCleanupError(err, cleanupError) - }) - res.status(agentRouteErrorStatus(responseError)).json(fail(responseError?.message || 'Failed to split pane')) + await shutdownUnownedCodexSidecar(unownedCodexSidecar) + if (rollbackPaneId) { + layoutStore.closePane?.(rollbackPaneId) + } + res.status(agentRouteErrorStatus(err)).json(fail(err?.message || 'Failed to split pane')) } }) @@ -1070,8 +1094,7 @@ export function createAgentApiRouter({ }) router.post('/panes/:id/respawn', async (req, res) => { - let launch: ResolvedSpawnProviderSettings | undefined - let createdTerminalId: string | undefined + let unownedCodexSidecar: CodexLaunchPlan['sidecar'] | undefined try { const resolved = resolvePaneTarget(req.params.id) if (rejectPaneTargetError(res, resolved)) return @@ -1081,39 +1104,38 @@ export function createAgentApiRouter({ if (!tabId) return res.status(404).json(fail('pane not found')) const effectiveMode = req.body?.mode || 'shell' assertTerminalAdmission() - launch = await resolveSpawnProviderSettings( + const preallocatedTerminalId = nanoid() + const launch = await resolveSpawnProviderSettings( effectiveMode, configStore, {}, { cwd: req.body?.cwd, - resumeSessionId: req.body?.resumeSessionId, + terminalId: preallocatedTerminalId, + envContext: { tabId, paneId }, + sessionRef: resolveRequestedSessionRef(effectiveMode, req.body?.sessionRef), codexLaunchPlanner, - assertTerminalCreateAccepted: assertTerminalAdmission, }, ) - assertTerminalAdmission() - const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, req.body?.resumeSessionId) + unownedCodexSidecar = launch.codexSidecar + const sessionBindingReason = getCodexSessionBindingReason(effectiveMode, launch.sessionRef?.sessionId) assertTerminalAdmission() const terminal = registry.create({ + terminalId: preallocatedTerminalId, mode: effectiveMode, shell: req.body?.shell, cwd: req.body?.cwd, - resumeSessionId: launch.resumeSessionId, + resumeSessionId: launch.sessionRef?.sessionId, ...(sessionBindingReason ? { sessionBindingReason } : {}), + ...(launch.codexSidecar ? { codexSidecar: launch.codexSidecar } : {}), + ...(launch.codexLaunchFactory ? { codexLaunchFactory: launch.codexLaunchFactory } : {}), + ...(launch.codexLaunchBaseProviderSettings + ? { codexLaunchBaseProviderSettings: launch.codexLaunchBaseProviderSettings } + : {}), providerSettings: launch.providerSettings, envContext: { tabId, paneId }, }) - createdTerminalId = terminal.terminalId - const launchResumeSessionId = launch.resumeSessionId - assertTerminalAdmission() - await adoptCodexLaunch(launch, terminal.terminalId) - assertTerminalAdmission() - await waitForCodexResumeReadiness(launch, req.body?.resumeSessionId) - assertCodexCreateTerminalRunning(terminal) - assertTerminalAdmission() - publishCodexLaunch(registry, launch, terminal.terminalId) - launch = undefined + unownedCodexSidecar = undefined const content = { kind: 'terminal', terminalId: terminal.terminalId, @@ -1121,18 +1143,14 @@ export function createAgentApiRouter({ mode: req.body?.mode || 'shell', shell: req.body?.shell || 'system', createRequestId: nanoid(), - ...(launchResumeSessionId ? { resumeSessionId: launchResumeSessionId } : {}), + ...(launch.sessionRef ? { sessionRef: launch.sessionRef } : {}), } layoutStore.attachPaneContent(tabId, paneId, content) wsHandler?.broadcastUiCommand({ command: 'pane.attach', payload: { tabId, paneId, content } }) - createdTerminalId = undefined res.json(ok({ terminalId: terminal.terminalId }, 'pane respawned')) } catch (err: any) { - let responseError = err - await cleanupFailedCodexCreate(registry, createdTerminalId ?? terminalIdFromCreateError(err), launch).catch((cleanupError) => { - responseError = combineWithCleanupError(err, cleanupError) - }) - res.status(agentRouteErrorStatus(responseError)).json(fail(responseError?.message || 'Failed to respawn pane')) + await shutdownUnownedCodexSidecar(unownedCodexSidecar) + res.status(agentRouteErrorStatus(err)).json(fail(err?.message || 'Failed to respawn pane')) } }) diff --git a/server/coding-cli/codex-app-server/client.ts b/server/coding-cli/codex-app-server/client.ts index c0154263d..72612d460 100644 --- a/server/coding-cli/codex-app-server/client.ts +++ b/server/coding-cli/codex-app-server/client.ts @@ -6,7 +6,6 @@ import { CodexFsWatchResultSchema, CodexInitializeParamsSchema, CodexInitializeResultSchema, - CodexLoadedThreadListResultSchema, CodexRpcErrorEnvelopeSchema, CodexRpcNotificationEnvelopeSchema, CodexRpcSuccessEnvelopeSchema, @@ -36,12 +35,7 @@ type PendingRequest = { timeout: NodeJS.Timeout } -export type CodexThreadLifecycleLossEvent = - | { method: 'thread/closed'; threadId?: string } - | { method: 'thread/status/changed'; threadId?: string; status: 'notLoaded' | 'systemError' } - const DEFAULT_REQUEST_TIMEOUT_MS = 5_000 -const LOSS_STATUSES = new Set(['notLoaded', 'systemError']) export type CodexThreadLifecycleEvent = { kind: 'thread_started' @@ -79,7 +73,6 @@ export class CodexAppServerClient { private readonly threadLifecycleHandlers = new Set<(event: CodexThreadLifecycleEvent) => void>() private readonly disconnectHandlers = new Set<(event: CodexAppServerDisconnectEvent) => void>() private readonly fsChangedHandlers = new Set<(event: { watchId: string; changedPaths: string[] }) => void>() - private lifecycleLossHandlers = new Set<(event: CodexThreadLifecycleLossEvent) => void>() constructor( private readonly endpoint: CodexAppServerEndpoint, @@ -96,12 +89,11 @@ export class CodexAppServerClient { capabilities: { experimentalApi: true, }, - })).then(async (result) => { + })).then((result) => { const parsed = CodexInitializeResultSchema.safeParse(result) if (!parsed.success) { throw new Error('Codex app-server returned an invalid initialize payload.') } - await this.notify('initialized') return parsed.data }).catch((error) => { this.initializePromise = null @@ -113,7 +105,7 @@ export class CodexAppServerClient { async startThread( params: Omit<CodexThreadStartParams, 'experimentalRawEvents' | 'persistExtendedHistory'>, - ): Promise<{ threadId: string }> { + ): Promise<CodexThreadOperationResult> { const result = await this.request('thread/start', { ...params, // Freshell attaches the visible TUI over `codex --remote`, so it does not @@ -125,12 +117,14 @@ export class CodexAppServerClient { if (!parsed.success) { throw new Error('Codex app-server returned an invalid thread/start payload.') } - return { threadId: parsed.data.thread.id } + return { + thread: normalizeThread(parsed.data.thread), + } } async resumeThread( params: Omit<CodexThreadResumeParams, 'persistExtendedHistory'>, - ): Promise<{ threadId: string }> { + ): Promise<CodexThreadOperationResult> { // Intentionally preserve Codex's default raw-event behavior for resume calls. const result = await this.request('thread/resume', { ...params, @@ -140,7 +134,9 @@ export class CodexAppServerClient { if (!parsed.success) { throw new Error('Codex app-server returned an invalid thread/resume payload.') } - return { threadId: parsed.data.thread.id } + return { + thread: normalizeThread(parsed.data.thread), + } } async watchPath(targetPath: string, watchId: string): Promise<{ path: string }> { @@ -161,15 +157,6 @@ export class CodexAppServerClient { })) } - async listLoadedThreads(): Promise<string[]> { - const result = await this.request('thread/loaded/list', {}) - const parsed = CodexLoadedThreadListResultSchema.safeParse(result) - if (!parsed.success) { - throw new Error('Codex app-server returned an invalid thread/loaded/list payload.') - } - return parsed.data.data - } - async close(): Promise<void> { const socket = this.socket this.socket = null @@ -229,13 +216,6 @@ export class CodexAppServerClient { } } - onThreadLifecycleLoss(handler: (event: CodexThreadLifecycleLossEvent) => void): () => void { - this.lifecycleLossHandlers.add(handler) - return () => { - this.lifecycleLossHandlers.delete(handler) - } - } - private async ensureSocket(): Promise<WebSocket> { if (this.socket && this.socket.readyState === WebSocket.OPEN) { return this.socket @@ -296,7 +276,6 @@ export class CodexAppServerClient { const lifecycle = CodexThreadLifecycleNotificationSchema.safeParse(notification.data) if (lifecycle.success) { this.emitThreadLifecycle(lifecycle.data) - this.handleNotification(notification.data) return } @@ -313,10 +292,7 @@ export class CodexAppServerClient { for (const handler of this.fsChangedHandlers) { handler(fsChanged.data.params) } - return } - - this.handleNotification(notification.data) return } } @@ -346,68 +322,6 @@ export class CodexAppServerClient { pending.reject(new Error(this.formatRpcError(pending.method, failure.data.error))) } - private handleNotification(notification: { method: string; params?: unknown }): void { - if (notification.method === 'thread/closed') { - this.emitLifecycleLoss({ - method: 'thread/closed', - threadId: this.extractThreadId(notification.params), - }) - return - } - - if (notification.method !== 'thread/status/changed') return - const status = this.extractThreadStatus(notification.params) - if (status !== 'notLoaded' && status !== 'systemError') return - - this.emitLifecycleLoss({ - method: 'thread/status/changed', - threadId: this.extractThreadId(notification.params), - status, - }) - } - - private emitLifecycleLoss(event: CodexThreadLifecycleLossEvent): void { - for (const handler of this.lifecycleLossHandlers) { - handler(event) - } - } - - private extractThreadId(params: unknown): string | undefined { - if (!params || typeof params !== 'object') return undefined - const object = params as Record<string, unknown> - if (typeof object.threadId === 'string') return object.threadId - const thread = object.thread - if (thread && typeof thread === 'object' && typeof (thread as Record<string, unknown>).id === 'string') { - return (thread as Record<string, string>).id - } - return undefined - } - - private extractThreadStatus(params: unknown): 'notLoaded' | 'systemError' | undefined { - if (!params || typeof params !== 'object') return undefined - const object = params as Record<string, unknown> - const status = this.extractLossStatus(object.status) - if (status) return status - const thread = object.thread - if (thread && typeof thread === 'object') { - return this.extractLossStatus((thread as Record<string, unknown>).status) - } - return undefined - } - - private extractLossStatus(status: unknown): 'notLoaded' | 'systemError' | undefined { - if (typeof status === 'string' && LOSS_STATUSES.has(status)) { - return status as 'notLoaded' | 'systemError' - } - if (status && typeof status === 'object') { - const statusType = (status as Record<string, unknown>).type - if (typeof statusType === 'string' && LOSS_STATUSES.has(statusType)) { - return statusType as 'notLoaded' | 'systemError' - } - } - return undefined - } - private handleSocketClose(socket: WebSocket, event: CodexAppServerDisconnectEvent): void { if (this.socket !== socket) { return @@ -493,23 +407,6 @@ export class CodexAppServerClient { }) } - private async notify<TParams extends object>(method: string, params?: TParams): Promise<void> { - const socket = await this.ensureSocket() - await new Promise<void>((resolve, reject) => { - socket.send(JSON.stringify({ - jsonrpc: '2.0', - method, - ...(params ? { params } : {}), - }), (error) => { - if (error) { - reject(error) - return - } - resolve() - }) - }) - } - private formatRpcError(method: string, error: CodexRpcError): string { return `Codex app-server ${method} failed: ${error.message}` } diff --git a/server/coding-cli/codex-app-server/durable-rollout-tracker.ts b/server/coding-cli/codex-app-server/durable-rollout-tracker.ts new file mode 100644 index 000000000..34fb4ea35 --- /dev/null +++ b/server/coding-cli/codex-app-server/durable-rollout-tracker.ts @@ -0,0 +1,250 @@ +import fsp from 'node:fs/promises' +import path from 'node:path' +import { logger } from '../../logger.js' +import type { CodexThreadHandle } from './protocol.js' + +const DEFAULT_INITIAL_PROBE_DELAY_MS = 250 +const DEFAULT_MAX_PROBE_DELAY_MS = 5_000 + +export type CodexFsChangedEvent = { + watchId: string + changedPaths: string[] +} + +export type CodexDurableRolloutTrackerOptions = { + watchPath: (targetPath: string, watchId: string) => Promise<{ path: string }> + unwatchPath: (watchId: string) => Promise<void> + subscribeToFsChanged: (handler: (event: CodexFsChangedEvent) => void) => () => void + onDurableRollout: (sessionId: string) => void + pathExists?: (targetPath: string) => Promise<boolean> + initialProbeDelayMs?: number + maxProbeDelayMs?: number + setTimeoutFn?: typeof setTimeout + clearTimeoutFn?: typeof clearTimeout + createWatchId?: (kind: 'rollout' | 'parent', threadId: string) => string + log?: Pick<typeof logger, 'warn'> +} + +type PendingRollout = { + thread: CodexThreadHandle + rolloutPath: string + parentPath: string + rolloutWatchId: string + parentWatchId: string + registeredWatchIds: Set<string> + nextProbeDelayMs: number + timer: ReturnType<typeof setTimeout> | null + probeInFlight: boolean + immediateProbeQueued: boolean +} + +async function defaultPathExists(targetPath: string): Promise<boolean> { + try { + await fsp.access(targetPath) + return true + } catch { + return false + } +} + +export class CodexDurableRolloutTracker { + private readonly pathExists: (targetPath: string) => Promise<boolean> + private readonly initialProbeDelayMs: number + private readonly maxProbeDelayMs: number + private readonly setTimeoutFn: typeof setTimeout + private readonly clearTimeoutFn: typeof clearTimeout + private readonly createWatchId: (kind: 'rollout' | 'parent', threadId: string) => string + private readonly log: Pick<typeof logger, 'warn'> + private readonly cleanupFsChangedSubscription: () => void + + private disposed = false + private promotedThreadId: string | null = null + private pending: PendingRollout | null = null + + constructor(private readonly options: CodexDurableRolloutTrackerOptions) { + this.pathExists = options.pathExists ?? defaultPathExists + this.initialProbeDelayMs = options.initialProbeDelayMs ?? DEFAULT_INITIAL_PROBE_DELAY_MS + this.maxProbeDelayMs = options.maxProbeDelayMs ?? DEFAULT_MAX_PROBE_DELAY_MS + this.setTimeoutFn = options.setTimeoutFn ?? setTimeout + this.clearTimeoutFn = options.clearTimeoutFn ?? clearTimeout + this.createWatchId = options.createWatchId ?? ((kind, threadId) => `freshell-codex-${kind}:${threadId}`) + this.log = options.log ?? logger + this.cleanupFsChangedSubscription = options.subscribeToFsChanged((event) => { + this.handleFsChanged(event) + }) + } + + trackThread(thread: CodexThreadHandle): void { + if (this.disposed || this.promotedThreadId || !thread.id) { + return + } + if (thread.ephemeral) { + return + } + if (!thread.path) { + this.log.warn({ threadId: thread.id }, 'Codex thread/started did not include a durable rollout path; promotion will stay pending') + return + } + + if (this.pending?.thread.id === thread.id && this.pending.rolloutPath === thread.path) { + return + } + + void this.replacePendingRollout({ + thread: { + id: thread.id, + path: thread.path, + ephemeral: thread.ephemeral ?? false, + }, + rolloutPath: thread.path, + parentPath: path.dirname(thread.path), + rolloutWatchId: this.createWatchId('rollout', thread.id), + parentWatchId: this.createWatchId('parent', thread.id), + registeredWatchIds: new Set<string>(), + nextProbeDelayMs: this.initialProbeDelayMs, + timer: null, + probeInFlight: false, + immediateProbeQueued: false, + }) + } + + async dispose(): Promise<void> { + this.disposed = true + this.cleanupFsChangedSubscription() + await this.clearPending(this.pending) + this.pending = null + } + + private async replacePendingRollout(nextPending: PendingRollout): Promise<void> { + const previousPending = this.pending + this.pending = nextPending + await this.clearPending(previousPending) + if (this.disposed || this.promotedThreadId) { + return + } + if (this.pending !== nextPending) { + return + } + void this.registerWatch(nextPending, nextPending.rolloutPath, nextPending.rolloutWatchId) + void this.registerWatch(nextPending, nextPending.parentPath, nextPending.parentWatchId) + this.requestImmediateProbe(nextPending) + } + + private handleFsChanged(event: CodexFsChangedEvent): void { + const pending = this.pending + if (!pending || this.disposed || this.promotedThreadId) { + return + } + const mentionsRolloutPath = event.changedPaths.includes(pending.rolloutPath) + const matchesWatchId = ( + event.watchId === pending.rolloutWatchId + || event.watchId === pending.parentWatchId + ) + if (!mentionsRolloutPath && !matchesWatchId) { + return + } + this.requestImmediateProbe(pending) + } + + private requestImmediateProbe(pending: PendingRollout): void { + if (this.pending !== pending || this.disposed || this.promotedThreadId) { + return + } + if (pending.timer) { + this.clearTimeoutFn(pending.timer) + pending.timer = null + } + if (pending.probeInFlight) { + pending.immediateProbeQueued = true + return + } + void this.runProbe(pending) + } + + private scheduleNextProbe(pending: PendingRollout): void { + if (this.pending !== pending || this.disposed || this.promotedThreadId || pending.timer) { + return + } + const delayMs = pending.nextProbeDelayMs + pending.nextProbeDelayMs = Math.min(this.maxProbeDelayMs, pending.nextProbeDelayMs * 2) + pending.timer = this.setTimeoutFn(() => { + pending.timer = null + void this.runProbe(pending) + }, delayMs) + } + + private async runProbe(pending: PendingRollout): Promise<void> { + if (this.pending !== pending || this.disposed || this.promotedThreadId) { + return + } + pending.probeInFlight = true + + try { + if (await this.pathExists(pending.rolloutPath)) { + this.promotedThreadId = pending.thread.id + await this.clearPending(pending) + if (!this.disposed) { + this.options.onDurableRollout(pending.thread.id) + } + return + } + } finally { + pending.probeInFlight = false + } + + if (this.pending !== pending || this.disposed || this.promotedThreadId) { + return + } + + if (pending.immediateProbeQueued) { + pending.immediateProbeQueued = false + void this.runProbe(pending) + return + } + + this.scheduleNextProbe(pending) + } + + private async registerWatch(pending: PendingRollout, targetPath: string, watchId: string): Promise<void> { + try { + await this.options.watchPath(targetPath, watchId) + if (this.pending === pending && !this.disposed) { + pending.registeredWatchIds.add(watchId) + } else { + await this.options.unwatchPath(watchId).catch(() => undefined) + } + } catch (error) { + this.log.warn( + { + err: error, + watchId, + targetPath, + threadId: pending.thread.id, + }, + 'Failed to register Codex rollout watch; falling back to exact-path probes.', + ) + } + } + + private async clearPending(pending: PendingRollout | null): Promise<void> { + if (!pending) { + return + } + if (pending.timer) { + this.clearTimeoutFn(pending.timer) + pending.timer = null + } + if (this.pending === pending) { + this.pending = null + } + const watchIds = [...pending.registeredWatchIds] + pending.registeredWatchIds.clear() + await Promise.all(watchIds.map(async (watchId) => { + try { + await this.options.unwatchPath(watchId) + } catch (error) { + this.log.warn({ err: error, watchId }, 'Failed to unregister Codex rollout watch during cleanup.') + } + })) + } +} diff --git a/server/coding-cli/codex-app-server/launch-planner.ts b/server/coding-cli/codex-app-server/launch-planner.ts index 020bf3278..f2040ef12 100644 --- a/server/coding-cli/codex-app-server/launch-planner.ts +++ b/server/coding-cli/codex-app-server/launch-planner.ts @@ -1,246 +1,132 @@ -import type { CodexAppServerRuntime } from './runtime.js' -import type { CodexThreadLifecycleLossEvent } from './client.js' -import { waitForAllSettledOrThrow } from '../../shutdown-join.js' - -type CodexRuntimeLike = Pick< - CodexAppServerRuntime, - 'ensureReady' | 'startThread' | 'listLoadedThreads' | 'shutdown' | 'updateOwnershipMetadata' | 'onThreadLifecycleLoss' -> - -export type CodexLaunchSidecar = { - adopt(input: { terminalId: string; generation: number }): Promise<void> - listLoadedThreads(): Promise<string[]> - onLifecycleLoss?(handler: (event: CodexThreadLifecycleLossEvent) => void): () => void - shutdown(): Promise<void> - waitForLoadedThread(threadId: string, options?: { timeoutMs?: number; pollMs?: number }): Promise<void> -} +import { CodexTerminalSidecar } from './sidecar.js' +import { generateMcpInjection } from '../../mcp/config-writer.js' +import type { TerminalEnvContext } from '../../terminal-registry.js' export type CodexLaunchPlan = { - sessionId: string + sessionId?: string remote: { wsUrl: string + processPid?: number } - sidecar: CodexLaunchSidecar -} - -export type CodexSidecarTeardownError = Error & { - codexSidecarTeardownFailed: true + sidecar: Pick<CodexTerminalSidecar, 'attachTerminal' | 'shutdown'> } type PlanCreateInput = { cwd?: string + terminalId: string + env: NodeJS.ProcessEnv resumeSessionId?: string model?: string sandbox?: 'read-only' | 'workspace-write' | 'danger-full-access' approvalPolicy?: string } -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) +export type CodexLaunchFactoryInput = { + terminalId: string + cwd?: string + envContext?: TerminalEnvContext + resumeSessionId?: string + providerSettings?: { + model?: string + sandbox?: string + permissionMode?: string + } } -function codexSidecarTeardownError(message: string, cause: unknown): CodexSidecarTeardownError { - const error = new Error(message) as CodexSidecarTeardownError - error.codexSidecarTeardownFailed = true - error.cause = cause - return error -} +export type CodexLaunchFactory = (input: CodexLaunchFactoryInput) => Promise<CodexLaunchPlan> -export function isCodexSidecarTeardownError(error: unknown): error is CodexSidecarTeardownError { - return (error as { codexSidecarTeardownFailed?: boolean } | null | undefined)?.codexSidecarTeardownFailed === true +type CodexLaunchRetryOptions = { + onFailedAttempt?: (input: { attempt: number; delayMs: number; error: Error }) => void + shouldRetry?: (error: Error) => boolean } -export class CodexLaunchPlanner { - private readonly activeSidecars = new Set<CodexLaunchSidecar>() - private readonly failedSidecarShutdowns = new Set<CodexLaunchSidecar>() - private readonly runtimeFactory: () => CodexRuntimeLike - private shutdownStarted = false - private shutdownPromise: Promise<void> | null = null - - constructor(runtimeOrFactory: CodexRuntimeLike | (() => CodexRuntimeLike)) { - this.runtimeFactory = typeof runtimeOrFactory === 'function' - ? runtimeOrFactory - : () => runtimeOrFactory - } - - async planCreate(input: PlanCreateInput): Promise<CodexLaunchPlan> { - this.assertAcceptingPlans() - await this.retryFailedSidecarShutdownsBeforePlan() - this.assertAcceptingPlans() +const INITIAL_LAUNCH_RETRY_DELAYS_MS = [0, 250, 1000, 2000, 5000] as const - const runtime = this.runtimeFactory() - const sidecar = this.createSidecar(runtime) - this.activeSidecars.add(sidecar) +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)) +} +export async function runCodexLaunchWithRetry<T>( + launch: (attempt: number) => Promise<T>, + options: CodexLaunchRetryOptions = {}, +): Promise<T> { + let lastError: Error | undefined + + for (let index = 0; index < INITIAL_LAUNCH_RETRY_DELAYS_MS.length; index += 1) { + const attempt = index + 1 + const delayMs = INITIAL_LAUNCH_RETRY_DELAYS_MS[index] + if (delayMs > 0) { + await sleep(delayMs) + } try { - if (input.resumeSessionId) { - const ready = await runtime.ensureReady() - this.assertAcceptingPlans() - return { - sessionId: input.resumeSessionId, - remote: { - wsUrl: ready.wsUrl, - }, - sidecar, - } - } - - const planResult = await runtime.startThread({ - cwd: input.cwd, - model: input.model, - sandbox: input.sandbox, - approvalPolicy: input.approvalPolicy, - }) - this.assertAcceptingPlans() - - return { - sessionId: planResult.threadId, - remote: { - wsUrl: planResult.wsUrl, - }, - sidecar, - } + return await launch(attempt) } catch (error) { - try { - await sidecar.shutdown() - } catch (shutdownError) { - throw codexSidecarTeardownError( - `Codex launch sidecar teardown failed after planning error: ${errorMessage(shutdownError)}`, - shutdownError, - ) + lastError = error instanceof Error ? error : new Error(String(error)) + if (options.shouldRetry && !options.shouldRetry(lastError)) { + throw lastError } - throw error - } - } - - async shutdown(): Promise<void> { - this.shutdownStarted = true - if (this.shutdownPromise) { - await this.shutdownPromise - return - } - const attempt = waitForAllSettledOrThrow( - [...this.activeSidecars].map((sidecar) => Promise.resolve().then(() => sidecar.shutdown())), - 'Codex launch planner shutdown failed.', - ) - this.shutdownPromise = attempt - try { - await attempt - } finally { - if (this.shutdownPromise === attempt) { - this.shutdownPromise = null + if (attempt < INITIAL_LAUNCH_RETRY_DELAYS_MS.length) { + options.onFailedAttempt?.({ attempt, delayMs: INITIAL_LAUNCH_RETRY_DELAYS_MS[index + 1], error: lastError }) } } } - private assertAcceptingPlans(): void { - if (this.shutdownStarted) { - throw new Error('Codex launch planner is shutting down; new Codex launch plans are not accepted.') - } - } + throw lastError ?? new Error('Codex launch failed before a terminal record could be created.') +} + +type SidecarCreateInput = PlanCreateInput & { + commandArgs: string[] +} + +function appServerMcpTarget(): 'unix' | 'windows' { + return process.platform === 'win32' ? 'windows' : 'unix' +} - private async retryFailedSidecarShutdownsBeforePlan(): Promise<void> { - const failedSidecars = [...this.failedSidecarShutdowns] - .filter((sidecar) => this.activeSidecars.has(sidecar)) - if (failedSidecars.length === 0) return +export class CodexLaunchPlanner { + constructor( + private readonly createSidecar: (input: SidecarCreateInput) => Pick<CodexTerminalSidecar, 'ensureReady' | 'attachTerminal' | 'shutdown'> + = (input) => new CodexTerminalSidecar({ + cwd: input.cwd, + commandArgs: input.commandArgs, + env: input.env, + }), + ) {} + async planCreate(input: PlanCreateInput): Promise<CodexLaunchPlan> { + const sidecar = this.createSidecar({ + ...input, + commandArgs: generateMcpInjection('codex', input.terminalId, input.cwd, appServerMcpTarget()).args, + }) + let ready: Awaited<ReturnType<typeof sidecar.ensureReady>> try { - await waitForAllSettledOrThrow( - failedSidecars.map((sidecar) => sidecar.shutdown()), - 'Codex launch planner failed to clear blocked sidecar shutdowns.', - ) + ready = await sidecar.ensureReady() } catch (error) { - throw codexSidecarTeardownError( - `Codex launch planner cannot create a new plan while sidecar teardown is blocked: ${errorMessage(error)}`, - error, - ) + await sidecar.shutdown().catch(() => undefined) + throw error } - } - private createSidecar(runtime: CodexRuntimeLike): CodexLaunchSidecar { - let shutdownPromise: Promise<void> | null = null - let shutdownAttemptStarted = false - let shutdownSucceeded = false - let notifyShutdownStarted!: () => void - const shutdownStarted = new Promise<void>((resolve) => { - notifyShutdownStarted = resolve - }) - const assertAdoptable = () => { - if (this.shutdownStarted || shutdownAttemptStarted) { - throw new Error('Codex launch sidecar is shutting down; it cannot be adopted.') - } - } - const assertReadable = () => { - if (this.shutdownStarted || shutdownAttemptStarted) { - throw new Error('Codex launch sidecar is shutting down; loaded-thread readiness polling stopped.') + if (input.resumeSessionId) { + return { + sessionId: input.resumeSessionId, + remote: { + wsUrl: ready.wsUrl, + processPid: ready.processPid, + }, + sidecar, } } - const waitForNextPoll = async (pollMs: number) => { - await Promise.race([ - new Promise((resolve) => setTimeout(resolve, pollMs)), - shutdownStarted, - ]) - assertReadable() - } - const sidecar: CodexLaunchSidecar = { - adopt: async ({ terminalId, generation }) => { - assertAdoptable() - await runtime.updateOwnershipMetadata({ terminalId, generation }) - assertAdoptable() - this.activeSidecars.delete(sidecar) - this.failedSidecarShutdowns.delete(sidecar) - }, - listLoadedThreads: async () => { - assertReadable() - const loaded = await runtime.listLoadedThreads() - assertReadable() - return loaded - }, - onLifecycleLoss: (handler) => runtime.onThreadLifecycleLoss(handler), - shutdown: async () => { - if (shutdownSucceeded) return - if (shutdownPromise) { - await shutdownPromise - return - } - if (!shutdownAttemptStarted) { - shutdownAttemptStarted = true - notifyShutdownStarted() - } - const attempt = Promise.resolve() - .then(() => runtime.shutdown()) - .then(() => { - shutdownSucceeded = true - this.activeSidecars.delete(sidecar) - this.failedSidecarShutdowns.delete(sidecar) - }) - .catch((error) => { - this.failedSidecarShutdowns.add(sidecar) - throw error - }) - shutdownPromise = attempt - try { - await attempt - } finally { - if (shutdownPromise === attempt) { - shutdownPromise = null - } - } - }, - waitForLoadedThread: async (threadId, options = {}) => { - const timeoutMs = options.timeoutMs ?? 10_000 - const pollMs = options.pollMs ?? 100 - const deadline = Date.now() + timeoutMs - - while (Date.now() < deadline) { - const loaded = await sidecar.listLoadedThreads() - if (loaded.includes(threadId)) return - await waitForNextPoll(pollMs) - } - - throw new Error(`Codex app-server did not load thread ${threadId} within ${timeoutMs}ms.`) + + return { + remote: { + wsUrl: ready.wsUrl, + processPid: ready.processPid, }, + sidecar, } - return sidecar + } + + async shutdown(): Promise<void> { + // Sidecars transfer to TerminalRegistry ownership immediately after create. + // Unowned planning failures are shut down by the create call sites. } } diff --git a/server/coding-cli/codex-app-server/protocol.ts b/server/coding-cli/codex-app-server/protocol.ts index ea3e6a906..a528d9e79 100644 --- a/server/coding-cli/codex-app-server/protocol.ts +++ b/server/coding-cli/codex-app-server/protocol.ts @@ -61,10 +61,6 @@ export const CodexFsUnwatchParamsSchema = z.object({ watchId: z.string().min(1), }) -export const CodexLoadedThreadListResultSchema = z.object({ - data: z.array(z.string().min(1)), -}) - export const CodexRpcErrorSchema = z.object({ code: z.number(), message: z.string().min(1), @@ -135,7 +131,6 @@ export type CodexThreadOperationResult = z.infer<typeof CodexThreadOperationResu export type CodexFsWatchParams = z.infer<typeof CodexFsWatchParamsSchema> export type CodexFsWatchResult = z.infer<typeof CodexFsWatchResultSchema> export type CodexFsUnwatchParams = z.infer<typeof CodexFsUnwatchParamsSchema> -export type CodexLoadedThreadListResult = z.infer<typeof CodexLoadedThreadListResultSchema> export type CodexRpcError = z.infer<typeof CodexRpcErrorSchema> export type CodexThreadStartedNotification = z.infer<typeof CodexThreadStartedNotificationSchema> export type CodexThreadClosedNotification = z.infer<typeof CodexThreadClosedNotificationSchema> diff --git a/server/coding-cli/codex-app-server/runtime.ts b/server/coding-cli/codex-app-server/runtime.ts index 96471533c..99878ea5b 100644 --- a/server/coding-cli/codex-app-server/runtime.ts +++ b/server/coding-cli/codex-app-server/runtime.ts @@ -1,25 +1,22 @@ +import { spawn, type ChildProcess } from 'node:child_process' import { randomUUID } from 'node:crypto' -import { spawn } from 'node:child_process' import fsp from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import { allocateLocalhostPort, type LoopbackServerEndpoint } from '../../local-port.js' import { logger } from '../../logger.js' -import { - CodexAppServerClient, - type CodexThreadLifecycleEvent, - type CodexThreadLifecycleLossEvent, -} from './client.js' +import { convertWindowsPathToWslPath, isWslEnvironment, sanitizeUserPathInput } from '../../path-utils.js' +import { CodexAppServerClient, type CodexAppServerDisconnectEvent, type CodexThreadLifecycleEvent } from './client.js' import type { CodexFsWatchResult, CodexInitializeResult, CodexThreadHandle, + CodexThreadOperationResult, CodexThreadResumeParams, CodexThreadStartParams, } from './protocol.js' type RuntimeStatus = 'running' | 'stopped' - export type CodexAppServerRuntimeFailureSource = | 'app_server_exit' | 'app_server_client_disconnect' @@ -46,9 +43,10 @@ export type CodexSidecarOwnershipMetadata = { codexHome?: string } -export type ReadyState = { +type ReadyState = { wsUrl: string processPid: number + codexHome: string ownershipId: string processGroupId: number metadataPath: string @@ -60,15 +58,15 @@ type ActiveOwnership = { metadata: CodexSidecarOwnershipMetadata } -type ChildProcessHandle = ReturnType<typeof spawn> - type RuntimeOptions = { command?: string commandArgs?: string[] + cwd?: string env?: NodeJS.ProcessEnv requestTimeoutMs?: number startupAttemptLimit?: number startupAttemptTimeoutMs?: number + terminateGraceMs?: number portAllocator?: () => Promise<LoopbackServerEndpoint> metadataDir?: string serverInstanceId?: string @@ -77,32 +75,74 @@ type RuntimeOptions = { processIdentityReader?: (pid: number) => Promise<WrapperIdentity | null> } -export type ReapOrphanedSidecarsOptions = { - metadataDir?: string - serverInstanceId: string - terminateGraceMs?: number -} - export type ReapOrphanedSidecarsResult = { + scanned: number reapedOwnershipIds: string[] - ignoredLegacyRecords: string[] skippedActiveOwnershipIds: string[] + ignoredLegacyRecords: string[] failedOwnershipIds: string[] } +type ReapOrphanedSidecarsOptions = { + metadataDir?: string + serverInstanceId?: string + terminateGraceMs?: number +} + const DEFAULT_STARTUP_ATTEMPT_LIMIT = 2 const DEFAULT_STARTUP_ATTEMPT_TIMEOUT_MS = 3_000 -const STARTUP_POLL_MS = 50 const DEFAULT_TERMINATE_GRACE_MS = 1_000 +const STARTUP_POLL_MS = 50 +const OUTPUT_TAIL_MAX_CHARS = 4 * 1024 +const OUTPUT_TAIL_MAX_LINES = 40 const OWNERSHIP_SCHEMA_VERSION = 1 +export const DEFAULT_CODEX_SIDECAR_METADATA_DIR = path.join(os.tmpdir(), 'freshell-codex-sidecars') + +class BoundedOutputTail { + private value = '' + + push(chunk: Buffer | string): void { + this.value += chunk.toString() + const lines = this.value.split(/\r?\n/) + if (lines.length > OUTPUT_TAIL_MAX_LINES) { + this.value = lines.slice(-OUTPUT_TAIL_MAX_LINES).join('\n') + } + if (this.value.length > OUTPUT_TAIL_MAX_CHARS) { + this.value = this.value.slice(-OUTPUT_TAIL_MAX_CHARS) + } + } + + snapshot(): string { + return this.value + } +} + +type RuntimeChildDiagnostics = { + wsUrl: string + wsPort: number + startedAt: number + stdoutTail: BoundedOutputTail + stderrTail: BoundedOutputTail + processError?: Error +} + function sleep(ms: number): Promise<void> { return new Promise((resolve) => setTimeout(resolve, ms)) } function defaultMetadataDir(): string { - return process.env.FRESHELL_CODEX_SIDECAR_DIR - || path.join(os.homedir(), '.freshell', 'codex-sidecars') + return process.env.FRESHELL_CODEX_SIDECAR_DIR || DEFAULT_CODEX_SIDECAR_METADATA_DIR +} + +function resolveAppServerCwd(cwd: string | undefined): string | undefined { + if (typeof cwd !== 'string') return undefined + const candidate = sanitizeUserPathInput(cwd) + if (!candidate) return undefined + if (isWslEnvironment()) { + return convertWindowsPathToWslPath(candidate) ?? candidate + } + return candidate } function assertUnixSidecarSupport(): void { @@ -220,25 +260,23 @@ async function processHasOwnershipEnv(pid: number, ownershipId: string): Promise async function processGroupMembers(processGroupId: number): Promise<number[]> { const entries = await fsp.readdir('/proc') const members: number[] = [] - - await Promise.all(entries.map(async (entry) => { - if (!/^\d+$/.test(entry)) return + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue const pid = Number(entry) const pgrp = await getProcessGroupId(pid) if (pgrp === processGroupId) members.push(pid) - })) - - return members.sort((a, b) => a - b) + } + return members } async function isProcessGroupGone(processGroupId: number): Promise<boolean> { if (!Number.isInteger(processGroupId) || processGroupId <= 0) return true try { process.kill(-processGroupId, 0) + return false } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return true + return (error as NodeJS.ErrnoException).code === 'ESRCH' } - return false } async function verifyOwnedProcessGroup(metadata: CodexSidecarOwnershipMetadata): Promise<boolean> { @@ -263,10 +301,17 @@ async function verifyOwnedProcessGroup(metadata: CodexSidecarOwnershipMetadata): return true } } - return false } +function hasRecordedWrapperProof(metadata: CodexSidecarOwnershipMetadata): boolean { + return metadata.wrapperIdentity.commandLine.length > 0 + && typeof metadata.wrapperIdentity.cwd === 'string' + && metadata.wrapperIdentity.cwd.length > 0 + && typeof metadata.wrapperIdentity.startTimeTicks === 'number' + && Number.isFinite(metadata.wrapperIdentity.startTimeTicks) +} + function signalProcessGroup(processGroupId: number, signal: NodeJS.Signals): void { try { process.kill(-processGroupId, signal) @@ -289,10 +334,15 @@ async function waitForProcessGroupGone(processGroupId: number, timeoutMs: number async function teardownOwnedProcessGroup( ownership: ActiveOwnership, terminateGraceMs: number, + options: { activeOwner?: boolean } = {}, ): Promise<boolean> { const { metadata } = ownership - if (!(await verifyOwnedProcessGroup(metadata))) { - logger.error( + const verified = await verifyOwnedProcessGroup(metadata) + || (options.activeOwner === true + && hasRecordedWrapperProof(metadata) + && (await getProcessGroupId('self')) !== metadata.processGroupId) + if (!verified) { + logger.warn( { ownershipId: metadata.ownershipId, terminalId: metadata.terminalId, @@ -311,8 +361,12 @@ async function teardownOwnedProcessGroup( signalProcessGroup(metadata.processGroupId, 'SIGTERM') } if (!(await waitForProcessGroupGone(metadata.processGroupId, terminateGraceMs))) { - if (!(await verifyOwnedProcessGroup(metadata))) { - logger.error( + const stillVerified = await verifyOwnedProcessGroup(metadata) + || (options.activeOwner === true + && hasRecordedWrapperProof(metadata) + && (await getProcessGroupId('self')) !== metadata.processGroupId) + if (!stillVerified) { + logger.warn( { ownershipId: metadata.ownershipId, terminalId: metadata.terminalId, @@ -331,7 +385,7 @@ async function teardownOwnedProcessGroup( const gone = await waitForProcessGroupGone(metadata.processGroupId, terminateGraceMs) if (!gone) { - logger.error( + logger.warn( { ownershipId: metadata.ownershipId, terminalId: metadata.terminalId, @@ -342,7 +396,7 @@ async function teardownOwnedProcessGroup( serverInstanceId: metadata.serverInstanceId, remainingPids: await processGroupMembers(metadata.processGroupId), }, - 'Codex app-server sidecar process group remained alive after shutdown', + 'Codex app-server sidecar process group did not exit after SIGKILL', ) return false } @@ -359,23 +413,10 @@ type ParsedMetadataRecord = | { kind: 'legacy' } | { kind: 'malformedNewSchema'; ownershipId: string } -function isWrapperIdentity(value: unknown): value is WrapperIdentity { - if (!value || typeof value !== 'object') return false - const candidate = value as Partial<WrapperIdentity> - return Array.isArray(candidate.commandLine) - && candidate.commandLine.every((arg) => typeof arg === 'string') - && (candidate.cwd === null || typeof candidate.cwd === 'string') - && (candidate.startTimeTicks === null || typeof candidate.startTimeTicks === 'number') -} - function isPositiveInteger(value: unknown): value is number { return typeof value === 'number' && Number.isInteger(value) && value > 0 } -function isNonNegativeInteger(value: unknown): value is number { - return typeof value === 'number' && Number.isInteger(value) && value >= 0 -} - function parseMetadataRecord(raw: string, metadataPath: string): ParsedMetadataRecord { let parsed: unknown try { @@ -383,20 +424,20 @@ function parseMetadataRecord(raw: string, metadataPath: string): ParsedMetadataR } catch { return { kind: 'legacy' } } + if (!parsed || typeof parsed !== 'object') return { kind: 'legacy' } const candidate = parsed as Partial<CodexSidecarOwnershipMetadata> if (candidate.schemaVersion !== OWNERSHIP_SCHEMA_VERSION) return { kind: 'legacy' } + const ownershipId = typeof candidate.ownershipId === 'string' ? candidate.ownershipId : metadataPath if ( typeof candidate.ownershipId !== 'string' || typeof candidate.serverInstanceId !== 'string' || !isPositiveInteger(candidate.ownerServerPid) - || (candidate.terminalId !== null && typeof candidate.terminalId !== 'string') - || (candidate.generation !== null && !isNonNegativeInteger(candidate.generation)) || typeof candidate.wsUrl !== 'string' || !isPositiveInteger(candidate.wrapperPid) || !isPositiveInteger(candidate.processGroupId) - || !isWrapperIdentity(candidate.wrapperIdentity) + || !candidate.wrapperIdentity || typeof candidate.createdAt !== 'string' || typeof candidate.updatedAt !== 'string' ) { @@ -406,28 +447,27 @@ function parseMetadataRecord(raw: string, metadataPath: string): ParsedMetadataR } export async function reapOrphanedCodexAppServerSidecars( - options: ReapOrphanedSidecarsOptions, + options: ReapOrphanedSidecarsOptions = {}, ): Promise<ReapOrphanedSidecarsResult> { + assertUnixSidecarSupport() const metadataDir = options.metadataDir ?? defaultMetadataDir() const result: ReapOrphanedSidecarsResult = { + scanned: 0, reapedOwnershipIds: [], - ignoredLegacyRecords: [], skippedActiveOwnershipIds: [], + ignoredLegacyRecords: [], failedOwnershipIds: [], } - let procOwnershipProofChecked = false - const ensureProcOwnershipProof = async () => { - if (procOwnershipProofChecked) return - await assertProcOwnershipProofAvailable() - procOwnershipProofChecked = true - } + const entries = await fsp.readdir(metadataDir).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [] + const code = (error as NodeJS.ErrnoException).code + if (code === 'ENOENT' || code === 'ENOTDIR') return [] throw error }) for (const entry of entries) { if (!entry.endsWith('.json')) continue + result.scanned += 1 const metadataPath = path.join(metadataDir, entry) const raw = await fsp.readFile(metadataPath, 'utf8') const parsed = parseMetadataRecord(raw, metadataPath) @@ -441,9 +481,7 @@ export async function reapOrphanedCodexAppServerSidecars( continue } - await ensureProcOwnershipProof() const metadata = parsed.metadata - if (await isPidAlive(metadata.ownerServerPid)) { result.skippedActiveOwnershipIds.push(metadata.ownershipId) continue @@ -467,32 +505,24 @@ export async function reapOrphanedCodexAppServerSidecars( return result } -export async function runCodexStartupReaper( - options: ReapOrphanedSidecarsOptions, +export async function reapOrphanedCodexAppServerSidecarsOnStartup( + options: ReapOrphanedSidecarsOptions = {}, ): Promise<ReapOrphanedSidecarsResult> { const result = await reapOrphanedCodexAppServerSidecars(options) - assertCodexStartupReaperSucceeded(result) - return result -} - -export const reapOrphanedCodexAppServerSidecarsOnStartup = runCodexStartupReaper - -export function assertCodexStartupReaperSucceeded(result: ReapOrphanedSidecarsResult): void { - const unreapedOwnershipIds = [ - ...result.failedOwnershipIds, - ...result.skippedActiveOwnershipIds, - ] - if (unreapedOwnershipIds.length === 0) return + const blockedOwnershipIds = result.failedOwnershipIds + if (blockedOwnershipIds.length === 0) return result - const blockedOwnershipIds = [...new Set(unreapedOwnershipIds)] throw new Error( `Codex app-server startup reaper failed to reap ${blockedOwnershipIds.length} ownership record(s): ${blockedOwnershipIds.join(', ')}. ` + 'Refusing to continue until the unreaped Codex sidecar ownership is verified gone or handled explicitly.', ) } +export const runCodexStartupReaper = reapOrphanedCodexAppServerSidecarsOnStartup + export class CodexAppServerRuntime { - private child: ChildProcessHandle | null = null + private child: ChildProcess | null = null + private childDiagnostics: RuntimeChildDiagnostics | null = null private client: CodexAppServerClient | null = null private ready: ReadyState | null = null private ensureReadyPromise: Promise<ReadyState> | null = null @@ -501,7 +531,6 @@ export class CodexAppServerRuntime { private ownershipTeardownPromise: Promise<void> | null = null private ownershipTeardownFailure: Error | null = null private shutdownRequested = false - private lifecycleLossHandlers = new Set<(event: CodexThreadLifecycleLossEvent) => void>() private readonly exitHandlers = new Set<(error?: Error, source?: CodexAppServerRuntimeFailureSource) => void>() private readonly threadStartedHandlers = new Set<(thread: CodexThreadHandle) => void>() private readonly threadLifecycleHandlers = new Set<(event: CodexThreadLifecycleEvent) => void>() @@ -509,10 +538,12 @@ export class CodexAppServerRuntime { private readonly command: string private readonly commandArgs: string[] + private readonly cwd?: string private readonly env?: NodeJS.ProcessEnv private readonly requestTimeoutMs?: number private readonly startupAttemptLimit: number private readonly startupAttemptTimeoutMs: number + private readonly terminateGraceMs: number private readonly portAllocator: () => Promise<LoopbackServerEndpoint> private readonly metadataDir: string private readonly serverInstanceId: string @@ -523,10 +554,12 @@ export class CodexAppServerRuntime { constructor(options: RuntimeOptions = {}) { this.command = options.command ?? (process.env.CODEX_CMD || 'codex') this.commandArgs = options.commandArgs ?? [] + this.cwd = resolveAppServerCwd(options.cwd) this.env = options.env this.requestTimeoutMs = options.requestTimeoutMs this.startupAttemptLimit = options.startupAttemptLimit ?? DEFAULT_STARTUP_ATTEMPT_LIMIT this.startupAttemptTimeoutMs = options.startupAttemptTimeoutMs ?? DEFAULT_STARTUP_ATTEMPT_TIMEOUT_MS + this.terminateGraceMs = options.terminateGraceMs ?? DEFAULT_TERMINATE_GRACE_MS this.portAllocator = options.portAllocator ?? allocateLocalhostPort this.metadataDir = options.metadataDir ?? defaultMetadataDir() this.serverInstanceId = options.serverInstanceId ?? process.env.FRESHELL_SERVER_INSTANCE_ID ?? `srv-${process.pid}` @@ -539,6 +572,34 @@ export class CodexAppServerRuntime { return this.statusValue } + onExit(handler: (error?: Error, source?: CodexAppServerRuntimeFailureSource) => void): () => void { + this.exitHandlers.add(handler) + return () => { + this.exitHandlers.delete(handler) + } + } + + onThreadStarted(handler: (thread: CodexThreadHandle) => void): () => void { + this.threadStartedHandlers.add(handler) + return () => { + this.threadStartedHandlers.delete(handler) + } + } + + onThreadLifecycle(handler: (event: CodexThreadLifecycleEvent) => void): () => void { + this.threadLifecycleHandlers.add(handler) + return () => { + this.threadLifecycleHandlers.delete(handler) + } + } + + onFsChanged(handler: (event: { watchId: string; changedPaths: string[] }) => void): () => void { + this.fsChangedHandlers.add(handler) + return () => { + this.fsChangedHandlers.delete(handler) + } + } + async ensureReady(): Promise<ReadyState> { if (this.shutdownRequested) { throw new Error('Codex app-server sidecar is shutting down.') @@ -551,14 +612,18 @@ export class CodexAppServerRuntime { this.ensureReadyPromise = null }) - this.ready = await this.ensureReadyPromise + return this.publishReady(await this.ensureReadyPromise) + } + + private publishReady(ready: ReadyState): ReadyState { + this.ready = ready this.statusValue = 'running' - return this.ready + return ready } async startThread( params: Omit<CodexThreadStartParams, 'experimentalRawEvents' | 'persistExtendedHistory'>, - ): Promise<{ threadId: string; wsUrl: string }> { + ): Promise<CodexThreadOperationResult & { wsUrl: string }> { const ready = await this.ensureReady() return { ...(await this.client!.startThread(params)), @@ -568,7 +633,7 @@ export class CodexAppServerRuntime { async resumeThread( params: Omit<CodexThreadResumeParams, 'persistExtendedHistory'>, - ): Promise<{ threadId: string; wsUrl: string }> { + ): Promise<CodexThreadOperationResult & { wsUrl: string }> { const ready = await this.ensureReady() return { ...(await this.client!.resumeThread(params)), @@ -576,9 +641,14 @@ export class CodexAppServerRuntime { } } - async listLoadedThreads(): Promise<string[]> { + async watchPath(targetPath: string, watchId: string): Promise<CodexFsWatchResult> { await this.ensureReady() - return this.client!.listLoadedThreads() + return this.client!.watchPath(targetPath, watchId) + } + + async unwatchPath(watchId: string): Promise<void> { + await this.ensureReady() + await this.client!.unwatchPath(watchId) } async updateOwnershipMetadata(input: { @@ -588,7 +658,7 @@ export class CodexAppServerRuntime { }): Promise<void> { await this.assertNoBlockedOwnership('update Codex app-server ownership metadata') if (!this.ownership) { - throw new Error('Cannot update Codex app-server ownership metadata because no active owned Codex app-server sidecar exists.') + return } this.ownership.metadata = { ...this.ownership.metadata, @@ -597,52 +667,7 @@ export class CodexAppServerRuntime { ...(input.codexHome !== undefined ? { codexHome: input.codexHome } : {}), updatedAt: new Date().toISOString(), } - await atomicWriteJson(this.ownership.metadataPath, this.ownership.metadata) - } - - onThreadLifecycleLoss(handler: (event: CodexThreadLifecycleLossEvent) => void): () => void { - this.lifecycleLossHandlers.add(handler) - return () => { - this.lifecycleLossHandlers.delete(handler) - } - } - - onExit(handler: (error?: Error, source?: CodexAppServerRuntimeFailureSource) => void): () => void { - this.exitHandlers.add(handler) - return () => { - this.exitHandlers.delete(handler) - } - } - - onThreadStarted(handler: (thread: CodexThreadHandle) => void): () => void { - this.threadStartedHandlers.add(handler) - return () => { - this.threadStartedHandlers.delete(handler) - } - } - - onThreadLifecycle(handler: (event: CodexThreadLifecycleEvent) => void): () => void { - this.threadLifecycleHandlers.add(handler) - return () => { - this.threadLifecycleHandlers.delete(handler) - } - } - - onFsChanged(handler: (event: { watchId: string; changedPaths: string[] }) => void): () => void { - this.fsChangedHandlers.add(handler) - return () => { - this.fsChangedHandlers.delete(handler) - } - } - - async watchPath(targetPath: string, watchId: string): Promise<CodexFsWatchResult> { - await this.ensureReady() - return this.client!.watchPath(targetPath, watchId) - } - - async unwatchPath(watchId: string): Promise<void> { - await this.ensureReady() - await this.client!.unwatchPath(watchId) + await this.writeOwnershipRecord(this.ownership) } async shutdown(): Promise<void> { @@ -682,10 +707,8 @@ export class CodexAppServerRuntime { if (this.shutdownRequested) { throw new Error('Codex app-server startup was cancelled because the sidecar is shutting down.') } + const endpoint = await this.portAllocator() - if (this.shutdownRequested) { - throw new Error('Codex app-server startup was cancelled because the sidecar is shutting down.') - } const wsUrl = `ws://${endpoint.hostname}:${endpoint.port}` const ownershipId = this.ownershipIdFactory() const child = spawn(this.command, [ @@ -695,6 +718,7 @@ export class CodexAppServerRuntime { wsUrl, ], { detached: true, + ...(this.cwd ? { cwd: this.cwd } : {}), env: { ...process.env, ...this.env, @@ -702,22 +726,31 @@ export class CodexAppServerRuntime { }, stdio: ['ignore', 'pipe', 'pipe'], }) - const childErrorPromise = this.watchChildError(child) - // Drain child stdio continuously so verbose app-server or MCP startup logs - // cannot fill the pipe buffer and stall JSON-RPC request handling. - child.stdout?.resume() - child.stderr?.resume() + const childDiagnostics: RuntimeChildDiagnostics = { + wsUrl, + wsPort: endpoint.port, + startedAt: Date.now(), + stdoutTail: new BoundedOutputTail(), + stderrTail: new BoundedOutputTail(), + } + + child.stdout?.on('data', (chunk) => childDiagnostics.stdoutTail.push(chunk)) + child.stderr?.on('data', (chunk) => childDiagnostics.stderrTail.push(chunk)) this.child = child - this.attachChildExitHandler(child) + this.childDiagnostics = childDiagnostics + this.attachChildErrorHandler(child, childDiagnostics) + this.attachChildExitHandler(child, childDiagnostics) let attemptOwnership: ActiveOwnership | null = null try { if (!child.pid) { const launchError = await Promise.race([ - childErrorPromise, - sleep(25).then(() => null), + new Promise<Error | null>((resolve) => { + child.once('error', (error) => resolve(error instanceof Error ? error : new Error(String(error)))) + setTimeout(() => resolve(null), 25) + }), ]) if (launchError) throw launchError throw new Error('Codex app-server sidecar spawn did not expose a wrapper PID.') @@ -740,8 +773,11 @@ export class CodexAppServerRuntime { { wsUrl }, this.requestTimeoutMs ? { requestTimeoutMs: this.requestTimeoutMs } : {}, ) - client.onThreadLifecycleLoss((event) => { - for (const handler of this.lifecycleLossHandlers) { + client.onDisconnect((event) => { + this.handleClientDisconnect(client, event) + }) + client.onThreadLifecycle((event) => { + for (const handler of this.threadLifecycleHandlers) { handler(event) } }) @@ -750,30 +786,19 @@ export class CodexAppServerRuntime { handler(thread) } }) - client.onThreadLifecycle((event) => { - for (const handler of this.threadLifecycleHandlers) { - handler(event) - } - }) client.onFsChanged((event) => { for (const handler of this.fsChangedHandlers) { handler(event) } }) - client.onDisconnect((event) => { - if (this.shutdownRequested) return - for (const handler of this.exitHandlers) { - handler(event.error, 'app_server_client_disconnect') - } - }) this.client = client - const initialized = await this.waitForInitialize(client, child, childErrorPromise) + const initialized = await this.waitForInitialize(client, child, childDiagnostics) await this.updateOwnershipMetadata({ codexHome: initialized.codexHome }) - this.statusValue = 'running' return { wsUrl, processPid: child.pid, + codexHome: initialized.codexHome, ownershipId, processGroupId: child.pid, metadataPath: ownership.metadataPath, @@ -789,24 +814,15 @@ export class CodexAppServerRuntime { } if (attemptOwnership) { await this.beginOwnershipTeardown(attemptOwnership).catch((teardownError) => { - if (lastError) { - lastError = new Error(`${lastError.message}; teardown failed: ${teardownError instanceof Error ? teardownError.message : String(teardownError)}`) - } else { - lastError = teardownError instanceof Error ? teardownError : new Error(String(teardownError)) - } + lastError = new Error(`${lastError?.message ?? 'startup failed'}; teardown failed: ${teardownError instanceof Error ? teardownError.message : String(teardownError)}`) throw lastError }) } else { await this.stopActiveChild() } - if (this.shutdownRequested) break } } - if (this.shutdownRequested) { - throw lastError ?? new Error('Codex app-server startup was cancelled because the sidecar is shutting down.') - } - throw new Error( `Failed to start Codex app-server on a loopback endpoint after ${this.startupAttemptLimit} attempts: ${lastError?.message ?? 'unknown error'}`, ) @@ -863,28 +879,28 @@ export class CodexAppServerRuntime { private async waitForInitialize( client: CodexAppServerClient, - child: ChildProcessHandle, - childErrorPromise: Promise<Error>, + child: ChildProcess, + diagnostics: RuntimeChildDiagnostics, ): Promise<CodexInitializeResult> { const deadline = Date.now() + this.startupAttemptTimeoutMs let lastError: Error | undefined while (Date.now() < deadline) { + if (diagnostics.processError) { + throw this.createUnexpectedExitError( + child, + diagnostics, + child.exitCode, + child.signalCode, + `Codex app-server runtime failed to start: ${diagnostics.processError.message}`, + ) + } if (child.exitCode !== null || child.signalCode !== null) { break } - const remainingMs = Math.max(0, deadline - Date.now()) try { - return await Promise.race([ - client.initialize(), - childErrorPromise.then((error) => { - throw error - }), - sleep(remainingMs).then(() => { - throw new Error(`Codex app-server did not finish initialize within ${this.startupAttemptTimeoutMs}ms.`) - }), - ]) + return await client.initialize() } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)) await sleep(STARTUP_POLL_MS) @@ -894,79 +910,167 @@ export class CodexAppServerRuntime { throw lastError ?? new Error('Codex app-server exited before it finished initializing.') } - private watchChildError(child: ChildProcessHandle): Promise<Error> { - return new Promise((resolve) => { - child.once('error', (error) => { - const base = error instanceof Error ? error : new Error(String(error)) - const launchError = new Error(`Failed to launch Codex app-server sidecar: ${base.message}`) - ;(launchError as Error & { code?: string; cause?: unknown }).code = - (base as NodeJS.ErrnoException).code - ;(launchError as Error & { code?: string; cause?: unknown }).cause = base - if (this.child === child) { - this.child = null - this.ready = null - this.ensureReadyPromise = null - this.statusValue = 'stopped' - } - resolve(launchError) - }) + private attachChildErrorHandler(child: ChildProcess, diagnostics: RuntimeChildDiagnostics): void { + child.once('error', (error) => { + diagnostics.processError = error instanceof Error ? error : new Error(String(error)) + if (this.child !== child) { + return + } + + const wasReady = this.ready !== null + const ownership = this.ownership + this.child = null + this.childDiagnostics = null + this.ready = null + this.statusValue = 'stopped' + + const client = this.client + this.client = null + const closeClient = client?.close().catch(() => undefined) + if (ownership) { + void this.beginOwnershipTeardown(ownership, closeClient).catch((teardownError) => { + logger.error({ err: teardownError, ownershipId: ownership.metadata.ownershipId }, 'Codex app-server sidecar teardown after wrapper error failed') + }) + } else { + void closeClient + } + + if (!wasReady || this.shutdownRequested) { + return + } + + const runtimeError = this.createUnexpectedExitError( + child, + diagnostics, + child.exitCode, + child.signalCode, + `Codex app-server runtime errored unexpectedly: ${diagnostics.processError.message}`, + ) + for (const handler of this.exitHandlers) { + handler(runtimeError, 'app_server_exit') + } }) } - private attachChildExitHandler(child: ChildProcessHandle): void { - child.once('exit', () => { + private attachChildExitHandler(child: ChildProcess, diagnostics: RuntimeChildDiagnostics): void { + child.once('exit', (code, signal) => { if (this.child !== child) { return } + const wasReady = this.ready !== null const ownership = this.ownership this.child = null + this.childDiagnostics = null this.ready = null - this.ensureReadyPromise = null this.statusValue = 'stopped' const client = this.client this.client = null const closeClient = client?.close().catch(() => undefined) if (ownership) { - void this.beginOwnershipTeardown(ownership, closeClient).catch((error) => { - logger.error( - { - err: error, - ownershipId: ownership.metadata.ownershipId, - terminalId: ownership.metadata.terminalId, - generation: ownership.metadata.generation, - wsUrl: ownership.metadata.wsUrl, - wrapperPid: ownership.metadata.wrapperPid, - processGroupId: ownership.metadata.processGroupId, - serverInstanceId: ownership.metadata.serverInstanceId, - }, - 'Codex app-server sidecar teardown after wrapper exit failed', - ) + void this.beginOwnershipTeardown(ownership, closeClient).catch((teardownError) => { + logger.error({ err: teardownError, ownershipId: ownership.metadata.ownershipId }, 'Codex app-server sidecar teardown after wrapper exit failed') }) } else { void closeClient } - if (!this.shutdownRequested) { - for (const handler of this.exitHandlers) { - handler(undefined, 'app_server_exit') - } + if (!wasReady || this.shutdownRequested) { + return + } + + const error = this.createUnexpectedExitError(child, diagnostics, code, signal) + for (const handler of this.exitHandlers) { + handler(error, 'app_server_exit') } }) } + private handleClientDisconnect(client: CodexAppServerClient, event: CodexAppServerDisconnectEvent): void { + if (this.client !== client) { + return + } + + const child = this.child + const diagnostics = this.childDiagnostics + const wasReady = this.ready !== null + this.client = null + this.ready = null + this.statusValue = 'stopped' + + if (!wasReady || this.shutdownRequested) { + void this.stopActiveChild().catch(() => undefined) + return + } + + const error = child && diagnostics + ? this.createUnexpectedExitError( + child, + diagnostics, + child.exitCode, + child.signalCode, + event.reason === 'error' + ? `Codex app-server client socket errored: ${event.error?.message ?? 'unknown error'}` + : 'Codex app-server client socket closed unexpectedly.', + ) + : new Error(event.reason === 'error' + ? `Codex app-server client socket errored: ${event.error?.message ?? 'unknown error'}` + : 'Codex app-server client socket closed unexpectedly.') + for (const handler of this.exitHandlers) { + handler(error, 'app_server_client_disconnect') + } + void this.stopActiveChild().catch(() => undefined) + } + + private createUnexpectedExitError( + child: ChildProcess, + diagnostics: RuntimeChildDiagnostics, + code: number | null, + signal: NodeJS.Signals | null, + prefix = 'Codex app-server runtime exited unexpectedly.', + ): Error { + const elapsedMs = Date.now() - diagnostics.startedAt + const stdoutTail = diagnostics.stdoutTail.snapshot() + const stderrTail = diagnostics.stderrTail.snapshot() + return new Error([ + prefix, + `pid ${child.pid ?? 'unknown'}`, + `ws port ${diagnostics.wsPort}`, + `ws url ${diagnostics.wsUrl}`, + `exit code ${code ?? 'unknown'}`, + `signal ${signal ?? 'none'}`, + `elapsed ${elapsedMs}ms`, + `stdout tail: ${stdoutTail || '(empty)'}`, + `stderr tail: ${stderrTail || '(empty)'}`, + ].join(' ')) + } + private async stopActiveChild(): Promise<void> { - const ownership = this.ownership const child = this.child + const ownership = this.ownership this.child = null + this.childDiagnostics = null this.ready = null this.statusValue = 'stopped' if (!ownership) { - if (child && child.exitCode === null && child.signalCode === null) { - child.kill('SIGTERM') + if (!child || child.exitCode !== null || child.signalCode !== null) { + return } + child.kill('SIGTERM') + await new Promise<void>((resolve) => { + const timeout = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL') + } + resolve() + }, this.terminateGraceMs) + child.once('exit', () => { + clearTimeout(timeout) + resolve() + }) + }) return } @@ -1015,13 +1119,13 @@ export class CodexAppServerRuntime { const teardown = (async () => { await beforeTeardown?.catch(() => undefined) try { - const stopped = await teardownOwnedProcessGroup(ownership, DEFAULT_TERMINATE_GRACE_MS) + const stopped = await teardownOwnedProcessGroup(ownership, this.terminateGraceMs, { activeOwner: true }) if (!stopped) { throw new Error( `Codex app-server sidecar process-group teardown failed for ownership ${ownership.metadata.ownershipId}.`, ) } - if (stopped && this.ownership === ownership) { + if (this.ownership === ownership) { this.ownership = null } this.ownershipTeardownFailure = null diff --git a/server/coding-cli/codex-app-server/sidecar.ts b/server/coding-cli/codex-app-server/sidecar.ts new file mode 100644 index 000000000..0c89e8380 --- /dev/null +++ b/server/coding-cli/codex-app-server/sidecar.ts @@ -0,0 +1,218 @@ +import { + CodexDurableRolloutTracker, + type CodexDurableRolloutTrackerOptions, +} from './durable-rollout-tracker.js' +import type { CodexThreadHandle } from './protocol.js' +import type { CodexThreadLifecycleEvent } from './client.js' +import { + CodexAppServerRuntime, + reapOrphanedCodexAppServerSidecarsOnStartup, + type CodexAppServerRuntimeFailureSource, +} from './runtime.js' + +const MAX_PENDING_LIFECYCLE_EVENTS = 10 + +type CodexSidecarReady = { + wsUrl: string + processPid: number + codexHome: string +} + +type CodexTerminalFatalSource = CodexAppServerRuntimeFailureSource | 'sidecar_fatal' + +type CodexTerminalAttachment = { + terminalId: string + onDurableSession: (sessionId: string) => void + onThreadLifecycle: (event: CodexThreadLifecycleEvent) => void + onFatal: (error: Error, source: CodexTerminalFatalSource) => void +} + +type CodexTerminalSidecarOptions = { + cwd?: string + commandArgs?: string[] + env?: NodeJS.ProcessEnv + runtime?: CodexAppServerRuntime + createDurableRolloutTracker?: ( + options: CodexDurableRolloutTrackerOptions, + ) => Pick<CodexDurableRolloutTracker, 'trackThread' | 'dispose'> +} + +export class CodexTerminalSidecar { + private readonly runtime: CodexAppServerRuntime + private readonly durableRolloutTracker: Pick<CodexDurableRolloutTracker, 'trackThread' | 'dispose'> + private readonly cleanupRuntimeExit: () => void + private readonly cleanupThreadStarted: () => void + private readonly cleanupThreadLifecycle: () => void + + private ready: CodexSidecarReady | null = null + private readyPromise: Promise<CodexSidecarReady> | null = null + private attachedTerminal: CodexTerminalAttachment | null = null + private shuttingDown = false + private pendingFatal: { error: Error; source: CodexTerminalFatalSource } | null = null + private durableSessionId: string | null = null + private readonly pendingLifecycleEvents: CodexThreadLifecycleEvent[] = [] + private readonly observedThreadStartedIds = new Set<string>() + + constructor(options: CodexTerminalSidecarOptions = {}) { + this.runtime = options.runtime ?? new CodexAppServerRuntime( + { + ...(options.cwd ? { cwd: options.cwd } : {}), + ...(options.commandArgs ? { commandArgs: options.commandArgs } : {}), + ...(options.env ? { env: options.env } : {}), + }, + ) + const createDurableRolloutTracker = options.createDurableRolloutTracker + ?? ((trackerOptions: CodexDurableRolloutTrackerOptions) => new CodexDurableRolloutTracker(trackerOptions)) + this.durableRolloutTracker = createDurableRolloutTracker({ + watchPath: (targetPath, watchId) => this.runtime.watchPath(targetPath, watchId), + unwatchPath: async (watchId) => { + if (this.runtime.status() !== 'running') { + return + } + await this.runtime.unwatchPath(watchId) + }, + subscribeToFsChanged: (handler) => this.runtime.onFsChanged(handler), + onDurableRollout: (sessionId) => { + this.promoteDurableSession(sessionId) + }, + }) + this.cleanupRuntimeExit = this.runtime.onExit((error, source) => { + if (this.shuttingDown) { + return + } + this.handleFatal( + error ?? new Error('Codex app-server sidecar exited unexpectedly.'), + source ?? 'app_server_exit', + ) + }) + this.cleanupThreadLifecycle = this.runtime.onThreadLifecycle((event) => { + this.noteThreadLifecycle(event) + }) + this.cleanupThreadStarted = this.runtime.onThreadStarted((thread) => { + this.noteThreadStarted(thread) + }) + } + + async ensureReady(): Promise<CodexSidecarReady> { + if (this.ready) { + return this.ready + } + if (this.readyPromise) { + return this.readyPromise + } + + this.readyPromise = this.runtime.ensureReady() + .then(async (ready) => { + this.ready = ready + await this.updateOwnershipMetadata() + return ready + }) + .finally(() => { + this.readyPromise = null + }) + + return this.readyPromise + } + + attachTerminal(input: CodexTerminalAttachment): void { + this.attachedTerminal = input + void this.updateOwnershipMetadata() + if (this.durableSessionId) { + input.onDurableSession(this.durableSessionId) + } + for (const event of this.pendingLifecycleEvents) { + input.onThreadLifecycle(event) + } + if (this.pendingFatal) { + input.onFatal(this.pendingFatal.error, this.pendingFatal.source) + return + } + } + + async shutdown(): Promise<void> { + this.shuttingDown = true + this.cleanupRuntimeExit() + this.cleanupThreadLifecycle() + this.cleanupThreadStarted() + this.pendingLifecycleEvents.length = 0 + await this.durableRolloutTracker.dispose() + await this.runtime.shutdown() + this.ready = null + this.readyPromise = null + this.attachedTerminal = null + } + + private noteThreadStarted(thread: CodexThreadHandle): void { + if (!this.recordThreadStarted(thread)) { + return + } + this.forwardThreadLifecycle({ + kind: 'thread_started', + thread, + }) + } + + private noteThreadLifecycle(event: CodexThreadLifecycleEvent): void { + if (this.shuttingDown) { + return + } + if (event.kind === 'thread_started') { + if (!this.recordThreadStarted(event.thread)) { + return + } + } + this.forwardThreadLifecycle(event) + } + + private recordThreadStarted(thread: CodexThreadHandle): boolean { + if (!thread.id || this.shuttingDown || this.observedThreadStartedIds.has(thread.id)) { + return false + } + this.observedThreadStartedIds.add(thread.id) + if (!this.durableSessionId) { + this.durableRolloutTracker.trackThread(thread) + } + return true + } + + private forwardThreadLifecycle(event: CodexThreadLifecycleEvent): void { + const terminal = this.attachedTerminal + if (terminal) { + terminal.onThreadLifecycle(event) + return + } + this.pendingLifecycleEvents.push(event) + if (this.pendingLifecycleEvents.length > MAX_PENDING_LIFECYCLE_EVENTS) { + this.pendingLifecycleEvents.splice(0, this.pendingLifecycleEvents.length - MAX_PENDING_LIFECYCLE_EVENTS) + } + } + + private promoteDurableSession(threadId: string): void { + if (this.durableSessionId || this.shuttingDown) { + return + } + this.durableSessionId = threadId + this.attachedTerminal?.onDurableSession(threadId) + } + + private handleFatal(error: Error, source: CodexTerminalFatalSource = 'sidecar_fatal'): void { + this.pendingFatal = { error, source } + this.attachedTerminal?.onFatal(error, source) + } + + private async updateOwnershipMetadata(): Promise<void> { + const ready = this.ready + if (!ready) { + return + } + + await this.runtime.updateOwnershipMetadata({ + codexHome: ready.codexHome, + terminalId: this.attachedTerminal?.terminalId ?? null, + }) + } + + static async reapOrphanedSidecars(): Promise<void> { + await reapOrphanedCodexAppServerSidecarsOnStartup() + } +} diff --git a/server/index.ts b/server/index.ts index d443fae17..0e10e6c01 100644 --- a/server/index.ts +++ b/server/index.ts @@ -78,6 +78,7 @@ import { runCodexStartupReaper, } from './coding-cli/codex-app-server/runtime.js' import { CodexLaunchPlanner } from './coding-cli/codex-app-server/launch-planner.js' +import { CodexTerminalSidecar } from './coding-cli/codex-app-server/sidecar.js' import { registerStaticClientRoutes } from './static-client-routes.js' import { joinCodexShutdownOwners } from './shutdown-join.js' @@ -298,7 +299,14 @@ async function main() { sdkBridge = new SdkBridge(agentHistorySource) const server = http.createServer(app) - const codexLaunchPlanner = new CodexLaunchPlanner(() => new CodexAppServerRuntime({ serverInstanceId })) + const codexLaunchPlanner = new CodexLaunchPlanner((input) => new CodexTerminalSidecar({ + runtime: new CodexAppServerRuntime({ + serverInstanceId, + cwd: input.cwd, + commandArgs: input.commandArgs, + env: input.env, + }), + })) const wsHandler = new WsHandler( server, registry, diff --git a/server/terminal-registry.ts b/server/terminal-registry.ts index 8e0395bda..c266ffe39 100644 --- a/server/terminal-registry.ts +++ b/server/terminal-registry.ts @@ -22,12 +22,26 @@ import type { TerminalSessionBoundEvent, TerminalSessionUnboundEvent, } from './terminal-stream/registry-events.js' +import type { CodexTerminalSidecar } from './coding-cli/codex-app-server/sidecar.js' +import type { CodexThreadLifecycleEvent } from './coding-cli/codex-app-server/client.js' +import type { CodexLaunchFactory, CodexLaunchPlan } from './coding-cli/codex-app-server/launch-planner.js' +import { + CODEX_RECOVERY_INPUT_BUFFER_TTL_MS, + CodexRecoveryPolicy, + type CodexRecoveryState, + type CodexWorkerCloseReason, + type CodexWorkerFailureSource, +} from './coding-cli/codex-app-server/recovery-policy.js' +import { CodexRemoteTuiFailureDetector } from './coding-cli/codex-app-server/remote-tui-failure-detector.js' import { getOpencodeEnvOverrides, resolveOpencodeLaunchModel } from './opencode-launch.js' import { generateMcpInjection, cleanupMcpConfig } from './mcp/config-writer.js' -import type { CodexLaunchPlan, CodexLaunchSidecar } from './coding-cli/codex-app-server/launch-planner.js' -import { isCodexSidecarTeardownError } from './coding-cli/codex-app-server/launch-planner.js' +import { + createTerminalStartupProbeState, + extractTerminalStartupProbes, + type TerminalStartupProbeColors, + type TerminalStartupProbeState, +} from '../shared/terminal-startup-probes.js' import { collectShutdownFailures, throwShutdownFailures } from './shutdown-join.js' -import { recordSessionLifecycleEvent } from './session-observability.js' const MAX_WS_BUFFERED_AMOUNT = Number(process.env.MAX_WS_BUFFERED_AMOUNT || 2 * 1024 * 1024) const DEFAULT_MAX_SCROLLBACK_CHARS = Number(process.env.MAX_SCROLLBACK_CHARS || 512 * 1024) @@ -40,6 +54,15 @@ const OUTPUT_FLUSH_MS = Number(process.env.OUTPUT_FLUSH_MS || process.env.MOBILE const MAX_OUTPUT_BUFFER_CHARS = Number(process.env.MAX_OUTPUT_BUFFER_CHARS || process.env.MAX_MOBILE_OUTPUT_BUFFER_CHARS || 256 * 1024) const MAX_OUTPUT_FRAME_CHARS = Math.max(1, Number(process.env.MAX_OUTPUT_FRAME_CHARS || 8192)) const perfConfig = getPerfConfig() +const PREATTACH_CODEX_STARTUP_PROBE_COLORS: TerminalStartupProbeColors = { + foreground: '#c9d1d9', + background: '#0d1117', + cursor: '#c9d1d9', +} +const CODEX_RECOVERY_READINESS_TIMEOUT_MS = Number(process.env.CODEX_RECOVERY_READINESS_TIMEOUT_MS || 5_000) +const CODEX_PRE_DURABLE_STABILITY_MS = Number(process.env.CODEX_PRE_DURABLE_STABILITY_MS || 1_500) +const CODEX_RECOVERY_INPUT_NOT_SENT_MESSAGE = + '\r\n[Freshell] Codex is reconnecting; input was not sent because recovery is still in progress.\r\n' // TerminalMode is now a wider type -- any string is valid as a mode name. // 'shell' is the only built-in; all CLI modes come from registered extensions. @@ -176,25 +199,25 @@ export type ProviderSettings = { sandbox?: string codexAppServer?: { wsUrl: string - sidecar?: CodexLaunchSidecar - recovery?: CodexRecoveryOptions - deferLifecycleUntilPublished?: boolean } opencodeServer?: LoopbackServerEndpoint } -export type CodexRecoveryLaunchInput = { - terminalId: string - generation: number - cwd?: string - resumeSessionId: string -} +export type TerminalEnvContext = { tabId?: string; paneId?: string } -export type CodexRecoveryOptions = { - planCreate(input: CodexRecoveryLaunchInput): Promise<CodexLaunchPlan> - retryDelayMs?: number - readinessTimeoutMs?: number - readinessPollMs?: number +export function buildFreshellTerminalEnv( + terminalId: string, + envContext?: TerminalEnvContext, +): Record<string, string> { + const port = Number(process.env.PORT || 3001) + return { + FRESHELL: '1', + FRESHELL_URL: process.env.FRESHELL_URL || `http://localhost:${port}`, + FRESHELL_TOKEN: process.env.AUTH_TOKEN || '', + FRESHELL_TERMINAL_ID: terminalId, + ...(envContext?.tabId ? { FRESHELL_TAB_ID: envContext.tabId } : {}), + ...(envContext?.paneId ? { FRESHELL_PANE_ID: envContext.paneId } : {}), + } } function resolveCodingCliCommand( @@ -364,34 +387,6 @@ function wrapTerminalSpawnError( return wrapped } -type CodexRecoveryTeardownError = Error & { - codexRecoveryTeardownFailed?: boolean -} - -function codexRecoveryTeardownError(message: string): CodexRecoveryTeardownError { - const error = new Error(message) as CodexRecoveryTeardownError - error.codexRecoveryTeardownFailed = true - return error -} - -export function terminalIdFromCreateError(error: unknown): string | undefined { - if (!error || (typeof error !== 'object' && typeof error !== 'function')) return undefined - const terminalId = (error as { terminalId?: unknown }).terminalId - return typeof terminalId === 'string' ? terminalId : undefined -} - -function attachTerminalIdToCreateError(error: unknown, terminalId: string): unknown { - const target: { terminalId?: string } = error && (typeof error === 'object' || typeof error === 'function') - ? error as { terminalId?: string } - : new Error(String(error)) as Error & { terminalId?: string } - try { - target.terminalId ??= terminalId - } catch { - // Preserve the original failure even if the thrown value rejects mutation. - } - return target -} - type PendingSnapshotQueue = { chunks: string[] queuedChars: number @@ -404,19 +399,12 @@ type PendingOutput = { queuedChars: number } -type SidecarShutdownEntry = { - promise: Promise<void> - status: 'pending' | 'failed' - terminalId: string - shutdownSidecar: () => Promise<void> - failureMessage: string -} - export type TerminalRecord = { terminalId: string title: string description?: string mode: TerminalMode + codexSidecar?: Pick<CodexTerminalSidecar, 'shutdown'> opencodeServer?: LoopbackServerEndpoint resumeSessionId?: string pendingResumeName?: string @@ -426,8 +414,6 @@ export type TerminalRecord = { status: 'running' | 'exited' exitCode?: number cwd?: string - shell: ShellType - envContext?: { tabId?: string; paneId?: string } /** Normalized cwd used for MCP config injection (may differ from raw cwd on WSL). */ mcpCwd?: string cols: number @@ -435,6 +421,7 @@ export type TerminalRecord = { clients: Set<WebSocket> suppressedOutputClients: Set<WebSocket> pendingSnapshotClients: Map<WebSocket, PendingSnapshotQueue> + preAttachStartupProbeState?: TerminalStartupProbeState buffer: ChunkRingBuffer pty: pty.IPty @@ -451,17 +438,80 @@ export type TerminalRecord = { lastInputToOutputMs?: number maxInputToOutputMs: number } - codexSidecar?: Pick<CodexLaunchSidecar, 'shutdown' | 'onLifecycleLoss'> - codexSidecarLifecycleUnsubscribe?: () => void - codexSidecarLifecyclePublished?: boolean - codexSidecarPrePublicationLoss?: unknown - codexSidecarGeneration?: number - codexRecovery?: CodexRecoveryOptions - codexRecoveryAttempt?: Promise<void> - codexRecoveryRetry?: { timer: NodeJS.Timeout; resolve: () => void } - codexRecoveryBlockedError?: Error - codexRecoveryFinalClose?: boolean - codexRecoveryRetiringPty?: pty.IPty + codex?: { + recoveryState: CodexRecoveryState + workerGeneration: number + nextWorkerGeneration: number + retiringGenerations: Set<number> + closeReasonByGeneration: Map<number, CodexWorkerCloseReason> + durableSessionId?: string + originalResumeSessionId?: string + currentWsUrl?: string + currentAppServerPid?: number + launchFactory?: CodexLaunchFactory + launchBaseProviderSettings?: { + model?: string + sandbox?: string + permissionMode?: string + } + envContext?: TerminalEnvContext + recoveryPolicy: CodexRecoveryPolicy + inputExpiryTimer?: NodeJS.Timeout + remoteTuiFailureDetector: CodexRemoteTuiFailureDetector + activeReplacement?: CodexActiveReplacement + } +} + +type TerminalLaunchSpec = { + terminalId: string + mode: TerminalMode + shell: ShellType + cwd?: string + cols: number + rows: number + resumeSessionId?: string + providerSettings?: ProviderSettings + envContext?: TerminalEnvContext + baseEnv: Record<string, string> +} + +type SpawnedTerminalWorker = { + pty: pty.IPty + procCwd?: string + mcpCwd?: string +} + +type TerminalRuntimeStatus = 'running' | 'recovering' + +type CodexActiveReplacement = { + id: string + attempt: number + source: CodexWorkerFailureSource + retiringGeneration: number + candidateGeneration: number + candidatePublished: boolean + aborted: boolean + retiringWsUrl?: string + retiringAppServerPid?: number + retiringPtyPid?: number + pendingReadinessSessionId?: string + pendingDurableSessionId?: string + readinessTimer?: NodeJS.Timeout + preDurableTimer?: NodeJS.Timeout + backoffTimer?: NodeJS.Timeout + candidateSidecar?: CodexLaunchPlan['sidecar'] + candidatePty?: pty.IPty + candidateMcpCwd?: string + candidateWsUrl?: string + candidateAppServerPid?: number +} + +type SidecarShutdownEntry = { + promise: Promise<void> + status: 'pending' | 'failed' + terminalId: string + shutdownSidecar: () => Promise<void> + failureMessage: string } export type BindSessionResult = @@ -1127,6 +1177,7 @@ export class TerminalRegistry extends EventEmitter { for (const term of this.terminals.values()) { if (term.status !== 'running') continue if (term.clients.size > 0) continue // only detached + if (term.mode === 'codex' && this.isCodexRecoveryProtected(term)) continue const idleMs = now - term.lastActivityAt const idleMinutes = idleMs / 60000 @@ -1151,32 +1202,12 @@ export class TerminalRegistry extends EventEmitter { return n } - private recordTerminalExitWithoutDurableSession( - record: TerminalRecord, - exitCode: number | undefined, - reason: 'pty_exit' | 'user_final_close', - ): void { - if (record.mode === 'shell' || record.resumeSessionId) { - return - } - const ptyPid = record.pty.pid - recordSessionLifecycleEvent({ - kind: 'terminal_exit_without_durable_session', - terminalId: record.terminalId, - mode: record.mode, - exitCode: exitCode ?? 0, - ageMs: Math.max(0, Date.now() - record.createdAt), - reason, - ...(ptyPid ? { ptyPid } : {}), - }) - } - private reapExitedTerminals(): void { const max = this.maxExitedTerminals if (!max || max <= 0) return const exited = Array.from(this.terminals.values()) - .filter((t) => t.status === 'exited' && !t.codexSidecar && this.sidecarShutdownPromisesForTerminal(t.terminalId).length === 0) + .filter((t) => t.status === 'exited') .sort((a, b) => (a.exitedAt ?? a.lastActivityAt) - (b.exitedAt ?? b.lastActivityAt)) const excess = exited.length - max @@ -1186,653 +1217,1070 @@ export class TerminalRegistry extends EventEmitter { } } - private buildTerminalBaseEnv( - terminalId: string, - envContext?: { tabId?: string; paneId?: string }, - ): Record<string, string> { - const port = Number(process.env.PORT || 3001) - return { - FRESHELL: '1', - FRESHELL_URL: process.env.FRESHELL_URL || `http://localhost:${port}`, - FRESHELL_TOKEN: process.env.AUTH_TOKEN || '', - FRESHELL_TERMINAL_ID: terminalId, - ...(envContext?.tabId ? { FRESHELL_TAB_ID: envContext.tabId } : {}), - ...(envContext?.paneId ? { FRESHELL_PANE_ID: envContext.paneId } : {}), - } - } - - create(opts: { - mode: TerminalMode - shell?: ShellType - cwd?: string - cols?: number - rows?: number - resumeSessionId?: string - sessionBindingReason?: SessionBindingReason - providerSettings?: ProviderSettings - envContext?: { tabId?: string; paneId?: string } - }): TerminalRecord { - this.reapExitedTerminals() - if (this.runningCount() >= this.maxTerminals) { - throw new Error(`Maximum terminal limit (${this.maxTerminals}) reached. Please close some terminals before creating new ones.`) - } - - const terminalId = nanoid() - const createdAt = Date.now() - const cols = opts.cols || 120 - const rows = opts.rows || 30 - - const cwd = opts.cwd || getDefaultCwd(this.settings) || (isWindows() ? undefined : os.homedir()) - const resumeForSpawn = normalizeResumeForSpawn(opts.mode, opts.resumeSessionId) - const resumeForBinding = normalizeResumeForBinding(opts.mode, opts.resumeSessionId) - const shell = opts.shell || 'system' - const baseEnv = this.buildTerminalBaseEnv(terminalId, opts.envContext) - + private spawnTerminalWorker(spec: TerminalLaunchSpec): SpawnedTerminalWorker { const { file, args, env, cwd: procCwd, mcpCwd } = buildSpawnSpec( - opts.mode, - cwd, - shell, - resumeForSpawn, - opts.providerSettings, - baseEnv, - terminalId, + spec.mode, + spec.cwd, + spec.shell, + spec.resumeSessionId, + spec.providerSettings, + spec.baseEnv, + spec.terminalId, ) const endSpawnTimer = startPerfTimer( 'terminal_spawn', - { terminalId, mode: opts.mode, shell }, + { terminalId: spec.terminalId, mode: spec.mode, shell: spec.shell }, { minDurationMs: perfConfig.slowTerminalCreateMs, level: 'warn' }, ) - logger.info({ terminalId, file, args, cwd: procCwd, mode: opts.mode, shell }, 'Spawning terminal') + logger.info({ + terminalId: spec.terminalId, + file, + args, + cwd: procCwd, + mode: spec.mode, + shell: spec.shell, + }, 'Spawning terminal') - let ptyProc: ReturnType<typeof pty.spawn> try { - ptyProc = pty.spawn(file, args, { + const ptyProc = pty.spawn(file, args, { name: 'xterm-256color', - cols, - rows, + cols: spec.cols, + rows: spec.rows, cwd: procCwd, env: env as any, }) + endSpawnTimer({ cwd: procCwd }) + return { + pty: ptyProc, + procCwd, + mcpCwd, + } } catch (err) { // Clean up MCP config temp files that were created before the spawn attempt. // Use mcpCwd (the Linux path passed to generateMcpInjection), not procCwd // (which may be undefined for WSL cmd/powershell paths). - cleanupMcpConfig(terminalId, opts.mode, mcpCwd) + cleanupMcpConfig(spec.terminalId, spec.mode, mcpCwd) throw wrapTerminalSpawnError(err, { - mode: opts.mode, + mode: spec.mode, file, - resumeSessionId: resumeForSpawn, + resumeSessionId: spec.resumeSessionId, }) } - endSpawnTimer({ cwd: procCwd }) - - const title = getModeLabel(opts.mode) - - const record: TerminalRecord = { - terminalId, - title, - description: undefined, - mode: opts.mode, - opencodeServer: opts.mode === 'opencode' ? opts.providerSettings?.opencodeServer : undefined, - resumeSessionId: undefined, - createdAt, - lastActivityAt: createdAt, - status: 'running', - cwd, - shell, - envContext: opts.envContext, - mcpCwd, - cols, - rows, - clients: new Set(), - suppressedOutputClients: new Set(), - pendingSnapshotClients: new Map(), - - buffer: new ChunkRingBuffer(this.scrollbackMaxChars), - pty: ptyProc, - codexSidecar: opts.mode === 'codex' ? opts.providerSettings?.codexAppServer?.sidecar : undefined, - codexSidecarLifecyclePublished: opts.mode === 'codex' - ? !opts.providerSettings?.codexAppServer?.deferLifecycleUntilPublished - : undefined, - codexSidecarGeneration: opts.mode === 'codex' ? 0 : undefined, - codexRecovery: opts.mode === 'codex' ? opts.providerSettings?.codexAppServer?.recovery : undefined, - perf: perfConfig.enabled - ? { - outBytes: 0, - outChunks: 0, - droppedMessages: 0, - inBytes: 0, - inChunks: 0, - pendingInputAt: undefined, - pendingInputBytes: 0, - pendingInputCount: 0, - lastInputBytes: undefined, - lastInputToOutputMs: undefined, - maxInputToOutputMs: 0, - } - : undefined, - } - - this.registerCodexSidecarLifecycle(record) + } - ptyProc.onData((data) => { - if (record.pty !== ptyProc) return - const now = Date.now() - record.lastActivityAt = now - record.buffer.append(data) - this.emit('terminal.output.raw', { - terminalId, - data, - at: now, - } satisfies TerminalOutputRawEvent) - if (record.perf) { - record.perf.outBytes += data.length - record.perf.outChunks += 1 - if (record.perf.pendingInputAt !== undefined) { - const lagMs = now - record.perf.pendingInputAt - record.perf.lastInputToOutputMs = lagMs - if (lagMs > record.perf.maxInputToOutputMs) { - record.perf.maxInputToOutputMs = lagMs - } - if (lagMs >= perfConfig.terminalInputLagMs) { - const key = `terminal_input_lag_${terminalId}` - if (shouldLog(key, perfConfig.rateLimitMs)) { - logPerfEvent( - 'terminal_input_lag', - { - terminalId, - mode: record.mode, - status: record.status, - lagMs, - pendingInputBytes: record.perf.pendingInputBytes, - pendingInputCount: record.perf.pendingInputCount, - lastInputBytes: record.perf.lastInputBytes, - }, - 'warn', - ) - } - } - record.perf.pendingInputAt = undefined - record.perf.pendingInputBytes = 0 - record.perf.pendingInputCount = 0 - } + private installTerminalWorkerHandlers(record: TerminalRecord, generation: number, attemptId?: string): void { + record.pty.onData((data) => { + if (record.mode === 'codex') { + if (!this.isCurrentCodexGeneration(record, generation)) return + if (record.codex?.retiringGenerations.has(generation)) return } - for (const client of record.clients) { - if (record.suppressedOutputClients.has(client)) continue - // Legacy snapshot ordering path. Broker cutover destination: - // - pendingSnapshotClients ordering -> broker attach-staging queue. - const pending = record.pendingSnapshotClients.get(client) - if (pending) { - const nextChars = pending.queuedChars + data.length - if (data.length > this.maxPendingSnapshotChars || nextChars > this.maxPendingSnapshotChars) { - // If a terminal spews output while we're sending a snapshot, queueing unboundedly can OOM the server. - // Prefer explicit resync: drop the client and let it reconnect/reattach for a fresh snapshot. - try { - client.close(4008, 'Attach snapshot queue overflow') - } catch { - // ignore - } - record.pendingSnapshotClients.delete(client) - record.clients.delete(client) - continue - } - pending.chunks.push(data) - pending.queuedChars = nextChars - continue + this.handleTerminalWorkerData(record, generation, data) + if (record.mode === 'codex') { + const fatal = record.codex?.remoteTuiFailureDetector.push(data) + if (fatal?.fatal) { + void this.handleCodexWorkerFailure( + record, + generation, + 'remote_tui_fatal_output', + new Error(`Codex remote TUI reported a fatal ${fatal.reason} condition.`), + attemptId, + ) } - this.sendTerminalOutput(client, terminalId, data, record.perf) } }) - ptyProc.onExit((e) => { - if (record.codexRecoveryRetiringPty === ptyProc) { - return - } - if (record.pty !== ptyProc) { + record.pty.onExit((e) => { + if (record.mode === 'codex') { + const codex = record.codex + if (!codex) return + if (!this.isCurrentCodexGeneration(record, generation)) return + if (codex.retiringGenerations.has(generation)) return + const closeReason = codex.closeReasonByGeneration.get(generation) + if (closeReason === 'recovery_retire') return + if (closeReason === 'user_final_close') { + this.finalizeTerminalExit(record, e.exitCode, 'user_final_close') + return + } + void this.handleCodexWorkerFailure( + record, + generation, + 'pty_exit', + new Error(`Codex worker PTY exited with code ${e.exitCode}.`), + attemptId, + ) return } - if (record.status === 'exited') { + this.finalizeTerminalExit(record, e.exitCode, 'pty_exit') + }) + } + + private isCurrentCodexGeneration(record: TerminalRecord, generation: number): boolean { + return record.codex?.workerGeneration === generation + } + + private isActiveCodexCandidate(record: TerminalRecord, generation: number, attemptId?: string): boolean { + const active = record.codex?.activeReplacement + return Boolean( + active + && !active.aborted + && active.candidateGeneration === generation + && attemptId !== undefined + && active.id === attemptId, + ) + } + + private isCodexRecoveryState(record: TerminalRecord): boolean { + return record.codex?.recoveryState === 'recovering_durable' + || record.codex?.recoveryState === 'recovering_pre_durable' + } + + private isCodexRecoveryProtected(record: TerminalRecord): boolean { + return this.isCodexRecoveryState(record) + } + + private clearCodexInputExpiryTimer(record: TerminalRecord): void { + const codex = record.codex + if (!codex?.inputExpiryTimer) return + clearTimeout(codex.inputExpiryTimer) + codex.inputExpiryTimer = undefined + } + + private scheduleCodexInputExpiryTimer(record: TerminalRecord): void { + const codex = record.codex + if (!codex || codex.inputExpiryTimer) return + codex.inputExpiryTimer = setTimeout(() => { + codex.inputExpiryTimer = undefined + if (record.status !== 'running' || !this.isCodexRecoveryState(record)) { + codex.recoveryPolicy.clearBufferedInput() return } - this.markCodexRecoveryFinalClose(record) - record.status = 'exited' - record.exitCode = e.exitCode - const now = Date.now() - record.lastActivityAt = now - record.exitedAt = now - cleanupMcpConfig(terminalId, opts.mode, record.mcpCwd) - for (const client of record.clients) { - this.flushOutputBuffer(client) - this.safeSend(client, { type: 'terminal.exit', terminalId, exitCode: e.exitCode }, { terminalId, perf: record.perf }) + const drain = codex.recoveryPolicy.drainBufferedInput() + if (!drain.ok && drain.reason === 'expired') { + this.appendLocalTerminalMessage(record, CODEX_RECOVERY_INPUT_NOT_SENT_MESSAGE) } - record.clients.clear() - record.suppressedOutputClients.clear() - record.pendingSnapshotClients.clear() - this.releaseBinding(terminalId, 'exit') - this.emit('terminal.exit', { terminalId, exitCode: e.exitCode }) - this.recordTerminalExitWithoutDurableSession(record, e.exitCode, 'pty_exit') - void this.releaseCodexSidecar(record).catch(() => undefined) - this.reapExitedTerminals() - }) + }, CODEX_RECOVERY_INPUT_BUFFER_TTL_MS + 1) + codex.inputExpiryTimer.unref?.() + } - this.terminals.set(terminalId, record) - const exactSessionId = resumeForBinding - if (modeSupportsResume(opts.mode) && exactSessionId) { - const bound = this.bindSession( - terminalId, - opts.mode as CodingCliProviderName, - exactSessionId, - opts.sessionBindingReason ?? 'resume', - ) - if (!bound.ok) { - logger.warn( - { terminalId, mode: opts.mode, sessionId: exactSessionId, reason: bound.reason }, - 'Failed to bind resume session during terminal create', - ) - } - } - if (resumeForSpawn && !resumeForBinding) { - record.pendingResumeName = resumeForSpawn - logger.info( - { terminalId, mode: opts.mode, pendingResumeName: resumeForSpawn }, - 'Terminal created with named resume; awaiting session association', - ) - } + private codexRecoveryLogContext(record: TerminalRecord, active?: CodexActiveReplacement): Record<string, unknown> { + const codex = record.codex + return { + terminalId: record.terminalId, + hasDurableSession: Boolean(codex?.durableSessionId), + oldWsUrl: active?.retiringWsUrl ?? codex?.currentWsUrl, + newWsUrl: active?.candidateWsUrl, + oldPtyPid: active?.retiringPtyPid ?? record.pty?.pid, + newPtyPid: active?.candidatePty?.pid, + oldAppServerPid: active?.retiringAppServerPid ?? codex?.currentAppServerPid, + newAppServerPid: active?.candidateAppServerPid, + } + } + + private resizePublishedCodexRecoveryCandidate(record: TerminalRecord, generation?: number): void { + const active = record.codex?.activeReplacement + if (!active || active.aborted || !active.candidatePublished) return + if (generation !== undefined && active.candidateGeneration !== generation) return + if (!this.isCurrentCodexGeneration(record, active.candidateGeneration)) return + const candidatePty = active.candidatePty ?? record.pty try { - this.emit('terminal.created', record) + candidatePty.resize(record.cols, record.rows) } catch (err) { - throw attachTerminalIdToCreateError(err, terminalId) + logger.debug({ err, terminalId: record.terminalId }, 'codex recovery resize failed') } - return record } - private registerCodexSidecarLifecycle(record: TerminalRecord): void { - record.codexSidecarLifecycleUnsubscribe?.() - record.codexSidecarLifecycleUnsubscribe = record.codexSidecar?.onLifecycleLoss?.((event) => { - this.handleCodexLifecycleLoss(record.terminalId, event) - }) + private getRuntimeStatus(record: TerminalRecord): TerminalRuntimeStatus | undefined { + if (record.status === 'exited') return undefined + if (record.mode !== 'codex') return 'running' + return this.isCodexRecoveryState(record) ? 'recovering' : 'running' } - publishCodexSidecar(terminalId: string): void { - const record = this.terminals.get(terminalId) - if (!record) { - throw new Error(`Cannot publish Codex sidecar for missing terminal ${terminalId}.`) + private async handleCodexWorkerFailure( + record: TerminalRecord, + generation: number, + source: CodexWorkerFailureSource, + error: Error, + attemptId?: string, + ): Promise<void> { + const codex = record.codex + if (!codex || record.status === 'exited') { + return } - if (!record.codexSidecar) return - if (record.codexSidecarPrePublicationLoss !== undefined) { - throw new Error('Codex app-server reported lifecycle loss before terminal create completed.') + const isCurrent = this.isCurrentCodexGeneration(record, generation) + const isActiveCandidate = this.isActiveCodexCandidate(record, generation, attemptId) + if (!isCurrent && !isActiveCandidate) { + logger.info({ + terminalId: record.terminalId, + source, + generation, + currentGeneration: codex.workerGeneration, + }, 'codex_recovery_abandoned_stale_generation') + return } - if (record.status !== 'running') { - throw new Error('Codex terminal PTY exited before create completed.') + if (codex.retiringGenerations.has(generation) && !isActiveCandidate) { + codex.recoveryPolicy.noteRecoveryRetireCallback() + return + } + if (codex.closeReasonByGeneration.get(generation) === 'user_final_close') { + this.finalizeTerminalExit(record, record.exitCode ?? 0, 'user_final_close') + return } - record.codexSidecarLifecyclePublished = true - } - - private handleCodexLifecycleLoss(terminalId: string, event: unknown): void { - const record = this.terminals.get(terminalId) - if (!record || record.status !== 'running' || record.codexRecoveryFinalClose) return - if (!record.codexSidecarLifecyclePublished) { - record.codexSidecarPrePublicationLoss = event - logger.warn( - { terminalId, event }, - 'Codex app-server reported lifecycle loss before terminal create completed', - ) + logger.warn({ + err: error, + terminalId: record.terminalId, + source, + generation, + recoveryState: codex.recoveryState, + hasDurableSession: Boolean(codex.durableSessionId), + }, 'codex_worker_failure') + + if (isActiveCandidate) { + await this.failActiveCodexReplacementAttempt(record, attemptId!, source, error) return } - const eventThreadId = typeof event === 'object' && event !== null && 'threadId' in event - ? (event as { threadId?: unknown }).threadId - : undefined + await this.startCodexBundleReplacement(record, source, error) + } + + private attachCodexSidecar( + record: TerminalRecord, + sidecar: Pick<CodexTerminalSidecar, 'attachTerminal' | 'shutdown'>, + generation: number, + attemptId?: string, + ): void { + sidecar.attachTerminal({ + terminalId: record.terminalId, + onDurableSession: (sessionId) => { + this.noteCodexDurableSession(record, sessionId, generation, attemptId) + }, + onThreadLifecycle: (event) => { + this.handleCodexThreadLifecycle(record, generation, attemptId, event) + }, + onFatal: (error, source = 'sidecar_fatal') => { + void this.handleCodexWorkerFailure(record, generation, source, error, attemptId) + }, + }) + } + + private handleCodexThreadLifecycle( + record: TerminalRecord, + generation: number, + attemptId: string | undefined, + event: CodexThreadLifecycleEvent, + ): void { + const codex = record.codex + if (!codex || record.status === 'exited') return + const isCurrent = this.isCurrentCodexGeneration(record, generation) + const isActiveCandidate = this.isActiveCodexCandidate(record, generation, attemptId) + if (!isCurrent && !isActiveCandidate) return + + const active = codex.activeReplacement + const expectedSessionId = codex.durableSessionId + ?? (isActiveCandidate ? active?.pendingDurableSessionId : undefined) if ( - typeof eventThreadId === 'string' - && record.resumeSessionId - && eventThreadId !== record.resumeSessionId + expectedSessionId + && event.kind === 'thread_closed' + && event.threadId === expectedSessionId ) { + void this.handleCodexWorkerFailure( + record, + generation, + 'provider_thread_lifecycle_loss', + new Error('Codex provider reported the active thread closed.'), + attemptId, + ) return } - if (!record.resumeSessionId || !record.codexRecovery) { - logger.warn( - { terminalId, event }, - 'Codex app-server reported terminal lifecycle loss without durable recovery; closing terminal', + if ( + expectedSessionId + && event.kind === 'thread_status_changed' + && event.threadId === expectedSessionId + && (event.status.type === 'notLoaded' || event.status.type === 'systemError') + ) { + void this.handleCodexWorkerFailure( + record, + generation, + 'provider_thread_lifecycle_loss', + new Error(`Codex provider reported the active thread status ${event.status.type}.`), + attemptId, ) - void this.killAndWait(terminalId).catch((err) => { - logger.error({ err, terminalId }, 'Failed to close terminal after Codex app-server lifecycle loss') - }) return } - if (record.codexRecoveryBlockedError) { - logger.error( - { err: record.codexRecoveryBlockedError, terminalId, event }, - 'Codex durable recovery is blocked by a previous sidecar teardown failure', - ) + if ( + event.kind === 'thread_started' + && (expectedSessionId ? event.thread.id === expectedSessionId : isActiveCandidate) + && this.isCodexRecoveryState(record) + ) { + this.noteCodexReadinessEvidence(record, generation, attemptId, event.thread.id) return } - if (record.codexRecoveryAttempt) return - - logger.warn( - { terminalId, event, resumeSessionId: record.resumeSessionId }, - 'Codex app-server reported terminal lifecycle loss; starting durable recovery', - ) - const attempt = this.runCodexRecoveryLoop(terminalId) - .catch((err) => { - logger.error({ err, terminalId }, 'Codex durable recovery loop failed') - }) - .finally(() => { - const latest = this.terminals.get(terminalId) - if (latest?.codexRecoveryAttempt === attempt) { - latest.codexRecoveryAttempt = undefined - } - }) - record.codexRecoveryAttempt = attempt - } - - private canContinueCodexRecovery(record: TerminalRecord | undefined, resumeSessionId?: string): record is TerminalRecord { - return !!record - && record.status === 'running' - && !record.codexRecoveryFinalClose - && !record.codexRecoveryBlockedError - && !!record.codexRecovery - && !!record.resumeSessionId - && (!resumeSessionId || record.resumeSessionId === resumeSessionId) + if ( + expectedSessionId + && event.kind === 'thread_status_changed' + && event.threadId === expectedSessionId + && event.status.type === 'idle' + && this.isCodexRecoveryState(record) + ) { + this.noteCodexReadinessEvidence(record, generation, attemptId, event.threadId) + } } - private async runCodexRecoveryLoop(terminalId: string): Promise<void> { - while (true) { - const record = this.terminals.get(terminalId) - if (!this.canContinueCodexRecovery(record)) return - const resumeSessionId = record.resumeSessionId! + private promoteCodexDurableSession(record: TerminalRecord, sessionId: string, generation: number): void { + const codex = record.codex + if (!codex || !this.isCurrentCodexGeneration(record, generation)) { + return + } + if (codex.retiringGenerations.has(generation)) { + return + } + if (codex.durableSessionId && codex.durableSessionId !== sessionId) { + logger.warn({ + terminalId: record.terminalId, + existingSessionId: codex.durableSessionId, + nextSessionId: sessionId, + generation, + }, 'Ignoring conflicting Codex durable session promotion') + return + } - try { - await this.runCodexRecoveryAttempt(record, resumeSessionId) - return - } catch (err) { - if ( - (err as { codexRecoveryTeardownFailed?: boolean })?.codexRecoveryTeardownFailed - || isCodexSidecarTeardownError(err) - ) { - this.blockCodexRecovery(record, err) - throw err + codex.durableSessionId = sessionId + if (codex.recoveryState === 'running_live_only') { + codex.recoveryState = 'running_durable' + } else if (codex.recoveryState === 'recovering_pre_durable') { + const active = codex.activeReplacement + if ( + active + && active.candidateGeneration === generation + && active.candidatePublished + && !active.aborted + ) { + if (active.preDurableTimer) { + clearTimeout(active.preDurableTimer) + active.preDurableTimer = undefined + } + codex.recoveryState = 'recovering_durable' + if (!active.readinessTimer) { + active.readinessTimer = setTimeout(() => { + void this.failActiveCodexReplacementAttempt( + record, + active.id, + 'readiness_timeout', + new Error('Timed out waiting for Codex durable session readiness evidence.'), + ) + }, CODEX_RECOVERY_READINESS_TIMEOUT_MS) + active.readinessTimer.unref?.() + } + if (active.pendingReadinessSessionId === sessionId) { + this.markCodexRecoveryReady(record, generation, active.id) } - logger.warn( - { err, terminalId, resumeSessionId: record.resumeSessionId }, - 'Codex durable recovery candidate failed; retrying after teardown', - ) } - - const latest = this.terminals.get(terminalId) - if (!this.canContinueCodexRecovery(latest, resumeSessionId)) return - await this.waitForCodexRecoveryRetry(latest, latest.codexRecovery?.retryDelayMs ?? 1_000) + } + const rebound = this.rebindSession(record.terminalId, 'codex', sessionId, 'association') + if (!rebound.ok) { + logger.warn( + { terminalId: record.terminalId, sessionId, reason: rebound.reason }, + 'Failed to promote Codex durable session from sidecar notification', + ) } } - private waitForCodexRecoveryRetry(record: TerminalRecord, delayMs: number): Promise<void> { - if (record.codexRecoveryFinalClose) return Promise.resolve() - return new Promise((resolve) => { - const timer = setTimeout(() => { - if (record.codexRecoveryRetry?.timer === timer) { - record.codexRecoveryRetry = undefined - } - resolve() - }, Math.max(0, delayMs)) - timer.unref?.() - record.codexRecoveryRetry = { - timer, - resolve: () => { - clearTimeout(timer) - if (record.codexRecoveryRetry?.timer === timer) { - record.codexRecoveryRetry = undefined - } - resolve() - }, - } - }) - } - - private blockCodexRecovery(record: TerminalRecord, err: unknown): void { - record.codexRecoveryBlockedError = err instanceof Error ? err : new Error(String(err)) - const retry = record.codexRecoveryRetry - if (retry) { - retry.resolve() + private noteCodexDurableSession( + record: TerminalRecord, + sessionId: string, + generation: number, + attemptId?: string, + ): void { + const codex = record.codex + if (!codex || record.status === 'exited') return + + const active = codex.activeReplacement + if ( + active + && active.id === attemptId + && active.candidateGeneration === generation + && !active.candidatePublished + ) { + if (codex.durableSessionId && codex.durableSessionId !== sessionId) { + logger.warn({ + terminalId: record.terminalId, + existingSessionId: codex.durableSessionId, + candidateSessionId: sessionId, + generation, + }, 'Ignoring conflicting unpublished Codex durable session promotion') + return + } + active.pendingDurableSessionId = sessionId + return } + + this.promoteCodexDurableSession(record, sessionId, generation) } - private markCodexRecoveryFinalClose(record: TerminalRecord): void { - record.codexRecoveryFinalClose = true - const retry = record.codexRecoveryRetry - if (retry) { - retry.resolve() + private emitTerminalStatus( + record: TerminalRecord, + status: TerminalRuntimeStatus, + reason?: string, + attempt?: number, + ): void { + const event = { + terminalId: record.terminalId, + status, + ...(reason ? { reason } : {}), + ...(attempt !== undefined ? { attempt } : {}), } + this.emit('terminal.status', event) } - private async runCodexRecoveryAttempt( + private async startCodexBundleReplacement( record: TerminalRecord, - resumeSessionId: string, + source: CodexWorkerFailureSource, + error: Error, ): Promise<void> { - const recovery = record.codexRecovery - if (!recovery) return - const generation = (record.codexSidecarGeneration ?? 0) + 1 - let plan: CodexLaunchPlan | undefined - let candidate: { pty: ReturnType<typeof pty.spawn>; mcpCwd?: string; exited: boolean; exitCode?: number } | undefined - let published = false - - const cleanupCandidate = async () => { - if (candidate && !published) { - try { - candidate.pty.kill() - } catch (err) { - logger.warn({ err, terminalId: record.terminalId }, 'Failed to kill unpublished Codex recovery PTY') - } - } - if (plan) { - try { - await this.trackSidecarShutdown( - record.terminalId, - `recovery-candidate:${generation}`, - () => plan!.sidecar.shutdown(), - 'Codex recovery candidate sidecar shutdown failed', - ) - } catch (err) { - throw codexRecoveryTeardownError( - `Codex recovery candidate teardown failed: ${err instanceof Error ? err.message : String(err)}`, - ) - } - } + const codex = record.codex + if (!codex || record.status === 'exited') return + const existing = codex.activeReplacement + if (existing && !existing.aborted) { + logger.warn({ + terminalId: record.terminalId, + source, + generation: codex.workerGeneration, + attempt: existing.attempt, + err: error, + }, 'codex_recovery_attempt_coalesced') + return + } + + const retiringGeneration = codex.workerGeneration + const attempt = codex.recoveryPolicy.nextAttempt() + + const recoveryState: CodexRecoveryState = codex.durableSessionId ? 'recovering_durable' : 'recovering_pre_durable' + codex.recoveryState = recoveryState + const candidateGeneration = codex.nextWorkerGeneration + codex.nextWorkerGeneration += 1 + const active: CodexActiveReplacement = { + id: nanoid(), + attempt: attempt.attempt, + source, + retiringGeneration, + candidateGeneration, + candidatePublished: false, + aborted: false, + retiringWsUrl: codex.currentWsUrl, + retiringAppServerPid: codex.currentAppServerPid, + retiringPtyPid: record.pty.pid, + } + codex.activeReplacement = active + + logger.warn({ + ...this.codexRecoveryLogContext(record, active), + terminalId: record.terminalId, + source, + state: recoveryState, + generation: retiringGeneration, + candidateGeneration, + attempt: attempt.attempt, + err: error, + }, 'codex_recovery_started') + this.emitTerminalStatus(record, 'recovering', source, attempt.attempt) + await this.retireCodexWorkerBundle(record, retiringGeneration) + + const launch = () => { + void this.runCodexReplacementAttempt(record, active.id).catch((err) => { + void this.failActiveCodexReplacementAttempt( + record, + active.id, + 'replacement_launch_failure', + err instanceof Error ? err : new Error(String(err)), + ) + }) + } + + if (attempt.delayMs > 0) { + active.backoffTimer = setTimeout(launch, attempt.delayMs) + active.backoffTimer.unref?.() + return + } + launch() + } + + private async runCodexReplacementAttempt(record: TerminalRecord, attemptId: string): Promise<void> { + const codex = record.codex + const active = codex?.activeReplacement + if (!codex || !active || active.id !== attemptId || active.aborted || record.status === 'exited') return + const launchFactory = codex.launchFactory + if (!launchFactory) { + await this.failActiveCodexReplacementAttempt( + record, + attemptId, + 'replacement_launch_failure', + new Error('Codex recovery cannot continue because no launch factory is stored for this terminal.'), + ) + return } + const resumeSessionId = codex.durableSessionId ?? codex.originalResumeSessionId + if (codex.recoveryState === 'recovering_durable' && !resumeSessionId) { + await this.failActiveCodexReplacementAttempt( + record, + attemptId, + 'replacement_launch_failure', + new Error('Codex durable recovery cannot continue without a durable session id.'), + ) + return + } + + logger.warn({ + ...this.codexRecoveryLogContext(record, active), + terminalId: record.terminalId, + attempt: active.attempt, + generation: active.retiringGeneration, + candidateGeneration: active.candidateGeneration, + }, 'codex_recovery_attempt') + + let plan: CodexLaunchPlan | undefined + let worker: SpawnedTerminalWorker | undefined + let spawnStarted = false try { - plan = await recovery.planCreate({ + plan = await launchFactory({ terminalId: record.terminalId, - generation, cwd: record.cwd, + envContext: codex.envContext, resumeSessionId, + providerSettings: codex.launchBaseProviderSettings, }) - if (!this.canContinueCodexRecovery(this.terminals.get(record.terminalId), resumeSessionId)) { - await cleanupCandidate() - return - } - candidate = this.spawnCodexRecoveryPty(record, plan, resumeSessionId) - await plan.sidecar.adopt({ terminalId: record.terminalId, generation }) - await plan.sidecar.waitForLoadedThread(resumeSessionId, { - ...(recovery.readinessTimeoutMs !== undefined ? { timeoutMs: recovery.readinessTimeoutMs } : {}), - ...(recovery.readinessPollMs !== undefined ? { pollMs: recovery.readinessPollMs } : {}), - }) - if (candidate.exited) { - throw new Error(`Codex recovery candidate PTY exited before publication with code ${candidate.exitCode ?? 'unknown'}.`) - } - - const latest = this.terminals.get(record.terminalId) - if (!this.canContinueCodexRecovery(latest, resumeSessionId) || latest !== record) { - await cleanupCandidate() + if (!this.isActiveAttempt(record, attemptId)) { + await plan.sidecar.shutdown().catch(() => undefined) return } - const oldPty = record.pty - const oldSidecar = record.codexSidecar - record.codexRecoveryRetiringPty = oldPty - if (oldSidecar) { - try { - await this.trackSidecarShutdown( - record.terminalId, - `recovery-retiring:${record.codexSidecarGeneration ?? 0}`, - () => oldSidecar.shutdown(), - 'Codex retiring sidecar shutdown failed', - ) - } catch (err) { - record.codexRecoveryRetiringPty = undefined - throw codexRecoveryTeardownError( - `Codex retiring sidecar teardown failed: ${err instanceof Error ? err.message : String(err)}`, + active.candidateSidecar = plan.sidecar + active.candidateWsUrl = plan.remote.wsUrl + active.candidateAppServerPid = plan.remote.processPid + this.attachCodexSidecar(record, plan.sidecar, active.candidateGeneration, attemptId) + + if (codex.durableSessionId) { + active.readinessTimer = setTimeout(() => { + void this.failActiveCodexReplacementAttempt( + record, + attemptId, + 'readiness_timeout', + new Error('Timed out waiting for Codex durable session readiness evidence.'), ) - } + }, CODEX_RECOVERY_READINESS_TIMEOUT_MS) + active.readinessTimer.unref?.() } - if (candidate.exited) { - throw new Error(`Codex recovery candidate PTY exited before publication with code ${candidate.exitCode ?? 'unknown'}.`) - } + spawnStarted = true + worker = this.spawnTerminalWorker({ + terminalId: record.terminalId, + mode: record.mode, + shell: 'system', + cwd: record.cwd, + cols: record.cols, + rows: record.rows, + resumeSessionId: plan.sessionId ?? resumeSessionId, + providerSettings: { + ...codex.launchBaseProviderSettings, + codexAppServer: { wsUrl: plan.remote.wsUrl }, + }, + envContext: codex.envContext, + baseEnv: buildFreshellTerminalEnv(record.terminalId, codex.envContext), + }) - const latestAfterRetire = this.terminals.get(record.terminalId) - if (!this.canContinueCodexRecovery(latestAfterRetire, resumeSessionId) || latestAfterRetire !== record) { - record.codexRecoveryRetiringPty = undefined - await cleanupCandidate() + if (!this.isActiveAttempt(record, attemptId)) { + try { worker.pty.kill() } catch {} + await plan.sidecar.shutdown().catch(() => undefined) + cleanupMcpConfig(record.terminalId, record.mode, worker.mcpCwd) return } - record.codexSidecarLifecycleUnsubscribe?.() - record.codexSidecarLifecycleUnsubscribe = undefined - record.pty = candidate.pty - record.mcpCwd = candidate.mcpCwd + active.candidatePty = worker.pty + active.candidateMcpCwd = worker.mcpCwd + record.pty = worker.pty + record.mcpCwd = worker.mcpCwd record.codexSidecar = plan.sidecar - record.codexSidecarLifecyclePublished = true - record.codexSidecarPrePublicationLoss = undefined - record.codexSidecarGeneration = generation - this.registerCodexSidecarLifecycle(record) - record.codexRecoveryRetiringPty = undefined - published = true - - try { - oldPty.kill('SIGTERM') - } catch (err) { - logger.warn({ err, terminalId: record.terminalId }, 'Failed to retire previous Codex recovery PTY') + codex.currentWsUrl = plan.remote.wsUrl + codex.currentAppServerPid = plan.remote.processPid + if (record.clients.size === 0) { + record.preAttachStartupProbeState = createTerminalStartupProbeState() + } + this.installTerminalWorkerHandlers(record, active.candidateGeneration, attemptId) + codex.workerGeneration = active.candidateGeneration + codex.remoteTuiFailureDetector.reset() + active.candidatePublished = true + codex.closeReasonByGeneration.delete(active.candidateGeneration) + + if (active.pendingDurableSessionId) { + this.promoteCodexDurableSession(record, active.pendingDurableSessionId, active.candidateGeneration) + } + if (codex.durableSessionId) { + if (active.pendingReadinessSessionId === codex.durableSessionId) { + this.markCodexRecoveryReady(record, active.candidateGeneration, attemptId) + } + } else { + this.startCodexPreDurableStabilityTimer(record, active.candidateGeneration, attemptId) } } catch (err) { - if (!published) { - record.codexRecoveryRetiringPty = undefined - await cleanupCandidate() + if (worker) { + try { worker.pty.kill() } catch {} + cleanupMcpConfig(record.terminalId, record.mode, worker.mcpCwd) } - throw err + if (plan) { + await plan.sidecar.shutdown().catch(() => undefined) + } + await this.failActiveCodexReplacementAttempt( + record, + attemptId, + spawnStarted ? 'replacement_spawn_failure' : 'replacement_launch_failure', + err instanceof Error ? err : new Error(String(err)), + ) } } - private spawnCodexRecoveryPty( - record: TerminalRecord, - plan: CodexLaunchPlan, - resumeSessionId: string, - ): { pty: ReturnType<typeof pty.spawn>; mcpCwd?: string; exited: boolean; exitCode?: number } { - const providerSettings: ProviderSettings = { - codexAppServer: { - ...plan.remote, - sidecar: plan.sidecar, - }, + private isActiveAttempt(record: TerminalRecord, attemptId: string): boolean { + const active = record.codex?.activeReplacement + return Boolean(active && active.id === attemptId && !active.aborted && record.status === 'running') + } + + private async retireCodexWorkerBundle(record: TerminalRecord, generation: number): Promise<void> { + const codex = record.codex + if (!codex || codex.retiringGenerations.has(generation)) return + codex.retiringGenerations.add(generation) + codex.closeReasonByGeneration.set(generation, 'recovery_retire') + const sidecar = record.codexSidecar + record.codexSidecar = undefined + if (sidecar) { + await this.trackSidecarShutdown( + record.terminalId, + `recovery-retiring:${generation}`, + () => sidecar.shutdown(), + 'Failed to shut down retiring Codex sidecar', + ).catch(() => undefined) } - const { file, args, env, cwd: procCwd, mcpCwd } = buildSpawnSpec( - record.mode, - record.cwd, - record.shell, - resumeSessionId, - providerSettings, - this.buildTerminalBaseEnv(record.terminalId, record.envContext), - record.terminalId, - ) + try { + record.pty.kill() + } catch (err) { + logger.warn({ err, terminalId: record.terminalId, generation }, 'Failed to kill retiring Codex PTY') + } + cleanupMcpConfig(record.terminalId, record.mode, record.mcpCwd) + logger.warn({ terminalId: record.terminalId, generation }, 'codex_recovery_bundle_retired') + } - const ptyProc = pty.spawn(file, args, { - name: 'xterm-256color', - cols: record.cols, - rows: record.rows, - cwd: procCwd, - env: env as any, - }) - const candidate = { pty: ptyProc, mcpCwd, exited: false, exitCode: undefined as number | undefined } - this.attachCodexRecoveryPtyHandlers(record, ptyProc, candidate) - return candidate + private async failActiveCodexReplacementAttempt( + record: TerminalRecord, + attemptId: string, + source: CodexWorkerFailureSource, + error: Error, + ): Promise<void> { + const codex = record.codex + const active = codex?.activeReplacement + if (!codex || !active || active.id !== attemptId || active.aborted) return + active.aborted = true + if (active.readinessTimer) clearTimeout(active.readinessTimer) + if (active.preDurableTimer) clearTimeout(active.preDurableTimer) + if (active.backoffTimer) clearTimeout(active.backoffTimer) + codex.retiringGenerations.add(active.candidateGeneration) + codex.closeReasonByGeneration.set(active.candidateGeneration, 'recovery_retire') + const candidateSidecar = active.candidatePublished ? record.codexSidecar : active.candidateSidecar + if (candidateSidecar) { + await this.trackSidecarShutdown( + record.terminalId, + `candidate:${active.candidateGeneration}`, + () => candidateSidecar.shutdown(), + 'Failed to shut down failed Codex recovery candidate sidecar', + ).catch(() => undefined) + } + const candidatePty = active.candidatePublished ? record.pty : active.candidatePty + if (candidatePty) { + try { candidatePty.kill() } catch {} + } + if (active.candidateMcpCwd) { + cleanupMcpConfig(record.terminalId, record.mode, active.candidateMcpCwd) + } + codex.activeReplacement = undefined + logger.warn({ + ...this.codexRecoveryLogContext(record, active), + err: error, + terminalId: record.terminalId, + source, + attempt: active.attempt, + generation: active.candidateGeneration, + }, 'codex_recovery_attempt_failed') + await this.startCodexBundleReplacement(record, source, error) } - private attachCodexRecoveryPtyHandlers( + private noteCodexReadinessEvidence( record: TerminalRecord, - ptyProc: ReturnType<typeof pty.spawn>, - candidate?: { exited: boolean; exitCode?: number }, + generation: number, + attemptId: string | undefined, + sessionId: string, ): void { - ptyProc.onData((data) => { - if (record.pty !== ptyProc || record.status !== 'running') return - const now = Date.now() - record.lastActivityAt = now - record.buffer.append(data) - this.emit('terminal.output.raw', { - terminalId: record.terminalId, - data, - at: now, - } satisfies TerminalOutputRawEvent) - for (const client of record.clients) { - if (record.suppressedOutputClients.has(client)) continue - const pending = record.pendingSnapshotClients.get(client) - if (pending) { - const nextChars = pending.queuedChars + data.length - if (data.length > this.maxPendingSnapshotChars || nextChars > this.maxPendingSnapshotChars) { - try { - client.close(4008, 'Attach snapshot queue overflow') - } catch { - // ignore - } - record.pendingSnapshotClients.delete(client) - record.clients.delete(client) - continue + const codex = record.codex + if (!codex) return + const active = codex.activeReplacement + if ( + active + && active.id === attemptId + && active.candidateGeneration === generation + && !active.candidatePublished + ) { + if ( + (codex.durableSessionId || active.pendingDurableSessionId) + && codex.durableSessionId !== sessionId + && active.pendingDurableSessionId !== sessionId + ) { + return + } + active.pendingReadinessSessionId = sessionId + return + } + if ( + active + && active.id === attemptId + && active.candidateGeneration === generation + && active.candidatePublished + && !codex.durableSessionId + ) { + active.pendingReadinessSessionId = sessionId + return + } + if (codex.durableSessionId !== sessionId) return + if (this.isCurrentCodexGeneration(record, generation)) { + this.markCodexRecoveryReady(record, generation, attemptId) + } + } + + private markCodexRecoveryReady(record: TerminalRecord, generation: number, attemptId: string | undefined): void { + const codex = record.codex + const active = codex?.activeReplacement + if (!codex || !active || active.id !== attemptId || !active.candidatePublished) return + if (!this.isCurrentCodexGeneration(record, generation)) return + if (active.readinessTimer) clearTimeout(active.readinessTimer) + if (active.preDurableTimer) clearTimeout(active.preDurableTimer) + this.resizePublishedCodexRecoveryCandidate(record, generation) + codex.activeReplacement = undefined + codex.recoveryState = 'running_durable' + codex.recoveryPolicy.markStableRunning() + this.emitTerminalStatus(record, 'running', 'codex_recovery_ready', active.attempt) + this.flushCodexBufferedInput(record) + logger.warn({ + ...this.codexRecoveryLogContext(record, active), + terminalId: record.terminalId, + generation, + attempt: active.attempt, + }, 'codex_recovery_ready') + } + + private startCodexPreDurableStabilityTimer( + record: TerminalRecord, + generation: number, + attemptId: string, + ): void { + const active = record.codex?.activeReplacement + if (!active || active.id !== attemptId || active.candidateGeneration !== generation) return + active.preDurableTimer = setTimeout(() => { + const codex = record.codex + if (!codex || codex.activeReplacement?.id !== attemptId || record.status !== 'running') return + if (!this.isCurrentCodexGeneration(record, generation)) return + if (codex.durableSessionId) return + this.resizePublishedCodexRecoveryCandidate(record, generation) + codex.activeReplacement = undefined + codex.recoveryState = 'running_live_only' + codex.recoveryPolicy.markStableRunning() + this.emitTerminalStatus(record, 'running', 'codex_recovery_ready', active.attempt) + this.flushCodexBufferedInput(record) + }, CODEX_PRE_DURABLE_STABILITY_MS) + active.preDurableTimer.unref?.() + } + + private flushCodexBufferedInput(record: TerminalRecord): void { + this.clearCodexInputExpiryTimer(record) + const drain = record.codex?.recoveryPolicy.drainBufferedInput() + if (!drain) return + if (!drain.ok) { + if (drain.reason === 'expired') { + this.appendLocalTerminalMessage(record, CODEX_RECOVERY_INPUT_NOT_SENT_MESSAGE) + } + return + } + record.pty.write(drain.data) + this.emit('terminal.input.raw', { + terminalId: record.terminalId, + data: drain.data, + at: Date.now(), + } satisfies TerminalInputRawEvent) + } + + private appendLocalTerminalMessage(record: TerminalRecord, message: string): void { + const terminalId = record.terminalId + const now = Date.now() + record.lastActivityAt = now + record.buffer.append(message) + this.emit('terminal.output.raw', { + terminalId, + data: message, + at: now, + } satisfies TerminalOutputRawEvent) + this.deliverTerminalOutputToClients(record, terminalId, message) + } + + private deliverTerminalOutputToClients(record: TerminalRecord, terminalId: string, data: string): void { + for (const client of record.clients) { + if (record.suppressedOutputClients.has(client)) continue + const pending = record.pendingSnapshotClients.get(client) + if (pending) { + const nextChars = pending.queuedChars + data.length + if (data.length > this.maxPendingSnapshotChars || nextChars > this.maxPendingSnapshotChars) { + try { + client.close(4008, 'Attach snapshot queue overflow') + } catch { + // ignore } - pending.chunks.push(data) - pending.queuedChars = nextChars + record.pendingSnapshotClients.delete(client) + record.clients.delete(client) continue } - this.sendTerminalOutput(client, record.terminalId, data, record.perf) + pending.chunks.push(data) + pending.queuedChars = nextChars + continue } - }) + this.sendTerminalOutput(client, terminalId, data, record.perf) + } + } - ptyProc.onExit((event) => { - if (candidate) { - candidate.exited = true - candidate.exitCode = event.exitCode - } - if (record.codexRecoveryRetiringPty === ptyProc) { - return - } - if (record.pty !== ptyProc || record.status === 'exited') return - this.markCodexRecoveryFinalClose(record) - record.status = 'exited' - record.exitCode = event.exitCode - const now = Date.now() - record.lastActivityAt = now - record.exitedAt = now - cleanupMcpConfig(record.terminalId, record.mode, record.mcpCwd) - for (const client of record.clients) { - this.flushOutputBuffer(client) - this.safeSend(client, { type: 'terminal.exit', terminalId: record.terminalId, exitCode: event.exitCode }, { terminalId: record.terminalId, perf: record.perf }) + private handleTerminalWorkerData(record: TerminalRecord, _generation: number, data: string): void { + const terminalId = record.terminalId + this.handlePreAttachStartupProbes(record, data) + const now = Date.now() + record.lastActivityAt = now + record.buffer.append(data) + this.emit('terminal.output.raw', { + terminalId, + data, + at: now, + } satisfies TerminalOutputRawEvent) + if (record.perf) { + record.perf.outBytes += data.length + record.perf.outChunks += 1 + if (record.perf.pendingInputAt !== undefined) { + const lagMs = now - record.perf.pendingInputAt + record.perf.lastInputToOutputMs = lagMs + if (lagMs > record.perf.maxInputToOutputMs) { + record.perf.maxInputToOutputMs = lagMs + } + if (lagMs >= perfConfig.terminalInputLagMs) { + const key = `terminal_input_lag_${terminalId}` + if (shouldLog(key, perfConfig.rateLimitMs)) { + logPerfEvent( + 'terminal_input_lag', + { + terminalId, + mode: record.mode, + status: record.status, + lagMs, + pendingInputBytes: record.perf.pendingInputBytes, + pendingInputCount: record.perf.pendingInputCount, + lastInputBytes: record.perf.lastInputBytes, + }, + 'warn', + ) + } + } + record.perf.pendingInputAt = undefined + record.perf.pendingInputBytes = 0 + record.perf.pendingInputCount = 0 } - record.clients.clear() - record.suppressedOutputClients.clear() - record.pendingSnapshotClients.clear() - this.releaseBinding(record.terminalId, 'exit') - this.emit('terminal.exit', { terminalId: record.terminalId, exitCode: event.exitCode }) - void this.releaseCodexSidecar(record).catch(() => undefined) - this.reapExitedTerminals() + } + this.deliverTerminalOutputToClients(record, terminalId, data) + } + + private finalizeTerminalExit( + record: TerminalRecord, + exitCode: number | undefined, + _reason: 'pty_exit' | 'user_final_close', + ): void { + if (record.status === 'exited') { + return + } + const terminalId = record.terminalId + const finalExitCode = exitCode ?? 0 + record.status = 'exited' + record.exitCode = finalExitCode + const now = Date.now() + record.lastActivityAt = now + record.exitedAt = now + cleanupMcpConfig(terminalId, record.mode, record.mcpCwd) + void this.releaseCodexSidecar(record).catch(() => undefined) + for (const client of record.clients) { + this.flushOutputBuffer(client) + this.safeSend(client, { type: 'terminal.exit', terminalId, exitCode: finalExitCode }, { terminalId, perf: record.perf }) + } + record.clients.clear() + record.suppressedOutputClients.clear() + record.pendingSnapshotClients.clear() + this.releaseBinding(terminalId, 'exit') + this.emit('terminal.exit', { terminalId, exitCode: finalExitCode }) + this.reapExitedTerminals() + } + + create(opts: { + terminalId?: string + mode: TerminalMode + shell?: ShellType + cwd?: string + cols?: number + rows?: number + resumeSessionId?: string + sessionBindingReason?: SessionBindingReason + providerSettings?: ProviderSettings + codexLaunchBaseProviderSettings?: { + model?: string + sandbox?: string + permissionMode?: string + } + codexSidecar?: Pick<CodexTerminalSidecar, 'attachTerminal' | 'shutdown'> + codexLaunchFactory?: CodexLaunchFactory + envContext?: TerminalEnvContext + }): TerminalRecord { + this.reapExitedTerminals() + if (this.runningCount() >= this.maxTerminals) { + throw new Error(`Maximum terminal limit (${this.maxTerminals}) reached. Please close some terminals before creating new ones.`) + } + + const terminalId = opts.terminalId ?? nanoid() + const createdAt = Date.now() + const cols = opts.cols || 120 + const rows = opts.rows || 30 + + const cwd = opts.cwd || getDefaultCwd(this.settings) || (isWindows() ? undefined : os.homedir()) + const resumeForSpawn = normalizeResumeForSpawn(opts.mode, opts.resumeSessionId) + const resumeForBinding = normalizeResumeForBinding(opts.mode, opts.resumeSessionId) + const baseEnv = buildFreshellTerminalEnv(terminalId, opts.envContext) + const worker = this.spawnTerminalWorker({ + terminalId, + mode: opts.mode, + shell: opts.shell || 'system', + cwd, + cols, + rows, + resumeSessionId: resumeForSpawn, + providerSettings: opts.providerSettings, + envContext: opts.envContext, + baseEnv, }) + + const title = getModeLabel(opts.mode) + + const record: TerminalRecord = { + terminalId, + title, + description: undefined, + mode: opts.mode, + codexSidecar: opts.mode === 'codex' ? opts.codexSidecar : undefined, + opencodeServer: opts.mode === 'opencode' ? opts.providerSettings?.opencodeServer : undefined, + resumeSessionId: undefined, + createdAt, + lastActivityAt: createdAt, + status: 'running', + cwd, + mcpCwd: worker.mcpCwd, + cols, + rows, + clients: new Set(), + suppressedOutputClients: new Set(), + pendingSnapshotClients: new Map(), + preAttachStartupProbeState: opts.mode === 'codex' ? createTerminalStartupProbeState() : undefined, + + buffer: new ChunkRingBuffer(this.scrollbackMaxChars), + pty: worker.pty, + perf: perfConfig.enabled + ? { + outBytes: 0, + outChunks: 0, + droppedMessages: 0, + inBytes: 0, + inChunks: 0, + pendingInputAt: undefined, + pendingInputBytes: 0, + pendingInputCount: 0, + lastInputBytes: undefined, + lastInputToOutputMs: undefined, + maxInputToOutputMs: 0, + } + : undefined, + codex: opts.mode === 'codex' + ? { + recoveryState: resumeForBinding ? 'running_durable' : 'running_live_only', + workerGeneration: 1, + nextWorkerGeneration: 2, + retiringGenerations: new Set(), + closeReasonByGeneration: new Map(), + durableSessionId: resumeForBinding, + originalResumeSessionId: resumeForBinding, + currentWsUrl: opts.providerSettings?.codexAppServer?.wsUrl, + launchFactory: opts.codexLaunchFactory, + launchBaseProviderSettings: opts.codexLaunchBaseProviderSettings + ? { + model: opts.codexLaunchBaseProviderSettings.model, + sandbox: opts.codexLaunchBaseProviderSettings.sandbox, + permissionMode: opts.codexLaunchBaseProviderSettings.permissionMode, + } + : { + model: opts.providerSettings?.model, + sandbox: opts.providerSettings?.sandbox, + permissionMode: opts.providerSettings?.permissionMode, + }, + envContext: opts.envContext, + recoveryPolicy: new CodexRecoveryPolicy(), + remoteTuiFailureDetector: new CodexRemoteTuiFailureDetector(), + } + : undefined, + } + + this.installTerminalWorkerHandlers(record, 1) + + this.terminals.set(terminalId, record) + if (opts.mode === 'codex' && opts.codexSidecar) { + const generation = record.codex?.workerGeneration ?? 1 + this.attachCodexSidecar(record, opts.codexSidecar, generation) + } + const exactSessionId = resumeForBinding + if (modeSupportsResume(opts.mode) && exactSessionId) { + const bound = this.bindSession( + terminalId, + opts.mode as CodingCliProviderName, + exactSessionId, + opts.sessionBindingReason ?? 'resume', + ) + if (!bound.ok) { + logger.warn( + { terminalId, mode: opts.mode, sessionId: exactSessionId, reason: bound.reason }, + 'Failed to bind resume session during terminal create', + ) + } + } + if (resumeForSpawn && !resumeForBinding) { + record.pendingResumeName = resumeForSpawn + logger.info( + { terminalId, mode: opts.mode, pendingResumeName: resumeForSpawn }, + 'Terminal created with named resume; awaiting session association', + ) + } + this.emit('terminal.created', record) + return record } attach(terminalId: string, client: WebSocket, opts?: { pendingSnapshot?: boolean; suppressOutput?: boolean }): TerminalRecord | null { const term = this.terminals.get(terminalId) if (!term) return null + term.preAttachStartupProbeState = undefined term.clients.add(client) if (opts?.pendingSnapshot) term.pendingSnapshotClients.set(client, { chunks: [], queuedChars: 0 }) if (opts?.suppressOutput) term.suppressedOutputClients.add(client) @@ -1866,6 +2314,16 @@ export class TerminalRegistry extends EventEmitter { if (!term || term.status !== 'running') return false const now = Date.now() term.lastActivityAt = now + if (term.mode === 'codex' && this.isCodexRecoveryState(term)) { + const buffered = term.codex?.recoveryPolicy.bufferInput(data) + if (!buffered?.ok) { + this.clearCodexInputExpiryTimer(term) + this.appendLocalTerminalMessage(term, CODEX_RECOVERY_INPUT_NOT_SENT_MESSAGE) + } else { + this.scheduleCodexInputExpiryTimer(term) + } + return true + } if (term.perf) { term.perf.inBytes += data.length term.perf.inChunks += 1 @@ -1885,12 +2343,40 @@ export class TerminalRegistry extends EventEmitter { return true } + private handlePreAttachStartupProbes(term: TerminalRecord, data: string): void { + if (term.mode !== 'codex') return + if (term.clients.size > 0) return + const state = term.preAttachStartupProbeState + if (!state) return + + const { replies } = extractTerminalStartupProbes(data, state, PREATTACH_CODEX_STARTUP_PROBE_COLORS) + if (!state.armed && !state.pending) { + term.preAttachStartupProbeState = undefined + } + if (replies.length === 0) { + return + } + + for (const reply of replies) { + try { + term.pty.write(reply) + } catch (err) { + logger.debug({ err, terminalId: term.terminalId }, 'pre-attach codex startup probe reply failed') + break + } + } + } + resize(terminalId: string, cols: number, rows: number): boolean { const term = this.terminals.get(terminalId) if (!term || term.status !== 'running') return false if (term.cols === cols && term.rows === rows) return true term.cols = cols term.rows = rows + if (term.mode === 'codex' && this.isCodexRecoveryProtected(term)) { + this.resizePublishedCodexRecoveryCandidate(term) + return true + } try { term.pty.resize(cols, rows) } catch (err) { @@ -1902,52 +2388,50 @@ export class TerminalRegistry extends EventEmitter { kill(terminalId: string): boolean { const term = this.terminals.get(terminalId) if (!term) return false - if (term.status === 'exited') { - void this.releaseCodexSidecar(term).catch(() => undefined) - return true - } - this.markCodexRecoveryFinalClose(term) - cleanupMcpConfig(terminalId, term.mode, term.mcpCwd) + if (term.status === 'exited') return true + this.markCodexFinalClose(term) try { term.pty.kill() } catch (err) { logger.warn({ err, terminalId }, 'kill failed') } - term.status = 'exited' - term.exitCode = term.exitCode ?? 0 - const now = Date.now() - term.lastActivityAt = now - term.exitedAt = now - for (const client of term.clients) { - this.flushOutputBuffer(client) - this.safeSend(client, { type: 'terminal.exit', terminalId, exitCode: term.exitCode }) - } - term.clients.clear() - term.suppressedOutputClients.clear() - term.pendingSnapshotClients.clear() - this.releaseBinding(terminalId, 'exit') - this.emit('terminal.exit', { terminalId, exitCode: term.exitCode }) - this.recordTerminalExitWithoutDurableSession(term, term.exitCode, 'user_final_close') - void this.releaseCodexSidecar(term).catch(() => undefined) - this.reapExitedTerminals() + this.finalizeTerminalExit(term, term.exitCode ?? 0, 'user_final_close') return true } - async killAndWait(terminalId: string): Promise<boolean> { - const term = this.terminals.get(terminalId) - const ok = this.kill(terminalId) - if (!ok) return false - const recoveryAttempt = term?.codexRecoveryAttempt - ? term.codexRecoveryAttempt.catch((err) => { - logger.error({ err, terminalId }, 'Codex recovery did not finish cleanly during terminal close') - throw err - }) - : undefined - const joins = [this.waitForSidecarShutdown(terminalId)] - if (recoveryAttempt) joins.push(recoveryAttempt) - const failures = await collectShutdownFailures(joins) - throwShutdownFailures(failures, 'Codex terminal final close failed.') - return true + private markCodexWorkerCloseReason(record: TerminalRecord, reason: CodexWorkerCloseReason): void { + const codex = record.codex + if (!codex) return + codex.closeReasonByGeneration.set(codex.workerGeneration, reason) + } + + private markCodexFinalClose(record: TerminalRecord): void { + const codex = record.codex + if (!codex) return + this.markCodexWorkerCloseReason(record, 'user_final_close') + this.clearCodexInputExpiryTimer(record) + const active = codex.activeReplacement + if (active) { + active.aborted = true + if (active.readinessTimer) clearTimeout(active.readinessTimer) + if (active.preDurableTimer) clearTimeout(active.preDurableTimer) + if (active.backoffTimer) clearTimeout(active.backoffTimer) + codex.closeReasonByGeneration.set(active.candidateGeneration, 'user_final_close') + if (active.candidateSidecar && !active.candidatePublished) { + const candidateSidecar = active.candidateSidecar + void this.trackSidecarShutdown( + record.terminalId, + `candidate:${active.candidateGeneration}`, + () => candidateSidecar.shutdown(), + 'Failed to shut down final-closed Codex recovery candidate sidecar', + ).catch(() => undefined) + } + if (active.candidatePty && !active.candidatePublished) { + try { active.candidatePty.kill() } catch {} + } + codex.activeReplacement = undefined + } + codex.recoveryPolicy.clearBufferedInput() } remove(terminalId: string): boolean { @@ -1962,8 +2446,6 @@ export class TerminalRegistry extends EventEmitter { const existing = this.sidecarShutdowns.get(this.sidecarShutdownKey(term.terminalId)) if (existing?.status === 'pending') return existing.promise - term.codexSidecarLifecycleUnsubscribe?.() - term.codexSidecarLifecycleUnsubscribe = undefined const sidecar = term.codexSidecar if (!sidecar) return existing?.promise ?? Promise.resolve() @@ -1974,8 +2456,6 @@ export class TerminalRegistry extends EventEmitter { await sidecar.shutdown() if (term.codexSidecar === sidecar) { term.codexSidecar = undefined - term.codexSidecarLifecyclePublished = undefined - term.codexSidecarPrePublicationLoss = undefined } }, 'Codex sidecar shutdown failed', @@ -2051,9 +2531,6 @@ export class TerminalRegistry extends EventEmitter { private async waitForCodexShutdownWork(records: Iterable<TerminalRecord>): Promise<void> { const recordList = Array.from(records) - const recoveryAttempts = recordList - .map((term) => term.codexRecoveryAttempt) - .filter((promise): promise is Promise<void> => !!promise) const sidecarShutdowns = new Set<Promise<void>>() for (const term of recordList) { sidecarShutdowns.add(this.releaseCodexSidecar(term)) @@ -2061,10 +2538,7 @@ export class TerminalRegistry extends EventEmitter { for (const [key, entry] of [...this.sidecarShutdowns.entries()]) { sidecarShutdowns.add(this.runSidecarShutdownEntry(key, entry)) } - const failures = [ - ...await collectShutdownFailures(recoveryAttempts), - ...await collectShutdownFailures([...sidecarShutdowns]), - ] + const failures = await collectShutdownFailures([...sidecarShutdowns]) throwShutdownFailures(failures, 'Codex registry shutdown work failed.') } @@ -2077,6 +2551,7 @@ export class TerminalRegistry extends EventEmitter { createdAt: number lastActivityAt: number status: 'running' | 'exited' + runtimeStatus?: TerminalRuntimeStatus hasClients: boolean cwd?: string }> { @@ -2089,6 +2564,7 @@ export class TerminalRegistry extends EventEmitter { createdAt: t.createdAt, lastActivityAt: t.lastActivityAt, status: t.status, + runtimeStatus: this.getRuntimeStatus(t), hasClients: t.clients.size > 0, cwd: t.cwd, })) @@ -2563,13 +3039,6 @@ export class TerminalRegistry extends EventEmitter { sessionId: normalized, reason, } satisfies TerminalSessionBoundEvent) - recordSessionLifecycleEvent({ - kind: 'terminal_session_bound', - terminalId, - provider, - sessionId: normalized, - reason, - }) return { ok: true, terminalId, sessionId: normalized } } @@ -2690,7 +3159,7 @@ export class TerminalRegistry extends EventEmitter { // Send SIGTERM (or plain kill on Windows where signal args are unsupported) const isWindows = process.platform === 'win32' for (const term of running) { - this.markCodexRecoveryFinalClose(term) + this.markCodexFinalClose(term) try { if (isWindows) { term.pty.kill() diff --git a/server/ws-handler.ts b/server/ws-handler.ts index 630ce9cf2..76fb3cd62 100644 --- a/server/ws-handler.ts +++ b/server/ws-handler.ts @@ -1,13 +1,13 @@ import type http from 'http' import { randomUUID } from 'crypto' +import { nanoid } from 'nanoid' import WebSocket, { WebSocketServer } from 'ws' import { z } from 'zod' import { logger } from './logger.js' -import { recordSessionLifecycleEvent } from './session-observability.js' import { getPerfConfig, logPerfEvent, shouldLog, startPerfTimer } from './perf-logger.js' import { getRequiredAuthToken, isLoopbackAddress, isOriginAllowed, timingSafeCompare } from './auth.js' -import { modeSupportsResume, terminalIdFromCreateError } from './terminal-registry.js' -import type { TerminalRecord, TerminalRegistry, TerminalMode } from './terminal-registry.js' +import { buildFreshellTerminalEnv, modeSupportsResume } from './terminal-registry.js' +import type { TerminalEnvContext, TerminalRecord, TerminalRegistry, TerminalMode } from './terminal-registry.js' import { configStore, type ConfigReadError } from './config-store.js' import type { CodingCliSessionManager } from './coding-cli/session-manager.js' import type { ProjectGroup } from './coding-cli/types.js' @@ -22,6 +22,7 @@ import type { OpencodeActivityRecord, SdkServerMessage, SdkSessionStatus, + TerminalStatusMessage, } from '../shared/ws-protocol.js' import type { ExtensionManager } from './extension-manager.js' import { allocateLocalhostPort } from './local-port.js' @@ -33,7 +34,7 @@ import { TabRegistryRecordBaseSchema, TabRegistryRecordSchema } from './tabs-reg import type { TabsRegistryStore } from './tabs-registry/store.js' import type { ServerSettings } from '../shared/settings.js' import { stripAnsi } from './ai-prompts.js' -import type { CodexLaunchPlan, CodexLaunchPlanner } from './coding-cli/codex-app-server/launch-planner.js' +import { runCodexLaunchWithRetry, type CodexLaunchFactory, type CodexLaunchPlanner } from './coding-cli/codex-app-server/launch-planner.js' import { CodexLaunchConfigError, getCodexSessionBindingReason, @@ -53,7 +54,6 @@ import { OpencodeActivityUpdatedSchema, HelloSchema, PingSchema, - ClientDiagnosticSchema, TerminalAttachSchema, TerminalDetachSchema, TerminalInputSchema, @@ -173,6 +173,20 @@ function isNonEmptyString(value: unknown): value is string { return typeof value === 'string' && value.trim().length > 0 } +function buildCanonicalTerminalSessionRef( + mode: TerminalMode, + resumeSessionId?: string, +): { provider: string; sessionId: string } | undefined { + if (mode === 'shell' || !isNonEmptyString(resumeSessionId)) return undefined + if (mode === 'claude' && !isValidClaudeSessionId(resumeSessionId)) { + return undefined + } + return { + provider: mode, + sessionId: resumeSessionId, + } +} + const TERMINAL_FAILURE_SUMMARY_MAX_CHARS = 200 function summarizeTerminalFailureOutput(snapshot: string): string | undefined { @@ -200,27 +214,17 @@ function formatExitedTerminalAttachMessage(record: Pick<TerminalRecord, 'title' return `${label} is no longer running${exitSuffix}.` } -function assertCodexCreateTerminalRunning(record: Pick<TerminalRecord, 'status'>): void { - if (record.status !== 'running') { - throw new Error('Codex terminal PTY exited before create completed.') - } -} - function normalizeUiSessionLocator(value: unknown): SidebarSessionLocator | undefined { if (!value || typeof value !== 'object') return undefined const candidate = value as { provider?: unknown sessionId?: unknown - serverInstanceId?: unknown } const provider = CodingCliProviderSchema.safeParse(candidate.provider) if (!provider.success || !isNonEmptyString(candidate.sessionId)) return undefined return { provider: provider.data, sessionId: candidate.sessionId, - ...(isNonEmptyString(candidate.serverInstanceId) - ? { serverInstanceId: candidate.serverInstanceId } - : {}), } } @@ -234,7 +238,7 @@ function extractSessionLocatorsFromUiContent(content: Record<string, unknown>): const kind = content.kind if (kind === 'agent-chat') { - if (isNonEmptyString(content.resumeSessionId)) { + if (isNonEmptyString(content.resumeSessionId) && isValidClaudeSessionId(content.resumeSessionId)) { locators.push({ provider: 'claude', sessionId: content.resumeSessionId }) } return locators @@ -243,7 +247,12 @@ function extractSessionLocatorsFromUiContent(content: Record<string, unknown>): if (kind !== 'terminal') return locators const mode = CodingCliProviderSchema.safeParse(content.mode) - if (!mode.success || !isNonEmptyString(content.resumeSessionId)) { + if ( + !mode.success + || mode.data !== 'claude' + || !isNonEmptyString(content.resumeSessionId) + || !isValidClaudeSessionId(content.resumeSessionId) + ) { return locators } @@ -359,12 +368,6 @@ function createScreenshotError(code: ScreenshotErrorCode, message: string): Erro return err } -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} - -class TerminalCreateAdmissionError extends Error {} - export class WsHandler { private readonly config: WsHandlerConfig private readonly authToken: string @@ -405,6 +408,13 @@ export class WsHandler { if (!payload?.terminalId) return this.forgetCreatedRequestIdsForTerminal(payload.terminalId) } + private onTerminalStatusBound = (payload: Omit<TerminalStatusMessage, 'type'>) => { + if (!payload?.terminalId) return + this.broadcast({ + type: 'terminal.status', + ...payload, + } satisfies TerminalStatusMessage) + } private sessionRepairListeners?: { scanned: (result: SessionScanResult) => void repaired: (result: SessionRepairResult) => void @@ -499,7 +509,6 @@ export class WsHandler { this.clientMessageSchema = z.discriminatedUnion('type', [ HelloSchema, PingSchema, - ClientDiagnosticSchema, dynamicTerminalCreateSchema, TerminalAttachSchema, TerminalDetachSchema, @@ -529,6 +538,7 @@ export class WsHandler { on?: (event: string, listener: (...args: any[]) => void) => void } registryWithEvents.on?.('terminal.exit', this.onTerminalExitBound) + registryWithEvents.on?.('terminal.status', this.onTerminalStatusBound) this.wss = new WebSocketServer({ server, path: '/ws', @@ -682,12 +692,16 @@ export class WsHandler { cwd: string | undefined, resumeSessionId: string | undefined, providerSettings: { model?: string; sandbox?: string; permissionMode?: string } | undefined, + terminalId: string, + envContext: TerminalEnvContext, ) { if (!this.codexLaunchPlanner) { - throw new Error('Codex terminal launch requires the app-server launch planner.') + throw new Error('Codex terminal launch requires the per-terminal app-server sidecar planner.') } return this.codexLaunchPlanner.planCreate({ cwd, + terminalId, + env: buildFreshellTerminalEnv(terminalId, envContext), resumeSessionId, model: providerSettings?.model, sandbox: normalizeCodexSandboxSetting(providerSettings?.sandbox), @@ -695,10 +709,41 @@ export class WsHandler { }) } - private assertTerminalCreateAccepted(): void { - if (this.closed) { - throw new TerminalCreateAdmissionError('Server is shutting down; terminal.create is no longer accepted.') - } + private createCodexLaunchFactory( + providerSettings: { model?: string; sandbox?: string; permissionMode?: string } | undefined, + ): CodexLaunchFactory { + return async (input) => this.planCodexLaunch( + input.cwd, + input.resumeSessionId, + input.providerSettings ?? providerSettings, + input.terminalId, + input.envContext ?? {}, + ) + } + + private async planCodexLaunchWithRetry( + cwd: string | undefined, + resumeSessionId: string | undefined, + providerSettings: { model?: string; sandbox?: string; permissionMode?: string } | undefined, + terminalId: string, + envContext: TerminalEnvContext, + requestId: string, + ): ReturnType<WsHandler['planCodexLaunch']> { + return runCodexLaunchWithRetry( + () => this.planCodexLaunch(cwd, resumeSessionId, providerSettings, terminalId, envContext), + { + shouldRetry: (error) => !(error instanceof CodexLaunchConfigError), + onFailedAttempt: ({ attempt, delayMs, error }) => { + log.warn({ + err: error, + requestId, + terminalId, + attempt, + nextDelayMs: delayMs, + }, 'Codex initial launch planning failed; retrying before terminal.create') + }, + }, + ) } private terminalCreateLockKey( @@ -1003,11 +1048,6 @@ export class WsHandler { } private onConnection(ws: LiveWebSocket, req: http.IncomingMessage) { - if (this.closed) { - ws.close(CLOSE_CODES.SERVER_SHUTDOWN, 'Server shutting down') - return - } - if (this.connections.size >= this.config.maxConnections) { ws.close(CLOSE_CODES.MAX_CONNECTIONS, 'Too many connections') return @@ -1285,13 +1325,6 @@ export class WsHandler { err: error instanceof Error ? error : new Error(String(error)), sessionId, }, 'sdk restore history resolution failed') - recordSessionLifecycleEvent({ - kind: 'client_restore_unavailable', - sessionId, - connectionId: ws.connectionId || 'unknown', - reason: 'restore_internal', - hasSessionRef: true, - }) this.send(ws, { type: 'sdk.error', sessionId, @@ -1631,17 +1664,18 @@ export class WsHandler { // Send terminal inventory so the client knows what's alive const terminals = this.registry.list().map((terminal) => { - const { resumeSessionId, ...rest } = terminal + const sessionRef = buildCanonicalTerminalSessionRef(terminal.mode, terminal.resumeSessionId) return { - ...rest, - ...(resumeSessionId - ? { - sessionRef: { - provider: terminal.mode, - sessionId: resumeSessionId, - }, - } - : {}), + terminalId: terminal.terminalId, + title: terminal.title, + description: terminal.description, + mode: terminal.mode, + ...(sessionRef ? { sessionRef } : {}), + createdAt: terminal.createdAt, + lastActivityAt: terminal.lastActivityAt, + status: terminal.status, + ...(terminal.runtimeStatus ? { runtimeStatus: terminal.runtimeStatus } : {}), + cwd: terminal.cwd, } }) const terminalMeta = this.terminalMetaListProvider?.() ?? [] @@ -1680,6 +1714,20 @@ export class WsHandler { this.sendError(ws, { code: 'INVALID_MESSAGE', message: 'Invalid JSON' }) return } + const rawSessionRef = ( + msg?.sessionRef + && typeof msg.sessionRef === 'object' + && typeof msg.sessionRef.provider === 'string' + && msg.sessionRef.provider.length > 0 + && typeof msg.sessionRef.sessionId === 'string' + && msg.sessionRef.sessionId.length > 0 + ) + ? { + provider: msg.sessionRef.provider, + sessionId: msg.sessionRef.sessionId, + } + : undefined + const rawRestoreRequested = msg?.restore === true if (msg?.type === 'hello' && msg?.protocolVersion !== WS_PROTOCOL_VERSION) { this.sendError(ws, { @@ -1763,31 +1811,7 @@ export class WsHandler { return } - if (this.closed && m.type === 'terminal.create') { - this.sendError(ws, { - code: 'INTERNAL_ERROR', - message: 'Server is shutting down; terminal.create is no longer accepted.', - requestId: m.requestId, - }) - return - } - switch (m.type) { - case 'client.diagnostic': { - if (m.event === 'restore_unavailable') { - recordSessionLifecycleEvent({ - kind: 'client_restore_unavailable', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - ...(m.tabId ? { tabId: m.tabId } : {}), - ...(m.paneId ? { paneId: m.paneId } : {}), - mode: m.mode, - reason: m.reason, - hasSessionRef: m.hasSessionRef, - }) - } - return - } case 'ui.screenshot.result': { const pending = this.screenshotRequests.get(m.requestId) if (!pending) return @@ -1822,11 +1846,13 @@ export class WsHandler { return } case 'terminal.create': { - const requestedSessionRef = m.sessionRef?.provider === m.mode - ? m.sessionRef - : undefined + const requestedSessionRef = ( + m.sessionRef?.provider === m.mode && typeof m.sessionRef?.sessionId === 'string' + ? m.sessionRef + : (rawSessionRef?.provider === m.mode ? rawSessionRef : undefined) + ) const canonicalSessionId = requestedSessionRef?.sessionId - const restoreRequested = m.restore === true + const restoreRequested = m.restore === true || rawRestoreRequested const localLiveTerminalId = ( m.liveTerminal?.serverInstanceId === this.serverInstanceId && typeof m.liveTerminal?.terminalId === 'string' @@ -1839,29 +1865,16 @@ export class WsHandler { mode: m.mode, sessionRef: requestedSessionRef, }, '[TRACE sessionRef] terminal.create received') - recordSessionLifecycleEvent({ - kind: 'terminal_create_requested', - requestId: m.requestId, - connectionId: ws.connectionId || 'unknown', - ...(m.tabId ? { tabId: m.tabId } : {}), - ...(m.paneId ? { paneId: m.paneId } : {}), - ...(m.cwd ? { cwd: m.cwd } : {}), - mode: m.mode as TerminalMode, - restoreRequested, - hasRequestedSessionRef: !!requestedSessionRef, - ...(canonicalSessionId ? { requestedSessionId: canonicalSessionId } : {}), - }) const endCreateTimer = startPerfTimer( 'terminal_create', { connectionId: ws.connectionId, mode: m.mode, shell: m.shell }, { minDurationMs: perfConfig.slowTerminalCreateMs, level: 'warn' }, ) let terminalId: string | undefined - let pendingCodexPlan: CodexLaunchPlan | undefined let reused = false let error = false let rateLimited = false - let effectiveResumeSessionId = canonicalSessionId + let restoreSessionId = canonicalSessionId try { await this.withTerminalCreateLock( this.terminalCreateLockKey(m.mode as TerminalMode, m.requestId, canonicalSessionId), @@ -1899,7 +1912,6 @@ export class WsHandler { const attachReusedTerminal = async ( reusedTerminalId: string, createdAt: number, - resumeSessionId?: string, ): Promise<boolean> => { const sent = await sendCreateResult({ ws, @@ -1914,18 +1926,6 @@ export class WsHandler { this.rememberCreatedRequestId(m.requestId, reusedTerminalId) terminalId = reusedTerminalId reused = true - recordSessionLifecycleEvent({ - kind: 'terminal_created', - requestId: m.requestId, - connectionId: ws.connectionId || 'unknown', - terminalId: reusedTerminalId, - ...(m.tabId ? { tabId: m.tabId } : {}), - ...(m.paneId ? { paneId: m.paneId } : {}), - ...(m.cwd ? { cwd: m.cwd } : {}), - mode: m.mode as TerminalMode, - reused: true, - hasSessionRef: !!resumeSessionId, - }) this.broadcastTerminalsChanged() return true } @@ -1939,7 +1939,7 @@ export class WsHandler { } const existing = this.registry.get(existingId) if (existing) { - await attachReusedTerminal(existing.terminalId, existing.createdAt, existing.resumeSessionId) + await attachReusedTerminal(existing.terminalId, existing.createdAt) return } // If it no longer exists, fall through and create a new one. @@ -1950,7 +1950,7 @@ export class WsHandler { if (localLiveTerminalId) { const liveTerminal = this.registry.get(localLiveTerminalId) if (liveTerminal?.status === 'running' && liveTerminal.mode === m.mode) { - await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt, liveTerminal.resumeSessionId) + await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt) return } } @@ -1971,7 +1971,7 @@ export class WsHandler { ) } if (existing) { - await attachReusedTerminal(existing.terminalId, existing.createdAt, existing.resumeSessionId) + await attachReusedTerminal(existing.terminalId, existing.createdAt) return } } @@ -1991,7 +1991,7 @@ export class WsHandler { } const existing = this.registry.get(existingAfterConfigId) if (existing) { - await attachReusedTerminal(existing.terminalId, existing.createdAt, existing.resumeSessionId) + await attachReusedTerminal(existing.terminalId, existing.createdAt) return } state.createdByRequestId.delete(m.requestId) @@ -2018,7 +2018,7 @@ export class WsHandler { if (localLiveTerminalId) { const liveTerminal = this.registry.get(localLiveTerminalId) if (liveTerminal?.status === 'running' && liveTerminal.mode === m.mode) { - await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt, liveTerminal.resumeSessionId) + await attachReusedTerminal(liveTerminal.terminalId, liveTerminal.createdAt) return } } @@ -2039,7 +2039,7 @@ export class WsHandler { ) } if (existing) { - await attachReusedTerminal(existing.terminalId, existing.createdAt, existing.resumeSessionId) + await attachReusedTerminal(existing.terminalId, existing.createdAt) return } } @@ -2047,12 +2047,12 @@ export class WsHandler { // Session repair is Claude-specific (uses JSONL session files). // Other providers (codex, opencode, etc.) don't use the same file // structure, so this block correctly remains gated on mode === 'claude'. - if (m.mode === 'claude' && effectiveResumeSessionId && isValidClaudeSessionId(effectiveResumeSessionId) && this.sessionRepairService) { - const sessionId = effectiveResumeSessionId + if (m.mode === 'claude' && restoreSessionId && isValidClaudeSessionId(restoreSessionId) && this.sessionRepairService) { + const sessionId = restoreSessionId const cached = this.sessionRepairService.getResult(sessionId) if (cached?.status === 'missing') { log.info({ sessionId, connectionId: ws.connectionId }, 'Session previously marked missing; resume will start fresh') - effectiveResumeSessionId = undefined + restoreSessionId = undefined } else { // Reserve requestId to prevent same-socket duplicate creates during async repair wait. state.createdByRequestId.set(m.requestId, REPAIR_PENDING_SENTINEL) @@ -2066,7 +2066,7 @@ export class WsHandler { endRepairTimer({ status: result.status }) if (result.status === 'missing') { log.info({ sessionId, connectionId: ws.connectionId }, 'Session file missing; resume will start fresh') - effectiveResumeSessionId = undefined + restoreSessionId = undefined } } catch (err) { endRepairTimer({ error: err instanceof Error ? err.message : String(err) }) @@ -2084,7 +2084,7 @@ export class WsHandler { return } - if (m.mode === 'claude' && restoreRequested && !isValidClaudeSessionId(effectiveResumeSessionId)) { + if (m.mode === 'claude' && restoreRequested && !isValidClaudeSessionId(restoreSessionId)) { this.sendError(ws, { code: 'RESTORE_UNAVAILABLE', message: 'Claude restore requires a canonical durable session id', @@ -2107,176 +2107,110 @@ export class WsHandler { requestId: m.requestId, connectionId: ws.connectionId, sessionRef: requestedSessionRef, - effectiveResumeSessionId, + restoreSessionId, }, '[TRACE sessionRef] about to create terminal') const requestedCodexResumeSessionId = m.mode === 'codex' - ? effectiveResumeSessionId + ? canonicalSessionId : undefined - this.assertTerminalCreateAccepted() - const codexPlan = m.mode === 'codex' - ? await this.planCodexLaunch(m.cwd, requestedCodexResumeSessionId, providerSettings) - : undefined - pendingCodexPlan = codexPlan - - if (codexPlan) { - effectiveResumeSessionId = codexPlan.sessionId - } - this.assertTerminalCreateAccepted() - - const codexRecovery = codexPlan - ? { - planCreate: (input: { cwd?: string; resumeSessionId: string }) => - this.planCodexLaunch(input.cwd ?? m.cwd, input.resumeSessionId, providerSettings), - } + let codexPlan: Awaited<ReturnType<WsHandler['planCodexLaunch']>> | undefined + const preallocatedTerminalId = nanoid() + const terminalEnvContext = { tabId: m.tabId, paneId: m.paneId } + const codexLaunchFactory = m.mode === 'codex' + ? this.createCodexLaunchFactory(providerSettings) : undefined + try { + codexPlan = m.mode === 'codex' + ? await this.planCodexLaunchWithRetry( + m.cwd, + requestedCodexResumeSessionId, + providerSettings, + preallocatedTerminalId, + terminalEnvContext, + m.requestId, + ) + : undefined - const spawnProviderSettings = ( - providerSettings - ? { - ...(m.mode === 'codex' - ? {} - : { - permissionMode: providerSettings.permissionMode, - model: providerSettings.model, - sandbox: providerSettings.sandbox, - }), - ...(m.mode === 'opencode' - ? { opencodeServer: await allocateLocalhostPort() } - : {}), - ...(codexPlan ? { - codexAppServer: { - ...codexPlan.remote, - sidecar: codexPlan.sidecar, - recovery: codexRecovery, - deferLifecycleUntilPublished: true, - }, - } : {}), - } - : (codexPlan + const spawnProviderSettings = ( + providerSettings ? { - codexAppServer: { - ...codexPlan.remote, - sidecar: codexPlan.sidecar, - recovery: codexRecovery, - deferLifecycleUntilPublished: true, - }, + ...(m.mode === 'codex' + ? {} + : { + permissionMode: providerSettings.permissionMode, + model: providerSettings.model, + sandbox: providerSettings.sandbox, + }), + ...(m.mode === 'opencode' + ? { opencodeServer: await allocateLocalhostPort() } + : {}), + ...(codexPlan ? { codexAppServer: codexPlan.remote } : {}), } - : undefined) - ) + : (codexPlan + ? { codexAppServer: codexPlan.remote } + : undefined) + ) - this.assertTerminalCreateAccepted() - const record = this.registry.create({ - mode: m.mode as TerminalMode, - shell: m.shell as 'system' | 'cmd' | 'powershell' | 'wsl', - cwd: m.cwd, - resumeSessionId: effectiveResumeSessionId, - ...(codexPlan - ? { - sessionBindingReason: getCodexSessionBindingReason(m.mode, requestedCodexResumeSessionId), - } - : {}), - envContext: { tabId: m.tabId, paneId: m.paneId }, - providerSettings: spawnProviderSettings, - }) - terminalId = record.terminalId - this.assertTerminalCreateAccepted() - if (codexPlan) { - await codexPlan.sidecar.adopt({ terminalId: record.terminalId, generation: 0 }) - this.assertTerminalCreateAccepted() - if (requestedCodexResumeSessionId) { - await codexPlan.sidecar.waitForLoadedThread(requestedCodexResumeSessionId) - this.assertTerminalCreateAccepted() - } - assertCodexCreateTerminalRunning(record) - this.assertTerminalCreateAccepted() - this.registry.publishCodexSidecar?.(record.terminalId) - pendingCodexPlan = undefined - if (effectiveResumeSessionId) { - recordSessionLifecycleEvent({ - kind: 'codex_durable_session_observed', - provider: 'codex', - terminalId: record.terminalId, - sessionId: effectiveResumeSessionId, - generation: 0, - source: 'sidecar', + const record = this.registry.create({ + terminalId: preallocatedTerminalId, + mode: m.mode as TerminalMode, + shell: m.shell as 'system' | 'cmd' | 'powershell' | 'wsl', + cwd: m.cwd, + resumeSessionId: restoreSessionId, + ...(requestedCodexResumeSessionId + ? { + sessionBindingReason: getCodexSessionBindingReason(m.mode, requestedCodexResumeSessionId), + } + : {}), + envContext: terminalEnvContext, + providerSettings: spawnProviderSettings, + ...(m.mode === 'codex' ? { codexLaunchBaseProviderSettings: providerSettings } : {}), + ...(codexPlan ? { codexSidecar: codexPlan.sidecar } : {}), + ...(codexLaunchFactory ? { codexLaunchFactory } : {}), + }) + if (m.mode !== 'shell' && typeof m.cwd === 'string' && m.cwd.trim()) { + const recentDirectory = m.cwd.trim() + void configStore.pushRecentDirectory(recentDirectory).catch((err) => { + log.warn({ err, recentDirectory }, 'Failed to record recent directory') }) } - } - this.assertTerminalCreateAccepted() - if (m.mode !== 'shell' && typeof m.cwd === 'string' && m.cwd.trim()) { - const recentDirectory = m.cwd.trim() - void configStore.pushRecentDirectory(recentDirectory).catch((err) => { - log.warn({ err, recentDirectory }, 'Failed to record recent directory') - }) - } + state.createdByRequestId.set(m.requestId, record.terminalId) + this.rememberCreatedRequestId(m.requestId, record.terminalId) + terminalId = record.terminalId - state.createdByRequestId.set(m.requestId, record.terminalId) - this.rememberCreatedRequestId(m.requestId, record.terminalId) + const sent = await sendCreateResult({ + ws, + requestId: m.requestId, + terminalId: record.terminalId, + createdAt: record.createdAt, + }) + if (!sent) { + // Terminal may still exist even if created delivery failed (for + // example: socket closed after create). Broadcast inventory so + // other clients can discover it. + this.broadcastTerminalsChanged() + return + } - const sent = await sendCreateResult({ - ws, - requestId: m.requestId, - terminalId: record.terminalId, - createdAt: record.createdAt, - }) - if (!sent) { - // Terminal may still exist even if created delivery failed (for - // example: socket closed after create). Broadcast inventory so - // other clients can discover it. + // Notify all clients that list changed this.broadcastTerminalsChanged() - return + } catch (error) { + await codexPlan?.sidecar.shutdown().catch(() => undefined) + throw error } - - recordSessionLifecycleEvent({ - kind: 'terminal_created', - requestId: m.requestId, - connectionId: ws.connectionId || 'unknown', - terminalId: record.terminalId, - ...(m.tabId ? { tabId: m.tabId } : {}), - ...(m.paneId ? { paneId: m.paneId } : {}), - ...(m.cwd ? { cwd: m.cwd } : {}), - mode: m.mode as TerminalMode, - reused: false, - hasSessionRef: !!effectiveResumeSessionId, - }) - - // Notify all clients that list changed - this.broadcastTerminalsChanged() }, ) } catch (err: any) { error = true - const cleanupErrors: string[] = [] - const cleanupTerminalId = terminalId ?? terminalIdFromCreateError(err) - if (typeof cleanupTerminalId === 'string') { - await this.registry.killAndWait(cleanupTerminalId).catch((killErr) => { - cleanupErrors.push(`created terminal cleanup failed: ${errorMessage(killErr)}`) - log.warn({ err: killErr, terminalId: cleanupTerminalId }, 'terminal.create cleanup failed') - }) - } - if (pendingCodexPlan) { - await pendingCodexPlan.sidecar.shutdown().catch((shutdownErr) => { - cleanupErrors.push(`Codex sidecar cleanup failed: ${errorMessage(shutdownErr)}`) - log.warn({ err: shutdownErr }, 'terminal.create pending Codex sidecar cleanup failed') - }) - } - const errorMessageText = cleanupErrors.length > 0 - ? `${err?.message || 'Failed to spawn PTY'}; cleanup failed: ${cleanupErrors.join('; ')}` - : err?.message || 'Failed to spawn PTY' // Clean up repair sentinel if terminal creation failed if (state.createdByRequestId.get(m.requestId) === REPAIR_PENDING_SENTINEL) { state.createdByRequestId.delete(m.requestId) } log.warn({ err, connectionId: ws.connectionId }, 'terminal.create failed') this.sendError(ws, { - code: err instanceof CodexLaunchConfigError - ? 'INVALID_MESSAGE' - : err instanceof TerminalCreateAdmissionError - ? 'INTERNAL_ERROR' - : 'PTY_SPAWN_FAILED', - message: errorMessageText, + code: err instanceof CodexLaunchConfigError ? 'INVALID_MESSAGE' : 'PTY_SPAWN_FAILED', + message: err?.message || 'Failed to spawn PTY', requestId: m.requestId, }) } finally { @@ -2288,22 +2222,10 @@ export class WsHandler { case 'terminal.attach': { const record = this.registry.get(m.terminalId) if (!record) { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.attach', - }) this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Terminal not running', terminalId: m.terminalId }) return } if (record.status !== 'running') { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.attach', - }) this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: formatExitedTerminalAttachMessage(record), @@ -2325,12 +2247,6 @@ export class WsHandler { if (attachResult === 'missing') { const latestRecord = this.registry.get(m.terminalId) if (latestRecord && latestRecord.status !== 'running') { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.attach', - }) this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: formatExitedTerminalAttachMessage(latestRecord), @@ -2338,12 +2254,6 @@ export class WsHandler { }) return } - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.attach', - }) this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Unknown terminalId', terminalId: m.terminalId }) return } @@ -2356,14 +2266,6 @@ export class WsHandler { const ok = this.terminalStreamBroker.detach(m.terminalId, ws) state.attachedTerminalIds.delete(m.terminalId) if (!ok) { - if (!this.registry.get(m.terminalId)) { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.detach', - }) - } this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Unknown terminalId', terminalId: m.terminalId }) return } @@ -2374,15 +2276,6 @@ export class WsHandler { case 'terminal.input': { const ok = this.registry.input(m.terminalId, m.data) if (!ok) { - if (!this.registry.get(m.terminalId)) { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.input', - attemptedInputBytes: typeof m.data === 'string' ? Buffer.byteLength(m.data) : 0, - }) - } this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Terminal not running', terminalId: m.terminalId }) } return @@ -2391,39 +2284,14 @@ export class WsHandler { case 'terminal.resize': { const ok = this.registry.resize(m.terminalId, m.cols, m.rows) if (!ok) { - if (!this.registry.get(m.terminalId)) { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.resize', - }) - } this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Terminal not running', terminalId: m.terminalId }) } return } case 'terminal.kill': { - let ok: boolean - try { - ok = await this.registry.killAndWait(m.terminalId) - } catch (err) { - log.warn({ err, terminalId: m.terminalId, connectionId: ws.connectionId }, 'terminal.kill failed') - this.sendError(ws, { - code: 'INTERNAL_ERROR', - message: `Failed to kill terminal: ${errorMessage(err)}`, - terminalId: m.terminalId, - }) - return - } + const ok = this.registry.kill(m.terminalId) if (!ok) { - recordSessionLifecycleEvent({ - kind: 'invalid_terminal_id_without_session_ref', - terminalId: m.terminalId, - connectionId: ws.connectionId || 'unknown', - operation: 'terminal.kill', - }) this.sendError(ws, { code: 'INVALID_TERMINAL_ID', message: 'Unknown terminalId', terminalId: m.terminalId }) return } @@ -3029,13 +2897,6 @@ export class WsHandler { return } if (resolved?.kind === 'missing') { - recordSessionLifecycleEvent({ - kind: 'client_restore_unavailable', - sessionId: m.sessionId, - connectionId: ws.connectionId || 'unknown', - reason: 'restore_not_found', - hasSessionRef: true, - }) this.send(ws, { type: 'sdk.error', sessionId: m.sessionId, @@ -3116,13 +2977,6 @@ export class WsHandler { } if (snapshotResult?.kind === 'missing') { attachSubscriptionOff?.() - recordSessionLifecycleEvent({ - kind: 'client_restore_unavailable', - sessionId: m.sessionId, - connectionId: ws.connectionId || 'unknown', - reason: 'restore_not_found', - hasSessionRef: true, - }) this.send(ws, { type: 'sdk.error', sessionId: m.sessionId, @@ -3337,6 +3191,7 @@ export class WsHandler { off?: (event: string, listener: (...args: any[]) => void) => void } registryWithEvents.off?.('terminal.exit', this.onTerminalExitBound) + registryWithEvents.off?.('terminal.status', this.onTerminalStatusBound) if (this.sessionRepairService && this.sessionRepairListeners) { this.sessionRepairService.off('scanned', this.sessionRepairListeners.scanned) diff --git a/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs b/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs index 0c741f60b..fb8749ba7 100644 --- a/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs +++ b/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs @@ -1,21 +1,8 @@ #!/usr/bin/env node -import { WebSocketServer } from 'ws' -import { spawn } from 'node:child_process' import fs from 'node:fs' - -if (process.argv[2] === 'fake-native-child') { - process.on('SIGTERM', () => { - if (process.env.FAKE_CODEX_NATIVE_CHILD_IGNORE_SIGTERM === '1') { - return - } - process.exit(0) - }) - - setInterval(() => undefined, 1_000) - process.stdin.resume() - await new Promise(() => undefined) -} +import path from 'node:path' +import { WebSocketServer } from 'ws' function parseListenUrl(argv) { const listenIndex = argv.indexOf('--listen') @@ -31,6 +18,42 @@ function loadBehavior() { return JSON.parse(raw) } +function getCodexHome() { + return process.env.CODEX_HOME || '/tmp/fake-codex-home' +} + +function getRolloutSessionDir() { + const now = new Date() + const year = String(now.getUTCFullYear()) + const month = String(now.getUTCMonth() + 1).padStart(2, '0') + const day = String(now.getUTCDate()).padStart(2, '0') + return path.join(getCodexHome(), 'sessions', year, month, day) +} + +function getThreadHandle(threadId) { + return { + id: threadId, + path: path.join(getRolloutSessionDir(), `rollout-${threadId}.jsonl`), + ephemeral: false, + } +} + +function ensureDurableArtifact(threadId) { + const thread = getThreadHandle(threadId) + const codexHome = process.env.CODEX_HOME || '/tmp/fake-codex-home' + const now = new Date() + const sessionDir = path.dirname(thread.path) + fs.mkdirSync(sessionDir, { recursive: true }) + fs.writeFileSync(thread.path, JSON.stringify({ + threadId, + createdAt: now.toISOString(), + }) + '\n', 'utf8') + return { + codexHome, + thread, + } +} + function writeBytes(stream, totalBytes, chunkSize = 16 * 1024) { if (!Number.isFinite(totalBytes) || totalBytes <= 0) { return Promise.resolve() @@ -63,16 +86,14 @@ function successResult(method, params) { if (method === 'initialize') { return { userAgent: 'freshell-fixture/1.0.0', - codexHome: '/tmp/fake-codex-home', + codexHome: getCodexHome(), platformFamily: 'unix', platformOs: 'linux', } } if (method === 'thread/start') { return { - thread: { - id: 'thread-new-1', - }, + thread: getThreadHandle('thread-new-1'), cwd: params?.cwd ?? process.cwd(), model: 'fixture-model', modelProvider: 'openai', @@ -85,10 +106,9 @@ function successResult(method, params) { } } if (method === 'thread/resume') { + const threadId = params?.threadId || 'thread-new-1' return { - thread: { - id: params?.threadId, - }, + thread: getThreadHandle(threadId), cwd: params?.cwd ?? process.cwd(), model: 'fixture-model', modelProvider: 'openai', @@ -100,50 +120,131 @@ function successResult(method, params) { }, } } - if (method === 'thread/loaded/list') { + if (method === 'turn/start') { return { - data: behavior.loadedThreadIds || [], + thread: getThreadHandle(params?.threadId || 'thread-new-1'), } } + if (method === 'fs/watch') { + return { + path: path.resolve(String(params?.path || '')), + } + } + if (method === 'fs/unwatch') { + return {} + } return {} } const listenUrl = parseListenUrl(process.argv.slice(2)) const behavior = loadBehavior() +if (process.env.FAKE_CODEX_APP_SERVER_ARG_LOG) { + fs.writeFileSync(process.env.FAKE_CODEX_APP_SERVER_ARG_LOG, JSON.stringify({ + argv: process.argv.slice(2), + env: { + FRESHELL: process.env.FRESHELL, + FRESHELL_URL: process.env.FRESHELL_URL, + FRESHELL_TOKEN: process.env.FRESHELL_TOKEN, + FRESHELL_TERMINAL_ID: process.env.FRESHELL_TERMINAL_ID, + FRESHELL_TAB_ID: process.env.FRESHELL_TAB_ID, + FRESHELL_PANE_ID: process.env.FRESHELL_PANE_ID, + }, + }), 'utf8') +} const closeSocketAfterMethodsOnce = new Set(behavior.closeSocketAfterMethodsOnce || []) +const exitProcessAfterMethodsOnce = new Set(behavior.exitProcessAfterMethodsOnce || []) +const threadClosedAfterMethodsOnce = new Set(behavior.threadClosedAfterMethodsOnce || []) const url = new URL(listenUrl) const host = url.hostname const port = Number(url.port) +const watches = new Map() +const activeThreadIds = new Set() -let nativeChild -if (behavior.spawnNativeChild) { - nativeChild = spawn(process.execPath, [new URL(import.meta.url).pathname, 'fake-native-child'], { - env: { - ...process.env, - FAKE_CODEX_NATIVE_CHILD_IGNORE_SIGTERM: behavior.nativeChildIgnoresSigterm ? '1' : '', - }, - stdio: 'ignore', +const wss = new WebSocketServer({ host, port }) + +function broadcastNotification(method, params) { + const payload = JSON.stringify({ + jsonrpc: '2.0', + method, + params, }) - nativeChild.unref() - if (behavior.nativePidFile) { - fs.writeFileSync(behavior.nativePidFile, `${nativeChild.pid}\n`, 'utf8') + for (const client of wss.clients) { + if (client.readyState === 1) { + client.send(payload) + } + } +} + +function emitConfiguredNotifications(method) { + const notifications = behavior.notifyAfterMethodsOnce?.[method] + if (!Array.isArray(notifications) || notifications.length === 0) { + return } - if (behavior.exitAfterSpawningNative) { - process.exit(Number(behavior.exitAfterSpawningNativeCode ?? 42)) + delete behavior.notifyAfterMethodsOnce[method] + for (const notification of notifications) { + broadcastNotification(notification.method, notification.params) } } -const wss = new WebSocketServer({ host, port }) +function appendThreadOperation(method, params, result) { + const logPath = behavior.appendThreadOperationLogPath + if (!logPath || !method.startsWith('thread/')) { + return + } + const threadId = result?.thread?.id || params?.threadId || null + fs.mkdirSync(path.dirname(logPath), { recursive: true }) + fs.appendFileSync(logPath, JSON.stringify({ + method, + threadId, + params, + listenUrl, + at: new Date().toISOString(), + }) + '\n', 'utf8') +} + +function claimCrossProcessOnce(markerPath, key) { + if (!markerPath) { + return true + } + + try { + fs.mkdirSync(path.dirname(markerPath), { recursive: true }) + fs.writeFileSync(markerPath, `${key}\n`, { flag: 'wx' }) + return true + } catch (error) { + if (error && error.code === 'EEXIST') { + return false + } + throw error + } +} + +function emitConfiguredThreadStatusChanges(method) { + const byMethod = behavior.threadStatusChangedAfterMethodsOnce + const entries = byMethod?.[method] + if (!Array.isArray(entries) || entries.length === 0) { + return + } + if (!claimCrossProcessOnce(behavior.threadStatusChangedAfterMethodsOnceMarkerPath, `thread-status:${method}`)) { + return + } + delete byMethod[method] + for (const entry of entries) { + broadcastNotification('thread/status/changed', { + threadId: entry.threadId, + status: entry.status, + }) + } +} + +function claimCrossProcessCloseSocketOnce(method) { + return claimCrossProcessOnce(behavior.closeSocketAfterMethodsOnceMarkerPath, `close-socket:${method}`) +} wss.on('connection', (socket) => { let initialized = false - let initializedNotification = false socket.on('message', (raw) => { const message = JSON.parse(raw.toString()) - if (message.method === 'initialized') { - initializedNotification = true - return - } if (behavior.requireJsonRpc && message.jsonrpc !== '2.0') { socket.send(JSON.stringify({ id: message.id, @@ -156,11 +257,7 @@ wss.on('connection', (socket) => { } const method = message.method - if ( - behavior.requireInitializeBeforeOtherMethods - && method !== 'initialize' - && (!initialized || (behavior.requireInitializedNotification && !initializedNotification)) - ) { + if (behavior.requireInitializeBeforeOtherMethods && method !== 'initialize' && !initialized) { socket.send(JSON.stringify({ id: message.id, error: { @@ -175,6 +272,17 @@ wss.on('connection', (socket) => { return } + if (behavior.assertNoDuplicateActiveThread && method === 'thread/start' && activeThreadIds.size > 0) { + socket.send(JSON.stringify({ + id: message.id, + error: { + code: -32001, + message: `Duplicate active thread start attempted while ${[...activeThreadIds].join(', ')} is active`, + }, + })) + return + } + const override = behavior.overrides?.[method] const delayMs = Number(behavior.delayMethodsMs?.[method] || 0) const floodStdoutBytes = Number(behavior.floodStdoutBeforeMethodsBytes?.[method] || 0) @@ -192,19 +300,80 @@ wss.on('connection', (socket) => { setTimeout(async () => { await writeBytes(process.stdout, floodStdoutBytes) await writeBytes(process.stderr, floodStderrBytes) + const result = override?.result ?? successResult(method, message.params) socket.send(JSON.stringify({ id: message.id, - result: override?.result ?? successResult(method, message.params), + result, })) - for (const notification of behavior.notificationsAfterMethods?.[method] || []) { - socket.send(JSON.stringify(notification)) - } + appendThreadOperation(method, message.params, result) if (method === 'initialize') { initialized = true } - if (closeSocketAfterMethodsOnce.delete(method)) { + if (method === 'thread/start') { + const thread = result?.thread || getThreadHandle(message.params?.threadId || 'thread-new-1') + activeThreadIds.add(thread.id) + broadcastNotification('thread/started', { + thread, + }) + } + if (method === 'thread/resume') { + const thread = result?.thread || getThreadHandle(message.params?.threadId || 'thread-new-1') + activeThreadIds.add(thread.id) + broadcastNotification('thread/started', { + thread, + }) + } + if (method === 'fs/watch') { + const watchId = message.params?.watchId + const watchedPath = result?.path + if (watchId && watchedPath) { + watches.set(watchId, watchedPath) + } + } + if (method === 'fs/unwatch') { + const watchId = message.params?.watchId + if (watchId) { + watches.delete(watchId) + } + } + if (method === 'turn/start' && message.params?.threadId) { + const { thread } = ensureDurableArtifact(message.params.threadId) + const rolloutPath = thread.path + const rolloutParent = path.dirname(rolloutPath) + for (const [watchId, watchedPath] of watches) { + if (watchedPath !== rolloutPath && watchedPath !== rolloutParent) { + continue + } + broadcastNotification('fs/changed', { + watchId, + changedPaths: [rolloutPath], + }) + } + } + emitConfiguredNotifications(method) + if ( + threadClosedAfterMethodsOnce.delete(method) + && claimCrossProcessOnce(behavior.threadClosedAfterMethodsOnceMarkerPath, `thread-closed:${method}`) + ) { + const threadId = result?.thread?.id || message.params?.threadId || 'thread-new-1' + activeThreadIds.delete(threadId) + broadcastNotification('thread/closed', { threadId }) + } + emitConfiguredThreadStatusChanges(method) + if (closeSocketAfterMethodsOnce.delete(method) && claimCrossProcessCloseSocketOnce(method)) { setTimeout(() => socket.close(), 0) } + if (exitProcessAfterMethodsOnce.delete(method)) { + setTimeout(() => { + if (behavior.stdoutBeforeExit) { + process.stdout.write(String(behavior.stdoutBeforeExit)) + } + if (behavior.stderrBeforeExit) { + process.stderr.write(String(behavior.stderrBeforeExit)) + } + process.exit(0) + }, 0) + } }, delayMs) }) }) @@ -213,17 +382,5 @@ process.on('SIGTERM', () => { if (process.env.FAKE_CODEX_APP_SERVER_IGNORE_SIGTERM === '1') { return } - if (behavior.signalFileOnSigterm) { - fs.writeFileSync(behavior.signalFileOnSigterm, `${process.pid}\n`, 'utf8') - } - if (!behavior.wrapperLeavesNativeOnSigterm) { - nativeChild?.kill('SIGTERM') - } - const exit = () => wss.close(() => process.exit(0)) - const delayExitMs = Number(behavior.delayExitOnSigtermMs || 0) - if (delayExitMs > 0) { - setTimeout(exit, delayExitMs) - return - } - exit() + wss.close(() => process.exit(0)) }) diff --git a/test/helpers/coding-cli/fake-codex-launch-planner.ts b/test/helpers/coding-cli/fake-codex-launch-planner.ts index f669d025f..900fc3b2b 100644 --- a/test/helpers/coding-cli/fake-codex-launch-planner.ts +++ b/test/helpers/coding-cli/fake-codex-launch-planner.ts @@ -1,66 +1,73 @@ export const DEFAULT_CODEX_REMOTE_WS_URL = 'ws://127.0.0.1:43123' -export class FakeCodexLaunchSidecar { - adoptCalls: Array<{ terminalId: string; generation: number }> = [] +export class FakeCodexTerminalSidecar { + attachedTerminalId?: string + durableSessionHandlers = new Set<(sessionId: string) => void>() + fatalHandlers = new Set<(error: Error, source?: 'sidecar_fatal' | 'app_server_exit' | 'app_server_client_disconnect') => void>() shutdownCalls = 0 - waitForLoadedThreadCalls: Array<{ threadId: string; options?: { timeoutMs?: number; pollMs?: number } }> = [] - waitForLoadedThreadError: Error | null = null - shutdownError: Error | null = null - shutdownStarted = false - private lifecycleLossHandlers = new Set<(event: unknown) => void>() - async adopt(input: { terminalId: string; generation: number }) { - this.adoptCalls.push(input) - } - - async listLoadedThreads() { - return ['thread-new-1'] + attachTerminal(input: { + terminalId: string + onDurableSession: (sessionId: string) => void + onThreadLifecycle?: (event: unknown) => void + onFatal: (error: Error, source?: 'sidecar_fatal' | 'app_server_exit' | 'app_server_client_disconnect') => void + }) { + this.attachedTerminalId = input.terminalId + this.durableSessionHandlers.add(input.onDurableSession) + this.fatalHandlers.add(input.onFatal) } async shutdown() { - if (this.shutdownStarted) return - this.shutdownStarted = true this.shutdownCalls += 1 - if (this.shutdownError) throw this.shutdownError } - async waitForLoadedThread(threadId: string, options?: { timeoutMs?: number; pollMs?: number }) { - this.waitForLoadedThreadCalls.push({ threadId, options }) - if (this.waitForLoadedThreadError) throw this.waitForLoadedThreadError - } - - onLifecycleLoss(handler: (event: unknown) => void) { - this.lifecycleLossHandlers.add(handler) - return () => this.lifecycleLossHandlers.delete(handler) + emitDurableSession(sessionId: string) { + for (const handler of this.durableSessionHandlers) { + handler(sessionId) + } } - emitLifecycleLoss(event: unknown) { - for (const handler of this.lifecycleLossHandlers) { - handler(event) + emitFatal( + message = 'fake codex sidecar failed', + source: 'sidecar_fatal' | 'app_server_exit' | 'app_server_client_disconnect' = 'sidecar_fatal', + ) { + const error = new Error(message) + for (const handler of this.fatalHandlers) { + handler(error, source) } } } export class FakeCodexLaunchPlanner { planCreateCalls: any[] = [] - sidecar = new FakeCodexLaunchSidecar() + readonly sidecar: FakeCodexTerminalSidecar + private failuresRemaining = 0 constructor( private readonly plan: { - sessionId: string + sessionId?: string remote: { wsUrl: string } - sidecar?: FakeCodexLaunchSidecar + sidecar?: FakeCodexTerminalSidecar } = { - sessionId: 'thread-new-1', remote: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL }, }, - ) {} + ) { + this.sidecar = this.plan.sidecar ?? new FakeCodexTerminalSidecar() + } + + failNext(count: number) { + this.failuresRemaining = count + } async planCreate(input: any) { this.planCreateCalls.push(input) + if (this.failuresRemaining > 0) { + this.failuresRemaining -= 1 + throw new Error('fake Codex launch failed') + } return { ...this.plan, - sidecar: this.plan.sidecar ?? this.sidecar, + sidecar: this.sidecar, } } } diff --git a/test/integration/server/codex-session-flow.test.ts b/test/integration/server/codex-session-flow.test.ts index b4df4837c..476057fa7 100644 --- a/test/integration/server/codex-session-flow.test.ts +++ b/test/integration/server/codex-session-flow.test.ts @@ -1,14 +1,16 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import fsp from 'fs/promises' import http from 'http' import os from 'os' import path from 'path' +import { createRequire } from 'node:module' import express from 'express' import WebSocket from 'ws' import { WsHandler } from '../../../server/ws-handler.js' import { TerminalRegistry } from '../../../server/terminal-registry.js' import { CodexAppServerRuntime } from '../../../server/coding-cli/codex-app-server/runtime.js' import { CodexLaunchPlanner } from '../../../server/coding-cli/codex-app-server/launch-planner.js' +import { CodexTerminalSidecar } from '../../../server/coding-cli/codex-app-server/sidecar.js' import { configStore } from '../../../server/config-store.js' import { WS_PROTOCOL_VERSION } from '../../../shared/ws-protocol.js' @@ -30,7 +32,7 @@ vi.mock('../../../server/logger', () => { child: vi.fn(), } logger.child.mockReturnValue(logger) - return { logger, sessionLifecycleLogger: logger } + return { logger } }) process.env.AUTH_TOKEN = 'test-token' @@ -40,61 +42,148 @@ const FAKE_APP_SERVER_PATH = path.resolve( process.cwd(), 'test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs', ) +const require = createRequire(import.meta.url) +const WS_MODULE_PATH = require.resolve('ws') async function writeFakeCodexExecutable(binaryPath: string) { const script = `#!/usr/bin/env node const fs = require('fs') - -function appendJsonLine(filePath, value) { - if (!filePath) return - fs.appendFileSync(filePath, JSON.stringify(value) + '\\n', 'utf8') -} +const WebSocket = require(${JSON.stringify(WS_MODULE_PATH)}) const argLogPath = process.env.FAKE_CODEX_ARG_LOG if (argLogPath) { fs.writeFileSync(argLogPath, JSON.stringify(process.argv.slice(2)), 'utf8') } -appendJsonLine(process.env.FAKE_CODEX_LAUNCH_LOG, { - pid: process.pid, - args: process.argv.slice(2), -}) +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} -let isFirstLaunch = false -if (process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH) { - try { - fs.writeFileSync(process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH, String(process.pid), { flag: 'wx' }) - isFirstLaunch = true - } catch { - isFirstLaunch = false +async function maybeDriveRemote() { + const rawBehavior = process.env.FAKE_CODEX_REMOTE_BEHAVIOR + if (!rawBehavior) { + return } -} -process.stdin.on('data', (chunk) => { - appendJsonLine(process.env.FAKE_CODEX_INPUT_LOG, { - pid: process.pid, - data: chunk.toString('utf8'), + const args = process.argv.slice(2) + const remoteIndex = args.indexOf('--remote') + if (remoteIndex === -1 || remoteIndex === args.length - 1) { + return + } + + const wsUrl = args[remoteIndex + 1] + const resumeIndex = args.indexOf('resume') + const resumeSessionId = resumeIndex === -1 ? undefined : args[resumeIndex + 1] + const behavior = JSON.parse(rawBehavior) + + if (behavior.recordStdinPath) { + process.stdin.on('data', (chunk) => { + fs.appendFileSync(behavior.recordStdinPath, chunk) + }) + process.stdin.resume() + } + + const socket = new WebSocket(wsUrl) + const pending = new Map() + let nextId = 1 + + const waitForOpen = new Promise((resolve, reject) => { + socket.once('open', resolve) + socket.once('error', reject) }) -}) -process.on('SIGTERM', () => process.exit(0)) -process.stdout.write('codex remote attached\\n') -if (process.env.FAKE_CODEX_STAY_ALIVE === '1') { - if ( - process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS - && (process.env.FAKE_CODEX_EXIT_WATCH_FIRST_LAUNCH_ONLY !== '1' || isFirstLaunch) - ) { - setInterval(() => { - if (fs.existsSync(process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS)) { - process.exit(0) - } - }, 10) + socket.on('message', (raw) => { + let message + try { + message = JSON.parse(raw.toString()) + } catch { + return + } + if (typeof message.id !== 'number') { + return + } + const pendingRequest = pending.get(message.id) + if (!pendingRequest) { + return + } + pending.delete(message.id) + if (message.error) { + pendingRequest.reject(new Error(message.error.message || 'remote app-server request failed')) + return + } + pendingRequest.resolve(message.result) + }) + + function request(method, params) { + return new Promise((resolve, reject) => { + const id = nextId++ + pending.set(id, { resolve, reject }) + socket.send(JSON.stringify({ + jsonrpc: '2.0', + id, + method, + params, + }), (error) => { + if (!error) { + return + } + pending.delete(id) + reject(error) + }) + }) + } + + await waitForOpen + await request('initialize', { + clientInfo: { name: 'fake-codex-cli', version: '1.0.0' }, + capabilities: { experimentalApi: true }, + }) + + let threadId = resumeSessionId + if (resumeSessionId) { + await request('thread/resume', { + threadId: resumeSessionId, + cwd: process.cwd(), + persistExtendedHistory: true, + }) + } else { + const started = await request('thread/start', { + cwd: process.cwd(), + experimentalRawEvents: false, + persistExtendedHistory: true, + }) + threadId = started?.thread?.id + } + + if ((behavior.sendTurnStart || (behavior.sendTurnStartOnFreshOnly && !resumeSessionId)) && threadId) { + await request('turn/start', { + threadId, + input: 'fake turn', + }) + } + + if (behavior.recordRemoteThreadIdPath && threadId) { + fs.writeFileSync(behavior.recordRemoteThreadIdPath, threadId, 'utf8') } - process.stdin.resume() - setInterval(() => undefined, 1000) -} else { - setTimeout(() => process.exit(0), 50) + + if (behavior.sleepMs) { + await sleep(behavior.sleepMs) + } + + await new Promise((resolve) => socket.close(() => resolve())) } + +Promise.resolve() + .then(() => maybeDriveRemote()) + .then(() => { + process.stdout.write('codex remote attached\\n') + setTimeout(() => process.exit(0), 50) + }) + .catch((error) => { + const message = error instanceof Error ? error.stack || error.message : String(error) + process.stderr.write(message + '\\n') + process.exit(1) + }) ` await fsp.writeFile(binaryPath, script, 'utf8') @@ -155,57 +244,31 @@ async function waitForFile(filePath: string, timeoutMs = 3_000): Promise<void> { throw new Error(`Timed out waiting for file: ${filePath}`) } -async function waitForPidFile(filePath: string, timeoutMs = 5_000): Promise<number> { +async function waitForCondition( + predicate: () => Promise<boolean> | boolean, + timeoutMs = 3_000, +): Promise<void> { const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { - const raw = await fsp.readFile(filePath, 'utf8').catch(() => '') - const pid = Number(raw.trim()) - if (Number.isInteger(pid) && pid > 0) return pid - await new Promise((resolve) => setTimeout(resolve, 25)) - } - throw new Error(`Timed out waiting for pid file: ${filePath}`) -} - -async function isProcessAlive(pid: number): Promise<boolean> { - try { - process.kill(pid, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false - throw error + if (await predicate()) { + return + } + await new Promise((resolve) => setTimeout(resolve, 50)) } + throw new Error('Timed out waiting for condition') } -async function waitForProcessExit(pid: number, timeoutMs = 5_000): Promise<void> { - const deadline = Date.now() + timeoutMs - while (Date.now() < deadline) { - if (!(await isProcessAlive(pid))) return - await new Promise((resolve) => setTimeout(resolve, 25)) - } - throw new Error(`Timed out waiting for process ${pid} to exit`) +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)) } -async function readJsonLines(filePath: string): Promise<any[]> { - const raw = await fsp.readFile(filePath, 'utf8').catch(() => '') - return raw +async function readThreadOperations(filePath: string): Promise<Array<{ method: string; threadId: string }>> { + await waitForFile(filePath) + return (await fsp.readFile(filePath, 'utf8')) + .trim() .split('\n') .filter(Boolean) - .map((line) => JSON.parse(line)) -} - -async function waitForJsonLine( - filePath: string, - predicate: (line: any) => boolean, - timeoutMs = 3_000, -): Promise<any> { - const deadline = Date.now() + timeoutMs - while (Date.now() < deadline) { - const lines = await readJsonLines(filePath) - const match = lines.find(predicate) - if (match) return match - await new Promise((resolve) => setTimeout(resolve, 25)) - } - throw new Error(`Timed out waiting for matching JSON line in ${filePath}`) + .map((line) => JSON.parse(line) as { method: string; threadId: string }) } async function createAuthenticatedWs(port: number): Promise<WebSocket> { @@ -258,47 +321,58 @@ describe('Codex Session Flow Integration', () => { let tempDir: string let fakeCodexPath: string let argLogPath: string + let appServerArgLogPath: string + let remoteThreadLogPath: string + let remoteInputLogPath: string + let threadOperationLogPath: string + let appServerCloseMarkerPath: string + let providerLossMarkerPath: string + let codexHomePath: string let previousCodexCmd: string | undefined let previousFakeCodexArgLog: string | undefined + let previousFakeCodexAppServerArgLog: string | undefined + let previousFakeCodexRemoteBehavior: string | undefined + let previousCodexHome: string | undefined let server: http.Server let port: number let wsHandler: WsHandler let registry: TerminalRegistry - let runtimes: Set<CodexAppServerRuntime> - let planner: CodexLaunchPlanner | null - - const createPlanner = () => new CodexLaunchPlanner(() => { - const runtime = new CodexAppServerRuntime({ - command: process.execPath, - commandArgs: [FAKE_APP_SERVER_PATH], - }) - runtimes.add(runtime) - return runtime - }) + let planner: CodexLaunchPlanner beforeAll(async () => { tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'freshell-codex-flow-')) fakeCodexPath = path.join(tempDir, 'fake-codex') argLogPath = path.join(tempDir, 'args.json') + appServerArgLogPath = path.join(tempDir, 'app-server-args.json') + remoteThreadLogPath = path.join(tempDir, 'remote-thread.txt') + remoteInputLogPath = path.join(tempDir, 'remote-input.txt') + threadOperationLogPath = path.join(tempDir, 'thread-ops.jsonl') + appServerCloseMarkerPath = path.join(tempDir, 'app-server-close-once.marker') + providerLossMarkerPath = path.join(tempDir, 'provider-loss-once.marker') + codexHomePath = path.join(tempDir, '.codex-home') await writeFakeCodexExecutable(fakeCodexPath) previousCodexCmd = process.env.CODEX_CMD previousFakeCodexArgLog = process.env.FAKE_CODEX_ARG_LOG + previousFakeCodexAppServerArgLog = process.env.FAKE_CODEX_APP_SERVER_ARG_LOG + previousFakeCodexRemoteBehavior = process.env.FAKE_CODEX_REMOTE_BEHAVIOR + previousCodexHome = process.env.CODEX_HOME process.env.CODEX_CMD = fakeCodexPath process.env.FAKE_CODEX_ARG_LOG = argLogPath + process.env.FAKE_CODEX_APP_SERVER_ARG_LOG = appServerArgLogPath + process.env.CODEX_HOME = codexHomePath const app = express() server = http.createServer(app) registry = new TerminalRegistry() - runtimes = new Set() - planner = createPlanner() - const plannerDelegate = { - planCreate: (input: Parameters<CodexLaunchPlanner['planCreate']>[0]) => { - if (!planner) throw new Error('Codex launch planner is not initialized') - return planner.planCreate(input) - }, - } as CodexLaunchPlanner - wsHandler = new WsHandler(server, registry, { codexLaunchPlanner: plannerDelegate }) + planner = new CodexLaunchPlanner((input) => new CodexTerminalSidecar({ + runtime: new CodexAppServerRuntime({ + command: process.execPath, + commandArgs: [FAKE_APP_SERVER_PATH, ...input.commandArgs], + env: input.env, + }), + })) + wsHandler = new WsHandler(server, registry, { codexLaunchPlanner: planner }) await new Promise<void>((resolve) => { server.listen(0, '127.0.0.1', () => { @@ -310,10 +384,9 @@ describe('Codex Session Flow Integration', () => { beforeEach(async () => { delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR - await planner?.shutdown() - await Promise.all([...runtimes].map((runtime) => runtime.shutdown())) - runtimes.clear() - planner = createPlanner() + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + await fsp.rm(codexHomePath, { recursive: true, force: true }) + await fsp.mkdir(codexHomePath, { recursive: true }) vi.mocked(configStore.snapshot).mockResolvedValue({ settings: { codingCli: { @@ -328,6 +401,18 @@ describe('Codex Session Flow Integration', () => { }, }) await fsp.rm(argLogPath, { force: true }) + await fsp.rm(appServerArgLogPath, { force: true }) + await fsp.rm(remoteThreadLogPath, { force: true }) + await fsp.rm(remoteInputLogPath, { force: true }) + await fsp.rm(threadOperationLogPath, { force: true }) + await fsp.rm(appServerCloseMarkerPath, { force: true }) + await fsp.rm(providerLossMarkerPath, { force: true }) + }) + + afterEach(() => { + for (const terminal of registry.list()) { + registry.remove(terminal.terminalId) + } }) afterAll(async () => { @@ -341,17 +426,32 @@ describe('Codex Session Flow Integration', () => { } else { process.env.FAKE_CODEX_ARG_LOG = previousFakeCodexArgLog } + if (previousFakeCodexAppServerArgLog === undefined) { + delete process.env.FAKE_CODEX_APP_SERVER_ARG_LOG + } else { + process.env.FAKE_CODEX_APP_SERVER_ARG_LOG = previousFakeCodexAppServerArgLog + } + if (previousFakeCodexRemoteBehavior === undefined) { + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } else { + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = previousFakeCodexRemoteBehavior + } + if (previousCodexHome === undefined) { + delete process.env.CODEX_HOME + } else { + process.env.CODEX_HOME = previousCodexHome + } - await planner?.shutdown() - await Promise.all([...runtimes].map((runtime) => runtime.shutdown())) - runtimes.clear() registry.shutdown() wsHandler.close() await new Promise<void>((resolve) => server.close(() => resolve())) await fsp.rm(tempDir, { recursive: true, force: true }) }) - it('starts the exact codex thread before PTY spawn and launches the TUI in remote mode', async () => { + it('launches a fresh codex terminal in remote mode without promoting a provisional thread id to durable identity', async () => { + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + recordRemoteThreadIdPath: remoteThreadLogPath, + }) const ws = await createAuthenticatedWs(port) try { @@ -376,7 +476,10 @@ describe('Codex Session Flow Integration', () => { expect(created).not.toHaveProperty('effectiveResumeSessionId') const record = registry.get(created.terminalId) - expect(record?.resumeSessionId).toBe('thread-new-1') + expect(record?.resumeSessionId).toBeUndefined() + await waitForFile(remoteThreadLogPath) + expect(await fsp.readFile(remoteThreadLogPath, 'utf8')).toBe('thread-new-1') + expect(record?.resumeSessionId).toBeUndefined() await waitForFile(argLogPath) const recordedArgs = JSON.parse(await fsp.readFile(argLogPath, 'utf8')) @@ -384,35 +487,177 @@ describe('Codex Session Flow Integration', () => { '--remote', expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), ]) - expect(recordedArgs).toContain('resume') - expect(recordedArgs).toContain('thread-new-1') + expect(recordedArgs).not.toContain('resume') + expect(recordedArgs).not.toContain('thread-new-1') expect(recordedArgs).toContain('tui.notification_method=bel') expect(recordedArgs).not.toContain('--model') expect(recordedArgs).not.toContain('--sandbox') + + await waitForFile(appServerArgLogPath) + const appServerLaunch = JSON.parse(await fsp.readFile(appServerArgLogPath, 'utf8')) + expect(appServerLaunch.argv).toContain('app-server') + expect(appServerLaunch.argv).toContain('mcp_servers.freshell.command="node"') + expect(appServerLaunch.argv.some((arg: string) => arg.startsWith('mcp_servers.freshell.args=['))).toBe(true) + expect(appServerLaunch.argv.indexOf('mcp_servers.freshell.command="node"')).toBeLessThan( + appServerLaunch.argv.indexOf('app-server'), + ) + expect(appServerLaunch.env.FRESHELL_TERMINAL_ID).toBe(created.terminalId) + expect(appServerLaunch.env.FRESHELL_TOKEN).toBe('test-token') } finally { await closeWebSocket(ws) } }) - it('restores a persisted Codex session without calling thread/resume on the app-server', async () => { + it('promotes a fresh codex terminal only after notification plus durable artifact proof', async () => { + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + sendTurnStart: true, + recordRemoteThreadIdPath: remoteThreadLogPath, + sleepMs: 500, + }) + const ws = await createAuthenticatedWs(port) + + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-promotion', + mode: 'codex', + cwd: tempDir, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-promotion' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + expect(created).not.toHaveProperty('effectiveResumeSessionId') + + await waitForFile(remoteThreadLogPath) + expect(await fsp.readFile(remoteThreadLogPath, 'utf8')).toBe('thread-new-1') + await waitForCondition(() => registry.get(created.terminalId)?.resumeSessionId === 'thread-new-1') + + const record = registry.get(created.terminalId) + expect(record?.resumeSessionId).toBe('thread-new-1') + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }) + + it('keeps the terminal alive when the owning Codex sidecar dies after launch', async () => { process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ - loadedThreadIds: ['thread-existing-1'], - overrides: { - 'thread/resume': { - error: { - code: -32600, - message: 'no rollout found for thread id thread-existing-1', - }, - }, - }, + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + exitProcessAfterMethodsOnce: ['turn/start'], + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + recordRemoteThreadIdPath: remoteThreadLogPath, + recordStdinPath: remoteInputLogPath, + sendTurnStartOnFreshOnly: true, + sleepMs: 5_000, + }) + const ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) }) + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-sidecar-dies', + mode: 'codex', + cwd: tempDir, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-sidecar-dies' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + await waitForCondition(() => registry.get(created.terminalId)?.resumeSessionId === 'thread-new-1') + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering' + ))) + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running' + ))) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' + && msg.terminalId === created.terminalId + ))).toBe(false) + + const record = registry.get(created.terminalId) + expect(record?.status).toBe('running') + expect(record?.codex?.durableSessionId).toBe('thread-new-1') + expect(record?.terminalId).toBe(created.terminalId) + + ws.send(JSON.stringify({ + type: 'terminal.input', + terminalId: created.terminalId, + data: 'after-recovery-input\n', + })) + await waitForCondition(async () => { + try { + return (await fsp.readFile(remoteInputLogPath, 'utf8')).includes('after-recovery-input') + } catch { + return false + } + }) + + await waitForCondition(async () => { + const operations = await readThreadOperations(threadOperationLogPath).catch(() => []) + return operations.some((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === 'thread-new-1' + )) + }) + const operations = await readThreadOperations(threadOperationLogPath) + expect(operations.some((entry) => entry.method === 'thread/start')).toBe(true) + expect(operations.some((entry) => entry.method === 'thread/resume' && entry.threadId === 'thread-new-1')).toBe(true) + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }) + + it('recovers when the Codex app-server client socket disconnects while the child stays alive', async () => { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + closeSocketAfterMethodsOnce: ['fs/watch'], + closeSocketAfterMethodsOnceMarkerPath: appServerCloseMarkerPath, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + recordStdinPath: remoteInputLogPath, + sleepMs: 5_000, + }) const ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) + }) try { ws.send(JSON.stringify({ type: 'terminal.create', - requestId: 'test-req-codex-restore', + requestId: 'test-req-codex-app-server-client-disconnect', mode: 'codex', cwd: tempDir, sessionRef: { @@ -424,7 +669,7 @@ describe('Codex Session Flow Integration', () => { const created = await waitForMessage( ws, (msg) => ( - msg.requestId === 'test-req-codex-restore' + msg.requestId === 'test-req-codex-app-server-client-disconnect' && (msg.type === 'terminal.created' || msg.type === 'error') ), ) @@ -432,155 +677,550 @@ describe('Codex Session Flow Integration', () => { throw new Error(`terminal.create failed: ${created.message}`) } - expect(created).not.toHaveProperty('effectiveResumeSessionId') + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering' + && msg.reason === 'app_server_client_disconnect' + ))) + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running' + ))) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' + && msg.terminalId === created.terminalId + ))).toBe(false) const record = registry.get(created.terminalId) - expect(record?.resumeSessionId).toBe('thread-existing-1') + expect(record?.status).toBe('running') + expect(record?.codex?.durableSessionId).toBe('thread-existing-1') + expect(record?.terminalId).toBe(created.terminalId) - await waitForFile(argLogPath) - const recordedArgs = JSON.parse(await fsp.readFile(argLogPath, 'utf8')) - expect(recordedArgs.slice(0, 2)).toEqual([ - '--remote', - expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), - ]) - expect(recordedArgs).toContain('resume') - expect(recordedArgs).toContain('thread-existing-1') + ws.send(JSON.stringify({ + type: 'terminal.input', + terminalId: created.terminalId, + data: 'after-client-disconnect-recovery\n', + })) + await waitForCondition(async () => { + try { + return (await fsp.readFile(remoteInputLogPath, 'utf8')).includes('after-client-disconnect-recovery') + } catch { + return false + } + }) + + const operations = await readThreadOperations(threadOperationLogPath) + expect(operations.filter((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === 'thread-existing-1' + )).length).toBeGreaterThanOrEqual(2) + expect(operations.some((entry) => entry.method === 'thread/start')).toBe(false) } finally { await closeWebSocket(ws) delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR } }) - it('retires the previous wrapper/native app-server during recovery replacement and routes later input only to the replacement', async () => { - const testDir = await fsp.mkdtemp(path.join(tempDir, 'recovery-retire-')) - const metadataDir = path.join(testDir, 'metadata') - const oldNativePidFile = path.join(testDir, 'old-native.pid') - const replacementNativePidFile = path.join(testDir, 'replacement-native.pid') - const launchLogPath = path.join(testDir, 'codex-launches.jsonl') - const inputLogPath = path.join(testDir, 'codex-input.jsonl') - const oldSidecarShutdownSignalPath = path.join(testDir, 'old-sidecar-shutdown.signal') - const firstLaunchClaimPath = path.join(testDir, 'first-tui.claim') - await fsp.mkdir(metadataDir, { recursive: true }) - - const previousStayAlive = process.env.FAKE_CODEX_STAY_ALIVE - const previousLaunchLog = process.env.FAKE_CODEX_LAUNCH_LOG - const previousInputLog = process.env.FAKE_CODEX_INPUT_LOG - const previousExitWhenFileExists = process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS - const previousFirstLaunchOnly = process.env.FAKE_CODEX_EXIT_WATCH_FIRST_LAUNCH_ONLY - const previousFirstLaunchClaim = process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH - process.env.FAKE_CODEX_STAY_ALIVE = '1' - process.env.FAKE_CODEX_LAUNCH_LOG = launchLogPath - process.env.FAKE_CODEX_INPUT_LOG = inputLogPath - process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS = oldSidecarShutdownSignalPath - process.env.FAKE_CODEX_EXIT_WATCH_FIRST_LAUNCH_ONLY = '1' - process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH = firstLaunchClaimPath - - const oldRuntime = new CodexAppServerRuntime({ - command: process.execPath, - commandArgs: [FAKE_APP_SERVER_PATH], - metadataDir, - serverInstanceId: 'srv-codex-recovery-old', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile: oldNativePidFile, - wrapperLeavesNativeOnSigterm: true, - signalFileOnSigterm: oldSidecarShutdownSignalPath, - delayExitOnSigtermMs: 200, - loadedThreadIds: ['thread-existing-1'], - }), - }, + it('recovers when the provider reports the active durable thread closed', async () => { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + threadClosedAfterMethodsOnce: ['thread/resume'], + threadClosedAfterMethodsOnceMarkerPath: providerLossMarkerPath, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + recordStdinPath: remoteInputLogPath, + sleepMs: 5_000, + }) + const ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) }) - const replacementRuntime = new CodexAppServerRuntime({ - command: process.execPath, - commandArgs: [FAKE_APP_SERVER_PATH], - metadataDir, - serverInstanceId: 'srv-codex-recovery-replacement', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile: replacementNativePidFile, - wrapperLeavesNativeOnSigterm: true, - loadedThreadIds: ['thread-existing-1'], - }), + + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-thread-closed-recovery', + mode: 'codex', + cwd: tempDir, + sessionRef: { + provider: 'codex', + sessionId: 'thread-existing-1', + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-thread-closed-recovery' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering' + && msg.reason === 'provider_thread_lifecycle_loss' + ))) + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running' + ))) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' + && msg.terminalId === created.terminalId + ))).toBe(false) + + ws.send(JSON.stringify({ + type: 'terminal.input', + terminalId: created.terminalId, + data: 'after-thread-closed-recovery\n', + })) + await waitForCondition(async () => { + try { + return (await fsp.readFile(remoteInputLogPath, 'utf8')).includes('after-thread-closed-recovery') + } catch { + return false + } + }) + + const record = registry.get(created.terminalId) + expect(record?.status).toBe('running') + expect(record?.codex?.durableSessionId).toBe('thread-existing-1') + expect(record?.terminalId).toBe(created.terminalId) + + const operations = await readThreadOperations(threadOperationLogPath) + expect(operations.filter((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === 'thread-existing-1' + ))).toHaveLength(2) + expect(operations.some((entry) => entry.method === 'thread/start')).toBe(false) + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }) + + it.each(['notLoaded', 'systemError'])( + 'recovers when the provider reports active durable thread status %s', + async (statusType) => { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + threadStatusChangedAfterMethodsOnceMarkerPath: providerLossMarkerPath, + threadStatusChangedAfterMethodsOnce: { + 'thread/resume': [ + { + threadId: 'thread-existing-1', + status: { type: statusType }, + }, + ], + }, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + recordStdinPath: remoteInputLogPath, + sleepMs: 5_000, + }) + const ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) + }) + + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: `test-req-codex-thread-status-${statusType}`, + mode: 'codex', + cwd: tempDir, + sessionRef: { + provider: 'codex', + sessionId: 'thread-existing-1', + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === `test-req-codex-thread-status-${statusType}` + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering' + && msg.reason === 'provider_thread_lifecycle_loss' + ))) + await waitForCondition(() => receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running' + ))) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' + && msg.terminalId === created.terminalId + ))).toBe(false) + + const operations = await readThreadOperations(threadOperationLogPath) + expect(operations.filter((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === 'thread-existing-1' + ))).toHaveLength(2) + expect(operations.some((entry) => entry.method === 'thread/start')).toBe(false) + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }, + ) + + it('ignores provider lifecycle-loss notifications for other durable threads', async () => { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + notifyAfterMethodsOnce: { + 'thread/resume': [ + { + method: 'thread/closed', + params: { threadId: 'thread-other-1' }, + }, + { + method: 'thread/status/changed', + params: { + threadId: 'thread-other-2', + status: { type: 'notLoaded' }, + }, + }, + ], }, }) - runtimes.add(oldRuntime) - runtimes.add(replacementRuntime) - const oldPlanner = new CodexLaunchPlanner(oldRuntime) - const replacementPlanner = new CodexLaunchPlanner(replacementRuntime) - let terminalId: string | undefined + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + sleepMs: 800, + }) + const ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) + }) try { - const oldPlan = await oldPlanner.planCreate({ resumeSessionId: 'thread-existing-1' }) - const oldNativePid = await waitForPidFile(oldNativePidFile) - const recovery = { - planCreate: vi.fn(() => replacementPlanner.planCreate({ resumeSessionId: 'thread-existing-1' })), - retryDelayMs: 0, - readinessTimeoutMs: 1_000, - readinessPollMs: 25, + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-ignore-other-thread-loss', + mode: 'codex', + cwd: tempDir, + sessionRef: { + provider: 'codex', + sessionId: 'thread-existing-1', + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-ignore-other-thread-loss' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + await waitForCondition(async () => { + const operations = await readThreadOperations(threadOperationLogPath) + return operations.some((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === 'thread-existing-1' + )) + }) + await sleep(350) + + const record = registry.get(created.terminalId) + expect(record?.status).toBe('running') + expect(record?.codex?.workerGeneration).toBe(1) + expect(record?.codex?.recoveryState).toBe('running_durable') + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering' + ))).toBe(false) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' + && msg.terminalId === created.terminalId + ))).toBe(false) + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }) + + it('recovers a durable Codex PTY exit by resuming the existing upstream thread', async () => { + const durableSessionId = 'thread-existing-1' + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + assertNoDuplicateActiveThread: true, + appendThreadOperationLogPath: threadOperationLogPath, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + sleepMs: 100, + }) + const ws = await createAuthenticatedWs(port) + const terminalStatusMessages: any[] = [] + const onMessage = (raw: WebSocket.Data) => { + const msg = JSON.parse(raw.toString()) + if (msg.type === 'terminal.status' || msg.type === 'terminal.exit') { + terminalStatusMessages.push(msg) } - const term = registry.create({ + } + ws.on('message', onMessage) + + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-durable-recovery', mode: 'codex', - resumeSessionId: 'thread-existing-1', cwd: tempDir, - providerSettings: { - codexAppServer: { - wsUrl: oldPlan.remote.wsUrl, - sidecar: oldPlan.sidecar, - recovery, - }, - } as any, + sessionRef: { + provider: 'codex', + sessionId: durableSessionId, + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-durable-recovery' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } + + await waitForMessage( + ws, + (msg) => msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'recovering', + ) + await waitForMessage( + ws, + (msg) => msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running', + ) + await waitForCondition(() => (registry.get(created.terminalId)?.codex?.workerGeneration ?? 0) >= 2) + + const record = registry.get(created.terminalId) + expect(record?.status).toBe('running') + expect(record?.codex?.durableSessionId).toBe(durableSessionId) + expect(record?.terminalId).toBe(created.terminalId) + + expect(terminalStatusMessages.some((msg) => ( + msg.type === 'terminal.status' && msg.status === 'recovery_failed' + ))).toBe(false) + expect(terminalStatusMessages.some((msg) => ( + msg.type === 'terminal.exit' && msg.terminalId === created.terminalId + ))).toBe(false) + + const operations = await readThreadOperations(threadOperationLogPath) + expect(operations.filter((entry) => entry.method === 'thread/resume')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ threadId: durableSessionId }), + ]), + ) + expect(operations.filter((entry) => ( + entry.method === 'thread/resume' && entry.threadId === durableSessionId + )).length).toBeGreaterThanOrEqual(2) + expect(operations.some((entry) => entry.method === 'thread/start')).toBe(false) + } finally { + ws.off('message', onMessage) + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }) + + it('keeps retrying replacement launch failures until durable resume succeeds', async () => { + const durableSessionId = 'thread-existing-1' + let planCreateSpy: ReturnType<typeof vi.spyOn> | undefined + let ws: WebSocket | undefined + + try { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + appendThreadOperationLogPath: threadOperationLogPath, + assertNoDuplicateActiveThread: true, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + sleepMs: 30_000, + }) + + ws = await createAuthenticatedWs(port) + const receivedMessages: any[] = [] + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString())) }) - terminalId = term.terminalId - const oldPtyPid = term.pty.pid - await waitForJsonLine(launchLogPath, (line) => line.pid === oldPtyPid) - await (registry as any).runCodexRecoveryAttempt( - registry.get(term.terminalId), - 'thread-existing-1', + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-retry-until-resume', + mode: 'codex', + cwd: tempDir, + sessionRef: { + provider: 'codex', + sessionId: durableSessionId, + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-retry-until-resume' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) + } - const replacementNativePid = await waitForPidFile(replacementNativePidFile) - await waitForProcessExit(oldNativePid) - await waitForProcessExit(oldPtyPid) - expect(await isProcessAlive(replacementNativePid)).toBe(true) + await waitForCondition(async () => { + const operations = await readThreadOperations(threadOperationLogPath).catch(() => []) + return operations.some((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === durableSessionId + )) + }) + + const originalPlanCreate = planner.planCreate.bind(planner) + let plannedFailures = 5 + planCreateSpy = vi.spyOn(planner, 'planCreate').mockImplementation(async (input) => { + if (input.resumeSessionId === durableSessionId && plannedFailures > 0) { + const failureNumber = 6 - plannedFailures + plannedFailures -= 1 + throw new Error(`planned replacement failure ${failureNumber}`) + } + return originalPlanCreate(input) + }) - const latest = registry.get(term.terminalId) - const replacementPtyPid = latest?.pty.pid - expect(replacementPtyPid).toEqual(expect.any(Number)) - expect(replacementPtyPid).not.toBe(oldPtyPid) + const record = registry.get(created.terminalId) + expect(record?.codex?.durableSessionId).toBe(durableSessionId) + record?.pty.kill() - expect(registry.input(term.terminalId, 'after recovery replacement\n')).toBe(true) - await waitForJsonLine( - inputLogPath, - (line) => line.pid === replacementPtyPid && line.data.includes('after recovery replacement'), + await waitForMessage( + ws, + (msg) => msg.type === 'terminal.status' + && msg.terminalId === created.terminalId + && msg.status === 'running', + 20_000, ) - const inputLines = await readJsonLines(inputLogPath) - expect(inputLines.some((line) => line.pid === oldPtyPid && line.data.includes('after recovery replacement'))).toBe(false) + + await waitForCondition(async () => { + const operations = await readThreadOperations(threadOperationLogPath).catch(() => []) + return operations.filter((entry) => ( + entry.method === 'thread/resume' + && entry.threadId === durableSessionId + )).length >= 2 + }, 20_000) + + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.status' && msg.status === 'recovery_failed' + ))).toBe(false) + expect(receivedMessages.some((msg) => ( + msg.type === 'terminal.exit' && msg.terminalId === created.terminalId + ))).toBe(false) + expect(receivedMessages).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: 'terminal.status', status: 'recovering' }), + expect.objectContaining({ type: 'terminal.status', status: 'running' }), + ])) + + const operations = await readThreadOperations(threadOperationLogPath) + const resumeOperations = operations.filter((entry) => ( + entry.method === 'thread/resume' && entry.threadId === durableSessionId + )) + const startOperations = operations.filter((entry) => entry.method === 'thread/start') + expect(resumeOperations.length).toBeGreaterThanOrEqual(2) + expect(resumeOperations.every((entry) => entry.threadId === durableSessionId)).toBe(true) + expect(startOperations).toHaveLength(0) + expect(planCreateSpy.mock.calls.filter(([input]) => ( + input.resumeSessionId === durableSessionId + )).length).toBeGreaterThanOrEqual(6) } finally { - if (terminalId) { - await registry.killAndWait(terminalId).catch(() => undefined) + planCreateSpy?.mockRestore() + if (ws) await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR + } + }, 25_000) + + it('restores a persisted Codex session through the exact durable CLI form', async () => { + process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR = JSON.stringify({ + overrides: { + 'thread/resume': { + error: { + code: -32600, + message: 'no rollout found for thread id thread-existing-1', + }, + }, + }, + }) + process.env.FAKE_CODEX_REMOTE_BEHAVIOR = JSON.stringify({ + sleepMs: 300, + }) + const ws = await createAuthenticatedWs(port) + + try { + ws.send(JSON.stringify({ + type: 'terminal.create', + requestId: 'test-req-codex-restore', + mode: 'codex', + cwd: tempDir, + sessionRef: { + provider: 'codex', + sessionId: 'thread-existing-1', + }, + })) + + const created = await waitForMessage( + ws, + (msg) => ( + msg.requestId === 'test-req-codex-restore' + && (msg.type === 'terminal.created' || msg.type === 'error') + ), + ) + if (created.type === 'error') { + throw new Error(`terminal.create failed: ${created.message}`) } - await replacementPlanner.shutdown().catch(() => undefined) - await oldPlanner.shutdown().catch(() => undefined) - await replacementRuntime.shutdown().catch(() => undefined) - await oldRuntime.shutdown().catch(() => undefined) - runtimes.delete(oldRuntime) - runtimes.delete(replacementRuntime) - if (previousStayAlive === undefined) delete process.env.FAKE_CODEX_STAY_ALIVE - else process.env.FAKE_CODEX_STAY_ALIVE = previousStayAlive - if (previousLaunchLog === undefined) delete process.env.FAKE_CODEX_LAUNCH_LOG - else process.env.FAKE_CODEX_LAUNCH_LOG = previousLaunchLog - if (previousInputLog === undefined) delete process.env.FAKE_CODEX_INPUT_LOG - else process.env.FAKE_CODEX_INPUT_LOG = previousInputLog - if (previousExitWhenFileExists === undefined) delete process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS - else process.env.FAKE_CODEX_EXIT_WHEN_FILE_EXISTS = previousExitWhenFileExists - if (previousFirstLaunchOnly === undefined) delete process.env.FAKE_CODEX_EXIT_WATCH_FIRST_LAUNCH_ONLY - else process.env.FAKE_CODEX_EXIT_WATCH_FIRST_LAUNCH_ONLY = previousFirstLaunchOnly - if (previousFirstLaunchClaim === undefined) delete process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH - else process.env.FAKE_CODEX_FIRST_LAUNCH_CLAIM_PATH = previousFirstLaunchClaim - await fsp.rm(testDir, { recursive: true, force: true }) + + expect(created).not.toHaveProperty('effectiveResumeSessionId') + + await waitForFile(argLogPath) + const recordedArgs = JSON.parse(await fsp.readFile(argLogPath, 'utf8')) + expect(recordedArgs.slice(0, 2)).toEqual([ + '--remote', + expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), + ]) + expect(recordedArgs).toContain('resume') + expect(recordedArgs).toContain('thread-existing-1') + } finally { + await closeWebSocket(ws) + delete process.env.FAKE_CODEX_APP_SERVER_BEHAVIOR + delete process.env.FAKE_CODEX_REMOTE_BEHAVIOR } }) }) diff --git a/test/server/agent-run.test.ts b/test/server/agent-run.test.ts index ebd24fbb4..499f65d6c 100644 --- a/test/server/agent-run.test.ts +++ b/test/server/agent-run.test.ts @@ -64,7 +64,7 @@ it('allocates and passes an OpenCode control endpoint for /api/run in opencode m })) }) -it('uses the Codex planner and marks fresh /api/run sessions as starts', async () => { +it('uses the shared Codex planner and marks fresh /api/run sessions as starts', async () => { const registry = { create: vi.fn((opts?: { terminalId?: string }) => ({ terminalId: opts?.terminalId ?? 'term1' })), input: vi.fn(() => true), @@ -115,129 +115,11 @@ it('uses the Codex planner and marks fresh /api/run sessions as starts', async ( resumeSessionId: undefined, sessionBindingReason: 'start', providerSettings: expect.objectContaining({ - codexAppServer: expect.objectContaining({ + codexAppServer: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL, - }), - }), - })) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([{ terminalId: 'term1', generation: 0 }]) -}) - -it('shuts down the pending Codex sidecar when /api/run fails after planning', async () => { - const registry = { - create: vi.fn(() => { - throw new Error('spawn failed after planning') - }), - input: vi.fn(() => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - - const app = express() - app.use(express.json()) - app.use('/api', createAgentApiRouter({ - layoutStore: { - createTab: () => ({ tabId: 't1', paneId: 'p1' }), - attachPaneContent: () => {}, - }, - registry, - codexLaunchPlanner, - })) - - const res = await request(app).post('/api/run').send({ command: 'echo done', mode: 'codex' }) - - expect(res.status).toBe(500) - expect(res.body.message).toBe('spawn failed after planning') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([]) -}) - -it('reports pending Codex sidecar shutdown failure when /api/run fails after planning', async () => { - const registry = { - create: vi.fn(() => { - throw new Error('spawn failed after planning') - }), - input: vi.fn(() => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - codexLaunchPlanner.sidecar.shutdownError = new Error('verified sidecar teardown failed') - - const app = express() - app.use(express.json()) - app.use('/api', createAgentApiRouter({ - layoutStore: { - createTab: () => ({ tabId: 't1', paneId: 'p1' }), - attachPaneContent: () => {}, - }, - registry, - codexLaunchPlanner, - })) - - const res = await request(app).post('/api/run').send({ command: 'echo done', mode: 'codex' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('spawn failed after planning') - expect(res.body.message).toContain('verified sidecar teardown failed') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([]) -}) - -it('kills the created terminal and sidecar when /api/run fails after registry.create', async () => { - const registry = { - create: vi.fn(() => ({ terminalId: 'term1' })), - input: vi.fn(() => true), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - vi.spyOn(codexLaunchPlanner.sidecar, 'adopt').mockRejectedValue(new Error('adopt failed after create')) - - const app = express() - app.use(express.json()) - app.use('/api', createAgentApiRouter({ - layoutStore: { - createTab: () => ({ tabId: 't1', paneId: 'p1' }), - attachPaneContent: () => {}, - }, - registry, - codexLaunchPlanner, - })) - - const res = await request(app).post('/api/run').send({ command: 'echo done', mode: 'codex' }) - - expect(res.status).toBe(500) - expect(res.body.message).toBe('adopt failed after create') - expect(registry.killAndWait).toHaveBeenCalledWith('term1') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) -}) - -it('reports created-terminal cleanup failure when /api/run fails after registry.create', async () => { - const registry = { - create: vi.fn(() => ({ terminalId: 'term1' })), - input: vi.fn(() => true), - killAndWait: vi.fn(async () => { - throw new Error('terminal cleanup failed') + }, }), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - vi.spyOn(codexLaunchPlanner.sidecar, 'adopt').mockRejectedValue(new Error('adopt failed after create')) - - const app = express() - app.use(express.json()) - app.use('/api', createAgentApiRouter({ - layoutStore: { - createTab: () => ({ tabId: 't1', paneId: 'p1' }), - attachPaneContent: () => {}, - }, - registry, - codexLaunchPlanner, })) - - const res = await request(app).post('/api/run').send({ command: 'echo done', mode: 'codex' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('adopt failed after create') - expect(res.body.message).toContain('terminal cleanup failed') - expect(registry.killAndWait).toHaveBeenCalledWith('term1') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) }) it('retries initial Codex launch before starting a detached /api/run session', async () => { @@ -391,47 +273,3 @@ it('rejects invalid Codex settings for /api/run before creating a tab', async () expect(createTab).not.toHaveBeenCalled() expect(registry.create).not.toHaveBeenCalled() }) - -it('rejects Codex /api/run without planning when shutdown admission closes while reading settings', async () => { - let acceptingCreates = true - const createTab = vi.fn(() => ({ tabId: 't1', paneId: 'p1' })) - const registry = { - create: vi.fn(() => ({ terminalId: 'term1' })), - input: vi.fn(() => true), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - - const app = express() - app.use(express.json()) - app.use('/api', createAgentApiRouter({ - layoutStore: { - createTab, - attachPaneContent: vi.fn(), - }, - registry, - codexLaunchPlanner, - configStore: { - getSettings: vi.fn(async () => { - acceptingCreates = false - return { codingCli: { providers: { codex: {} } } } - }), - }, - assertTerminalCreateAccepted: () => { - if (!acceptingCreates) { - throw new Error('Server is shutting down; terminal creation is not accepted.') - } - }, - })) - - const res = await request(app).post('/api/run').send({ command: 'echo done', mode: 'codex' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Server is shutting down') - expect(codexLaunchPlanner.planCreateCalls).toEqual([]) - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(0) - expect(createTab).not.toHaveBeenCalled() - expect(registry.create).not.toHaveBeenCalled() - expect(registry.input).not.toHaveBeenCalled() - expect(registry.killAndWait).not.toHaveBeenCalled() -}) diff --git a/test/server/agent-tabs-write.test.ts b/test/server/agent-tabs-write.test.ts index 68e455c3f..c5769a7a5 100644 --- a/test/server/agent-tabs-write.test.ts +++ b/test/server/agent-tabs-write.test.ts @@ -313,298 +313,6 @@ describe('tab endpoints', () => { expect(registry.create).not.toHaveBeenCalled() }) - it('rejects Codex tab creation without planning when shutdown admission closes while reading settings', async () => { - const app = express() - app.use(express.json()) - let acceptingCreates = true - const registry = { - create: vi.fn(() => { - throw new Error('registry.create should not run') - }), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - const layoutStore = { - createTab: vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - const configStore = { - getSettings: vi.fn(async () => { - acceptingCreates = false - return { codingCli: { providers: { codex: {} } } } - }), - } - app.use('/api', createAgentApiRouter({ - layoutStore, - registry, - configStore, - codexLaunchPlanner, - assertTerminalCreateAccepted: () => { - if (!acceptingCreates) { - throw new Error('Server is shutting down; terminal creation is not accepted.') - } - }, - })) - - const res = await request(app).post('/api/tabs').send({ mode: 'codex', name: 'shutdown before planning' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Server is shutting down') - expect(configStore.getSettings).toHaveBeenCalledTimes(1) - expect(codexLaunchPlanner.planCreateCalls).toEqual([]) - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(0) - expect(registry.create).not.toHaveBeenCalled() - expect(registry.killAndWait).not.toHaveBeenCalled() - expect(layoutStore.createTab).not.toHaveBeenCalled() - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('kills the created Codex terminal when tab creation fails after registry.create', async () => { - const app = express() - app.use(express.json()) - const registry = { - create: vi.fn(() => ({ terminalId: 'term_1' })), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - vi.spyOn(codexLaunchPlanner.sidecar, 'adopt').mockRejectedValue(new Error('adopt failed after tab create')) - const layoutStore = { - createTab: () => ({ tabId: 'tab_1', paneId: 'pane_1' }), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ layoutStore, registry, codexLaunchPlanner })) - - const res = await request(app).post('/api/tabs').send({ mode: 'codex', name: 'resume tab' }) - - expect(res.status).toBe(500) - expect(res.body.message).toBe('adopt failed after tab create') - expect(registry.killAndWait).toHaveBeenCalledWith('term_1') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('kills the inserted Codex terminal when registry.create fails after insertion', async () => { - const app = express() - app.use(express.json()) - const createError = new Error('terminal.created listener failed') as Error & { terminalId?: string } - createError.terminalId = 'term_inserted' - const registry = { - create: vi.fn(() => { - throw createError - }), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - const layoutStore = { - createTab: () => ({ tabId: 'tab_1', paneId: 'pane_1' }), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ layoutStore, registry, codexLaunchPlanner })) - - const res = await request(app).post('/api/tabs').send({ mode: 'codex', name: 'emit failure tab' }) - - expect(res.status).toBe(500) - expect(res.body.message).toBe('terminal.created listener failed') - expect(registry.killAndWait).toHaveBeenCalledWith('term_inserted') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('kills the created Codex terminal when resume readiness returns after the PTY exited', async () => { - const app = express() - app.use(express.json()) - const terminal = { terminalId: 'term_exited_before_publish', status: 'running' } - const registry = { - create: vi.fn(() => terminal), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - vi.spyOn(codexLaunchPlanner.sidecar, 'waitForLoadedThread').mockImplementation(async (threadId, options) => { - codexLaunchPlanner.sidecar.waitForLoadedThreadCalls.push({ threadId, options }) - terminal.status = 'exited' - }) - const layoutStore = { - createTab: () => ({ tabId: 'tab_1', paneId: 'pane_1' }), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ layoutStore, registry, codexLaunchPlanner })) - - const res = await request(app).post('/api/tabs').send({ - mode: 'codex', - name: 'resume tab', - resumeSessionId: 'thread-resume-exits', - }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Codex terminal PTY exited before create completed') - expect(registry.killAndWait).toHaveBeenCalledWith('term_exited_before_publish') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('kills the created Codex terminal without waiting for readiness when shutdown admission closes after adoption', async () => { - const app = express() - app.use(express.json()) - let acceptingCreates = true - const terminal = { terminalId: 'term_shutdown_after_adopt', status: 'running' } - const registry = { - create: vi.fn(() => terminal), - killAndWait: vi.fn(async () => true), - publishCodexSidecar: vi.fn(), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - const originalAdopt = codexLaunchPlanner.sidecar.adopt.bind(codexLaunchPlanner.sidecar) - vi.spyOn(codexLaunchPlanner.sidecar, 'adopt').mockImplementation(async (input) => { - await originalAdopt(input) - acceptingCreates = false - }) - const layoutStore = { - createTab: vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ - layoutStore, - registry, - codexLaunchPlanner, - assertTerminalCreateAccepted: () => { - if (!acceptingCreates) { - throw new Error('Server is shutting down; terminal creation is not accepted.') - } - }, - })) - - const res = await request(app).post('/api/tabs').send({ - mode: 'codex', - name: 'resume tab', - resumeSessionId: 'thread-resume-shutdown', - }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Server is shutting down') - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([{ terminalId: 'term_shutdown_after_adopt', generation: 0 }]) - expect(codexLaunchPlanner.sidecar.waitForLoadedThreadCalls).toEqual([]) - expect(registry.publishCodexSidecar).not.toHaveBeenCalled() - expect(registry.killAndWait).toHaveBeenCalledWith('term_shutdown_after_adopt') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('rejects Codex tab creation when shutdown admission closes after planning', async () => { - const app = express() - app.use(express.json()) - let acceptingCreates = true - const registry = { - create: vi.fn(() => ({ terminalId: 'term_1', status: 'running' })), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - const originalPlanCreate = codexLaunchPlanner.planCreate.bind(codexLaunchPlanner) - vi.spyOn(codexLaunchPlanner, 'planCreate').mockImplementation(async (input) => { - const plan = await originalPlanCreate(input) - acceptingCreates = false - return plan - }) - const layoutStore = { - createTab: vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ - layoutStore, - registry, - codexLaunchPlanner, - assertTerminalCreateAccepted: () => { - if (!acceptingCreates) { - throw new Error('Server is shutting down; terminal creation is not accepted.') - } - }, - })) - - const res = await request(app).post('/api/tabs').send({ mode: 'codex', name: 'shutdown after plan' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Server is shutting down') - expect(registry.create).not.toHaveBeenCalled() - expect(registry.killAndWait).not.toHaveBeenCalled() - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - - it('kills the created Codex terminal when shutdown admission closes before adoption', async () => { - const app = express() - app.use(express.json()) - const registry = { - create: vi.fn(() => ({ terminalId: 'term_1', status: 'running' })), - killAndWait: vi.fn(async () => true), - } - const codexLaunchPlanner = new FakeCodexLaunchPlanner() - const layoutStore = { - createTab: vi.fn(() => ({ tabId: 'tab_1', paneId: 'pane_1' })), - attachPaneContent: vi.fn(), - selectTab: () => ({}), - renameTab: () => ({}), - closeTab: () => ({}), - hasTab: () => true, - selectNextTab: () => ({ tabId: 'tab_1' }), - selectPrevTab: () => ({ tabId: 'tab_1' }), - } - app.use('/api', createAgentApiRouter({ - layoutStore, - registry, - codexLaunchPlanner, - assertTerminalCreateAccepted: () => { - if (registry.create.mock.calls.length > 0) { - throw new Error('Server is shutting down; terminal creation is not accepted.') - } - }, - })) - - const res = await request(app).post('/api/tabs').send({ mode: 'codex', name: 'shutdown before adopt' }) - - expect(res.status).toBe(500) - expect(res.body.message).toContain('Server is shutting down') - expect(registry.create).toHaveBeenCalledTimes(1) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([]) - expect(registry.killAndWait).toHaveBeenCalledWith('term_1') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(layoutStore.attachPaneContent).not.toHaveBeenCalled() - }) - it('rejects blank tab rename payloads', async () => { const app = express() app.use(express.json()) diff --git a/test/server/session-association.test.ts b/test/server/session-association.test.ts index 1db22992c..26bf8b896 100644 --- a/test/server/session-association.test.ts +++ b/test/server/session-association.test.ts @@ -1,11 +1,10 @@ -import { beforeEach, describe, it, expect, vi } from 'vitest' +import { describe, it, expect, vi } from 'vitest' import { TerminalRegistry } from '../../server/terminal-registry' import { CodingCliSessionIndexer } from '../../server/coding-cli/session-indexer' import { makeSessionKey, type CodingCliSession, type ProjectGroup } from '../../server/coding-cli/types' import { SessionAssociationCoordinator } from '../../server/session-association-coordinator' import { TerminalMetadataService } from '../../server/terminal-metadata-service' import { collectAppliedSessionAssociations } from '../../server/session-association-updates' -import { recordSessionLifecycleEvent } from '../../server/session-observability' vi.mock('node-pty', () => ({ spawn: vi.fn(() => ({ @@ -22,10 +21,6 @@ vi.mock('../../server/mcp/config-writer.js', () => ({ cleanupMcpConfig: vi.fn(), })) -vi.mock('../../server/session-observability.js', () => ({ - recordSessionLifecycleEvent: vi.fn(), -})) - const SESSION_ID_ONE = '550e8400-e29b-41d4-a716-446655440000' const SESSION_ID_TWO = '6f1c2b3a-4d5e-6f70-8a9b-0c1d2e3f4a5b' const SESSION_ID_THREE = 'f47ac10b-58cc-4372-a567-0e02b2c3d479' @@ -54,10 +49,6 @@ function createIndexer(): CodingCliSessionIndexer { return new CodingCliSessionIndexer([]) } -beforeEach(() => { - vi.mocked(recordSessionLifecycleEvent).mockClear() -}) - describe('SessionAssociationCoordinator integration', () => { it('associates a Claude terminal created with a human-readable resume name after UUID discovery', () => { const registry = new TerminalRegistry() @@ -151,77 +142,6 @@ describe('SessionAssociationCoordinator integration', () => { registry.shutdown() }) - - it('records a lifecycle event when Codex durable identity is explicitly bound', () => { - const registry = new TerminalRegistry() - const terminal = registry.create({ mode: 'codex', cwd: '/home/user/project' }) - - registry.rebindSession(terminal.terminalId, 'codex', 'codex-thread-1', 'association') - - expect(recordSessionLifecycleEvent).toHaveBeenCalledWith({ - kind: 'terminal_session_bound', - provider: 'codex', - terminalId: terminal.terminalId, - sessionId: 'codex-thread-1', - reason: 'association', - }) - - registry.shutdown() - }) - - it('records a lifecycle warning when a Codex terminal exits before durable identity exists', () => { - const registry = new TerminalRegistry() - const terminal = registry.create({ mode: 'codex', cwd: '/home/user/project' }) - const pty = terminal.pty as unknown as { onExit: ReturnType<typeof vi.fn> } - const onExit = pty.onExit.mock.calls[0][0] - - onExit({ exitCode: 0, signal: 0 }) - - expect(recordSessionLifecycleEvent).toHaveBeenCalledWith(expect.objectContaining({ - kind: 'terminal_exit_without_durable_session', - terminalId: terminal.terminalId, - mode: 'codex', - exitCode: 0, - reason: 'pty_exit', - })) - - registry.shutdown() - }) - - it('records a lifecycle event when the Codex sidecar reports durable identity', () => { - let onDurableSession: ((sessionId: string) => void) | undefined - const sidecar = { - attachTerminal: vi.fn((callbacks: { onDurableSession: (sessionId: string) => void }) => { - onDurableSession = callbacks.onDurableSession - }), - shutdown: vi.fn(async () => undefined), - } - const registry = new TerminalRegistry() - const terminal = registry.create({ - mode: 'codex', - cwd: '/home/user/project', - codexSidecar: sidecar, - }) - - onDurableSession?.('codex-thread-1') - onDurableSession?.('codex-thread-1') - - const durableObservationCalls = vi.mocked(recordSessionLifecycleEvent).mock.calls.filter(([event]) => - event.kind === 'codex_durable_session_observed' - ) - expect(durableObservationCalls).toEqual([[ - { - kind: 'codex_durable_session_observed', - provider: 'codex', - terminalId: terminal.terminalId, - sessionId: 'codex-thread-1', - generation: 1, - source: 'sidecar', - }, - ]]) - - registry.shutdown() - }) }) describe('Session-Terminal metadata broadcasts', () => { diff --git a/test/server/ws-protocol.test.ts b/test/server/ws-protocol.test.ts index 11e2c5a0f..3408a22f1 100644 --- a/test/server/ws-protocol.test.ts +++ b/test/server/ws-protocol.test.ts @@ -1,15 +1,16 @@ import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest' import http from 'http' import WebSocket from 'ws' -import { WS_PROTOCOL_VERSION } from '../../shared/ws-protocol.js' import { - FakeCodexLaunchPlanner, - FakeCodexLaunchSidecar, - DEFAULT_CODEX_REMOTE_WS_URL, -} from '../helpers/coding-cli/fake-codex-launch-planner.js' + HelloSchema, + TerminalCreateSchema, + WS_PROTOCOL_VERSION, +} from '../../shared/ws-protocol.js' +import { FakeCodexLaunchPlanner, DEFAULT_CODEX_REMOTE_WS_URL } from '../helpers/coding-cli/fake-codex-launch-planner.js' const TEST_TIMEOUT_MS = 30_000 const HOOK_TIMEOUT_MS = 30_000 +const VALID_SESSION_ID = '550e8400-e29b-41d4-a716-446655440000' vi.setConfig({ testTimeout: TEST_TIMEOUT_MS, hookTimeout: HOOK_TIMEOUT_MS }) // Mock the config-store module before importing ws-handler @@ -31,16 +32,6 @@ function defaultConfigSnapshot() { } } -function deferred<T = void>() { - let resolve!: (value: T | PromiseLike<T>) => void - let reject!: (reason?: unknown) => void - const promise = new Promise<T>((res, rej) => { - resolve = res - reject = rej - }) - return { promise, resolve, reject } -} - function listen(server: http.Server, timeoutMs = HOOK_TIMEOUT_MS): Promise<{ port: number }> { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { @@ -129,7 +120,6 @@ class FakeRegistry { inputCalls: { terminalId: string; data: string }[] = [] resizeCalls: { terminalId: string; cols: number; rows: number }[] = [] killCalls: string[] = [] - publishCalls: string[] = [] create(opts: any) { this.createCalls.push(opts) @@ -191,14 +181,6 @@ class FakeRegistry { return true } - async killAndWait(terminalId: string) { - return this.kill(terminalId) - } - - publishCodexSidecar(terminalId: string) { - this.publishCalls.push(terminalId) - } - list() { return Array.from(this.records.values()).map((r) => ({ terminalId: r.terminalId, @@ -229,38 +211,13 @@ class FakeRegistry { return undefined } - repairLegacySessionOwners() { - return { repaired: false, clearedTerminalIds: [] } - } -} - -function createAuthenticatedState() { - return { - authenticated: true, - supportsUiScreenshotV1: false, - attachedTerminalIds: new Set(), - createdByRequestId: new Map(), - terminalCreateTimestamps: [], - codingCliSessions: new Set(), - codingCliSubscriptions: new Map(), - sdkSessions: new Set(), - sdkSubscriptions: new Map(), - sdkSessionTargets: new Map(), - interestedSessions: new Set(), - sidebarOpenSessionKeys: new Set(), - } -} - -function createOpenFakeWs(connectionId: string, sent: any[]) { - return { - readyState: WebSocket.OPEN, - bufferedAmount: 0, - connectionId, - send: vi.fn((payload: string, cb?: (err?: Error) => void) => { - sent.push(JSON.parse(payload)) - cb?.() - }), - close: vi.fn(), + repairLegacySessionOwners(mode: string, sessionId: string) { + const canonical = this.getCanonicalRunningTerminalBySession(mode, sessionId) + return { + repaired: false, + canonicalTerminalId: canonical?.terminalId, + clearedTerminalIds: [] as string[], + } } } @@ -334,14 +291,8 @@ describe('ws protocol', () => { registry.inputCalls = [] registry.resizeCalls = [] registry.killCalls = [] - registry.publishCalls = [] codexLaunchPlanner.planCreateCalls = [] - codexLaunchPlanner.sidecar.adoptCalls = [] - codexLaunchPlanner.sidecar.shutdownCalls = 0 - codexLaunchPlanner.sidecar.shutdownStarted = false - codexLaunchPlanner.sidecar.shutdownError = null - codexLaunchPlanner.sidecar.waitForLoadedThreadCalls = [] - codexLaunchPlanner.sidecar.waitForLoadedThreadError = null + codexLaunchPlanner.failNext(0) }) afterAll(async () => { @@ -375,6 +326,52 @@ describe('ws protocol', () => { await closeWebSocket(ws) }) + it('rejects serverInstanceId inside hello sidebarOpenSessions durable identity', () => { + const parsed = HelloSchema.safeParse({ + type: 'hello', + token: 'testtoken-testtoken', + protocolVersion: WS_PROTOCOL_VERSION, + sidebarOpenSessions: [{ + provider: 'codex', + sessionId: 'codex-session-1', + serverInstanceId: 'srv-local', + }], + }) + + expect(parsed.success).toBe(false) + }) + + it('accepts terminal.create canonical sessionRef and rejects raw durable resumeSessionId', () => { + const parsed = TerminalCreateSchema.safeParse({ + type: 'terminal.create', + requestId: 'req-1', + mode: 'codex', + restore: true, + sessionRef: { + provider: 'codex', + sessionId: 'codex-session-1', + }, + }) + + expect(parsed.success).toBe(true) + if (!parsed.success) return + + expect((parsed.data as any).sessionRef).toEqual({ + provider: 'codex', + sessionId: 'codex-session-1', + }) + + const legacy = TerminalCreateSchema.safeParse({ + type: 'terminal.create', + requestId: 'req-legacy', + mode: 'claude', + restore: true, + resumeSessionId: '550e8400-e29b-41d4-a716-446655440000', + }) + + expect(legacy.success).toBe(false) + }) + it('accepts hello with capabilities', async () => { const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) await new Promise<void>((resolve) => ws.on('open', () => resolve())) @@ -491,462 +488,123 @@ describe('ws protocol', () => { const requestId = 'req-codex-settings' ws.send(JSON.stringify({ type: 'terminal.create', requestId, mode: 'codex' })) - await waitForMessage( + const created = await waitForMessage( ws, (msg) => msg.type === 'terminal.created' && msg.requestId === requestId, 5000, ) expect(registry.createCalls).toHaveLength(1) - expect(codexLaunchPlanner.planCreateCalls).toEqual([{ + expect(codexLaunchPlanner.planCreateCalls).toHaveLength(1) + const planCreate = codexLaunchPlanner.planCreateCalls[0] + expect(planCreate).toEqual(expect.objectContaining({ approvalPolicy: undefined, cwd: undefined, model: 'gpt-5-codex', resumeSessionId: undefined, sandbox: 'workspace-write', - }]) - expect(registry.createCalls[0]?.resumeSessionId).toBe('thread-new-1') + terminalId: expect.any(String), + env: expect.objectContaining({ + FRESHELL: '1', + FRESHELL_TERMINAL_ID: expect.any(String), + FRESHELL_TOKEN: 'testtoken-testtoken', + FRESHELL_URL: 'http://localhost:3001', + }), + })) + expect(planCreate.env.FRESHELL_TERMINAL_ID).toBe(planCreate.terminalId) + expect(registry.createCalls[0]?.terminalId).toBe(planCreate.terminalId) + expect(registry.createCalls[0]?.resumeSessionId).toBeUndefined() expect(registry.createCalls[0]?.providerSettings).toEqual({ - codexAppServer: expect.objectContaining({ + codexAppServer: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL, - }), + }, }) + expect(registry.createCalls[0]?.codexLaunchBaseProviderSettings).toEqual({ + model: 'gpt-5-codex', + sandbox: 'workspace-write', + permissionMode: undefined, + }) + expect(created).not.toHaveProperty('effectiveResumeSessionId') await closeWebSocket(ws) }) - it('shuts down a pending Codex sidecar when terminal.create fails after planning', async () => { - const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) - await new Promise<void>((resolve) => ws.on('open', () => resolve())) - ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) - await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - - const originalCreate = registry.create.bind(registry) - registry.create = vi.fn((opts: any) => { - registry.createCalls.push(opts) - throw new Error('spawn failed after planning') - }) as any - - try { - ws.send(JSON.stringify({ type: 'terminal.create', requestId: 'codex-create-fails', mode: 'codex' })) - const error = await waitForMessage( - ws, - (msg) => msg.type === 'error' && msg.requestId === 'codex-create-fails', - 5000, - ) - - expect(error.message).toContain('spawn failed after planning') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([]) - } finally { - registry.create = originalCreate as any - await closeWebSocket(ws) - } - }) - - it('reports terminal.create cleanup failure when pending Codex sidecar shutdown fails', async () => { - codexLaunchPlanner.sidecar.shutdownError = new Error('verified sidecar teardown failed') + it('retries initial Codex launch before terminal.created', async () => { + codexLaunchPlanner.failNext(2) const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) await new Promise<void>((resolve) => ws.on('open', () => resolve())) ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - const originalCreate = registry.create.bind(registry) - registry.create = vi.fn((opts: any) => { - registry.createCalls.push(opts) - throw new Error('spawn failed after planning') - }) as any - - try { - ws.send(JSON.stringify({ type: 'terminal.create', requestId: 'codex-cleanup-fails', mode: 'codex' })) - const error = await waitForMessage( - ws, - (msg) => msg.type === 'error' && msg.requestId === 'codex-cleanup-fails', - 5000, - ) - - expect(error.message).toContain('spawn failed after planning') - expect(error.message).toContain('verified sidecar teardown failed') - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(codexLaunchPlanner.sidecar.adoptCalls).toEqual([]) - } finally { - registry.create = originalCreate as any - await closeWebSocket(ws) - } - }) - - it('kills the inserted terminal when terminal.create fails before registry.create returns', async () => { - const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) - await new Promise<void>((resolve) => ws.on('open', () => resolve())) - ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) - await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - - const originalCreate = registry.create.bind(registry) - registry.create = vi.fn((opts: any) => { - registry.createCalls.push(opts) - const terminalId = 'term_inserted_before_emit_failure' - registry.records.set(terminalId, { - terminalId, - createdAt: Date.now(), - buffer: new FakeBuffer(), - title: 'Codex', - mode: opts.mode || 'codex', - shell: opts.shell || 'system', - status: 'running', - resumeSessionId: opts.resumeSessionId, - clients: new Set(), - }) - const error = new Error('terminal.created listener failed') as Error & { terminalId?: string } - error.terminalId = terminalId - throw error - }) as any - - try { - ws.send(JSON.stringify({ type: 'terminal.create', requestId: 'codex-create-emit-fails', mode: 'codex' })) - const error = await waitForMessage( - ws, - (msg) => msg.type === 'error' && msg.requestId === 'codex-create-emit-fails', - 5000, - ) - - expect(error.message).toContain('terminal.created listener failed') - expect(registry.killCalls).toContain('term_inserted_before_emit_failure') - expect(registry.records.has('term_inserted_before_emit_failure')).toBe(false) - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - } finally { - registry.create = originalCreate as any - await closeWebSocket(ws) - } - }) - - it('rejects late terminal.create after the WebSocket handler starts closing', async () => { - const localServer = http.createServer((_req, res) => { - res.statusCode = 404 - res.end() - }) - const localRegistry = new FakeRegistry() - const localPlanner = new FakeCodexLaunchPlanner() - const localHandler = new WsHandler(localServer, localRegistry as any, { codexLaunchPlanner: localPlanner }) - const sent: any[] = [] - const ws = { - readyState: WebSocket.OPEN, - bufferedAmount: 0, - connectionId: 'late-create-after-close', - send: vi.fn((payload: string, cb?: (err?: Error) => void) => { - sent.push(JSON.parse(payload)) - cb?.() - }), - close: vi.fn(), - } - const state = { - authenticated: true, - supportsUiScreenshotV1: false, - attachedTerminalIds: new Set(), - createdByRequestId: new Map(), - terminalCreateTimestamps: [], - codingCliSessions: new Set(), - codingCliSubscriptions: new Map(), - sdkSessions: new Set(), - sdkSubscriptions: new Map(), - sdkSessionTargets: new Map(), - interestedSessions: new Set(), - sidebarOpenSessionKeys: new Set(), - } - - localHandler.close() - await (localHandler as any).onMessage( - ws, - state, - Buffer.from(JSON.stringify({ type: 'terminal.create', requestId: 'after-close', mode: 'codex' })), - ) - - expect(sent).toContainEqual(expect.objectContaining({ - type: 'error', - requestId: 'after-close', - })) - expect(localRegistry.createCalls).toEqual([]) - expect(localPlanner.planCreateCalls).toEqual([]) - }) - - it('aborts in-flight Codex terminal.create when shutdown starts after planning before registry create', async () => { - const localServer = http.createServer((_req, res) => { - res.statusCode = 404 - res.end() - }) - const localRegistry = new FakeRegistry() - const sidecar = new FakeCodexLaunchSidecar() - const plan = deferred<any>() - const localPlanner = { - planCreateCalls: [] as any[], - planCreate: vi.fn((input: any) => { - localPlanner.planCreateCalls.push(input) - return plan.promise - }), - } - const localHandler = new WsHandler(localServer, localRegistry as any, { codexLaunchPlanner: localPlanner as any }) - const sent: any[] = [] - const ws = createOpenFakeWs('shutdown-after-plan', sent) - const state = createAuthenticatedState() - - const message = (localHandler as any).onMessage( - ws, - state, - Buffer.from(JSON.stringify({ type: 'terminal.create', requestId: 'shutdown-after-plan', mode: 'codex' })), - ) - await vi.waitFor(() => expect(localPlanner.planCreate).toHaveBeenCalledTimes(1)) - localHandler.close() - plan.resolve({ - sessionId: 'thread-after-plan', - remote: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL }, - sidecar, - }) - await message - - expect(sent).toContainEqual(expect.objectContaining({ - type: 'error', - requestId: 'shutdown-after-plan', - })) - expect(localRegistry.createCalls).toEqual([]) - expect(sidecar.shutdownCalls).toBe(1) - }) - - it('aborts in-flight Codex terminal.create when shutdown starts after registry create before adoption', async () => { - const localServer = http.createServer((_req, res) => { - res.statusCode = 404 - res.end() - }) - const localRegistry = new FakeRegistry() - const sidecar = new FakeCodexLaunchSidecar() - const localPlanner = new FakeCodexLaunchPlanner({ - sessionId: 'thread-after-registry-create', - remote: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL }, - sidecar, - }) - const localHandler = new WsHandler(localServer, localRegistry as any, { codexLaunchPlanner: localPlanner }) - const originalCreate = localRegistry.create.bind(localRegistry) - localRegistry.create = vi.fn((opts: any) => { - const record = originalCreate(opts) - localHandler.close() - return record - }) as any - const sent: any[] = [] - const ws = createOpenFakeWs('shutdown-after-registry-create', sent) - const state = createAuthenticatedState() - - await (localHandler as any).onMessage( - ws, - state, - Buffer.from(JSON.stringify({ type: 'terminal.create', requestId: 'shutdown-after-registry-create', mode: 'codex' })), - ) - - expect(sent).toContainEqual(expect.objectContaining({ - type: 'error', - requestId: 'shutdown-after-registry-create', - })) - expect(sidecar.adoptCalls).toEqual([]) - expect(sidecar.shutdownCalls).toBe(1) - expect(localRegistry.killCalls).toHaveLength(1) - expect(localRegistry.records.size).toBe(0) - }) - - it('aborts in-flight Codex terminal.create when shutdown starts after adoption before publication', async () => { - const localServer = http.createServer((_req, res) => { - res.statusCode = 404 - res.end() - }) - const localRegistry = new FakeRegistry() - const sidecar = new FakeCodexLaunchSidecar() - const originalAdopt = sidecar.adopt.bind(sidecar) - const localPlanner = new FakeCodexLaunchPlanner({ - sessionId: 'thread-after-adoption', - remote: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL }, - sidecar, - }) - const localHandler = new WsHandler(localServer, localRegistry as any, { codexLaunchPlanner: localPlanner }) - vi.spyOn(sidecar, 'adopt').mockImplementation(async (input) => { - await originalAdopt(input) - localHandler.close() - }) - const sent: any[] = [] - const ws = createOpenFakeWs('shutdown-after-adoption', sent) - const state = createAuthenticatedState() - - await (localHandler as any).onMessage( - ws, - state, - Buffer.from(JSON.stringify({ type: 'terminal.create', requestId: 'shutdown-after-adoption', mode: 'codex' })), - ) - - expect(sent).toContainEqual(expect.objectContaining({ - type: 'error', - requestId: 'shutdown-after-adoption', - })) - expect(sidecar.adoptCalls).toHaveLength(1) - expect(localRegistry.publishCalls).toEqual([]) - expect(localRegistry.killCalls).toHaveLength(1) - expect(sidecar.shutdownCalls).toBe(1) - expect(localRegistry.records.size).toBe(0) - }) - - it('aborts in-flight Codex resume terminal.create when shutdown starts during loaded-list readiness', async () => { - const localServer = http.createServer((_req, res) => { - res.statusCode = 404 - res.end() - }) - const localRegistry = new FakeRegistry() - const sidecar = new FakeCodexLaunchSidecar() - const readiness = deferred() - const originalWaitForLoadedThread = sidecar.waitForLoadedThread.bind(sidecar) - const localPlanner = new FakeCodexLaunchPlanner({ - sessionId: 'thread-during-readiness', - remote: { wsUrl: DEFAULT_CODEX_REMOTE_WS_URL }, - sidecar, - }) - const localHandler = new WsHandler(localServer, localRegistry as any, { codexLaunchPlanner: localPlanner }) - vi.spyOn(sidecar, 'waitForLoadedThread').mockImplementation(async (threadId, options) => { - await originalWaitForLoadedThread(threadId, options) - localHandler.close() - await readiness.promise - }) - const sent: any[] = [] - const ws = createOpenFakeWs('shutdown-during-readiness', sent) - const state = createAuthenticatedState() + const requestId = 'req-codex-launch-retry' + ws.send(JSON.stringify({ type: 'terminal.create', requestId, mode: 'codex' })) - const message = (localHandler as any).onMessage( + const created = await waitForMessage( ws, - state, - Buffer.from(JSON.stringify({ - type: 'terminal.create', - requestId: 'shutdown-during-readiness', - mode: 'codex', - sessionRef: { - provider: 'codex', - sessionId: 'thread-during-readiness', - }, - })), + (msg) => msg.type === 'terminal.created' && msg.requestId === requestId, + 5000, ) - await vi.waitFor(() => expect(sidecar.waitForLoadedThreadCalls).toHaveLength(1)) - readiness.resolve() - await message - expect(sent).toContainEqual(expect.objectContaining({ - type: 'error', - requestId: 'shutdown-during-readiness', - })) - expect(sidecar.adoptCalls).toHaveLength(1) - expect(localRegistry.publishCalls).toEqual([]) - expect(localRegistry.killCalls).toHaveLength(1) - expect(sidecar.shutdownCalls).toBe(1) - expect(localRegistry.records.size).toBe(0) + expect(created.terminalId).toMatch(/^term_/) + expect(codexLaunchPlanner.planCreateCalls).toHaveLength(3) + expect(registry.createCalls).toHaveLength(1) + await closeWebSocket(ws) }) - it('waits for candidate-local loaded-thread readiness before reporting Codex resume create success', async () => { + it('returns one create error and creates no record when initial Codex launch retries are exhausted', async () => { + codexLaunchPlanner.failNext(5) const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) await new Promise<void>((resolve) => ws.on('open', () => resolve())) ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - const requestId = 'codex-resume-loaded-list' - ws.send(JSON.stringify({ - type: 'terminal.create', - requestId, - mode: 'codex', - sessionRef: { - provider: 'codex', - sessionId: 'thread-resume-1', - }, - })) - const created = await waitForMessage( + const requestId = 'req-codex-launch-exhausted' + ws.send(JSON.stringify({ type: 'terminal.create', requestId, mode: 'codex' })) + + const error = await waitForMessage( ws, - (msg) => msg.type === 'terminal.created' && msg.requestId === requestId, - 5000, + (msg) => msg.type === 'error' && msg.requestId === requestId, + 12_000, ) - expect(codexLaunchPlanner.planCreateCalls[0]).toEqual(expect.objectContaining({ - resumeSessionId: 'thread-resume-1', - })) - expect(codexLaunchPlanner.sidecar.waitForLoadedThreadCalls).toEqual([{ - threadId: 'thread-resume-1', - options: undefined, - }]) - expect(registry.publishCalls).toEqual([created.terminalId]) - + expect(error.code).toBe('PTY_SPAWN_FAILED') + expect(error.message).toContain('fake Codex launch failed') + expect(codexLaunchPlanner.planCreateCalls).toHaveLength(5) + expect(registry.createCalls).toHaveLength(0) await closeWebSocket(ws) - }) + }, 15_000) - it('kills the created terminal and sidecar when Codex resume loaded-list readiness fails', async () => { - codexLaunchPlanner.sidecar.waitForLoadedThreadError = new Error('resume thread never loaded') + it('passes canonical Claude sessionRef through to registry.create without echoing a legacy durable id', async () => { const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) await new Promise<void>((resolve) => ws.on('open', () => resolve())) ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) + await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - const requestId = 'codex-resume-loaded-list-fails' + const requestId = 'req-claude-restore' ws.send(JSON.stringify({ type: 'terminal.create', requestId, - mode: 'codex', + mode: 'claude', + restore: true, sessionRef: { - provider: 'codex', - sessionId: 'thread-missing', + provider: 'claude', + sessionId: VALID_SESSION_ID, }, })) - const error = await waitForMessage( + + const created = await waitForMessage( ws, - (msg) => msg.type === 'error' && msg.requestId === requestId, + (msg) => msg.type === 'terminal.created' && msg.requestId === requestId, 5000, ) - expect(error.message).toContain('resume thread never loaded') - expect(codexLaunchPlanner.sidecar.waitForLoadedThreadCalls).toHaveLength(1) - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(registry.killCalls).toHaveLength(1) - expect(registry.records.size).toBe(0) + expect(registry.createCalls[0]?.resumeSessionId).toBe(VALID_SESSION_ID) + expect(created).not.toHaveProperty('effectiveResumeSessionId') await closeWebSocket(ws) }) - it('kills the created terminal and sidecar when the Codex resume PTY exits before publication', async () => { - const originalWaitForLoadedThread = codexLaunchPlanner.sidecar.waitForLoadedThread.bind(codexLaunchPlanner.sidecar) - const waitSpy = vi.spyOn(codexLaunchPlanner.sidecar, 'waitForLoadedThread').mockImplementation(async (threadId, options) => { - await originalWaitForLoadedThread(threadId, options) - const terminalId = codexLaunchPlanner.sidecar.adoptCalls[0]?.terminalId - const record = terminalId ? registry.get(terminalId) : null - if (record) record.status = 'exited' - }) - const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`) - await new Promise<void>((resolve) => ws.on('open', () => resolve())) - ws.send(JSON.stringify({ type: 'hello', token: 'testtoken-testtoken', protocolVersion: WS_PROTOCOL_VERSION })) - await waitForMessage(ws, (msg) => msg.type === 'ready', 5000) - - const requestId = 'codex-resume-pty-exits-before-publication' - try { - ws.send(JSON.stringify({ - type: 'terminal.create', - requestId, - mode: 'codex', - sessionRef: { - provider: 'codex', - sessionId: 'thread-resume-exits', - }, - })) - const error = await waitForMessage( - ws, - (msg) => msg.type === 'error' && msg.requestId === requestId, - 5000, - ) - - expect(error.message).toContain('Codex terminal PTY exited before create completed') - expect(codexLaunchPlanner.sidecar.waitForLoadedThreadCalls).toHaveLength(1) - expect(codexLaunchPlanner.sidecar.shutdownCalls).toBe(1) - expect(registry.killCalls).toHaveLength(1) - expect(registry.records.size).toBe(0) - } finally { - waitSpy.mockRestore() - await closeWebSocket(ws) - } - }) - it('returns INVALID_MESSAGE when persisted Codex settings are invalid', async () => { mockConfigStore.snapshot.mockResolvedValue({ ...defaultConfigSnapshot(), @@ -1218,6 +876,35 @@ describe('ws protocol', () => { await close() }) + it('terminal.input does not send INVALID_TERMINAL_ID when Codex recovery input is handled locally', async () => { + const { ws, close } = await createAuthenticatedConnection() + const terminalId = await createTerminal(ws, 'create-for-recovery-input') + const record = registry.get(terminalId) + record.mode = 'codex' + record.codex = { recoveryState: 'recovering_durable' } + + const observed: any[] = [] + const onMessage = (data: WebSocket.Data) => { + observed.push(JSON.parse(data.toString())) + } + ws.on('message', onMessage) + + ws.send(JSON.stringify({ type: 'terminal.input', terminalId, data: 'while recovering' })) + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(registry.inputCalls).toContainEqual({ terminalId, data: 'while recovering' }) + expect(observed).not.toEqual(expect.arrayContaining([ + expect.objectContaining({ + type: 'error', + code: 'INVALID_TERMINAL_ID', + terminalId, + }), + ])) + + ws.off('message', onMessage) + await close() + }) + it('terminal.input returns error for non-existent terminal', async () => { const { ws, close } = await createAuthenticatedConnection() @@ -1317,30 +1004,6 @@ describe('ws protocol', () => { await close() }) - it('terminal.kill returns a protocol error when verified Codex teardown fails', async () => { - const { ws, close } = await createAuthenticatedConnection() - const originalKillAndWait = registry.killAndWait.bind(registry) - registry.killAndWait = vi.fn(async () => { - throw new Error('verified Codex teardown failed') - }) as any - - try { - ws.send(JSON.stringify({ type: 'terminal.kill', terminalId: 'codex-terminal-with-failed-teardown' })) - - const error = await waitForMessage( - ws, - (msg) => msg.type === 'error' && msg.terminalId === 'codex-terminal-with-failed-teardown', - 5000, - ) - - expect(error.code).toBe('INTERNAL_ERROR') - expect(error.message).toContain('verified Codex teardown failed') - } finally { - registry.killAndWait = originalKillAndWait as any - await close() - } - }) - it('rejects legacy terminal.list commands', async () => { const { ws, close } = await createAuthenticatedConnection() diff --git a/test/unit/server/coding-cli/codex-app-server/client.test.ts b/test/unit/server/coding-cli/codex-app-server/client.test.ts index ea8bc325f..543ae9e56 100644 --- a/test/unit/server/coding-cli/codex-app-server/client.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/client.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -14,11 +14,9 @@ type FakeServerBehavior = { closeSocketAfterMethodsOnce?: string[] delayMethodsMs?: Record<string, number> ignoreMethods?: string[] - loadedThreadIds?: string[] - notificationsAfterMethods?: Record<string, unknown[]> + notifyAfterMethodsOnce?: Record<string, Array<{ method: string; params?: unknown }>> requireJsonRpc?: boolean requireInitializeBeforeOtherMethods?: boolean - requireInitializedNotification?: boolean overrides?: Record<string, { result?: unknown; error?: { code: number; message: string } }> } @@ -98,6 +96,24 @@ async function stopFakeCodexAppServer(handle: FakeServerHandle): Promise<void> { }) } +async function waitFor(assertion: () => void | Promise<void>, timeoutMs = 1_000): Promise<void> { + const deadline = Date.now() + timeoutMs + let lastError: unknown + + while (Date.now() < deadline) { + try { + await assertion() + return + } catch (error) { + lastError = error + await new Promise((resolve) => setTimeout(resolve, 10)) + } + } + + if (lastError instanceof Error) throw lastError + throw new Error('Timed out waiting for assertion') +} + afterEach(async () => { await Promise.all([...fakeServers].map((server) => stopFakeCodexAppServer(server))) }) @@ -121,8 +137,128 @@ describe('CodexAppServerClient', () => { await client.initialize() await expect(client.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }, + }) + }) + + it('surfaces thread/started notifications to sidecar consumers', async () => { + const server = await startFakeCodexAppServer() + const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) + + const startedThread = new Promise<{ id: string; path: string | null; ephemeral: boolean }>((resolve) => { + client.onThreadStarted((thread) => resolve(thread)) + }) + + await client.initialize() + await client.startThread({ cwd: '/repo/worktree' }) + + await expect(startedThread).resolves.toEqual({ + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }) + }) + + it('emits thread lifecycle notifications from app-server notifications', async () => { + const server = await startFakeCodexAppServer({ + notifyAfterMethodsOnce: { + initialize: [ + { + method: 'thread/started', + params: { + thread: { + id: 'thread-resume-1', + path: '/tmp/codex/rollout-thread-resume-1.jsonl', + ephemeral: false, + }, + }, + }, + ], + }, }) + const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) + const lifecycle = vi.fn() + client.onThreadLifecycle(lifecycle) + + await client.initialize() + + await waitFor(() => expect(lifecycle).toHaveBeenCalledWith({ + kind: 'thread_started', + thread: { + id: 'thread-resume-1', + path: '/tmp/codex/rollout-thread-resume-1.jsonl', + ephemeral: false, + }, + })) + }) + + it('emits thread closed lifecycle notifications from app-server notifications', async () => { + const server = await startFakeCodexAppServer({ + notifyAfterMethodsOnce: { + initialize: [ + { + method: 'thread/closed', + params: { + threadId: 'thread-resume-1', + }, + }, + ], + }, + }) + const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) + const lifecycle = vi.fn() + client.onThreadLifecycle(lifecycle) + + await client.initialize() + + await waitFor(() => expect(lifecycle).toHaveBeenCalledWith({ + kind: 'thread_closed', + threadId: 'thread-resume-1', + })) + }) + + it('emits thread status lifecycle notifications from app-server notifications', async () => { + const server = await startFakeCodexAppServer({ + notifyAfterMethodsOnce: { + initialize: [ + { + method: 'thread/status/changed', + params: { + threadId: 'thread-resume-1', + status: { type: 'notLoaded' }, + }, + }, + ], + }, + }) + const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) + const lifecycle = vi.fn() + client.onThreadLifecycle(lifecycle) + + await client.initialize() + + await waitFor(() => expect(lifecycle).toHaveBeenCalledWith({ + kind: 'thread_status_changed', + threadId: 'thread-resume-1', + status: { type: 'notLoaded' }, + })) + }) + + it('emits a disconnect callback when the app-server client socket closes unexpectedly', async () => { + const server = await startFakeCodexAppServer({ closeSocketAfterMethodsOnce: ['initialize'] }) + const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) + const onDisconnect = vi.fn() + client.onDisconnect(onDisconnect) + + await client.initialize() + + await waitFor(() => expect(onDisconnect).toHaveBeenCalledWith(expect.objectContaining({ + reason: 'close', + }))) }) it('sends JSON-RPC 2.0 envelopes to the app-server', async () => { @@ -131,7 +267,11 @@ describe('CodexAppServerClient', () => { await client.initialize() await expect(client.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }, }) }) @@ -144,7 +284,11 @@ describe('CodexAppServerClient', () => { threadId: '019d9859-5670-72b1-851f-794ad7fef112', cwd: '/repo/worktree', })).resolves.toEqual({ - threadId: '019d9859-5670-72b1-851f-794ad7fef112', + thread: { + id: '019d9859-5670-72b1-851f-794ad7fef112', + path: expect.stringMatching(/rollout-019d9859-5670-72b1-851f-794ad7fef112\.jsonl$/), + ephemeral: false, + }, }) }) @@ -156,7 +300,11 @@ describe('CodexAppServerClient', () => { await new Promise((resolve) => setTimeout(resolve, 25)) await expect(client.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }, }) }) @@ -177,75 +325,44 @@ describe('CodexAppServerClient', () => { platformOs: expect.any(String), }) await expect(startThreadPromise).resolves.toEqual({ - threadId: 'thread-new-1', - }) - }) - - it('sends initialized after initialize before later requests', async () => { - const server = await startFakeCodexAppServer({ - requireInitializeBeforeOtherMethods: true, - requireInitializedNotification: true, - }) - const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) - - await client.initialize() - - await expect(client.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }, }) }) - it('lists loaded in-memory thread ids', async () => { + it('sends fs/watch and fs/unwatch envelopes and surfaces fs/changed notifications with the original watchId', async () => { + const rolloutPath = '/repo/worktree/.codex/sessions/2026/04/23/rollout-thread-new-1.jsonl' const server = await startFakeCodexAppServer({ - loadedThreadIds: ['019d9859-5670-72b1-851f-794ad7fef112', 'thread-new-1'], - }) - const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) - - await client.initialize() - - await expect(client.listLoadedThreads()).resolves.toEqual([ - '019d9859-5670-72b1-851f-794ad7fef112', - 'thread-new-1', - ]) - }) - - it('emits lifecycle-loss events for closed, notLoaded, and systemError notifications', async () => { - const server = await startFakeCodexAppServer({ - notificationsAfterMethods: { - 'thread/loaded/list': [ - { - method: 'thread/closed', - params: { threadId: 'thread-closed' }, - }, + notifyAfterMethodsOnce: { + 'fs/watch': [ { - method: 'thread/status/changed', - params: { threadId: 'thread-not-loaded', status: { type: 'notLoaded' } }, - }, - { - method: 'thread/status/changed', - params: { thread: { id: 'thread-system-error', status: { type: 'systemError' } } }, - }, - { - method: 'thread/status/changed', - params: { threadId: 'thread-running', status: { type: 'running' } }, + method: 'fs/changed', + params: { + watchId: 'watch-rollout', + changedPaths: [rolloutPath], + }, }, ], }, }) const client = new CodexAppServerClient({ wsUrl: server.wsUrl }) - const events: unknown[] = [] - const unsubscribe = client.onThreadLifecycleLoss((event) => events.push(event)) - await client.initialize() - await client.listLoadedThreads() - await new Promise((resolve) => setTimeout(resolve, 25)) - unsubscribe() + const changedEvent = new Promise<{ watchId: string; changedPaths: string[] }>((resolve) => { + client.onFsChanged((event) => resolve(event)) + }) - expect(events).toEqual([ - { method: 'thread/closed', threadId: 'thread-closed' }, - { method: 'thread/status/changed', threadId: 'thread-not-loaded', status: 'notLoaded' }, - { method: 'thread/status/changed', threadId: 'thread-system-error', status: 'systemError' }, - ]) + await client.initialize() + await expect(client.watchPath(rolloutPath, 'watch-rollout')).resolves.toEqual({ + path: rolloutPath, + }) + await expect(changedEvent).resolves.toEqual({ + watchId: 'watch-rollout', + changedPaths: [rolloutPath], + }) + await expect(client.unwatchPath('watch-rollout')).resolves.toBeUndefined() }) it('fails clearly when the app-server never answers a request', async () => { diff --git a/test/unit/server/coding-cli/codex-app-server/durable-rollout-tracker.test.ts b/test/unit/server/coding-cli/codex-app-server/durable-rollout-tracker.test.ts new file mode 100644 index 000000000..b490102ba --- /dev/null +++ b/test/unit/server/coding-cli/codex-app-server/durable-rollout-tracker.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { CodexDurableRolloutTracker } from '../../../../../server/coding-cli/codex-app-server/durable-rollout-tracker.js' + +describe('CodexDurableRolloutTracker', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('checks only the exact rollout path when a matching fs/changed event arrives', async () => { + const rolloutPath = '/tmp/codex/sessions/2026/04/23/rollout-thread-new-1.jsonl' + const existsCalls: string[] = [] + let rolloutExists = false + let fsChangedHandler: ((event: { watchId: string; changedPaths: string[] }) => void) | null = null + const onDurableRollout = vi.fn() + + const tracker = new CodexDurableRolloutTracker({ + watchPath: vi.fn(async (targetPath) => ({ path: targetPath })), + unwatchPath: vi.fn(async () => undefined), + subscribeToFsChanged: (handler) => { + fsChangedHandler = handler + return () => { + fsChangedHandler = null + } + }, + pathExists: vi.fn(async (targetPath) => { + existsCalls.push(targetPath) + return rolloutExists + }), + onDurableRollout, + }) + + tracker.trackThread({ + id: 'thread-new-1', + path: rolloutPath, + ephemeral: false, + }) + await Promise.resolve() + + rolloutExists = true + fsChangedHandler?.({ + watchId: 'freshell-codex-rollout:thread-new-1', + changedPaths: [rolloutPath], + }) + await vi.advanceTimersByTimeAsync(250) + + expect(onDurableRollout).toHaveBeenCalledWith('thread-new-1') + expect(new Set(existsCalls)).toEqual(new Set([rolloutPath])) + + await tracker.dispose() + }) + + it('keeps retrying exact-path probes after the old 10 second cutoff until the rollout exists', async () => { + const rolloutPath = '/tmp/codex/sessions/2026/04/23/rollout-thread-late.jsonl' + const onDurableRollout = vi.fn() + + const tracker = new CodexDurableRolloutTracker({ + watchPath: vi.fn(async (targetPath) => ({ path: targetPath })), + unwatchPath: vi.fn(async () => undefined), + subscribeToFsChanged: () => () => undefined, + pathExists: vi.fn(async () => Date.now() >= 11_000), + onDurableRollout, + initialProbeDelayMs: 1_000, + maxProbeDelayMs: 5_000, + }) + + tracker.trackThread({ + id: 'thread-late', + path: rolloutPath, + ephemeral: false, + }) + await Promise.resolve() + + await vi.advanceTimersByTimeAsync(10_000) + expect(onDurableRollout).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(3_000) + expect(onDurableRollout).toHaveBeenCalledWith('thread-late') + + await tracker.dispose() + }) + + it('falls back to exact-path probes when fs/watch registration fails', async () => { + const rolloutPath = '/tmp/codex/sessions/2026/04/23/rollout-thread-fallback.jsonl' + const log = { warn: vi.fn() } + const onDurableRollout = vi.fn() + const unwatchPath = vi.fn(async () => undefined) + + const tracker = new CodexDurableRolloutTracker({ + watchPath: vi.fn(async () => { + throw new Error('watch registration failed') + }), + unwatchPath, + subscribeToFsChanged: () => () => undefined, + pathExists: vi.fn(async () => Date.now() >= 6_000), + onDurableRollout, + initialProbeDelayMs: 1_000, + maxProbeDelayMs: 5_000, + log, + }) + + tracker.trackThread({ + id: 'thread-fallback', + path: rolloutPath, + ephemeral: false, + }) + await Promise.resolve() + + await vi.advanceTimersByTimeAsync(7_000) + expect(onDurableRollout).toHaveBeenCalledWith('thread-fallback') + expect(log.warn).toHaveBeenCalled() + expect(unwatchPath).not.toHaveBeenCalled() + + await tracker.dispose() + }) + + it('lets a later trackThread replace an earlier pending rollout without overlapping state', async () => { + const firstPath = '/tmp/codex/sessions/2026/04/23/rollout-thread-first.jsonl' + const secondPath = '/tmp/codex/sessions/2026/04/23/rollout-thread-second.jsonl' + const onDurableRollout = vi.fn() + + const tracker = new CodexDurableRolloutTracker({ + watchPath: vi.fn(async (targetPath) => ({ path: targetPath })), + unwatchPath: vi.fn(async () => undefined), + subscribeToFsChanged: () => () => undefined, + pathExists: vi.fn(async (targetPath) => targetPath === secondPath && Date.now() >= 1_000), + onDurableRollout, + initialProbeDelayMs: 500, + maxProbeDelayMs: 500, + }) + + tracker.trackThread({ + id: 'thread-first', + path: firstPath, + ephemeral: false, + }) + tracker.trackThread({ + id: 'thread-second', + path: secondPath, + ephemeral: false, + }) + await Promise.resolve() + + await vi.advanceTimersByTimeAsync(1_000) + expect(onDurableRollout).toHaveBeenCalledTimes(1) + expect(onDurableRollout).toHaveBeenCalledWith('thread-second') + + await tracker.dispose() + }) +}) diff --git a/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts b/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts index 8e8061986..d7c841d2d 100644 --- a/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/launch-planner.test.ts @@ -1,333 +1,95 @@ import { describe, expect, it, vi } from 'vitest' import { CodexLaunchPlanner } from '../../../../../server/coding-cli/codex-app-server/launch-planner.js' -function deferred<T = void>() { - let resolve!: (value: T | PromiseLike<T>) => void - let reject!: (reason?: unknown) => void - const promise = new Promise<T>((res, rej) => { - resolve = res - reject = rej - }) - return { promise, resolve, reject } -} - -class FakeRuntime { - shutdownCalls = 0 - startThreadCalls = 0 - adopted: Array<{ terminalId: string; generation: number }> = [] - loadedThreadListCalls = 0 - adoptError?: Error - startThreadBlocker?: Promise<void> - shutdownBlocker?: Promise<void> - shutdownError?: Error - - constructor( - readonly wsUrl: string, - private readonly threadId: string, - private readonly startError?: Error, - private readonly loadedThreadLists: string[][] = [], - ) {} - - async ensureReady() { - return { - wsUrl: this.wsUrl, - processPid: 100, - ownershipId: `ownership-${this.threadId}`, - processGroupId: 100, - metadataPath: `/tmp/${this.threadId}.json`, - } - } - - async startThread() { - this.startThreadCalls += 1 - await this.startThreadBlocker - if (this.startError) throw this.startError +describe('CodexLaunchPlanner', () => { + function createSidecar() { return { - threadId: this.threadId, - wsUrl: this.wsUrl, + ensureReady: vi.fn().mockResolvedValue({ + wsUrl: 'ws://127.0.0.1:43123', + }), + attachTerminal: vi.fn(), + shutdown: vi.fn(), } } - async updateOwnershipMetadata(input: { terminalId?: string | null; generation?: number | null }) { - if (this.adoptError) throw this.adoptError - if (input.terminalId && typeof input.generation === 'number') { - this.adopted.push({ terminalId: input.terminalId, generation: input.generation }) - } - } - - async listLoadedThreads() { - const index = Math.min(this.loadedThreadListCalls, Math.max(0, this.loadedThreadLists.length - 1)) - this.loadedThreadListCalls += 1 - return this.loadedThreadLists[index] ?? [] - } - - async shutdown() { - this.shutdownCalls += 1 - await this.shutdownBlocker - if (this.shutdownError) throw this.shutdownError - } -} - -describe('CodexLaunchPlanner', () => { - it('creates a distinct owned sidecar for each launch plan', async () => { - const runtimes: FakeRuntime[] = [] - const planner = new CodexLaunchPlanner(() => { - const index = runtimes.length + 1 - const runtime = new FakeRuntime(`ws://127.0.0.1:${43000 + index}`, `thread-${index}`) - runtimes.push(runtime) - return runtime as any + it('starts a fresh Codex terminal without preallocating a thread id', async () => { + const sidecar = createSidecar() + const createSidecarWithInput = vi.fn(() => sidecar as any) + const planner = new CodexLaunchPlanner(createSidecarWithInput) + + const plan = await planner.planCreate({ + cwd: '/repo/worktree', + terminalId: 'term-codex-1', + env: { + FRESHELL_TERMINAL_ID: 'term-codex-1', + }, + model: 'codex-default', + sandbox: 'workspace-write', }) - const first = await planner.planCreate({ cwd: '/repo/one' }) - const second = await planner.planCreate({ cwd: '/repo/two' }) - - expect(runtimes).toHaveLength(2) - expect(first.remote.wsUrl).toBe('ws://127.0.0.1:43001') - expect(second.remote.wsUrl).toBe('ws://127.0.0.1:43002') - - await first.sidecar.adopt({ terminalId: 'term-one', generation: 1 }) - await second.sidecar.shutdown() - - expect(runtimes[0].adopted).toEqual([{ terminalId: 'term-one', generation: 1 }]) - expect(runtimes[0].shutdownCalls).toBe(0) - expect(runtimes[1].shutdownCalls).toBe(1) + expect(createSidecarWithInput).toHaveBeenCalledWith({ + cwd: '/repo/worktree', + terminalId: 'term-codex-1', + env: { + FRESHELL_TERMINAL_ID: 'term-codex-1', + }, + commandArgs: [ + '-c', + expect.stringMatching(/^mcp_servers\.freshell\.command=/), + '-c', + expect.stringMatching(/^mcp_servers\.freshell\.args=\[/), + ], + model: 'codex-default', + sandbox: 'workspace-write', + }) + expect(sidecar.ensureReady).toHaveBeenCalledTimes(1) + expect(plan.sessionId).toBeUndefined() + expect(plan.remote.wsUrl).toBe('ws://127.0.0.1:43123') + expect(plan.sidecar).toBe(sidecar) }) - it('shuts down the owned sidecar when planning fails before adoption', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43010', 'thread-fail', new Error('start failed')) - const planner = new CodexLaunchPlanner(() => runtime as any) - - await expect(planner.planCreate({ cwd: '/repo/fail' })).rejects.toThrow('start failed') + it('reuses an existing Codex session id and only ensures the remote runtime is ready', async () => { + const sidecar = createSidecar() + const planner = new CodexLaunchPlanner(() => sidecar as any) + + const plan = await planner.planCreate({ + cwd: '/repo/worktree', + terminalId: 'term-codex-restore', + env: { + FRESHELL_TERMINAL_ID: 'term-codex-restore', + }, + resumeSessionId: '019d9859-5670-72b1-851f-794ad7fef112', + }) - expect(runtime.shutdownCalls).toBe(1) + expect(sidecar.ensureReady).toHaveBeenCalledTimes(1) + expect(plan.sessionId).toBe('019d9859-5670-72b1-851f-794ad7fef112') + expect(plan.remote.wsUrl).toBe('ws://127.0.0.1:43123') + expect(plan.sidecar).toBe(sidecar) }) - it('marks planning cleanup teardown failures as sidecar teardown failures', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43022', 'thread-fail', new Error('start failed')) - runtime.shutdownError = new Error('verified runtime teardown failed') - const planner = new CodexLaunchPlanner(() => runtime as any) - - let rejection: unknown - try { - await planner.planCreate({ cwd: '/repo/fail-teardown' }) - } catch (err) { - rejection = err + it('uses the ready runtime wsUrl for fresh launch handoff', async () => { + const sidecar = { + ensureReady: vi.fn().mockResolvedValue({ + wsUrl: 'ws://127.0.0.1:43199', + }), + attachTerminal: vi.fn(), + shutdown: vi.fn(), } - - expect(rejection).toBeInstanceOf(Error) - expect((rejection as Error).message).toContain('verified runtime teardown failed') - expect(rejection).toMatchObject({ codexSidecarTeardownFailed: true }) - expect(runtime.shutdownCalls).toBe(1) - }) - - it('transfers sidecar ownership to the registry on adoption so planner shutdown only cleans unadopted plans', async () => { - const adoptedRuntime = new FakeRuntime('ws://127.0.0.1:43011', 'thread-adopted') - const pendingRuntime = new FakeRuntime('ws://127.0.0.1:43012', 'thread-pending') - const runtimes = [adoptedRuntime, pendingRuntime] - const planner = new CodexLaunchPlanner(() => runtimes.shift()! as any) - - const adopted = await planner.planCreate({ cwd: '/repo/adopted' }) - const pending = await planner.planCreate({ cwd: '/repo/pending' }) - await adopted.sidecar.adopt({ terminalId: 'term-adopted', generation: 1 }) - - await planner.shutdown() - - expect(adoptedRuntime.adopted).toEqual([{ terminalId: 'term-adopted', generation: 1 }]) - expect(adoptedRuntime.shutdownCalls).toBe(0) - expect(pendingRuntime.shutdownCalls).toBe(1) - - await pending.sidecar.shutdown() - expect(pendingRuntime.shutdownCalls).toBe(1) - }) - - it('keeps a failed-adoption sidecar planner-owned so shutdown can clean it up', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43013', 'thread-adopt-fails') - runtime.adoptError = new Error('no active owned Codex app-server sidecar') - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ cwd: '/repo/adopt-fails' }) - await expect(plan.sidecar.adopt({ terminalId: 'term-adopt-fails', generation: 1 })) - .rejects.toThrow('no active owned Codex app-server sidecar') - - await planner.shutdown() - - expect(runtime.adopted).toEqual([]) - expect(runtime.shutdownCalls).toBe(1) - }) - - it('rejects new plans after shutdown begins without creating another sidecar', async () => { - const shutdownGate = deferred() - const firstRuntime = new FakeRuntime('ws://127.0.0.1:43014', 'thread-before-shutdown') - firstRuntime.shutdownBlocker = shutdownGate.promise - const runtimes = [firstRuntime] - const planner = new CodexLaunchPlanner(() => { - const runtime = runtimes.shift() - if (!runtime) throw new Error('unexpected runtime allocation') - return runtime as any + const planner = new CodexLaunchPlanner(() => sidecar as any) + + const plan = await planner.planCreate({ + cwd: '/repo/worktree', + terminalId: 'term-codex-handoff', + env: { + FRESHELL_TERMINAL_ID: 'term-codex-handoff', + }, }) - await planner.planCreate({ cwd: '/repo/before-shutdown' }) - const shutdown = planner.shutdown() - await new Promise((resolve) => setImmediate(resolve)) - - await expect(planner.planCreate({ cwd: '/repo/after-shutdown' })).rejects.toThrow(/shutting down/i) - expect(runtimes).toHaveLength(0) - - shutdownGate.resolve() - await shutdown - await expect(planner.planCreate({ cwd: '/repo/after-shutdown-complete' })).rejects.toThrow(/shutting down/i) - }) - - it('rejects and cleans up an in-flight launch plan when shutdown starts before thread creation returns', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43018', 'thread-after-shutdown') - const startThreadGate = deferred() - runtime.startThreadBlocker = startThreadGate.promise - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = planner.planCreate({ cwd: '/repo/in-flight' }) - await vi.waitFor(() => expect(runtime.startThreadCalls).toBe(1)) - - const shutdown = planner.shutdown() - await vi.waitFor(() => expect(runtime.shutdownCalls).toBe(1)) - - startThreadGate.resolve() - - await expect(plan).rejects.toThrow(/shutting down/i) - await expect(shutdown).resolves.toBeUndefined() - expect(runtime.shutdownCalls).toBe(1) - }) - - it('rejects adoption after planner shutdown has started sidecar teardown', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43019', 'thread-adopt-after-shutdown') - const shutdownGate = deferred() - runtime.shutdownBlocker = shutdownGate.promise - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ cwd: '/repo/adopt-after-shutdown' }) - const shutdown = planner.shutdown() - await vi.waitFor(() => expect(runtime.shutdownCalls).toBe(1)) - - await expect(plan.sidecar.adopt({ terminalId: 'term-after-shutdown', generation: 1 })) - .rejects.toThrow(/shutting down/i) - expect(runtime.adopted).toEqual([]) - - shutdownGate.resolve() - await shutdown - }) - - it('keeps failed unadopted sidecar teardown planner-owned and joinable by planner shutdown', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43015', 'thread-teardown-fails') - runtime.shutdownError = new Error('verified runtime teardown failed') - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ cwd: '/repo/unadopted' }) - - await expect(plan.sidecar.shutdown()).rejects.toThrow('verified runtime teardown failed') - await expect(planner.shutdown()).rejects.toThrow('verified runtime teardown failed') - expect(runtime.shutdownCalls).toBe(2) - }) - - it('retries a failed planner-owned sidecar teardown on a later shutdown join', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43023', 'thread-teardown-retry') - runtime.shutdownError = new Error('transient metadata cleanup failure') - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ cwd: '/repo/unadopted-retry' }) - - await expect(plan.sidecar.shutdown()).rejects.toThrow('transient metadata cleanup failure') - expect(runtime.shutdownCalls).toBe(1) - - runtime.shutdownError = undefined - - await expect(planner.shutdown()).resolves.toBeUndefined() - expect(runtime.shutdownCalls).toBe(2) - }) - - it('blocks new plans behind a failed planner-owned sidecar teardown until retry succeeds', async () => { - const runtimes: FakeRuntime[] = [] - const planner = new CodexLaunchPlanner(() => { - const index = runtimes.length + 1 - const runtime = new FakeRuntime(`ws://127.0.0.1:${43030 + index}`, `thread-${index}`) - runtimes.push(runtime) - return runtime as any + expect(plan).toEqual({ + remote: { + wsUrl: 'ws://127.0.0.1:43199', + }, + sidecar, }) - - const first = await planner.planCreate({ cwd: '/repo/one' }) - runtimes[0].shutdownError = new Error('transient teardown failure') - - await expect(first.sidecar.shutdown()).rejects.toThrow('transient teardown failure') - expect(runtimes[0].shutdownCalls).toBe(1) - - await expect(planner.planCreate({ cwd: '/repo/two' })).rejects.toThrow('transient teardown failure') - expect(runtimes).toHaveLength(1) - expect(runtimes[0].shutdownCalls).toBe(2) - - runtimes[0].shutdownError = undefined - - const second = await planner.planCreate({ cwd: '/repo/two' }) - - expect(second.sessionId).toBe('thread-2') - expect(runtimes).toHaveLength(2) - expect(runtimes[0].shutdownCalls).toBe(3) - }) - - it('waits for every planner-owned sidecar shutdown before reporting a teardown failure', async () => { - const firstRuntime = new FakeRuntime('ws://127.0.0.1:43016', 'thread-fast-fails') - firstRuntime.shutdownError = new Error('fast verified runtime teardown failed') - const secondRuntime = new FakeRuntime('ws://127.0.0.1:43017', 'thread-slow-shutdown') - const slowShutdown = deferred() - secondRuntime.shutdownBlocker = slowShutdown.promise - const runtimes = [firstRuntime, secondRuntime] - const planner = new CodexLaunchPlanner(() => runtimes.shift()! as any) - - await planner.planCreate({ cwd: '/repo/fast-fails' }) - await planner.planCreate({ cwd: '/repo/slow-shutdown' }) - - const shutdown = planner.shutdown() - let settled = false - void shutdown.then( - () => { settled = true }, - () => { settled = true }, - ) - - await vi.waitFor(() => expect(firstRuntime.shutdownCalls).toBe(1)) - await vi.waitFor(() => expect(secondRuntime.shutdownCalls).toBe(1)) - await new Promise((resolve) => setImmediate(resolve)) - expect(settled).toBe(false) - - slowShutdown.resolve() - await expect(shutdown).rejects.toThrow('fast verified runtime teardown failed') - }) - - it('waits for candidate-local loaded-thread readiness', async () => { - const runtime = new FakeRuntime( - 'ws://127.0.0.1:43020', - 'thread-ready', - undefined, - [[], ['other-thread'], ['thread-ready']], - ) - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ resumeSessionId: 'thread-ready' }) - - await expect(plan.sidecar.waitForLoadedThread('thread-ready', { timeoutMs: 1_000, pollMs: 1 })) - .resolves.toBeUndefined() - expect(runtime.loadedThreadListCalls).toBe(3) - }) - - it('stops loaded-thread readiness polling after sidecar shutdown starts', async () => { - const runtime = new FakeRuntime('ws://127.0.0.1:43021', 'thread-never-loads') - const planner = new CodexLaunchPlanner(() => runtime as any) - - const plan = await planner.planCreate({ resumeSessionId: 'thread-never-loads' }) - const readiness = plan.sidecar.waitForLoadedThread('thread-never-loads', { timeoutMs: 250, pollMs: 20 }) - await vi.waitFor(() => expect(runtime.loadedThreadListCalls).toBeGreaterThan(0)) - - await plan.sidecar.shutdown() - await expect(readiness).rejects.toThrow(/shutting down/i) - - const callsAfterShutdown = runtime.loadedThreadListCalls - await new Promise((resolve) => setTimeout(resolve, 50)) - expect(runtime.loadedThreadListCalls).toBe(callsAfterShutdown) }) }) diff --git a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts index b2108bd4f..84cb15e07 100644 --- a/test/unit/server/coding-cli/codex-app-server/runtime.test.ts +++ b/test/unit/server/coding-cli/codex-app-server/runtime.test.ts @@ -4,12 +4,7 @@ import fsp from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -import { - assertCodexStartupReaperSucceeded, - CodexAppServerRuntime, - reapOrphanedCodexAppServerSidecars, - runCodexStartupReaper, -} from '../../../../../server/coding-cli/codex-app-server/runtime.js' +import { CodexAppServerRuntime } from '../../../../../server/coding-cli/codex-app-server/runtime.js' import { allocateLocalhostPort, type LoopbackServerEndpoint } from '../../../../../server/local-port.js' const __filename = fileURLToPath(import.meta.url) @@ -18,7 +13,6 @@ const FAKE_SERVER_PATH = path.resolve(__dirname, '../../../../fixtures/coding-cl const runtimes = new Set<CodexAppServerRuntime>() const blockers = new Set<http.Server>() -const tempDirs = new Set<string>() async function closeBlocker(server: http.Server): Promise<void> { blockers.delete(server) @@ -31,18 +25,8 @@ afterEach(async () => { await runtime.shutdown() })) await Promise.all([...blockers].map((blocker) => closeBlocker(blocker))) - await Promise.all([...tempDirs].map(async (dir) => { - tempDirs.delete(dir) - await fsp.rm(dir, { recursive: true, force: true }) - })) }) -async function makeTempDir(): Promise<string> { - const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'freshell-codex-runtime-')) - tempDirs.add(dir) - return dir -} - async function occupyLoopbackPort(): Promise<{ blocker: http.Server; endpoint: LoopbackServerEndpoint }> { const blocker = http.createServer((_req, res) => { res.statusCode = 404 @@ -88,113 +72,54 @@ async function waitForProcessExit(pid: number, timeoutMs = 5_000): Promise<void> throw new Error(`Timed out waiting for process ${pid} to exit`) } -async function killProcessGroupForTest(processGroupId: number): Promise<void> { - try { - process.kill(-processGroupId, 'SIGKILL') - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error - } - await waitForProcessExit(processGroupId).catch(() => undefined) +function createRuntime(options: ConstructorParameters<typeof CodexAppServerRuntime>[0] = {}): CodexAppServerRuntime { + const runtime = new CodexAppServerRuntime({ + command: process.execPath, + commandArgs: [FAKE_SERVER_PATH], + ...options, + }) + runtimes.add(runtime) + return runtime } -async function waitForMetadataRecord(metadataDir: string, timeoutMs = 5_000): Promise<any> { +async function waitFor(assertion: () => void | Promise<void>, timeoutMs = 1_000): Promise<void> { const deadline = Date.now() + timeoutMs + let lastError: unknown while (Date.now() < deadline) { - const entries = await fsp.readdir(metadataDir).catch(() => []) - for (const entry of entries) { - if (!entry.endsWith('.json')) continue - const raw = await fsp.readFile(path.join(metadataDir, entry), 'utf8') - return JSON.parse(raw) + try { + await assertion() + return + } catch (error) { + lastError = error + await new Promise((resolve) => setTimeout(resolve, 10)) } - await new Promise((resolve) => setTimeout(resolve, 25)) - } - - throw new Error(`Timed out waiting for metadata record in ${metadataDir}`) -} - -async function waitForPidFile(pidFile: string, timeoutMs = 5_000): Promise<number> { - const deadline = Date.now() + timeoutMs - while (Date.now() < deadline) { - const raw = await fsp.readFile(pidFile, 'utf8').catch(() => '') - const pid = Number(raw.trim()) - if (Number.isInteger(pid) && pid > 0) return pid - await new Promise((resolve) => setTimeout(resolve, 25)) - } - throw new Error(`Timed out waiting for pid file ${pidFile}`) -} - -async function readProcessEnvironment(pid: number): Promise<Record<string, string>> { - const raw = await fsp.readFile(`/proc/${pid}/environ`) - const pairs = raw.toString('utf8').split('\0').filter(Boolean) - return Object.fromEntries(pairs.map((pair) => { - const index = pair.indexOf('=') - return index === -1 ? [pair, ''] : [pair.slice(0, index), pair.slice(index + 1)] - })) -} - -async function readCurrentProcessGroupId(): Promise<number> { - const stat = await fsp.readFile('/proc/self/stat', 'utf8') - const closeParen = stat.lastIndexOf(')') - const fields = stat.slice(closeParen + 2).trim().split(/\s+/) - return Number(fields[2]) -} - -async function markOwnershipRecordStale( - metadataPath: string, - overrides: Record<string, unknown> = {}, -): Promise<any> { - const raw = await fsp.readFile(metadataPath, 'utf8') - const metadata = JSON.parse(raw) - const stale = { - ...metadata, - ownerServerPid: 999_999_999, - serverInstanceId: 'srv-previous', - updatedAt: new Date().toISOString(), - ...overrides, } - await fsp.writeFile(metadataPath, JSON.stringify(stale, null, 2), 'utf8') - return stale -} -async function isProcessGroupAlive(processGroupId: number): Promise<boolean> { - try { - process.kill(-processGroupId, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false - throw error - } + if (lastError instanceof Error) throw lastError + throw new Error('Timed out waiting for assertion') } -async function readWrapperIdentityForTest(pid: number) { - const [cmdline, cwd, stat] = await Promise.all([ - fsp.readFile(`/proc/${pid}/cmdline`).catch(() => Buffer.from('')), - fsp.readlink(`/proc/${pid}/cwd`).catch(() => null), - fsp.readFile(`/proc/${pid}/stat`, 'utf8'), - ]) - const closeParen = stat.lastIndexOf(')') - const fields = stat.slice(closeParen + 2).trim().split(/\s+/) - const startTimeTicks = Number(fields[19]) - return { - commandLine: cmdline.toString('utf8').split('\0').filter(Boolean), - cwd, - startTimeTicks: Number.isFinite(startTimeTicks) ? startTimeTicks : null, - } +function deferred<T = void>(): { + promise: Promise<T> + resolve: (value: T | PromiseLike<T>) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T | PromiseLike<T>) => void + let reject!: (reason?: unknown) => void + const promise = new Promise<T>((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } } -function createRuntime(options: ConstructorParameters<typeof CodexAppServerRuntime>[0] = {}): CodexAppServerRuntime { - const runtime = new CodexAppServerRuntime({ - command: process.execPath, - commandArgs: [FAKE_SERVER_PATH], - ...options, - }) - runtimes.add(runtime) - return runtime +type RuntimeCleanupHook = { + stopActiveChild(): Promise<void> } describe('CodexAppServerRuntime', () => { - it('starts one owned loopback app-server sidecar on first use', async () => { + it('starts one loopback app-server runtime on first use', async () => { const runtime = createRuntime() const ready = await runtime.ensureReady() @@ -204,66 +129,36 @@ describe('CodexAppServerRuntime', () => { expect(runtime.status()).toBe('running') }) - it('rejects before spawning on platforms without Linux /proc ownership support', async () => { - const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform') - if (!originalPlatform?.configurable) { - throw new Error('process.platform descriptor is not configurable in this test environment') + it('starts the app-server process in the requested cwd', async () => { + if (process.platform !== 'linux') { + return } - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - startupAttemptLimit: 1, - }) + + const runtimeCwd = await fsp.mkdtemp(path.join(os.tmpdir(), 'freshell-codex-runtime-cwd-')) + const runtime = createRuntime({ cwd: runtimeCwd }) try { - Object.defineProperty(process, 'platform', { value: 'darwin' }) - - await expect(runtime.ensureReady()).rejects.toThrow(/linux.*\/proc/i) - const entries = await fsp.readdir(metadataDir).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [] - throw error - }) - expect(entries).toEqual([]) + const ready = await runtime.ensureReady() + await expect(fsp.readlink(`/proc/${ready.processPid}/cwd`)).resolves.toBe(runtimeCwd) } finally { - Object.defineProperty(process, 'platform', originalPlatform) + await fsp.rm(runtimeCwd, { recursive: true, force: true }) } }) - it('rejects before spawning when Linux /proc ownership proof is unavailable', async () => { - const metadataDir = await makeTempDir() - let ownershipIdCalls = 0 - const originalReaddir = fsp.readdir.bind(fsp) - const readdirSpy = vi.spyOn(fsp, 'readdir').mockImplementation(((target: any, options?: any) => { - if (String(target) === '/proc') { - const error = new Error('simulated /proc read failure') as NodeJS.ErrnoException - error.code = 'EACCES' - return Promise.reject(error) - } - return originalReaddir(target, options as any) as any - }) as typeof fsp.readdir) - const runtime = createRuntime({ - metadataDir, - startupAttemptLimit: 1, - ownershipIdFactory: () => { - ownershipIdCalls += 1 - return `ownership-${ownershipIdCalls}` - }, - }) + it('keeps separate runtime instances isolated for concurrent codex terminals', async () => { + const firstRuntime = createRuntime() + const secondRuntime = createRuntime() - try { - await expect(runtime.ensureReady()).rejects.toThrow(/\/proc.*ownership proof/i) - expect(ownershipIdCalls).toBe(0) - const entries = await originalReaddir(metadataDir).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [] - throw error - }) - expect(entries).toEqual([]) - } finally { - readdirSpy.mockRestore() - } + const [first, second] = await Promise.all([ + firstRuntime.ensureReady(), + secondRuntime.ensureReady(), + ]) + + expect(first.processPid).not.toBe(second.processPid) + expect(first.wsUrl).not.toBe(second.wsUrl) }) - it('reuses the same process for repeated ensureReady calls on one runtime', async () => { + it('reuses the same process for repeated ensureReady calls', async () => { const runtime = createRuntime() const first = await runtime.ensureReady() @@ -296,994 +191,274 @@ describe('CodexAppServerRuntime', () => { expect(runtime.status()).toBe('stopped') }) - it('writes ownership metadata immediately after spawn before initialize completes', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - delayMethodsMs: { - initialize: 250, - }, - }), - }, - }) - - const readyPromise = runtime.ensureReady() - const metadata = await waitForMetadataRecord(metadataDir) - const ready = await readyPromise - - expect(metadata.schemaVersion).toBe(1) - expect(metadata.ownershipId).toBe(ready.ownershipId) - expect(metadata.serverInstanceId).toBe('srv-runtime-test') - expect(metadata.ownerServerPid).toBe(process.pid) - expect(metadata.terminalId).toBeNull() - expect(metadata.generation).toBeNull() - expect(metadata.wsUrl).toBe(ready.wsUrl) - expect(metadata.wrapperPid).toBe(ready.processPid) - expect(metadata.processGroupId).toBe(ready.processGroupId) - expect(metadata.wrapperIdentity.startTimeTicks).toEqual(expect.any(Number)) - }) + it('proxies thread/start through the runtime client after boot', async () => { + const runtime = createRuntime() - it('writes durable ownership metadata before wrapper identity lookup resolves', async () => { - const metadataDir = await makeTempDir() - let identityLookupStarted!: () => void - let releaseIdentityLookup!: (identity: null) => void - const identityStarted = new Promise<void>((resolve) => { - identityLookupStarted = resolve - }) - const identityReleased = new Promise<null>((resolve) => { - releaseIdentityLookup = resolve - }) - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - processIdentityReader: async () => { - identityLookupStarted() - return identityReleased + await expect(runtime.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, }, + wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), }) - - const readyPromise = runtime.ensureReady() - try { - await identityStarted - const metadata = await waitForMetadataRecord(metadataDir, 500) - - expect(metadata.schemaVersion).toBe(1) - expect(metadata.serverInstanceId).toBe('srv-runtime-test') - expect(metadata.wrapperIdentity).toEqual({ - commandLine: [], - cwd: null, - startTimeTicks: null, - }) - } finally { - releaseIdentityLookup(null) - await readyPromise.catch(() => undefined) - } }) - it('tears down both the wrapper and native child in its process group', async () => { - const metadataDir = await makeTempDir() - const nativePidFile = path.join(metadataDir, 'native.pid') - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - wrapperLeavesNativeOnSigterm: true, - }), + it('proxies thread/resume through the runtime client after boot', async () => { + const runtime = createRuntime() + + await expect(runtime.resumeThread({ + threadId: '019d9859-5670-72b1-851f-794ad7fef112', + cwd: '/repo/worktree', + })).resolves.toEqual({ + thread: { + id: '019d9859-5670-72b1-851f-794ad7fef112', + path: expect.stringMatching(/rollout-019d9859-5670-72b1-851f-794ad7fef112\.jsonl$/), + ephemeral: false, }, + wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), }) - - const ready = await runtime.ensureReady() - const nativePid = await waitForPidFile(nativePidFile) - - expect(nativePid).not.toBe(ready.processPid) - - await runtime.shutdown() - - await waitForProcessExit(ready.processPid) - await waitForProcessExit(nativePid) - await expect(fsp.readdir(metadataDir)).resolves.not.toContain(path.basename(ready.metadataPath)) }) - it('tears down an owned native child after the wrapper exits hard before restarting', async () => { - const metadataDir = await makeTempDir() - const nativePidFile = path.join(metadataDir, 'native.pid') - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - wrapperLeavesNativeOnSigterm: true, - }), - }, - }) + it('drops cached state after an unexpected child exit and starts a fresh process on the next call', async () => { + const runtime = createRuntime() const first = await runtime.ensureReady() - const oldNativePid = await waitForPidFile(nativePidFile) - - process.kill(first.processPid, 'SIGKILL') - await waitForProcessExit(first.processPid) - + await runtime.simulateChildExitForTest() const second = await runtime.ensureReady() expect(second.processPid).not.toBe(first.processPid) - await waitForProcessExit(oldNativePid) - }) - - it('tears down a native child when the wrapper exits before initialize', async () => { - const metadataDir = await makeTempDir() - const nativePidFile = path.join(metadataDir, 'native.pid') - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - startupAttemptLimit: 1, - startupAttemptTimeoutMs: 100, - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - wrapperLeavesNativeOnSigterm: true, - exitAfterSpawningNative: true, - }), - }, - }) - - await expect(runtime.ensureReady()).rejects.toThrow(/failed to start codex app-server/i) - - const nativePid = await waitForPidFile(nativePidFile) - await waitForProcessExit(nativePid) + expect(second.wsUrl).not.toBe(first.wsUrl) }) - it('uses the startup attempt timeout to tear down an initialize hang before retrying', async () => { - const tempDir = await makeTempDir() - const metadataDir = path.join(tempDir, 'metadata') - const processGroups: number[] = [] - const seenProcessGroups = new Set<number>() - let previousAttemptGoneBeforeRetry = false - const start = Date.now() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - startupAttemptLimit: 2, - startupAttemptTimeoutMs: 120, - requestTimeoutMs: 1_000, - metadataWriter: async (filePath, metadata) => { - if (!seenProcessGroups.has(metadata.processGroupId)) { - if (processGroups.length > 0) { - previousAttemptGoneBeforeRetry = !(await isProcessGroupAlive(processGroups[0])) - } - seenProcessGroups.add(metadata.processGroupId) - processGroups.push(metadata.processGroupId) - } - await fsp.mkdir(path.dirname(filePath), { recursive: true }) - await fsp.writeFile(filePath, JSON.stringify(metadata), 'utf8') - }, - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - ignoreMethods: ['initialize'], - }), - }, - }) - - await expect(runtime.ensureReady()).rejects.toThrow(/initialize|failed to start codex app-server/i) - - expect(processGroups).toHaveLength(2) - expect(previousAttemptGoneBeforeRetry).toBe(true) - expect(Date.now() - start).toBeLessThan(1_500) - }, 3_000) - - it('tears down the owned process group before retry when wrapper identity cannot be read', async () => { - const tempDir = await makeTempDir() - const metadataDir = path.join(tempDir, 'metadata') - const processGroups: number[] = [] - const seenProcessGroups = new Set<number>() - let previousAttemptGoneBeforeRetry = false - let identityReadAttempts = 0 - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - startupAttemptLimit: 2, - startupAttemptTimeoutMs: 120, - requestTimeoutMs: 1_000, - processIdentityReader: async (pid) => { - identityReadAttempts += 1 - if (identityReadAttempts === 1) return null - return readWrapperIdentityForTest(pid) - }, - metadataWriter: async (filePath, metadata) => { - if (!seenProcessGroups.has(metadata.processGroupId)) { - if (processGroups.length > 0) { - previousAttemptGoneBeforeRetry = !(await isProcessGroupAlive(processGroups[0])) - } - seenProcessGroups.add(metadata.processGroupId) - processGroups.push(metadata.processGroupId) - } - await fsp.mkdir(path.dirname(filePath), { recursive: true }) - await fsp.writeFile(filePath, JSON.stringify(metadata), 'utf8') - }, - }) - - const ready = await runtime.ensureReady() - const record = JSON.parse(await fsp.readFile(ready.metadataPath, 'utf8')) - - expect(processGroups).toHaveLength(2) - expect(identityReadAttempts).toBe(2) - expect(previousAttemptGoneBeforeRetry).toBe(true) - expect(record.processGroupId).toBe(processGroups[1]) - expect(record.wrapperIdentity.startTimeTicks).toEqual(expect.any(Number)) - }, 3_000) - - it('tears down the owned process group before retry when wrapper identity is incomplete', async () => { - const tempDir = await makeTempDir() - const metadataDir = path.join(tempDir, 'metadata') - const processGroups: number[] = [] - const seenProcessGroups = new Set<number>() - let previousAttemptGoneBeforeRetry = false - let identityReadAttempts = 0 + it('retries startup when the preallocated loopback port is lost before Codex binds', async () => { + const { blocker, endpoint } = await occupyLoopbackPort() + let first = true const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - startupAttemptLimit: 2, - startupAttemptTimeoutMs: 120, - requestTimeoutMs: 1_000, - processIdentityReader: async (pid) => { - identityReadAttempts += 1 - if (identityReadAttempts === 1) { - return { commandLine: [], cwd: null, startTimeTicks: null } - } - return readWrapperIdentityForTest(pid) - }, - metadataWriter: async (filePath, metadata) => { - if (!seenProcessGroups.has(metadata.processGroupId)) { - if (processGroups.length > 0) { - previousAttemptGoneBeforeRetry = !(await isProcessGroupAlive(processGroups[0])) - } - seenProcessGroups.add(metadata.processGroupId) - processGroups.push(metadata.processGroupId) + startupAttemptLimit: 3, + startupAttemptTimeoutMs: 1_000, + portAllocator: async () => { + if (first) { + first = false + return endpoint } - await fsp.mkdir(path.dirname(filePath), { recursive: true }) - await fsp.writeFile(filePath, JSON.stringify(metadata), 'utf8') - }, - }) - - const ready = await runtime.ensureReady() - const record = JSON.parse(await fsp.readFile(ready.metadataPath, 'utf8')) - - expect(processGroups).toHaveLength(2) - expect(identityReadAttempts).toBe(2) - expect(previousAttemptGoneBeforeRetry).toBe(true) - expect(record.processGroupId).toBe(processGroups[1]) - expect(record.wrapperIdentity.commandLine.length).toBeGreaterThan(0) - expect(record.wrapperIdentity.cwd).toEqual(expect.any(String)) - expect(record.wrapperIdentity.startTimeTicks).toEqual(expect.any(Number)) - }, 3_000) - - it('escalates to SIGKILL when the native child ignores SIGTERM', async () => { - const metadataDir = await makeTempDir() - const nativePidFile = path.join(metadataDir, 'native.pid') - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - nativeChildIgnoresSigterm: true, - wrapperLeavesNativeOnSigterm: true, - }), - }, - }) - - const ready = await runtime.ensureReady() - const nativePid = await waitForPidFile(nativePidFile) - - await runtime.shutdown() - - await waitForProcessExit(ready.processPid) - await waitForProcessExit(nativePid) - await expect(fsp.stat(ready.metadataPath)).rejects.toMatchObject({ code: 'ENOENT' }) - }) - - it('sets the ownership id in the fake native child environment', async () => { - const metadataDir = await makeTempDir() - const nativePidFile = path.join(metadataDir, 'native.pid') - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - }), + return allocateLocalhostPort() }, }) + const onExit = vi.fn() + runtime.onExit(onExit) const ready = await runtime.ensureReady() - const nativePid = await waitForPidFile(nativePidFile) - const nativeEnv = await readProcessEnvironment(nativePid) - expect(ready.ownershipId).toEqual(expect.any(String)) - expect(nativeEnv.FRESHELL_CODEX_SIDECAR_ID).toBe(ready.ownershipId) - }) - - it('rejects adoption metadata updates when no active owned sidecar exists', async () => { - const runtime = createRuntime() - - await expect(runtime.updateOwnershipMetadata({ - terminalId: 'term-missing', - generation: 1, - })).rejects.toThrow(/no active owned codex app-server sidecar/i) + expect(ready.wsUrl).toMatch(/^ws:\/\/127\.0\.0\.1:\d+$/) + expect(ready.wsUrl).not.toBe(`ws://${endpoint.hostname}:${endpoint.port}`) + expect(onExit).not.toHaveBeenCalled() + await closeBlocker(blocker) }) - it('tears down the process group and fails startup when ownership metadata cannot be written', async () => { - const tempDir = await makeTempDir() - const metadataDir = path.join(tempDir, 'metadata') - const nativePidFile = path.join(tempDir, 'native.pid') + it('coalesces ensureReady callers while startup spawn-error cleanup is still in progress', async () => { + const attemptedPorts: number[] = [] const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', + command: path.join(os.tmpdir(), `missing-codex-app-server-coalesce-${process.pid}`), + requestTimeoutMs: 50, startupAttemptLimit: 1, - metadataWriter: async () => { - await waitForPidFile(nativePidFile) - throw new Error('simulated metadata write failure') - }, - env: { - FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - spawnNativeChild: true, - nativePidFile, - wrapperLeavesNativeOnSigterm: true, - }), - }, - }) - - await expect(runtime.ensureReady()).rejects.toThrow(/ownership metadata/i) - - const nativePid = await waitForPidFile(nativePidFile) - await waitForProcessExit(nativePid) - }) - - it('does not retry startup when failed-attempt teardown cannot be verified', async () => { - const tempDir = await makeTempDir() - const metadataDir = path.join(tempDir, 'metadata') - const spawnedProcessGroups: number[] = [] - let metadataWriteAttempts = 0 - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - startupAttemptLimit: 3, - metadataWriter: async (_filePath, metadata) => { - metadataWriteAttempts += 1 - spawnedProcessGroups.push(metadata.processGroupId) - metadata.processGroupId = await readCurrentProcessGroupId() - throw new Error('simulated metadata write failure') + startupAttemptTimeoutMs: 500, + portAllocator: async () => { + const endpoint = await allocateLocalhostPort() + attemptedPorts.push(endpoint.port) + return endpoint }, }) - - try { - await expect(runtime.ensureReady()).rejects.toThrow(/teardown failed|process-group teardown failed/i) - expect(metadataWriteAttempts).toBe(1) - } finally { - runtimes.delete(runtime) - for (const processGroupId of spawnedProcessGroups) { - await killProcessGroupForTest(processGroupId) - } - } - }) - - it('rejects shutdown when owned process-group teardown cannot be verified', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', + const cleanupStarted = deferred() + const allowCleanup = deferred() + const cleanupHook = runtime as unknown as RuntimeCleanupHook + const originalStopActiveChild = cleanupHook.stopActiveChild.bind(runtime) + let cleanupCalls = 0 + cleanupHook.stopActiveChild = vi.fn(async () => { + cleanupCalls += 1 + cleanupStarted.resolve() + await allowCleanup.promise + return originalStopActiveChild() }) - const ready = await runtime.ensureReady() - const ownership = (runtime as any).ownership - ownership.metadata.processGroupId = await readCurrentProcessGroupId() - - await expect(runtime.shutdown()).rejects.toThrow(/could not be verified|failed/i) - - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - }) - - it('does not use matching wrapper identity to authorize teardown of a different process group', async () => { - const firstRuntime = createRuntime({ - metadataDir: await makeTempDir(), - serverInstanceId: 'srv-runtime-test', - }) - const secondRuntime = createRuntime({ - metadataDir: await makeTempDir(), - serverInstanceId: 'srv-runtime-test', - }) - let firstReady: Awaited<ReturnType<CodexAppServerRuntime['ensureReady']>> | undefined - let secondReady: Awaited<ReturnType<CodexAppServerRuntime['ensureReady']>> | undefined - + let first: Promise<unknown> | undefined + let second: Promise<unknown> | undefined try { - firstReady = await firstRuntime.ensureReady() - secondReady = await secondRuntime.ensureReady() - const ownership = (firstRuntime as any).ownership - ownership.metadata.processGroupId = secondReady.processGroupId - - await expect(firstRuntime.shutdown()).rejects.toThrow(/could not be verified|failed|ownership/i) - expect(await isProcessGroupAlive(secondReady.processGroupId)).toBe(true) - expect(await isProcessGroupAlive(firstReady.processGroupId)).toBe(true) + first = runtime.ensureReady() + void first.catch(() => undefined) + await cleanupStarted.promise + second = runtime.ensureReady() + void second.catch(() => undefined) + + allowCleanup.resolve() + await expect(Promise.allSettled([first, second])).resolves.toEqual([ + expect.objectContaining({ status: 'rejected' }), + expect.objectContaining({ status: 'rejected' }), + ]) + expect(cleanupCalls).toBe(1) + expect(attemptedPorts).toHaveLength(1) } finally { - runtimes.delete(firstRuntime) - runtimes.delete(secondRuntime) - if (firstReady) await killProcessGroupForTest(firstReady.processGroupId) - if (secondReady) await killProcessGroupForTest(secondReady.processGroupId) + allowCleanup.resolve() + cleanupHook.stopActiveChild = originalStopActiveChild + await Promise.allSettled([first, second].filter((promise): promise is Promise<unknown> => Boolean(promise))) } }) - it('does not use wrapper start ticks alone when command line and cwd no longer match', async () => { - const metadataDir = await makeTempDir() + it('rejects through the startup retry path when the app-server command cannot spawn', async () => { + const attemptedPorts: number[] = [] const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - }) - const ready = await runtime.ensureReady() - const ownership = (runtime as any).ownership - ownership.metadata.wrapperIdentity = { - commandLine: ['not-the-recorded-wrapper-command'], - cwd: '/not/the/recorded/cwd', - startTimeTicks: ownership.metadata.wrapperIdentity.startTimeTicks, - } - const originalReadFile = fsp.readFile.bind(fsp) - const readFileSpy = vi.spyOn(fsp, 'readFile').mockImplementation(((target: any, options?: any) => { - if (String(target) === `/proc/${ready.processPid}/environ`) { - return Promise.resolve(Buffer.from('')) as any - } - return originalReadFile(target, options as any) as any - }) as typeof fsp.readFile) - - try { - await expect(runtime.shutdown()).rejects.toThrow(/could not be verified|failed|ownership/i) - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - } finally { - readFileSpy.mockRestore() - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - } - }) - - it('keeps failed teardown ownership sticky and refuses a later startup', async () => { - const metadataDir = await makeTempDir() - let metadataWriteAttempts = 0 - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - metadataWriter: async (filePath, metadata) => { - metadataWriteAttempts += 1 - await fsp.mkdir(path.dirname(filePath), { recursive: true }) - await fsp.writeFile(filePath, JSON.stringify(metadata), 'utf8') + command: path.join(os.tmpdir(), `missing-codex-app-server-${process.pid}`), + requestTimeoutMs: 50, + startupAttemptLimit: 2, + startupAttemptTimeoutMs: 500, + portAllocator: async () => { + const endpoint = await allocateLocalhostPort() + attemptedPorts.push(endpoint.port) + return endpoint }, }) + const onExit = vi.fn() + runtime.onExit(onExit) - const ready = await runtime.ensureReady() - const metadataWriteAttemptsAfterReady = metadataWriteAttempts - const ownership = (runtime as any).ownership - ownership.metadata.processGroupId = await readCurrentProcessGroupId() - - await expect(runtime.shutdown()).rejects.toThrow(/could not be verified|failed/i) - await expect(runtime.ensureReady()).rejects.toThrow(/teardown failed|blocked/i) - expect(metadataWriteAttempts).toBe(metadataWriteAttemptsAfterReady) - - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - }) - - it('does not treat a live process group as gone when /proc member scanning returns no members', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - }) - const ready = await runtime.ensureReady() - const ownership = (runtime as any).ownership - ownership.metadata.wrapperIdentity = { - ...ownership.metadata.wrapperIdentity, - startTimeTicks: -1, - } - const originalReaddir = fsp.readdir.bind(fsp) - const readdirSpy = vi.spyOn(fsp, 'readdir').mockImplementation(((target: any, options?: any) => { - if (String(target) === '/proc') { - return Promise.resolve([]) as any - } - return originalReaddir(target, options as any) as any - }) as typeof fsp.readdir) + await expect(runtime.ensureReady()).rejects.toThrow( + /Failed to start Codex app-server on a loopback endpoint after 2 attempts: .*ENOENT/, + ) - try { - await expect(runtime.shutdown()).rejects.toThrow(/could not be verified|failed|ownership/i) - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - } finally { - readdirSpy.mockRestore() - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - } + expect(attemptedPorts).toHaveLength(2) + expect(runtime.status()).toBe('stopped') + expect(onExit).not.toHaveBeenCalled() }) - it('sets sticky failed ownership when process-group teardown throws', async () => { - const metadataDir = await makeTempDir() - let ownershipIdCalls = 0 + it('keeps child stdio drained so large app-server logs do not stall thread/start replies', async () => { const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - ownershipIdFactory: () => { - ownershipIdCalls += 1 - return `ownership-throws-${ownershipIdCalls}` + env: { + FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ + floodStdoutBeforeMethodsBytes: { + 'thread/start': 512 * 1024, + }, + }), }, - }) - const ready = await runtime.ensureReady() - const originalUnlink = fsp.unlink.bind(fsp) - const unlinkSpy = vi.spyOn(fsp, 'unlink').mockImplementation(((target: any) => { - if (String(target) === ready.metadataPath) { - return Promise.reject(new Error('simulated metadata unlink failure')) - } - return originalUnlink(target) as any - }) as typeof fsp.unlink) - - try { - await expect(runtime.shutdown()).rejects.toThrow('simulated metadata unlink failure') - await expect(runtime.ensureReady()).rejects.toThrow(/simulated metadata unlink failure|blocked/i) - expect(ownershipIdCalls).toBe(1) - } finally { - unlinkSpy.mockRestore() - await runtime.shutdown().catch(() => undefined) - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - } - }) - - it('retries a failed live process-group teardown on a later shutdown join', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-runtime-test', - }) - const ready = await runtime.ensureReady() - const originalKill = process.kill - let injected = false - const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: NodeJS.Signals | number) => { - if (!injected && pid === -ready.processGroupId && signal === 'SIGTERM') { - injected = true - const error = new Error('simulated transient SIGTERM failure') as NodeJS.ErrnoException - error.code = 'EPERM' - throw error - } - return originalKill(pid, signal as any) - }) as typeof process.kill) - - try { - await expect(runtime.shutdown()).rejects.toThrow('simulated transient SIGTERM failure') - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - - killSpy.mockRestore() - - await expect(runtime.shutdown()).resolves.toBeUndefined() - await waitForProcessExit(ready.processPid) - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(false) - await expect(fsp.stat(ready.metadataPath)).rejects.toMatchObject({ code: 'ENOENT' }) - } finally { - killSpy.mockRestore() - runtimes.delete(runtime) - await killProcessGroupForTest(ready.processGroupId) - } - }) - - it('rejects with a launch error instead of crashing when the command is missing', async () => { - const runtime = new CodexAppServerRuntime({ - command: '/tmp/definitely-missing-freshell-codex-binary', - startupAttemptLimit: 1, - startupAttemptTimeoutMs: 100, - }) - runtimes.add(runtime) - - await expect(runtime.ensureReady()).rejects.toThrow(/failed to launch codex app-server sidecar|enoent/i) - }) - - it('reaps only verified stale new-schema sidecar groups on startup', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - const raw = await fsp.readFile(ready.metadataPath, 'utf8') - const metadata = JSON.parse(raw) - await fsp.writeFile(ready.metadataPath, JSON.stringify({ - ...metadata, - ownerServerPid: 999_999_999, - serverInstanceId: 'srv-previous', - updatedAt: new Date().toISOString(), - }, null, 2), 'utf8') - - const result = await reapOrphanedCodexAppServerSidecars({ - metadataDir, - serverInstanceId: 'srv-current', - }) - - expect(result.reapedOwnershipIds).toContain(ready.ownershipId) - await waitForProcessExit(ready.processPid) - await expect(fsp.stat(ready.metadataPath)).rejects.toMatchObject({ code: 'ENOENT' }) - }) - - it('treats unreaped new-schema ownership records as a startup-blocking reaper failure', () => { - expect(() => assertCodexStartupReaperSucceeded({ - reapedOwnershipIds: [], - ignoredLegacyRecords: [], - skippedActiveOwnershipIds: [], - failedOwnershipIds: ['ownership-alpha', 'ownership-beta'], - })).toThrow(/startup reaper failed.*ownership-alpha.*ownership-beta/i) - }) - - it('blocks startup when a new-schema ownership record is skipped because the owner pid is live', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - await markOwnershipRecordStale(ready.metadataPath, { - ownerServerPid: process.pid, + requestTimeoutMs: 1_500, }) - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - terminateGraceMs: 1, - })).rejects.toThrow(new RegExp(ready.ownershipId)) - await expect(fsp.stat(ready.metadataPath)).resolves.toBeDefined() - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - }) - - it('propagates thrown startup reaper failures instead of treating them as warning fallbacks', async () => { - const metadataDir = await makeTempDir() - const originalReaddir = fsp.readdir.bind(fsp) - const readdirSpy = vi.spyOn(fsp, 'readdir').mockImplementation(((target: any, options?: any) => { - if (String(target) === metadataDir) { - return Promise.reject(new Error('simulated startup reaper metadata scan failure')) - } - return originalReaddir(target, options as any) as any - }) as typeof fsp.readdir) - - try { - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - })).rejects.toThrow('simulated startup reaper metadata scan failure') - } finally { - readdirSpy.mockRestore() - } - }) - - it('propagates startup reaper ownership verification failures', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - const raw = await fsp.readFile(ready.metadataPath, 'utf8') - const metadata = JSON.parse(raw) - await markOwnershipRecordStale(ready.metadataPath, { - wrapperIdentity: { - ...metadata.wrapperIdentity, - startTimeTicks: -1, + await expect(runtime.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ + thread: { + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, }, + wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), }) - const originalReaddir = fsp.readdir.bind(fsp) - let procReaddirCalls = 0 - const readdirSpy = vi.spyOn(fsp, 'readdir').mockImplementation(((target: any, options?: any) => { - if (String(target) === '/proc') { - procReaddirCalls += 1 - if (procReaddirCalls > 1) { - return Promise.reject(new Error('simulated ownership verification proc failure')) - } - } - return originalReaddir(target, options as any) as any - }) as typeof fsp.readdir) - - try { - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - terminateGraceMs: 1, - })).rejects.toThrow('simulated ownership verification proc failure') - } finally { - readdirSpy.mockRestore() - } - }) - - it('propagates startup reaper process-group signaling failures', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - await markOwnershipRecordStale(ready.metadataPath) - const originalKill = process.kill - const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: NodeJS.Signals | number) => { - if (pid === -ready.processGroupId && signal === 'SIGTERM') { - const error = new Error('simulated SIGTERM failure') as NodeJS.ErrnoException - error.code = 'EPERM' - throw error - } - return originalKill(pid, signal as any) - }) as typeof process.kill) - - try { - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - terminateGraceMs: 1, - })).rejects.toThrow('simulated SIGTERM failure') - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - } finally { - killSpy.mockRestore() - } - }) - - it('propagates startup reaper wait-for-gone diagnostic failures', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - await markOwnershipRecordStale(ready.metadataPath) - const originalKill = process.kill - let throwRemainingScan = false - const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: NodeJS.Signals | number) => { - if (pid === -ready.processGroupId && (signal === 'SIGTERM' || signal === 'SIGKILL')) { - if (signal === 'SIGKILL') throwRemainingScan = true - return true - } - return originalKill(pid, signal as any) - }) as typeof process.kill) - const originalReaddir = fsp.readdir.bind(fsp) - const readdirSpy = vi.spyOn(fsp, 'readdir').mockImplementation(((target: any, options?: any) => { - if (String(target) === '/proc' && throwRemainingScan) { - return Promise.reject(new Error('simulated wait-for-gone process scan failure')) - } - return originalReaddir(target, options as any) as any - }) as typeof fsp.readdir) - - try { - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - terminateGraceMs: 1, - })).rejects.toThrow('simulated wait-for-gone process scan failure') - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - } finally { - readdirSpy.mockRestore() - killSpy.mockRestore() - } - }) - - it('propagates startup reaper metadata removal failures', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - await markOwnershipRecordStale(ready.metadataPath) - const originalKill = process.kill - const killSpy = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: NodeJS.Signals | number) => { - if (pid === -ready.processGroupId && signal === 0) { - const error = new Error('simulated process group gone') as NodeJS.ErrnoException - error.code = 'ESRCH' - throw error - } - return originalKill(pid, signal as any) - }) as typeof process.kill) - const originalUnlink = fsp.unlink.bind(fsp) - const unlinkSpy = vi.spyOn(fsp, 'unlink').mockImplementation(((target: any) => { - if (String(target) === ready.metadataPath) { - return Promise.reject(new Error('simulated metadata removal failure')) - } - return originalUnlink(target) as any - }) as typeof fsp.unlink) - - try { - await expect(runCodexStartupReaper({ - metadataDir, - serverInstanceId: 'srv-current', - terminateGraceMs: 1, - })).rejects.toThrow('simulated metadata removal failure') - await expect(fsp.stat(ready.metadataPath)).resolves.toBeDefined() - } finally { - unlinkSpy.mockRestore() - killSpy.mockRestore() - } - }) - - it('removes legacy sidecar records without process-name cleanup', async () => { - const metadataDir = await makeTempDir() - const legacyPath = path.join(metadataDir, 'legacy.json') - await fsp.writeFile(legacyPath, JSON.stringify({ - pid: 12345, - wsUrl: 'ws://127.0.0.1:55555', - }), 'utf8') - - const result = await reapOrphanedCodexAppServerSidecars({ - metadataDir, - serverInstanceId: 'srv-current', - }) - - expect(result.ignoredLegacyRecords).toContain(legacyPath) - await expect(fsp.stat(legacyPath)).rejects.toMatchObject({ code: 'ENOENT' }) - }) - - it('retains malformed new-schema ownership records and reports them as startup-blocking failures', async () => { - const metadataDir = await makeTempDir() - const malformedPath = path.join(metadataDir, 'damaged-new-schema.json') - await fsp.writeFile(malformedPath, JSON.stringify({ - schemaVersion: 1, - ownershipId: 'damaged-ownership', - serverInstanceId: 'srv-previous', - ownerServerPid: 999_999_999, - }), 'utf8') - - const result = await reapOrphanedCodexAppServerSidecars({ - metadataDir, - serverInstanceId: 'srv-current', - }) - - expect(result.ignoredLegacyRecords).not.toContain(malformedPath) - expect(result.failedOwnershipIds).toContain('damaged-ownership') - await expect(fsp.stat(malformedPath)).resolves.toBeDefined() - expect(() => assertCodexStartupReaperSucceeded(result)).toThrow(/damaged-ownership/) - }) - - it('retains schema-v1 ownership records with invalid numeric ownership fields', async () => { - const metadataDir = await makeTempDir() - const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - await markOwnershipRecordStale(ready.metadataPath, { - processGroupId: 0, - }) - - const result = await reapOrphanedCodexAppServerSidecars({ - metadataDir, - serverInstanceId: 'srv-current', - }) - - expect(result.reapedOwnershipIds).not.toContain(ready.ownershipId) - expect(result.failedOwnershipIds).toContain(ready.ownershipId) - await expect(fsp.stat(ready.metadataPath)).resolves.toBeDefined() - expect(await isProcessGroupAlive(ready.processGroupId)).toBe(true) - expect(() => assertCodexStartupReaperSucceeded(result)).toThrow(new RegExp(ready.ownershipId)) }) - it('does not reap new-schema records for the current process group', async () => { - const metadataDir = await makeTempDir() + it('keeps child stderr drained so large app-server error logs do not stall thread/resume replies', async () => { const runtime = createRuntime({ - metadataDir, - serverInstanceId: 'srv-previous', - }) - const ready = await runtime.ensureReady() - const raw = await fsp.readFile(ready.metadataPath, 'utf8') - const metadata = JSON.parse(raw) - await fsp.writeFile(ready.metadataPath, JSON.stringify({ - ...metadata, - ownerServerPid: 999_999_999, - processGroupId: await readCurrentProcessGroupId(), - updatedAt: new Date().toISOString(), - }, null, 2), 'utf8') - - const result = await reapOrphanedCodexAppServerSidecars({ - metadataDir, - serverInstanceId: 'srv-current', - }) - - expect(result.skippedActiveOwnershipIds).toContain(ready.ownershipId) - await expect(fsp.stat(ready.metadataPath)).resolves.toBeDefined() - }) - - it('proxies thread/start through the sidecar client after boot', async () => { - const runtime = createRuntime() - - await expect(runtime.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', - wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), + env: { + FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ + floodStderrBeforeMethodsBytes: { + 'thread/resume': 512 * 1024, + }, + }), + }, + requestTimeoutMs: 1_500, }) - }) - - it('proxies thread/resume through the sidecar client after boot', async () => { - const runtime = createRuntime() await expect(runtime.resumeThread({ threadId: '019d9859-5670-72b1-851f-794ad7fef112', cwd: '/repo/worktree', })).resolves.toEqual({ - threadId: '019d9859-5670-72b1-851f-794ad7fef112', + thread: { + id: '019d9859-5670-72b1-851f-794ad7fef112', + path: expect.stringMatching(/rollout-019d9859-5670-72b1-851f-794ad7fef112\.jsonl$/), + ephemeral: false, + }, wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), }) }) - it('drops cached state after an unexpected child exit and starts a fresh process on the next call', async () => { - const runtime = createRuntime() - - const first = await runtime.ensureReady() - await runtime.simulateChildExitForTest() - const second = await runtime.ensureReady() - - expect(second.processPid).not.toBe(first.processPid) - expect(second.wsUrl).not.toBe(first.wsUrl) - }) - - it('retries startup when the preallocated loopback port is lost before Codex binds', async () => { - const { blocker, endpoint } = await occupyLoopbackPort() - let first = true + it('passes thread and fs watch notifications through runtime subscribers', async () => { + const rolloutPath = '/repo/worktree/.codex/sessions/2026/04/23/rollout-thread-new-1.jsonl' const runtime = createRuntime({ - startupAttemptLimit: 3, - startupAttemptTimeoutMs: 200, - portAllocator: async () => { - if (first) { - first = false - return endpoint - } - return allocateLocalhostPort() + env: { + FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ + notifyAfterMethodsOnce: { + 'fs/watch': [ + { + method: 'fs/changed', + params: { + watchId: 'watch-rollout', + changedPaths: [rolloutPath], + }, + }, + ], + }, + }), }, }) - const ready = await runtime.ensureReady() + const startedThread = new Promise<{ id: string; path: string | null; ephemeral: boolean }>((resolve) => { + runtime.onThreadStarted((thread) => resolve(thread)) + }) + const changedEvent = new Promise<{ watchId: string; changedPaths: string[] }>((resolve) => { + runtime.onFsChanged((event) => resolve(event)) + }) - expect(ready.wsUrl).toMatch(/^ws:\/\/127\.0\.0\.1:\d+$/) - expect(ready.wsUrl).not.toBe(`ws://${endpoint.hostname}:${endpoint.port}`) - await closeBlocker(blocker) + await runtime.startThread({ cwd: '/repo/worktree' }) + await expect(startedThread).resolves.toEqual({ + id: 'thread-new-1', + path: expect.stringMatching(/\/sessions\/\d{4}\/\d{2}\/\d{2}\/rollout-thread-new-1\.jsonl$/), + ephemeral: false, + }) + await expect(runtime.watchPath(rolloutPath, 'watch-rollout')).resolves.toEqual({ + path: rolloutPath, + }) + await expect(changedEvent).resolves.toEqual({ + watchId: 'watch-rollout', + changedPaths: [rolloutPath], + }) + await expect(runtime.unwatchPath('watch-rollout')).resolves.toBeUndefined() }) - it('keeps child stdio drained so large app-server logs do not stall thread/start replies', async () => { + it('notifies runtime exit handlers when the app-server client socket disconnects while the child is alive', async () => { const runtime = createRuntime({ env: { FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - floodStdoutBeforeMethodsBytes: { - 'thread/start': 512 * 1024, - }, + closeSocketAfterMethodsOnce: ['initialize'], }), }, - requestTimeoutMs: 1_500, }) + const onExit = vi.fn() + runtime.onExit(onExit) - await expect(runtime.startThread({ cwd: '/repo/worktree' })).resolves.toEqual({ - threadId: 'thread-new-1', - wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), - }) + await runtime.ensureReady() + + await waitFor(() => expect(onExit).toHaveBeenCalledWith( + expect.any(Error), + 'app_server_client_disconnect', + )) }) - it('keeps child stderr drained so large app-server error logs do not stall thread/resume replies', async () => { + it('includes pid, websocket port, exit code, signal, and stderr tail when a child exits unexpectedly', async () => { const runtime = createRuntime({ env: { FAKE_CODEX_APP_SERVER_BEHAVIOR: JSON.stringify({ - floodStderrBeforeMethodsBytes: { - 'thread/resume': 512 * 1024, - }, + stderrBeforeExit: 'queue full diagnostic', + exitProcessAfterMethodsOnce: ['initialize'], }), }, - requestTimeoutMs: 1_500, }) + const onExit = vi.fn() + runtime.onExit(onExit) - await expect(runtime.resumeThread({ - threadId: '019d9859-5670-72b1-851f-794ad7fef112', - cwd: '/repo/worktree', - })).resolves.toEqual({ - threadId: '019d9859-5670-72b1-851f-794ad7fef112', - wsUrl: expect.stringMatching(/^ws:\/\/127\.0\.0\.1:\d+$/), - }) + await runtime.ensureReady() + await waitFor(() => expect(onExit.mock.calls[0]?.[0]).toBeInstanceOf(Error)) + + const message = String(onExit.mock.calls[0]?.[0]?.message ?? '') + expect(message).toContain('pid ') + expect(message).toContain('ws port ') + expect(message).toContain('exit code ') + expect(message).toContain('signal ') + expect(message).toContain('stderr tail') + expect(message).toContain('queue full diagnostic') }) }) diff --git a/test/unit/server/coding-cli/codex-app-server/sidecar.test.ts b/test/unit/server/coding-cli/codex-app-server/sidecar.test.ts new file mode 100644 index 000000000..1e4a32b31 --- /dev/null +++ b/test/unit/server/coding-cli/codex-app-server/sidecar.test.ts @@ -0,0 +1,423 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import fsp from 'node:fs/promises' +import { spawn, type ChildProcess } from 'node:child_process' +import path from 'node:path' +import { randomUUID } from 'node:crypto' +import { CodexTerminalSidecar } from '../../../../../server/coding-cli/codex-app-server/sidecar.js' +import { + DEFAULT_CODEX_SIDECAR_METADATA_DIR, + type CodexSidecarOwnershipMetadata, +} from '../../../../../server/coding-cli/codex-app-server/runtime.js' + +const SIDECAR_OWNERSHIP_DIR = DEFAULT_CODEX_SIDECAR_METADATA_DIR +const children = new Set<ChildProcess>() + +async function waitForProcessExit(pid: number, timeoutMs = 5_000): Promise<void> { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + process.kill(pid, 0) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') { + return + } + throw error + } + await new Promise((resolve) => setTimeout(resolve, 25)) + } + throw new Error(`Timed out waiting for process ${pid} to exit`) +} + +async function readLinuxProcessIdentity(pid: number): Promise<{ + commandLine: string[] + cwd: string | null + startTimeTicks: number | null +}> { + const [cmdlineRaw, cwd, statRaw] = await Promise.all([ + fsp.readFile(`/proc/${pid}/cmdline`, 'utf8'), + fsp.readlink(`/proc/${pid}/cwd`), + fsp.readFile(`/proc/${pid}/stat`, 'utf8'), + ]) + const closeParen = statRaw.lastIndexOf(')') + const statFields = closeParen >= 0 + ? statRaw.slice(closeParen + 2).trim().split(/\s+/) + : statRaw.trim().split(/\s+/) + return { + commandLine: cmdlineRaw.split('\0').filter(Boolean), + cwd, + startTimeTicks: Number(statFields[19]), + } +} + +afterEach(async () => { + await Promise.all([...children].map(async (child) => { + children.delete(child) + if (child.exitCode !== null || child.signalCode !== null) { + return + } + child.kill('SIGKILL') + await new Promise<void>((resolve) => child.once('exit', () => resolve())) + })) + vi.restoreAllMocks() + + const entries = await fsp.readdir(SIDECAR_OWNERSHIP_DIR, { withFileTypes: true }).catch(() => []) + await Promise.all(entries.map(async (entry) => { + if (entry.isFile() && entry.name.endsWith('.json')) { + await fsp.rm(path.join(SIDECAR_OWNERSHIP_DIR, entry.name), { force: true }).catch(() => undefined) + } + })) +}) + +describe('CodexTerminalSidecar orphan reaper', () => { + it('refuses to SIGTERM a live pid when ownership metadata lacks a verified process match', async () => { + await fsp.mkdir(SIDECAR_OWNERSHIP_DIR, { recursive: true }) + const metadataPath = path.join(SIDECAR_OWNERSHIP_DIR, `${randomUUID()}.json`) + await fsp.writeFile(metadataPath, JSON.stringify({ + pid: process.pid, + wsUrl: 'ws://127.0.0.1:4545', + codexHome: '/tmp/fake-codex-home', + terminalId: 'term-mismatch', + createdAt: new Date().toISOString(), + }), 'utf8') + + const killSpy = vi.spyOn(process, 'kill') + + await CodexTerminalSidecar.reapOrphanedSidecars() + + expect(killSpy).not.toHaveBeenCalledWith(process.pid, 'SIGTERM') + }) + + it('SIGTERMs only a process group whose command line, cwd, and start time still match the recorded sidecar', async () => { + const wsUrl = 'ws://127.0.0.1:4546' + const ownershipId = `test-sidecar-${randomUUID()}` + const child = spawn(process.execPath, [ + '-e', + 'setInterval(() => {}, 1000)', + 'app-server', + '--listen', + wsUrl, + ], { + cwd: process.cwd(), + detached: true, + env: { + ...process.env, + FRESHELL_CODEX_SIDECAR_ID: ownershipId, + }, + stdio: 'ignore', + }) + children.add(child) + + if (!child.pid) { + throw new Error('Failed to spawn test sidecar process') + } + + const identity = await readLinuxProcessIdentity(child.pid) + await fsp.mkdir(SIDECAR_OWNERSHIP_DIR, { recursive: true }) + const metadataPath = path.join(SIDECAR_OWNERSHIP_DIR, `${ownershipId}.json`) + const now = new Date().toISOString() + const metadata: CodexSidecarOwnershipMetadata = { + schemaVersion: 1, + ownershipId, + serverInstanceId: 'test-dead-server', + ownerServerPid: 999_999_999, + wsUrl, + wrapperPid: child.pid, + processGroupId: child.pid, + codexHome: '/tmp/test-codex-home', + terminalId: 'term-owned', + generation: 2, + wrapperIdentity: identity, + createdAt: now, + updatedAt: now, + } + await fsp.writeFile(metadataPath, JSON.stringify(metadata), 'utf8') + + await CodexTerminalSidecar.reapOrphanedSidecars() + await waitForProcessExit(child.pid) + }) +}) + +describe('CodexTerminalSidecar durable rollout tracking', () => { + it('forwards thread handles into the exact-path tracker and promotes when the tracker confirms durability', () => { + let threadStartedHandler: ((thread: { id: string; path: string | null; ephemeral: boolean }) => void) | null = null + let trackerOptions: + | { + onDurableRollout: (sessionId: string) => void + } + | null = null + + const runtime = { + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn((handler) => { + threadStartedHandler = handler + return () => { + threadStartedHandler = null + } + }), + onThreadLifecycle: vi.fn(() => () => undefined), + shutdown: vi.fn(async () => undefined), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + + const tracker = { + trackThread: vi.fn(), + dispose: vi.fn(async () => undefined), + } + + const sidecar = new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: (options) => { + trackerOptions = { + onDurableRollout: options.onDurableRollout, + } + return tracker + }, + }) + + const onDurableSession = vi.fn() + sidecar.attachTerminal({ + terminalId: 'term-1', + onDurableSession, + onThreadLifecycle: vi.fn(), + onFatal: vi.fn(), + }) + + const thread = { + id: 'thread-new-1', + path: '/tmp/fake-codex-home/sessions/2026/04/23/rollout-thread-new-1.jsonl', + ephemeral: false, + } + threadStartedHandler?.(thread) + + expect(tracker.trackThread).toHaveBeenCalledWith(thread) + trackerOptions?.onDurableRollout('thread-new-1') + expect(onDurableSession).toHaveBeenCalledWith('thread-new-1') + }) + + it('forwards current thread lifecycle evidence to the attached terminal', () => { + let threadLifecycleHandler: ((event: { + kind: 'thread_started' + thread: { id: string; path: string | null; ephemeral: boolean } + }) => void) | null = null + const runtime = { + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn(() => () => undefined), + onThreadLifecycle: vi.fn((handler) => { + threadLifecycleHandler = handler + return () => { + threadLifecycleHandler = null + } + }), + shutdown: vi.fn(async () => undefined), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + const sidecar = new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: () => ({ + trackThread: vi.fn(), + dispose: vi.fn(async () => undefined), + }), + }) + const onThreadLifecycle = vi.fn() + sidecar.attachTerminal({ + terminalId: 'term-1', + onDurableSession: vi.fn(), + onThreadLifecycle, + onFatal: vi.fn(), + }) + + threadLifecycleHandler?.({ + kind: 'thread_started', + thread: { id: 'thread-1', path: '/tmp/rollout.jsonl', ephemeral: false }, + }) + + expect(onThreadLifecycle).toHaveBeenCalledWith({ + kind: 'thread_started', + thread: expect.objectContaining({ id: 'thread-1' }), + }) + }) + + it('replays lifecycle evidence observed before terminal attachment', () => { + let threadLifecycleHandler: ((event: { + kind: 'thread_started' + thread: { id: string; path: string | null; ephemeral: boolean } + }) => void) | null = null + const runtime = { + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn(() => () => undefined), + onThreadLifecycle: vi.fn((handler) => { + threadLifecycleHandler = handler + return () => { + threadLifecycleHandler = null + } + }), + shutdown: vi.fn(async () => undefined), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + const sidecar = new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: () => ({ + trackThread: vi.fn(), + dispose: vi.fn(async () => undefined), + }), + }) + + threadLifecycleHandler?.({ + kind: 'thread_started', + thread: { id: 'thread-1', path: '/tmp/rollout.jsonl', ephemeral: false }, + }) + + const onThreadLifecycle = vi.fn() + sidecar.attachTerminal({ + terminalId: 'term-1', + onDurableSession: vi.fn(), + onThreadLifecycle, + onFatal: vi.fn(), + }) + + expect(onThreadLifecycle).toHaveBeenCalledWith({ + kind: 'thread_started', + thread: expect.objectContaining({ id: 'thread-1' }), + }) + }) + + it('replays durable promotion before pending lifecycle evidence on terminal attachment', () => { + let threadLifecycleHandler: ((event: { + kind: 'thread_started' + thread: { id: string; path: string | null; ephemeral: boolean } + }) => void) | null = null + let trackerOptions: + | { + onDurableRollout: (sessionId: string) => void + } + | null = null + const runtime = { + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn(() => () => undefined), + onThreadLifecycle: vi.fn((handler) => { + threadLifecycleHandler = handler + return () => { + threadLifecycleHandler = null + } + }), + shutdown: vi.fn(async () => undefined), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + const sidecar = new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: (options) => { + trackerOptions = { onDurableRollout: options.onDurableRollout } + return { + trackThread: vi.fn(), + dispose: vi.fn(async () => undefined), + } + }, + }) + + threadLifecycleHandler?.({ + kind: 'thread_started', + thread: { id: 'thread-fast', path: '/tmp/rollout.jsonl', ephemeral: false }, + }) + trackerOptions?.onDurableRollout('thread-fast') + + const replayOrder: string[] = [] + sidecar.attachTerminal({ + terminalId: 'term-1', + onDurableSession: (sessionId) => replayOrder.push(`durable:${sessionId}`), + onThreadLifecycle: (event) => { + if (event.kind === 'thread_started') { + replayOrder.push(`lifecycle:${event.thread.id}`) + } + }, + onFatal: vi.fn(), + }) + + expect(replayOrder).toEqual(['durable:thread-fast', 'lifecycle:thread-fast']) + }) + + it('disposes the rollout tracker before shutting the runtime down', async () => { + const lifecycle: string[] = [] + const runtime = { + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn(() => () => undefined), + onThreadLifecycle: vi.fn(() => () => undefined), + shutdown: vi.fn(async () => { + lifecycle.push('runtime') + }), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + const tracker = { + trackThread: vi.fn(), + dispose: vi.fn(async () => { + lifecycle.push('tracker') + }), + } + + const sidecar = new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: () => tracker, + }) + + await sidecar.shutdown() + + expect(lifecycle).toEqual(['tracker', 'runtime']) + }) + + it('does not boot a stopped runtime just to unwatch during tracker cleanup', async () => { + let trackerOptions: + | { + unwatchPath: (watchId: string) => Promise<void> + } + | null = null + + const runtime = { + status: vi.fn(() => 'stopped'), + unwatchPath: vi.fn(async () => undefined), + onExit: vi.fn(() => () => undefined), + onThreadStarted: vi.fn(() => () => undefined), + onThreadLifecycle: vi.fn(() => () => undefined), + shutdown: vi.fn(async () => undefined), + ensureReady: vi.fn(async () => ({ + wsUrl: 'ws://127.0.0.1:4567', + processPid: 101, + codexHome: '/tmp/fake-codex-home', + })), + } + + new CodexTerminalSidecar({ + runtime: runtime as any, + createDurableRolloutTracker: (options) => { + trackerOptions = { + unwatchPath: options.unwatchPath, + } + return { + trackThread: vi.fn(), + dispose: vi.fn(async () => undefined), + } + }, + }) + + await trackerOptions?.unwatchPath('watch-rollout') + expect(runtime.unwatchPath).not.toHaveBeenCalled() + }) +}) diff --git a/test/unit/server/terminal-registry.codex-recovery.test.ts b/test/unit/server/terminal-registry.codex-recovery.test.ts index c544d66dc..e8a26d1c1 100644 --- a/test/unit/server/terminal-registry.codex-recovery.test.ts +++ b/test/unit/server/terminal-registry.codex-recovery.test.ts @@ -38,7 +38,7 @@ vi.mock('../../../server/logger.js', () => { child: vi.fn(), } logger.child.mockReturnValue(logger) - return { logger, sessionLifecycleLogger: logger } + return { logger } }) vi.mock('../../../server/mcp/config-writer.js', () => ({ diff --git a/test/unit/server/terminal-registry.codex-sidecar.test.ts b/test/unit/server/terminal-registry.codex-sidecar.test.ts deleted file mode 100644 index 0597f8a89..000000000 --- a/test/unit/server/terminal-registry.codex-sidecar.test.ts +++ /dev/null @@ -1,1065 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { EventEmitter } from 'node:events' - -const mockPtyProcess = vi.hoisted(() => { - const createMockPty = () => { - const emitter = new EventEmitter() - const pty = { - pid: Math.floor(Math.random() * 100000) + 1000, - cols: 120, - rows: 30, - process: 'mock-shell', - handleFlowControl: false, - autoExitOnKill: true, - onData: vi.fn((handler: (data: string) => void) => { - emitter.on('data', handler) - return { dispose: () => emitter.off('data', handler) } - }), - onExit: vi.fn((handler: (e: { exitCode: number; signal?: number }) => void) => { - emitter.on('exit', handler) - return { dispose: () => emitter.off('exit', handler) } - }), - write: vi.fn(), - resize: vi.fn(), - kill: vi.fn(() => { - if (pty.autoExitOnKill) { - emitter.emit('exit', { exitCode: 0 }) - } - }), - _emitExit: (exitCode: number, signal?: number) => emitter.emit('exit', { exitCode, signal }), - } - return pty - } - return { createMockPty, instances: [] as ReturnType<typeof createMockPty>[] } -}) - -vi.mock('node-pty', () => ({ - spawn: vi.fn(() => { - const pty = mockPtyProcess.createMockPty() - mockPtyProcess.instances.push(pty) - return pty - }), -})) - -vi.mock('../../../server/logger', () => { - const logger = { - info: vi.fn(), - warn: vi.fn(), - debug: vi.fn(), - error: vi.fn(), - trace: vi.fn(), - fatal: vi.fn(), - child: vi.fn(), - } - logger.child.mockReturnValue(logger) - return { logger } -}) - -import { TerminalRegistry } from '../../../server/terminal-registry.js' -import { logger } from '../../../server/logger.js' - -function deferred<T = void>() { - let resolve!: (value: T | PromiseLike<T>) => void - let reject!: (reason?: unknown) => void - const promise = new Promise<T>((res, rej) => { - resolve = res - reject = rej - }) - return { promise, resolve, reject } -} - -function createFakeSidecar(options: { - waitForLoadedThread?: () => Promise<void> - shutdown?: () => Promise<void> -} = {}) { - const lifecycleLossHandlers = new Set<(event: unknown) => void>() - return { - adopt: vi.fn(async () => undefined), - listLoadedThreads: vi.fn(async () => ['thread-1']), - waitForLoadedThread: vi.fn(options.waitForLoadedThread ?? (async () => undefined)), - shutdown: vi.fn(options.shutdown ?? (async () => undefined)), - onLifecycleLoss: vi.fn((handler: (event: unknown) => void) => { - lifecycleLossHandlers.add(handler) - return () => lifecycleLossHandlers.delete(handler) - }), - emitLifecycleLoss(event: unknown) { - for (const handler of lifecycleLossHandlers) { - handler(event) - } - }, - } -} - -describe('TerminalRegistry Codex sidecar ownership', () => { - beforeEach(() => { - mockPtyProcess.instances = [] - vi.clearAllMocks() - }) - - it('awaits Codex sidecar teardown when killing a terminal', async () => { - const registry = new TerminalRegistry() - const shutdown = vi.fn(async () => undefined) - const term = registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { shutdown }, - }, - }, - }) - - await expect(registry.killAndWait(term.terminalId)).resolves.toBe(true) - - expect(shutdown).toHaveBeenCalledTimes(1) - }) - - it('joins current sidecar shutdown before reporting a recovery-attempt failure on final close', async () => { - const registry = new TerminalRegistry() - const recoveryAttempt = deferred() - const currentShutdown = deferred() - const currentSidecar = createFakeSidecar({ - shutdown: () => currentShutdown.promise, - }) - const term = registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - }, - } as any, - }) - term.codexRecoveryAttempt = recoveryAttempt.promise - - const close = registry.killAndWait(term.terminalId) - let closeSettled = false - void close.then( - () => { closeSettled = true }, - () => { closeSettled = true }, - ) - await vi.waitFor(() => expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1)) - - recoveryAttempt.reject(new Error('durable recovery failed during close')) - await new Promise((resolve) => setImmediate(resolve)) - expect(closeSettled).toBe(false) - - currentShutdown.resolve() - await expect(close).rejects.toThrow('durable recovery failed during close') - }) - - it('recovers a durable Codex terminal when its sidecar reports lifecycle loss', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - expect(currentSidecar.onLifecycleLoss).toHaveBeenCalledTimes(1) - currentSidecar.emitLifecycleLoss({ method: 'thread/status/changed', threadId: 'thread-1', status: 'notLoaded' }) - await vi.waitFor(() => expect(replacementSidecar.waitForLoadedThread).toHaveBeenCalledWith('thread-1', expect.any(Object))) - - expect(registry.get(term.terminalId)?.status).toBe('running') - expect(planCreate).toHaveBeenCalledWith(expect.objectContaining({ - terminalId: term.terminalId, - resumeSessionId: 'thread-1', - })) - expect(replacementSidecar.adopt).toHaveBeenCalledWith({ terminalId: term.terminalId, generation: 1 }) - expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1) - expect(mockPtyProcess.instances[0].kill).toHaveBeenCalled() - expect(mockPtyProcess.instances[1].write).toBeDefined() - - expect(registry.input(term.terminalId, 'after recovery')).toBe(true) - expect(mockPtyProcess.instances[0].write).not.toHaveBeenCalled() - expect(mockPtyProcess.instances[1].write).toHaveBeenCalledWith('after recovery') - }) - - it('treats lifecycle loss before initial Codex publication as a create failure instead of recovery', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: createFakeSidecar(), - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - deferLifecycleUntilPublished: true, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await new Promise((resolve) => setTimeout(resolve, 25)) - - expect(planCreate).not.toHaveBeenCalled() - expect(() => registry.publishCodexSidecar(term.terminalId)).toThrow( - 'Codex app-server reported lifecycle loss before terminal create completed.', - ) - await expect(registry.killAndWait(term.terminalId)).resolves.toBe(true) - expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1) - }) - - it('starts durable recovery only after deferred initial Codex publication succeeds', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - deferLifecycleUntilPublished: true, - }, - } as any, - }) - - registry.publishCodexSidecar(term.terminalId) - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.waitForLoadedThread).toHaveBeenCalledWith('thread-1', expect.any(Object))) - - expect(planCreate).toHaveBeenCalledTimes(1) - }) - - it('keeps the old Codex generation current when retiring sidecar teardown fails', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar({ - shutdown: async () => { - throw new Error('retiring sidecar teardown failed') - }, - }) - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - expect(planCreate).toHaveBeenCalledTimes(1) - expect(registry.input(term.terminalId, 'still old generation')).toBe(true) - expect(mockPtyProcess.instances[0].write).toHaveBeenCalledWith('still old generation') - expect(mockPtyProcess.instances[1].write).not.toHaveBeenCalled() - }) - - it('blocks repeated lifecycle-loss recovery after retiring sidecar teardown fails', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar({ - shutdown: async () => { - throw new Error('retiring sidecar teardown failed') - }, - }) - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await new Promise((resolve) => setTimeout(resolve, 25)) - - expect(planCreate).toHaveBeenCalledTimes(1) - expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1) - }) - - it('blocks repeated lifecycle-loss recovery after candidate sidecar teardown fails', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const replacementSidecar = createFakeSidecar({ - waitForLoadedThread: async () => { - throw new Error('candidate never became ready') - }, - shutdown: async () => { - throw new Error('candidate sidecar teardown failed') - }, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await new Promise((resolve) => setTimeout(resolve, 25)) - - expect(planCreate).toHaveBeenCalledTimes(1) - expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1) - }) - - it('blocks durable recovery when candidate planning fails from sidecar teardown', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const teardownError = new Error('planner-owned sidecar teardown failed') as Error & { - codexSidecarTeardownFailed?: boolean - } - teardownError.codexSidecarTeardownFailed = true - const planCreate = vi.fn(async () => { - throw teardownError - }) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - try { - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(planCreate).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect(registry.get(term.terminalId)?.codexRecoveryBlockedError).toBe(teardownError)) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await new Promise((resolve) => setTimeout(resolve, 25)) - - expect(planCreate).toHaveBeenCalledTimes(1) - } finally { - await registry.killAndWait(term.terminalId).catch(() => undefined) - } - }) - - it('keeps unpublished candidate teardown failure retryable for final close', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const candidateShutdown = vi.fn() - .mockRejectedValueOnce(new Error('candidate verified teardown failed')) - .mockResolvedValueOnce(undefined) - const replacementSidecar = createFakeSidecar({ - waitForLoadedThread: async () => { - throw new Error('candidate never became ready') - }, - shutdown: candidateShutdown, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect((registry.get(term.terminalId) as any)?.codexRecoveryAttempt).toBeUndefined()) - - await expect(registry.killAndWait(term.terminalId)).resolves.toBe(true) - await expect(registry.shutdownGracefully(1_000)).resolves.toBeUndefined() - expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(2) - }) - - it('keeps unpublished candidate teardown failure retryable for graceful shutdown', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const candidateShutdown = vi.fn() - .mockRejectedValueOnce(new Error('candidate verified teardown failed')) - .mockResolvedValueOnce(undefined) - const replacementSidecar = createFakeSidecar({ - waitForLoadedThread: async () => { - throw new Error('candidate never became ready') - }, - shutdown: candidateShutdown, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - await expect(registry.shutdownGracefully(1_000)).resolves.toBeUndefined() - expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(2) - }) - - it('does not publish a recovery candidate whose PTY exited before readiness completed', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const readiness = deferred() - const firstCandidate = createFakeSidecar({ - waitForLoadedThread: () => readiness.promise, - }) - const secondCandidate = createFakeSidecar() - const planCreate = vi.fn() - .mockResolvedValueOnce({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: firstCandidate, - }) - .mockResolvedValueOnce({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43125' }, - sidecar: secondCandidate, - }) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(firstCandidate.waitForLoadedThread).toHaveBeenCalledTimes(1)) - mockPtyProcess.instances[1]._emitExit(42) - readiness.resolve() - - await vi.waitFor(() => expect(firstCandidate.shutdown).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect(secondCandidate.adopt).toHaveBeenCalledTimes(1)) - - expect(registry.get(term.terminalId)?.status).toBe('running') - expect(planCreate).toHaveBeenCalledTimes(2) - expect(currentSidecar.shutdown).toHaveBeenCalledTimes(1) - expect(firstCandidate.shutdown).toHaveBeenCalledTimes(1) - expect(registry.input(term.terminalId, 'after retry')).toBe(true) - expect(mockPtyProcess.instances[1].write).not.toHaveBeenCalled() - expect(mockPtyProcess.instances[2].write).toHaveBeenCalledWith('after retry') - }) - - it('publishes a ready recovery candidate even if the old PTY exits during retiring sidecar teardown', async () => { - const registry = new TerminalRegistry() - let oldPtyExitedDuringShutdown = false - const currentSidecar = createFakeSidecar({ - shutdown: async () => { - mockPtyProcess.instances[0]._emitExit(0) - oldPtyExitedDuringShutdown = true - }, - }) - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - - await vi.waitFor(() => expect(replacementSidecar.adopt).toHaveBeenCalledTimes(1)) - expect(oldPtyExitedDuringShutdown).toBe(true) - expect(registry.get(term.terminalId)?.status).toBe('running') - expect(replacementSidecar.shutdown).not.toHaveBeenCalled() - expect(registry.input(term.terminalId, 'after atomic handoff')).toBe(true) - expect(mockPtyProcess.instances[0].write).not.toHaveBeenCalled() - expect(mockPtyProcess.instances[1].write).toHaveBeenCalledWith('after atomic handoff') - }) - - it('waits for a failed recovery candidate to shut down before retrying', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const firstShutdown = deferred() - const firstCandidate = createFakeSidecar({ - waitForLoadedThread: async () => { - throw new Error('candidate not ready') - }, - shutdown: () => firstShutdown.promise, - }) - const secondCandidate = createFakeSidecar() - const planCreate = vi.fn() - .mockResolvedValueOnce({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: firstCandidate, - }) - .mockResolvedValueOnce({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43125' }, - sidecar: secondCandidate, - }) - registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(firstCandidate.shutdown).toHaveBeenCalledTimes(1)) - await Promise.resolve() - - expect(planCreate).toHaveBeenCalledTimes(1) - firstShutdown.resolve() - await vi.waitFor(() => expect(planCreate).toHaveBeenCalledTimes(2)) - await vi.waitFor(() => expect(secondCandidate.adopt).toHaveBeenCalled()) - }) - - it('does not grow active recovery candidates across repeated readiness failures', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - let activeCandidates = 0 - let maxActiveCandidates = 0 - const planCreate = vi.fn(async () => { - const attempt = planCreate.mock.calls.length - activeCandidates += 1 - maxActiveCandidates = Math.max(maxActiveCandidates, activeCandidates) - return { - sessionId: 'thread-1', - remote: { wsUrl: `ws://127.0.0.1:${43124 + attempt}` }, - sidecar: createFakeSidecar({ - waitForLoadedThread: async () => { - if (attempt >= 3) return - throw new Error('candidate not ready') - }, - shutdown: async () => { - activeCandidates -= 1 - }, - }), - } - }) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 1 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(planCreate.mock.calls.length).toBeGreaterThanOrEqual(3)) - - expect(maxActiveCandidates).toBe(1) - await registry.killAndWait(term.terminalId) - }) - - it('final close during a pending recovery launch prevents later recovery', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const launch = deferred<any>() - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(() => launch.promise) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(planCreate).toHaveBeenCalledTimes(1)) - const close = registry.killAndWait(term.terminalId) - launch.resolve({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - }) - await close - - expect(registry.get(term.terminalId)?.status).toBe('exited') - expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1) - expect(replacementSidecar.adopt).not.toHaveBeenCalled() - }) - - it('final close with an unpublished recovery candidate awaits candidate shutdown', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const readiness = deferred() - const shutdown = deferred() - const replacementSidecar = createFakeSidecar({ - waitForLoadedThread: () => readiness.promise, - shutdown: () => shutdown.promise, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.waitForLoadedThread).toHaveBeenCalledTimes(1)) - const close = registry.killAndWait(term.terminalId) - readiness.resolve() - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - let closed = false - void close.then(() => { closed = true }) - await Promise.resolve() - expect(closed).toBe(false) - shutdown.resolve() - await close - expect(closed).toBe(true) - }) - - it('final close with a published recovery candidate awaits replacement shutdown', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const replacementShutdown = deferred() - const replacementSidecar = createFakeSidecar({ - shutdown: () => replacementShutdown.promise, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.adopt).toHaveBeenCalledTimes(1)) - const close = registry.killAndWait(term.terminalId) - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - let closed = false - void close.then(() => { closed = true }) - await Promise.resolve() - expect(closed).toBe(false) - replacementShutdown.resolve() - await close - expect(closed).toBe(true) - }) - - it('awaits Codex sidecar teardown after natural PTY exit during graceful shutdown', async () => { - const registry = new TerminalRegistry() - let releaseShutdown: (() => void) | undefined - const shutdownStarted = vi.fn() - const shutdown = vi.fn(async () => { - shutdownStarted() - await new Promise<void>((resolve) => { - releaseShutdown = resolve - }) - }) - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { shutdown }, - }, - }, - }) - - const graceful = registry.shutdownGracefully(1_000) - mockPtyProcess.instances[0]._emitExit(0) - await vi.waitFor(() => expect(shutdownStarted).toHaveBeenCalledTimes(1)) - - let finished = false - void graceful.then(() => { - finished = true - }) - await Promise.resolve() - expect(finished).toBe(false) - - releaseShutdown?.() - await graceful - expect(finished).toBe(true) - }) - - it('prevents Codex lifecycle-loss recovery from starting during graceful shutdown', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const replacementSidecar = createFakeSidecar() - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const term = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - mockPtyProcess.instances[0].autoExitOnKill = false - - const graceful = registry.shutdownGracefully(1_000) - await vi.waitFor(() => expect(mockPtyProcess.instances[0].kill).toHaveBeenCalledTimes(1)) - currentSidecar.emitLifecycleLoss({ method: 'thread/status/changed', threadId: 'thread-1', status: 'notLoaded' }) - await Promise.resolve() - - expect(planCreate).not.toHaveBeenCalled() - mockPtyProcess.instances[0]._emitExit(0) - await graceful - expect(registry.get(term.terminalId)?.status).toBe('exited') - }) - - it('awaits in-flight Codex sidecar teardown when no terminals are still running', async () => { - const registry = new TerminalRegistry() - let releaseShutdown: (() => void) | undefined - const shutdownStarted = vi.fn() - const shutdown = vi.fn(async () => { - shutdownStarted() - await new Promise<void>((resolve) => { - releaseShutdown = resolve - }) - }) - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { shutdown }, - }, - }, - }) - mockPtyProcess.instances[0]._emitExit(0) - await vi.waitFor(() => expect(shutdownStarted).toHaveBeenCalledTimes(1)) - - const graceful = registry.shutdownGracefully(1_000) - let finished = false - void graceful.then(() => { - finished = true - }) - await Promise.resolve() - - expect(finished).toBe(false) - releaseShutdown?.() - await graceful - expect(finished).toBe(true) - }) - - it('awaits recovery candidate teardown for exited Codex terminals while shutting down other running terminals', async () => { - const registry = new TerminalRegistry() - const currentSidecar = createFakeSidecar() - const readiness = deferred() - const candidateShutdown = deferred() - const replacementSidecar = createFakeSidecar({ - waitForLoadedThread: () => readiness.promise, - shutdown: () => candidateShutdown.promise, - }) - const planCreate = vi.fn(async () => ({ - sessionId: 'thread-1', - remote: { wsUrl: 'ws://127.0.0.1:43124' }, - sidecar: replacementSidecar, - })) - const codexTerm = registry.create({ - mode: 'codex', - resumeSessionId: 'thread-1', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: currentSidecar, - recovery: { planCreate, retryDelayMs: 0 }, - }, - } as any, - }) - - currentSidecar.emitLifecycleLoss({ method: 'thread/closed', threadId: 'thread-1' }) - await vi.waitFor(() => expect(replacementSidecar.waitForLoadedThread).toHaveBeenCalledTimes(1)) - registry.kill(codexTerm.terminalId) - readiness.resolve() - await vi.waitFor(() => expect(replacementSidecar.shutdown).toHaveBeenCalledTimes(1)) - - registry.create({ mode: 'shell' }) - const runningPty = mockPtyProcess.instances[2] - runningPty.autoExitOnKill = false - - const graceful = registry.shutdownGracefully(1_000) - let finished = false - void graceful.then(() => { - finished = true - }) - await vi.waitFor(() => expect(runningPty.kill).toHaveBeenCalledTimes(1)) - runningPty._emitExit(0) - await new Promise((resolve) => setImmediate(resolve)) - - expect(finished).toBe(false) - candidateShutdown.resolve() - await graceful - expect(finished).toBe(true) - }) - - it('observes Codex sidecar shutdown rejection after natural PTY exit and keeps it joinable for shutdown', async () => { - const registry = new TerminalRegistry() - const shutdownError = new Error('verified sidecar teardown failed') - const unhandledRejection = vi.fn() - process.once('unhandledRejection', unhandledRejection) - - const term = registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { - shutdown: vi.fn(async () => { - throw shutdownError - }), - }, - }, - }, - }) - - try { - mockPtyProcess.instances[0]._emitExit(0) - await vi.waitFor(() => expect(logger.error).toHaveBeenCalledWith( - { err: shutdownError, terminalId: term.terminalId }, - 'Codex sidecar shutdown failed', - )) - await new Promise((resolve) => setImmediate(resolve)) - expect(unhandledRejection).not.toHaveBeenCalled() - await expect(registry.shutdownGracefully(1_000)).rejects.toThrow('verified sidecar teardown failed') - } finally { - process.off('unhandledRejection', unhandledRejection) - } - }) - - it('retries a failed current sidecar shutdown on later terminal close joins', async () => { - const registry = new TerminalRegistry() - const shutdown = vi.fn() - .mockRejectedValueOnce(new Error('verified sidecar teardown failed')) - .mockResolvedValueOnce(undefined) - const term = registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { shutdown }, - }, - }, - }) - - await expect(registry.killAndWait(term.terminalId)).rejects.toThrow('verified sidecar teardown failed') - await expect(registry.killAndWait(term.terminalId)).resolves.toBe(true) - - expect(shutdown).toHaveBeenCalledTimes(2) - }) - - it('retries a failed natural-exit sidecar shutdown during graceful shutdown', async () => { - const registry = new TerminalRegistry() - const shutdown = vi.fn() - .mockRejectedValueOnce(new Error('verified sidecar teardown failed')) - .mockResolvedValueOnce(undefined) - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: { shutdown }, - }, - }, - }) - - mockPtyProcess.instances[0]._emitExit(0) - await vi.waitFor(() => expect(shutdown).toHaveBeenCalledTimes(1)) - - await expect(registry.shutdownGracefully(1_000)).resolves.toBeUndefined() - expect(shutdown).toHaveBeenCalledTimes(2) - }) - - it('exposes the inserted terminal id when terminal.created listeners throw', async () => { - const registry = new TerminalRegistry() - const sidecar = createFakeSidecar() - registry.on('terminal.created', () => { - throw new Error('terminal.created listener failed') - }) - - let createdTerminalId: string | undefined - try { - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar, - }, - } as any, - }) - } catch (err) { - createdTerminalId = (err as { terminalId?: string }).terminalId - expect(err).toBeInstanceOf(Error) - expect((err as Error).message).toBe('terminal.created listener failed') - } - - expect(createdTerminalId).toEqual(expect.any(String)) - expect(registry.get(createdTerminalId!)).not.toBeNull() - await expect(registry.killAndWait(createdTerminalId!)).resolves.toBe(true) - expect(sidecar.shutdown).toHaveBeenCalledTimes(1) - }) - - it('waits for every tracked Codex sidecar shutdown before reporting a graceful-shutdown failure', async () => { - const registry = new TerminalRegistry() - const fastFailure = new Error('fast verified sidecar teardown failed') - const slowShutdown = deferred() - const fastSidecar = createFakeSidecar({ - shutdown: async () => { - throw fastFailure - }, - }) - const slowSidecar = createFakeSidecar({ - shutdown: () => slowShutdown.promise, - }) - - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43123', - sidecar: fastSidecar, - }, - } as any, - }) - registry.create({ - mode: 'codex', - providerSettings: { - codexAppServer: { - wsUrl: 'ws://127.0.0.1:43124', - sidecar: slowSidecar, - }, - } as any, - }) - mockPtyProcess.instances[0]._emitExit(0) - mockPtyProcess.instances[1]._emitExit(0) - await vi.waitFor(() => expect(fastSidecar.shutdown).toHaveBeenCalledTimes(1)) - await vi.waitFor(() => expect(slowSidecar.shutdown).toHaveBeenCalledTimes(1)) - - const graceful = registry.shutdownGracefully(1_000) - let settled = false - void graceful.then( - () => { settled = true }, - () => { settled = true }, - ) - await new Promise((resolve) => setImmediate(resolve)) - expect(settled).toBe(false) - - slowShutdown.resolve() - await expect(graceful).rejects.toThrow('fast verified sidecar teardown failed') - }) -}) diff --git a/test/unit/server/terminal-registry.test.ts b/test/unit/server/terminal-registry.test.ts index ba2b915df..e87287060 100644 --- a/test/unit/server/terminal-registry.test.ts +++ b/test/unit/server/terminal-registry.test.ts @@ -54,7 +54,7 @@ vi.mock('../../../server/logger', () => { child: vi.fn(), } logger.child.mockReturnValue(logger) - return { logger, sessionLifecycleLogger: logger } + return { logger } }) // Mock MCP config writer From c2e57b7de7e33b67203e135b980aa500cc636fa6 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:06:04 -0700 Subject: [PATCH 20/25] docs: update dev queue with parity PRs --- docs/development/branch-model.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 36c60f47a..cf89a1631 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -53,7 +53,7 @@ Use an explicit queue. Do not blindly apply every open PR. Example: ```bash -npm run dev:queue -- plan --prs 323,321,309,319,322 +npm run dev:queue -- plan --prs 323,321,309,319,324,326,325,322 ``` The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. @@ -62,7 +62,7 @@ To rebuild local `dev`: ```bash git switch dev -npm run dev:queue -- assemble --prs 323,321,309,319,322 +npm run dev:queue -- assemble --prs 323,321,309,319,324,326,325,322 ``` Use replacement PR numbers instead of external or superseded PRs. If the script stops on a conflict, do not resolve the conflict on `dev`. Abort the merge, fix the PR branch, and rerun the queue. @@ -75,7 +75,10 @@ Initial migration queue: | #321 | `7eae9acf13d2ecf36de6ecade8354cb22b944f7b` | Sidebar reopen corner behavior | | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | -| #322 | `0a334be42553929aed033c3aa3920d0bf58a2a65` | Replacement for externally-owned factory terminal orchestration PR | +| #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | +| #326 | `b34c7ae9ac91c4773927fd768bdaa3de6696622d` | Codex sidecar resilience parity | +| #325 | `e15a86db992ab5d70048dec9c45da8e8ff0b7652` | Intentional removal of broken Codex notification launch args | +| #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | Initial migration exclusions: From 37b6ffa3ec74057b6d06633c27ffa33a258e854c Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:42:33 -0700 Subject: [PATCH 21/25] docs: refresh queued sidecar PR head --- docs/development/branch-model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index cf89a1631..20374781b 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -76,7 +76,7 @@ Initial migration queue: | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | | #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | -| #326 | `b34c7ae9ac91c4773927fd768bdaa3de6696622d` | Codex sidecar resilience parity | +| #326 | `4f37b649d5f1aad162847935cb46ad10774a4722` | Codex sidecar resilience parity | | #325 | `e15a86db992ab5d70048dec9c45da8e8ff0b7652` | Intentional removal of broken Codex notification launch args | | #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | From 2a9f2b8e5c592fd6c2029300d674bd6298584866 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:06:04 -0700 Subject: [PATCH 22/25] docs: update dev queue with parity PRs --- docs/development/branch-model.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 36c60f47a..f9c4f1ae0 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -53,7 +53,7 @@ Use an explicit queue. Do not blindly apply every open PR. Example: ```bash -npm run dev:queue -- plan --prs 323,321,309,319,322 +npm run dev:queue -- plan --prs 323,321,309,319,324,326,325,322 ``` The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. @@ -62,7 +62,7 @@ To rebuild local `dev`: ```bash git switch dev -npm run dev:queue -- assemble --prs 323,321,309,319,322 +npm run dev:queue -- assemble --prs 323,321,309,319,324,326,325,322 ``` Use replacement PR numbers instead of external or superseded PRs. If the script stops on a conflict, do not resolve the conflict on `dev`. Abort the merge, fix the PR branch, and rerun the queue. @@ -75,7 +75,10 @@ Initial migration queue: | #321 | `7eae9acf13d2ecf36de6ecade8354cb22b944f7b` | Sidebar reopen corner behavior | | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | -| #322 | `0a334be42553929aed033c3aa3920d0bf58a2a65` | Replacement for externally-owned factory terminal orchestration PR | +| #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | +| #326 | Current PR head | Codex sidecar resilience parity | +| #325 | `e15a86db992ab5d70048dec9c45da8e8ff0b7652` | Intentional removal of broken Codex notification launch args | +| #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | Initial migration exclusions: From 662c66c4c770fa9ce8dc96609b887ab55bf8cc61 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:43:15 -0700 Subject: [PATCH 23/25] docs: avoid queued PR hash churn --- docs/development/branch-model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 20374781b..f9c4f1ae0 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -76,7 +76,7 @@ Initial migration queue: | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | | #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | -| #326 | `4f37b649d5f1aad162847935cb46ad10774a4722` | Codex sidecar resilience parity | +| #326 | Current PR head | Codex sidecar resilience parity | | #325 | `e15a86db992ab5d70048dec9c45da8e8ff0b7652` | Intentional removal of broken Codex notification launch args | | #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | From 7f6ef8c24a6382e779c7c752b2242e0742f17ee1 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:06:04 -0700 Subject: [PATCH 24/25] docs: update dev queue with parity PRs --- docs/development/branch-model.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index 36c60f47a..b035bcbe5 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -53,7 +53,7 @@ Use an explicit queue. Do not blindly apply every open PR. Example: ```bash -npm run dev:queue -- plan --prs 323,321,309,319,322 +npm run dev:queue -- plan --prs 323,321,309,319,324,326,325,322 ``` The queue script must fail if a PR is draft, closed, not targeting `main`, or cannot be applied cleanly. Fix PR branches before rebuilding `dev`. @@ -62,7 +62,7 @@ To rebuild local `dev`: ```bash git switch dev -npm run dev:queue -- assemble --prs 323,321,309,319,322 +npm run dev:queue -- assemble --prs 323,321,309,319,324,326,325,322 ``` Use replacement PR numbers instead of external or superseded PRs. If the script stops on a conflict, do not resolve the conflict on `dev`. Abort the merge, fix the PR branch, and rerun the queue. @@ -75,7 +75,10 @@ Initial migration queue: | #321 | `7eae9acf13d2ecf36de6ecade8354cb22b944f7b` | Sidebar reopen corner behavior | | #309 | `93c0e15f8b3e04d7e1bbd8ab312619ae28cfefa2` | Codex startup cwd fix | | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | -| #322 | `0a334be42553929aed033c3aa3920d0bf58a2a65` | Replacement for externally-owned factory terminal orchestration PR | +| #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | +| #326 | Current PR head | Codex sidecar resilience parity | +| #325 | Current PR head | Intentional removal of broken Codex notification launch args | +| #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | Initial migration exclusions: From fa6f1914b90563934e934c7e28f1e5e066d33e3a Mon Sep 17 00:00:00 2001 From: Dan Shapiro <dan@example.com> Date: Wed, 6 May 2026 17:46:05 -0700 Subject: [PATCH 25/25] docs: refresh notification PR queue note --- docs/development/branch-model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/branch-model.md b/docs/development/branch-model.md index f9c4f1ae0..b035bcbe5 100644 --- a/docs/development/branch-model.md +++ b/docs/development/branch-model.md @@ -77,7 +77,7 @@ Initial migration queue: | #319 | `48927eef6b46a2232ebe31d1e1dea38d2203eb72` | OpenCode native scroll behavior | | #324 | `fc8a953565c8c4e416fc7bc0e951b0888c8ed421` | Durable session restore identity parity | | #326 | Current PR head | Codex sidecar resilience parity | -| #325 | `e15a86db992ab5d70048dec9c45da8e8ff0b7652` | Intentional removal of broken Codex notification launch args | +| #325 | Current PR head | Intentional removal of broken Codex notification launch args | | #322 | `26601cec20434790936af3a3f9cc823c8c19f984` | Replacement for externally-owned factory terminal orchestration PR | Initial migration exclusions: