From 7276310e1008a99641687123fabc92987b365b63 Mon Sep 17 00:00:00 2001 From: Alex Alecu Date: Fri, 12 Jun 2026 14:33:18 +0300 Subject: [PATCH 1/3] fix(code-review): add model diagnostics --- .../[reviewId]/route.test.ts | 69 ++++++- .../code-review-status/[reviewId]/route.ts | 145 +++++++++++++- pnpm-lock.yaml | 8 + .../cloud-agent-next/src/callbacks/types.ts | 2 + .../session/message-settlement-outbox.test.ts | 46 +++++ .../src/session/message-settlement-outbox.ts | 7 +- .../src/session/session-message-state.ts | 15 ++ .../src/session/wrapper-supervisor.test.ts | 27 +++ .../src/session/wrapper-supervisor.ts | 153 ++++++++++++++- services/cloud-agent-next/src/shared/index.ts | 9 + .../src/shared/runtime-model-diagnostics.ts | 179 ++++++++++++++++++ .../src/websocket/ingest.test.ts | 106 +++++++++++ .../cloud-agent-next/src/websocket/ingest.ts | 39 ++++ .../test/unit/wrapper/kilo-api.test.ts | 44 +++++ .../unit/wrapper/model-diagnostics.test.ts | 52 +++++ .../test/unit/wrapper/reconnection.test.ts | 175 ++++++++++++++++- .../test/unit/wrapper/server.test.ts | 3 + .../test/unit/wrapper/snapshot.test.ts | 1 + .../cloud-agent-next/wrapper/package.json | 3 +- .../wrapper/src/connection.ts | 89 ++++++++- .../cloud-agent-next/wrapper/src/kilo-api.ts | 67 +++++++ .../wrapper/src/model-diagnostics.ts | 92 +++++++++ 22 files changed, 1316 insertions(+), 15 deletions(-) create mode 100644 services/cloud-agent-next/src/shared/runtime-model-diagnostics.ts create mode 100644 services/cloud-agent-next/test/unit/wrapper/model-diagnostics.test.ts create mode 100644 services/cloud-agent-next/wrapper/src/model-diagnostics.ts diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts index 200dbbfac0..14c3066501 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts @@ -876,14 +876,25 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { ); }); - it('reclassifies failed model-not-found callbacks as cancelled while preserving the error message', async () => { + it('reclassifies failed model-not-found callbacks as cancelled while preserving dashboard diagnostics', async () => { mockGetCodeReviewById.mockResolvedValue(makeReview()); mockFindKiloReviewComment.mockResolvedValue(null); + const diagnostics = { + requestedModel: 'kilo/retired-model', + availableModelCount: 3, + availableModels: ['vendor/alpha', 'vendor/beta', 'vendor/gamma'], + suggestedModels: ['vendor/alpha', 'vendor/beta'], + suggestionSource: 'fuzzy', + }; + const detailedErrorMessage = + 'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.'; const response = await POST( makeRequest({ status: 'failed', - errorMessage: 'Model not found: kilo/retired-model', + cloudAgentSessionId: 'agent_runtime_model_diagnostics', + errorMessage: detailedErrorMessage, + modelNotFoundRuntimeDiagnostics: diagnostics, }), makeParams(REVIEW_ID) ); @@ -892,7 +903,7 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith( expect.objectContaining({ status: 'cancelled', - errorMessage: 'Model not found: kilo/retired-model', + errorMessage: detailedErrorMessage, terminalReason: 'model_not_found', }) ); @@ -900,10 +911,38 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { REVIEW_ID, 'cancelled', expect.objectContaining({ - errorMessage: 'Model not found: kilo/retired-model', + errorMessage: detailedErrorMessage, terminalReason: 'model_not_found', }) ); + expect(mockCaptureMessage).toHaveBeenCalledTimes(1); + expect(mockCaptureMessage).toHaveBeenCalledWith( + 'Code review runtime model not found', + expect.objectContaining({ + level: 'warning', + tags: expect.objectContaining({ + source: 'code-review-runtime-model-not-found', + review_id: REVIEW_ID, + cloud_agent_session_id: 'agent_runtime_model_diagnostics', + }), + extra: expect.objectContaining({ + requestedModel: 'kilo/retired-model', + availableModelCount: 3, + availableModels: ['vendor/alpha', 'vendor/beta', 'vendor/gamma'], + suggestedModels: ['vendor/alpha', 'vendor/beta'], + suggestionSource: 'fuzzy', + }), + }) + ); + const publicOutputs = JSON.stringify({ + githubCheck: mockUpdateCheckRun.mock.calls, + githubSummary: mockCreatePRComment.mock.calls, + gitlabStatus: mockSetCommitStatus.mock.calls, + gitlabSummary: mockCreateMRNote.mock.calls, + }); + expect(publicOutputs).not.toContain('vendor/alpha'); + expect(publicOutputs).not.toContain('Available runtime models'); + expect(publicOutputs).not.toContain('retired-model'); expect(mockCreateInfraRetryAttemptIfMissing).not.toHaveBeenCalled(); expect(mockRetryReviewFresh).not.toHaveBeenCalled(); }); @@ -2148,9 +2187,11 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { it('updates GitHub check runs with actionable cancelled copy', async () => { mockGetCodeReviewById.mockResolvedValue(makeReview()); mockFindKiloReviewComment.mockResolvedValue(null); + const detailedErrorMessage = + 'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.'; await POST( - makeRequest({ status: 'failed', errorMessage: 'Model not found: kilo/retired-model' }), + makeRequest({ status: 'failed', errorMessage: detailedErrorMessage }), makeParams(REVIEW_ID) ); @@ -2169,6 +2210,13 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { }), 'standard' ); + const publicOutputs = JSON.stringify({ + githubCheck: mockUpdateCheckRun.mock.calls, + githubSummary: mockCreatePRComment.mock.calls, + }); + expect(publicOutputs).not.toContain('retired-model'); + expect(publicOutputs).not.toContain('vendor/alpha'); + expect(publicOutputs).not.toContain('Available runtime models'); }); it('updates GitLab commit status with actionable cancelled copy', async () => { @@ -2176,9 +2224,11 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { makeReview({ platform: 'gitlab', platform_project_id: 42, check_run_id: null }) ); mockFindKiloReviewNote.mockResolvedValue(null); + const detailedErrorMessage = + 'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.'; await POST( - makeRequest({ status: 'failed', errorMessage: 'Model not found: kilo/retired-model' }), + makeRequest({ status: 'failed', errorMessage: detailedErrorMessage }), makeParams(REVIEW_ID) ); @@ -2192,6 +2242,13 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => { }), 'https://gitlab.com' ); + const publicOutputs = JSON.stringify({ + gitlabStatus: mockSetCommitStatus.mock.calls, + gitlabSummary: mockCreateMRNote.mock.calls, + }); + expect(publicOutputs).not.toContain('retired-model'); + expect(publicOutputs).not.toContain('vendor/alpha'); + expect(publicOutputs).not.toContain('Available runtime models'); }); it('creates the canonical GitHub summary when absent', async () => { diff --git a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts index 2535abcf82..34e116fda5 100644 --- a/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts +++ b/apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts @@ -104,6 +104,7 @@ type CloudAgentNextCallbackPayload = { status: 'completed' | 'failed' | 'interrupted'; errorMessage?: string; terminalReason?: CodeReviewTerminalReason; + modelNotFoundRuntimeDiagnostics?: unknown; lastSeenBranch?: string; gateResult?: 'pass' | 'fail'; }; @@ -115,6 +116,135 @@ type TerminalOwnerResolution = { canDispatch: boolean; }; +type ModelNotFoundRuntimeDiagnostics = { + requestedModel: string; + availableModelCount: number; + availableModels: string[]; + suggestedModels: string[]; + suggestionSource: ModelNotFoundSuggestionSource; +}; + +type ModelNotFoundSuggestionSource = 'fuzzy' | 'first-five' | 'none'; + +const MODEL_DIAGNOSTIC_MAX_MODEL_ID_LENGTH = 512; +const MODEL_DIAGNOSTIC_MAX_SUGGESTIONS = 5; +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isValidDiagnosticModelId(value: unknown): value is string { + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= MODEL_DIAGNOSTIC_MAX_MODEL_ID_LENGTH + ); +} + +function hasUniqueEntries(values: string[]): boolean { + return new Set(values).size === values.length; +} + +function isModelDiagnosticSuggestionSource(value: unknown): value is ModelNotFoundSuggestionSource { + return value === 'fuzzy' || value === 'first-five' || value === 'none'; +} + +function parseModelNotFoundRuntimeDiagnostics( + value: unknown +): ModelNotFoundRuntimeDiagnostics | undefined { + if (!isRecord(value)) return undefined; + const requestedModel = value.requestedModel; + const availableModelCount = value.availableModelCount; + const availableModels = value.availableModels; + const suggestedModels = value.suggestedModels; + const suggestionSource = value.suggestionSource; + + if (!isValidDiagnosticModelId(requestedModel)) return undefined; + if ( + typeof availableModelCount !== 'number' || + !Number.isInteger(availableModelCount) || + availableModelCount < 0 + ) { + return undefined; + } + if (!Array.isArray(availableModels) || !availableModels.every(isValidDiagnosticModelId)) { + return undefined; + } + if (availableModels.length !== availableModelCount || !hasUniqueEntries(availableModels)) { + return undefined; + } + if ( + !Array.isArray(suggestedModels) || + suggestedModels.length > MODEL_DIAGNOSTIC_MAX_SUGGESTIONS || + !suggestedModels.every(isValidDiagnosticModelId) || + !hasUniqueEntries(suggestedModels) + ) { + return undefined; + } + if (!isModelDiagnosticSuggestionSource(suggestionSource)) { + return undefined; + } + if (suggestionSource === 'none' && suggestedModels.length > 0) return undefined; + if (availableModelCount === 0 && (availableModels.length > 0 || suggestedModels.length > 0)) { + return undefined; + } + + return { + requestedModel, + availableModelCount, + availableModels, + suggestedModels, + suggestionSource, + }; +} + +function getModelNotFoundRuntimeDiagnostics( + payload: StatusUpdatePayload, + terminalReason?: CodeReviewTerminalReason +): ModelNotFoundRuntimeDiagnostics | undefined { + if (terminalReason !== 'model_not_found') return undefined; + if (!('modelNotFoundRuntimeDiagnostics' in payload)) return undefined; + return parseModelNotFoundRuntimeDiagnostics(payload.modelNotFoundRuntimeDiagnostics); +} + +function getLoggableStatusErrorMessage( + errorMessage: string | undefined, + terminalReason: CodeReviewTerminalReason | undefined +): string | undefined { + if (!errorMessage) return undefined; + if (terminalReason === 'model_not_found') return 'Model not found'; + return errorMessage; +} + +function captureRuntimeModelNotFoundDiagnostics(params: { + reviewId: string; + sessionId?: string; + diagnostics: ModelNotFoundRuntimeDiagnostics; +}): void { + const { reviewId, sessionId, diagnostics } = params; + const tags = { + source: 'code-review-runtime-model-not-found', + review_id: reviewId, + cloud_agent_session_id: sessionId ?? '', + }; + const extra = { + requestedModel: diagnostics.requestedModel, + availableModelCount: diagnostics.availableModelCount, + availableModels: diagnostics.availableModels, + suggestedModels: diagnostics.suggestedModels, + suggestionSource: diagnostics.suggestionSource, + }; + captureMessage('Code review runtime model not found', { + level: 'warning', + tags, + extra, + }); + logExceptInTest('[code-review-status] Code review runtime model not found', { + reviewId, + sessionId, + ...extra, + }); +} + /** * Normalize a payload from either the orchestrator or cloud-agent-next callback * into the common format expected by the update logic. @@ -846,6 +976,7 @@ export async function POST( }); } + const loggableErrorMessage = getLoggableStatusErrorMessage(errorMessage, terminalReason); logExceptInTest('[code-review-status] Received status update', { reviewId, attemptId, @@ -853,7 +984,7 @@ export async function POST( cliSessionId, status, hasError: !!errorMessage, - ...(errorMessage ? { errorMessage } : {}), + ...(loggableErrorMessage ? { errorMessage: loggableErrorMessage } : {}), }); // Get current review to check if update is needed @@ -943,6 +1074,11 @@ export async function POST( }); } + const modelNotFoundRuntimeDiagnostics = getModelNotFoundRuntimeDiagnostics( + rawPayload, + terminalReason + ); + let terminalOwnerResolution: TerminalOwnerResolution | undefined; const getTerminalOwnerResolution = async () => { terminalOwnerResolution ??= await resolveTerminalOwner(review, reviewId); @@ -1118,6 +1254,13 @@ export async function POST( message: 'Review already in terminal state', }); } + if (modelNotFoundRuntimeDiagnostics) { + captureRuntimeModelNotFoundDiagnostics({ + reviewId, + sessionId, + diagnostics: modelNotFoundRuntimeDiagnostics, + }); + } } else { await updateCodeReviewStatus(reviewId, status, parentStatusUpdates); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e208522daa..217dde2e0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1698,6 +1698,9 @@ importers: '@kilocode/sdk': specifier: 7.3.21 version: 7.3.21 + fuzzysort: + specifier: 3.1.0 + version: 3.1.0 devDependencies: '@types/bun': specifier: 1.3.14 @@ -11625,6 +11628,9 @@ packages: functional-red-black-tree@1.0.1: resolution: {integrity: sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==} + fuzzysort@3.1.0: + resolution: {integrity: sha512-sR9BNCjBg6LNgwvxlBd0sBABvQitkLzoVY9MYYROQVX/FvfJ4Mai9LsGhDgd8qYdds0bY77VzYd5iuB+v5rwQQ==} + gaxios@7.1.4: resolution: {integrity: sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==} engines: {node: '>=18'} @@ -26953,6 +26959,8 @@ snapshots: functional-red-black-tree@1.0.1: {} + fuzzysort@3.1.0: {} + gaxios@7.1.4: dependencies: extend: 3.0.2 diff --git a/services/cloud-agent-next/src/callbacks/types.ts b/services/cloud-agent-next/src/callbacks/types.ts index 24e32d3e50..feb4010f11 100644 --- a/services/cloud-agent-next/src/callbacks/types.ts +++ b/services/cloud-agent-next/src/callbacks/types.ts @@ -1,4 +1,5 @@ import type { SafeFailureProjection } from '../session/safe-failure-projection.js'; +import type { ModelNotFoundRuntimeDiagnostics } from '../shared/runtime-model-diagnostics.js'; export type CallbackTarget = { url: string; @@ -20,6 +21,7 @@ export type ExecutionCallbackPayload = { status: 'completed' | 'failed' | 'interrupted'; errorMessage?: string; failure?: SafeFailureProjection; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; /** Present when errorMessage was shortened to fit the callback queue. */ errorMessageTruncation?: CallbackTextTruncation; lastSeenBranch?: string; diff --git a/services/cloud-agent-next/src/session/message-settlement-outbox.test.ts b/services/cloud-agent-next/src/session/message-settlement-outbox.test.ts index d7bcea7990..ab21e014e3 100644 --- a/services/cloud-agent-next/src/session/message-settlement-outbox.test.ts +++ b/services/cloud-agent-next/src/session/message-settlement-outbox.test.ts @@ -384,6 +384,52 @@ describe('MessageSettlementOutbox', () => { ); }); + it('sends model diagnostics privately in callbacks while keeping failure text generic', async () => { + const harness = createHarness(); + const diagnostics = { + requestedModel: 'kilo/retired-model', + availableModelCount: 3, + availableModels: ['vendor/alpha', 'vendor/beta', 'vendor/gamma'], + suggestedModels: ['vendor/alpha', 'vendor/beta'], + suggestionSource: 'fuzzy' as const, + }; + await putSessionMessageState( + harness.storage, + acceptedMessageState(firstMessageId, { url: 'https://example.com/model-diagnostics' }) + ); + + await harness.outbox.terminalizeSessionMessageOnce(firstMessageId, { + kind: 'failed', + reason: 'assistant_error', + error: 'Model not found: kilo/retired-model', + completionSource: 'wrapper_failure', + failureStage: 'agent_activity', + failureCode: 'assistant_error', + safeFailureMessage: 'Assistant request failed: model not found', + modelNotFoundRuntimeDiagnostics: diagnostics, + }); + + expect(harness.events).toHaveLength(1); + expect(JSON.parse(harness.events[0].payload)).toMatchObject({ + error: 'Assistant request failed: model not found', + failure: { + code: 'assistant_error', + message: 'Assistant request failed: model not found', + }, + }); + expect(harness.events[0].payload).not.toContain('vendor/alpha'); + expect(harness.callbackJobs).toHaveLength(1); + expect(harness.callbackJobs[0].payload).toMatchObject({ + errorMessage: + 'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.', + modelNotFoundRuntimeDiagnostics: diagnostics, + failure: { + code: 'assistant_error', + message: 'Assistant request failed: model not found', + }, + }); + }); + it('preserves allowlisted legacy reasons and replaces arbitrary reasons with status text', () => { const state = { ...acceptedMessageState(firstMessageId), diff --git a/services/cloud-agent-next/src/session/message-settlement-outbox.ts b/services/cloud-agent-next/src/session/message-settlement-outbox.ts index 3dded0c3b8..68b47bf190 100644 --- a/services/cloud-agent-next/src/session/message-settlement-outbox.ts +++ b/services/cloud-agent-next/src/session/message-settlement-outbox.ts @@ -20,6 +20,7 @@ import { } from './session-message-state.js'; import { projectSafeFailure, type SafeFailureProjection } from './safe-failure-projection.js'; import type { AssistantMessagePart, LatestAssistantMessage } from './types.js'; +import { formatModelNotFoundDashboardError } from '../shared/runtime-model-diagnostics.js'; const CURRENT_IDLE_BATCH_CALLBACK_KEY = 'idle_batch_callback_current'; const IDLE_BATCH_CALLBACK_PREFIX = 'idle_batch_callback:'; @@ -441,14 +442,18 @@ export function createMessageSettlementOutbox( ? undefined : (failure?.message ?? (status === 'failed' ? 'The message failed' : 'The message was interrupted')); + const modelNotFoundDashboardError = state.modelNotFoundRuntimeDiagnostics + ? formatModelNotFoundDashboardError(state.modelNotFoundRuntimeDiagnostics) + : undefined; const payload: CallbackJob['payload'] = { sessionId, cloudAgentSessionId: sessionId, executionId: state.messageId, messageId: state.messageId, status, - errorMessage: legacyErrorMessage, + errorMessage: modelNotFoundDashboardError ?? legacyErrorMessage, failure, + modelNotFoundRuntimeDiagnostics: state.modelNotFoundRuntimeDiagnostics, lastSeenBranch: metadata?.repository?.upstreamBranch, kiloSessionId: metadata?.auth.kiloSessionId, gateResult: state.gateResult, diff --git a/services/cloud-agent-next/src/session/session-message-state.ts b/services/cloud-agent-next/src/session/session-message-state.ts index 2f626491f0..d0e7438d4e 100644 --- a/services/cloud-agent-next/src/session/session-message-state.ts +++ b/services/cloud-agent-next/src/session/session-message-state.ts @@ -20,10 +20,19 @@ import { getWrapperRuntimeState, hasCompleteWrapperRunMessageIndex, } from './wrapper-runtime-state.js'; +import { + parseModelNotFoundRuntimeDiagnostics, + type ModelNotFoundRuntimeDiagnostics, +} from '../shared/runtime-model-diagnostics.js'; const SESSION_MESSAGE_STATE_PREFIX = 'session_message:'; const WRAPPER_RUN_MESSAGE_INDEX_PREFIX = 'session_message_wrapper_run:'; +const ModelNotFoundRuntimeDiagnosticsSchema = z.custom( + value => parseModelNotFoundRuntimeDiagnostics(value).success, + 'Invalid model-not-found runtime diagnostics' +); + export type SessionMessageStatus = 'queued' | 'accepted' | 'completed' | 'failed' | 'interrupted'; export const SessionMessageCompletionSourceSchema = z.enum([ @@ -101,6 +110,7 @@ export type SessionMessageState = { failureCode?: SessionMessageFailureCode; failureSubtype?: WorkspaceFailureSubtype; safeFailureMessage?: string; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; error?: string; failureReason?: string; attempts?: number; @@ -206,6 +216,7 @@ export const SessionMessageStateSchema = z failureCode: SessionMessageFailureCodeSchema.optional(), failureSubtype: WorkspaceFailureSubtypeSchema.optional(), safeFailureMessage: z.string().max(WRAPPER_READY_ERROR_DETAIL_MAX_LENGTH).optional(), + modelNotFoundRuntimeDiagnostics: ModelNotFoundRuntimeDiagnosticsSchema.optional(), error: z.string().optional(), failureReason: z.string().optional(), attempts: z.number().int().nonnegative().optional(), @@ -510,6 +521,7 @@ export type MarkMessageFailedParams = { failureCode?: SessionMessageFailureCode; failureSubtype?: WorkspaceFailureSubtype; safeFailureMessage?: string; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; attempts?: number; }; @@ -533,6 +545,7 @@ export async function markMessageFailed( failureCode: params.failureCode, failureSubtype: params.failureSubtype, safeFailureMessage: params.safeFailureMessage, + modelNotFoundRuntimeDiagnostics: params.modelNotFoundRuntimeDiagnostics, attempts: params.attempts, }; await putSessionMessageState(storage, updated); @@ -722,6 +735,7 @@ export type TerminalizeParams = failureCode?: SessionMessageFailureCode; failureSubtype?: WorkspaceFailureSubtype; safeFailureMessage?: string; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; attempts?: number; } | { @@ -784,6 +798,7 @@ export async function terminalizeMessageOnce( failureCode: params.failureCode, failureSubtype: params.failureSubtype, safeFailureMessage: params.safeFailureMessage, + modelNotFoundRuntimeDiagnostics: params.modelNotFoundRuntimeDiagnostics, attempts: params.attempts, terminalEffects, }; diff --git a/services/cloud-agent-next/src/session/wrapper-supervisor.test.ts b/services/cloud-agent-next/src/session/wrapper-supervisor.test.ts index 05530266d6..6d16db599b 100644 --- a/services/cloud-agent-next/src/session/wrapper-supervisor.test.ts +++ b/services/cloud-agent-next/src/session/wrapper-supervisor.test.ts @@ -605,6 +605,33 @@ describe('WrapperSupervisor', () => { }); }); + it('does not persist model diagnostics for non-model-not-found assistant failures', async () => { + const harness = createHarness([liveRuntimeState(), OWNED_WRAPPER_LEASE]); + await putSessionMessageState(harness.storage, acceptedMessage()); + + await harness.supervisor.onTerminalEvent({ + wrapperRunId: WRAPPER_RUN_ID, + status: 'failed', + error: 'Rate limit exceeded for provider request', + errorSource: 'assistant', + modelNotFoundRuntimeDiagnostics: { + requestedModel: 'kilo/retired-model', + availableModelCount: 2, + availableModels: ['vendor/alpha-model', 'vendor/beta-model'], + suggestedModels: ['vendor/alpha-model'], + suggestionSource: 'fuzzy', + }, + }); + + const message = await getSessionMessageState(harness.storage, MESSAGE_ID); + expect(message).toMatchObject({ + status: 'failed', + failureReason: 'assistant_error', + safeFailureMessage: 'Assistant request was rate limited', + }); + expect(message?.modelNotFoundRuntimeDiagnostics).toBeUndefined(); + }); + it.each([ { label: 'before activity', diff --git a/services/cloud-agent-next/src/session/wrapper-supervisor.ts b/services/cloud-agent-next/src/session/wrapper-supervisor.ts index 5869fe919e..c3cdbf62de 100644 --- a/services/cloud-agent-next/src/session/wrapper-supervisor.ts +++ b/services/cloud-agent-next/src/session/wrapper-supervisor.ts @@ -19,6 +19,12 @@ import { type SessionMessageStorage, } from './session-message-state.js'; import type { LatestAssistantMessage } from './types.js'; +import { + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE, + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SERIALIZED_BYTES, + isModelNotFoundRuntimeDiagnosticsWithinQueueBudget, + type ModelNotFoundRuntimeDiagnostics, +} from '../shared/runtime-model-diagnostics.js'; import { clearCurrentWrapperRuntimeFailureState, clearCurrentWrapperRuntimeLivenessState, @@ -47,6 +53,7 @@ const WRAPPER_PING_TIMEOUT_MS = 30_000; const WRAPPER_STOP_ATTEMPT_TIMEOUT_MS = 45_000; const WRAPPER_STOP_RETRY_DELAYS_MS = [5_000, 30_000, 120_000, 300_000]; const DISCONNECT_GRACE_KEY = 'disconnect_grace'; +const MODEL_NOT_FOUND_SAFE_ERROR_MESSAGE = 'Assistant request failed: model not found'; const disconnectGraceStateSchema = z.object({ wrapperRunId: z.string(), @@ -89,6 +96,7 @@ export type WrapperTerminalEvent = { status: 'completed' | 'failed' | 'interrupted'; error?: string; errorSource?: 'assistant'; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; interruptionSource?: 'container_shutdown'; gateResult?: 'pass' | 'fail'; messageIds?: string[]; @@ -216,6 +224,112 @@ function getWrapperInterruptionFailureCode( : 'system_interrupt'; } +function parseCodeReviewCallbackTarget( + metadata: SessionMetadata | null +): { reviewId: string; attemptId?: string } | undefined { + const callbackUrl = metadata?.callback?.target?.url; + if (!callbackUrl) return undefined; + + try { + const url = new URL(callbackUrl); + const segments = url.pathname.split('/').filter(Boolean); + const markerIndex = segments.findIndex( + (segment, index) => + segment === 'code-review-status' && + segments[index - 2] === 'api' && + segments[index - 1] === 'internal' + ); + const reviewId = markerIndex === -1 ? undefined : segments[markerIndex + 1]; + if (!reviewId) return undefined; + const attemptId = url.searchParams.get('attemptId') ?? undefined; + return { reviewId, ...(attemptId ? { attemptId } : {}) }; + } catch { + return undefined; + } +} + +function serializedDiagnosticsByteLength( + diagnostics: ModelNotFoundRuntimeDiagnostics +): number | undefined { + try { + return new TextEncoder().encode(JSON.stringify(diagnostics)).byteLength; + } catch { + return undefined; + } +} + +function logCodeReviewRuntimeModelDiagnostics(params: { + diagnostics: ModelNotFoundRuntimeDiagnostics; + metadata: SessionMetadata; + reviewId?: string; + attemptId?: string; + wrapperRunId: string; + wrapperGeneration: number; + wrapperConnectionId: string; +}): void { + const { + diagnostics, + metadata, + reviewId, + attemptId, + wrapperRunId, + wrapperGeneration, + wrapperConnectionId, + } = params; + const serializedByteLength = serializedDiagnosticsByteLength(diagnostics); + const fitsQueueBudget = isModelNotFoundRuntimeDiagnosticsWithinQueueBudget(diagnostics); + const baseFields = { + logTag: 'code-review-runtime-model-not-found', + reviewId, + attemptId, + sessionId: metadata.identity.sessionId, + wrapperRunId, + wrapperGeneration, + wrapperConnectionId, + requestedModel: diagnostics.requestedModel, + availableModelCount: diagnostics.availableModelCount, + suggestedModels: diagnostics.suggestedModels, + suggestionSource: diagnostics.suggestionSource, + serializedByteLength, + }; + + if (fitsQueueBudget) { + logger + .withFields({ + ...baseFields, + availableModels: diagnostics.availableModels, + }) + .warn('Code review runtime model not found'); + return; + } + + const chunkCount = Math.ceil( + diagnostics.availableModels.length / MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE + ); + logger + .withFields({ + ...baseFields, + maxSerializedByteLength: MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SERIALIZED_BYTES, + chunkCount, + }) + .warn('Code review runtime model diagnostics exceeded callback budget'); + + for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex += 1) { + const start = chunkIndex * MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE; + logger + .withFields({ + ...baseFields, + chunkIndex, + chunkCount, + availableModels: diagnostics.availableModels.slice( + start, + start + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE + ), + }) + .warn('Code review runtime model not found model-list chunk'); + } +} + export function createWrapperSupervisor( dependencies: WrapperSupervisorDependencies ): WrapperSupervisor { @@ -926,8 +1040,16 @@ export function createWrapperSupervisor( } async function onTerminalEvent(params: WrapperTerminalEvent): Promise { - const { wrapperRunId, status, error, errorSource, interruptionSource, gateResult, messageIds } = - params; + const { + wrapperRunId, + status, + error, + errorSource, + modelNotFoundRuntimeDiagnostics, + interruptionSource, + gateResult, + messageIds, + } = params; const sessionId = getSessionIdForLogs(); const state = await getWrapperRuntimeState(storage); if ( @@ -954,6 +1076,30 @@ export function createWrapperSupervisor( }) .info('Wrapper terminal event received by supervisor'); + let persistedModelNotFoundDiagnostics: ModelNotFoundRuntimeDiagnostics | undefined; + const canPersistModelNotFoundDiagnostics = + status === 'failed' && + errorSource === 'assistant' && + classifyAssistantFailureMessage(error) === MODEL_NOT_FOUND_SAFE_ERROR_MESSAGE; + if (modelNotFoundRuntimeDiagnostics && canPersistModelNotFoundDiagnostics) { + const metadata = await getMetadata(); + if (metadata?.identity.createdOnPlatform === 'code-review') { + const reviewTarget = parseCodeReviewCallbackTarget(metadata); + logCodeReviewRuntimeModelDiagnostics({ + diagnostics: modelNotFoundRuntimeDiagnostics, + metadata, + reviewId: reviewTarget?.reviewId, + attemptId: reviewTarget?.attemptId, + wrapperRunId, + wrapperGeneration: state.wrapperGeneration, + wrapperConnectionId: state.wrapperConnectionId, + }); + if (isModelNotFoundRuntimeDiagnosticsWithinQueueBudget(modelNotFoundRuntimeDiagnostics)) { + persistedModelNotFoundDiagnostics = modelNotFoundRuntimeDiagnostics; + } + } + } + if (status === 'failed' || status === 'interrupted') { await requestPhysicalWrapperStop( status === 'failed' ? 'terminal-failed' : 'terminal-interrupted' @@ -973,6 +1119,9 @@ export function createWrapperSupervisor( failureStage: 'agent_activity', failureCode: 'assistant_error', safeFailureMessage: classifyAssistantFailureMessage(error), + ...(persistedModelNotFoundDiagnostics + ? { modelNotFoundRuntimeDiagnostics: persistedModelNotFoundDiagnostics } + : {}), }); continue; } diff --git a/services/cloud-agent-next/src/shared/index.ts b/services/cloud-agent-next/src/shared/index.ts index f1508d5f1d..7ea6aa3c7a 100644 --- a/services/cloud-agent-next/src/shared/index.ts +++ b/services/cloud-agent-next/src/shared/index.ts @@ -6,3 +6,12 @@ export { type KilocodeEventData, SESSION_ID_RE, } from './protocol.js'; +export { + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE, + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SERIALIZED_BYTES, + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS, + type ModelNotFoundRuntimeDiagnostics, + formatModelNotFoundDashboardError, + isModelNotFoundRuntimeDiagnosticsWithinQueueBudget, + parseModelNotFoundRuntimeDiagnostics, +} from './runtime-model-diagnostics.js'; diff --git a/services/cloud-agent-next/src/shared/runtime-model-diagnostics.ts b/services/cloud-agent-next/src/shared/runtime-model-diagnostics.ts new file mode 100644 index 0000000000..f7569377fe --- /dev/null +++ b/services/cloud-agent-next/src/shared/runtime-model-diagnostics.ts @@ -0,0 +1,179 @@ +export const MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_MODEL_ID_LENGTH = 512; +export const MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS = 5; +export const MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SERIALIZED_BYTES = 96_000; +export const MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_LOG_CHUNK_SIZE = 50; + +export type ModelNotFoundSuggestionSource = 'fuzzy' | 'first-five' | 'none'; + +export type ModelNotFoundRuntimeDiagnostics = { + requestedModel: string; + availableModelCount: number; + availableModels: string[]; + suggestedModels: string[]; + suggestionSource: ModelNotFoundSuggestionSource; +}; + +export type ModelNotFoundRuntimeDiagnosticsParseResult = + | { success: true; data: ModelNotFoundRuntimeDiagnostics; serializedByteLength: number } + | { success: false; reason: string; serializedByteLength?: number }; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function serializedByteLength(value: unknown): number | undefined { + try { + return new TextEncoder().encode(JSON.stringify(value)).byteLength; + } catch { + return undefined; + } +} + +function isValidModelId(value: unknown): value is string { + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_MODEL_ID_LENGTH + ); +} + +function hasUniqueEntries(values: string[]): boolean { + return new Set(values).size === values.length; +} + +function isSuggestionSource(value: unknown): value is ModelNotFoundSuggestionSource { + return value === 'fuzzy' || value === 'first-five' || value === 'none'; +} + +export function parseModelNotFoundRuntimeDiagnostics( + value: unknown +): ModelNotFoundRuntimeDiagnosticsParseResult { + const serializedLength = serializedByteLength(value); + if (!isRecord(value)) { + return { + success: false, + reason: 'diagnostic is not an object', + serializedByteLength: serializedLength, + }; + } + + const requestedModel = value.requestedModel; + const availableModelCount = value.availableModelCount; + const availableModels = value.availableModels; + const suggestedModels = value.suggestedModels; + const suggestionSource = value.suggestionSource; + + if (!isValidModelId(requestedModel)) { + return { + success: false, + reason: 'requested model is invalid', + serializedByteLength: serializedLength, + }; + } + if ( + typeof availableModelCount !== 'number' || + !Number.isInteger(availableModelCount) || + availableModelCount < 0 + ) { + return { + success: false, + reason: 'available model count is invalid', + serializedByteLength: serializedLength, + }; + } + if (!Array.isArray(availableModels) || !availableModels.every(isValidModelId)) { + return { + success: false, + reason: 'available model list is invalid', + serializedByteLength: serializedLength, + }; + } + if (availableModels.length !== availableModelCount) { + return { + success: false, + reason: 'available model count does not match list length', + serializedByteLength: serializedLength, + }; + } + if (!hasUniqueEntries(availableModels)) { + return { + success: false, + reason: 'available model list contains duplicates', + serializedByteLength: serializedLength, + }; + } + if ( + !Array.isArray(suggestedModels) || + suggestedModels.length > MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS || + !suggestedModels.every(isValidModelId) + ) { + return { + success: false, + reason: 'suggested model list is invalid', + serializedByteLength: serializedLength, + }; + } + if (!hasUniqueEntries(suggestedModels)) { + return { + success: false, + reason: 'suggested model list contains duplicates', + serializedByteLength: serializedLength, + }; + } + if (!isSuggestionSource(suggestionSource)) { + return { + success: false, + reason: 'suggestion source is invalid', + serializedByteLength: serializedLength, + }; + } + if (suggestionSource === 'none' && suggestedModels.length > 0) { + return { + success: false, + reason: 'none suggestion source cannot include suggestions', + serializedByteLength: serializedLength, + }; + } + if (availableModelCount === 0 && (availableModels.length > 0 || suggestedModels.length > 0)) { + return { + success: false, + reason: 'zero model count cannot include models', + serializedByteLength: serializedLength, + }; + } + + return { + success: true, + data: { + requestedModel, + availableModelCount, + availableModels, + suggestedModels, + suggestionSource, + }, + serializedByteLength: serializedLength ?? 0, + }; +} + +export function isModelNotFoundRuntimeDiagnosticsWithinQueueBudget( + diagnostics: ModelNotFoundRuntimeDiagnostics +): boolean { + const byteLength = serializedByteLength(diagnostics); + return ( + byteLength !== undefined && + byteLength <= MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SERIALIZED_BYTES + ); +} + +export function formatModelNotFoundDashboardError( + diagnostics: ModelNotFoundRuntimeDiagnostics +): string { + const prefix = `Model not found: ${diagnostics.requestedModel}. Available runtime models: ${diagnostics.availableModelCount}.`; + if (diagnostics.availableModelCount === 0 || diagnostics.suggestedModels.length === 0) { + return prefix; + } + + const label = + diagnostics.suggestionSource === 'fuzzy' ? 'Closest matches' : 'Available models include'; + return `${prefix} ${label}: ${diagnostics.suggestedModels.join(', ')}.`; +} diff --git a/services/cloud-agent-next/src/websocket/ingest.test.ts b/services/cloud-agent-next/src/websocket/ingest.test.ts index 22530aac49..719590259e 100644 --- a/services/cloud-agent-next/src/websocket/ingest.test.ts +++ b/services/cloud-agent-next/src/websocket/ingest.test.ts @@ -1330,6 +1330,112 @@ describe('createIngestHandler', () => { }); }); + it('keeps model diagnostics private while forwarding them to the supervisor', async () => { + const doContext = createNewPathDOContext(); + const eventQueries = createFakeEventQueries(); + const broadcast = vi.fn(); + const handler = createIngestHandler( + createFakeState(), + eventQueries, + SESSION_ID, + broadcast, + doContext + ); + const ws = createFakeWebSocket(makeNewPathAttachment()); + const diagnostics = { + requestedModel: 'kilo/retired-model', + availableModelCount: 2, + availableModels: ['vendor/alpha-model', 'vendor/beta-model'], + suggestedModels: ['vendor/alpha-model'], + suggestionSource: 'fuzzy' as const, + }; + + await handler.handleIngestMessage( + ws, + makeStreamMessage('error', { + fatal: true, + error: 'Model not found: kilo/retired-model', + errorSource: 'assistant', + modelNotFoundRuntimeDiagnostics: diagnostics, + }) + ); + + const safeMessage = 'Assistant request failed: model not found'; + const safePayload = JSON.stringify({ + fatal: true, + errorSource: 'assistant', + error: safeMessage, + message: safeMessage, + }); + expect(eventQueries.insert).toHaveBeenCalledWith( + expect.objectContaining({ payload: safePayload }) + ); + expect(broadcast).toHaveBeenCalledWith(expect.objectContaining({ payload: safePayload })); + expect(broadcast).toHaveBeenCalledWith( + expect.objectContaining({ + stream_event_type: 'cloud.status', + payload: JSON.stringify({ cloudStatus: { type: 'error', message: safeMessage } }), + }) + ); + const publicCalls = JSON.stringify({ + persisted: vi.mocked(eventQueries.insert).mock.calls, + broadcast: vi.mocked(broadcast).mock.calls, + }); + expect(publicCalls).not.toContain('retired-model'); + expect(publicCalls).not.toContain('vendor/alpha-model'); + expect(doContext.wrapperSupervisor.onTerminalEvent).toHaveBeenCalledWith({ + wrapperRunId: WRAPPER_RUN_ID, + status: 'failed', + error: 'Model not found: kilo/retired-model', + errorSource: 'assistant', + modelNotFoundRuntimeDiagnostics: diagnostics, + }); + }); + + it('drops model diagnostics attached to non-model-not-found fatal events', async () => { + const doContext = createNewPathDOContext(); + const eventQueries = createFakeEventQueries(); + const broadcast = vi.fn(); + const handler = createIngestHandler( + createFakeState(), + eventQueries, + SESSION_ID, + broadcast, + doContext + ); + const ws = createFakeWebSocket(makeNewPathAttachment()); + const diagnostics = { + requestedModel: 'kilo/retired-model', + availableModelCount: 2, + availableModels: ['vendor/alpha-model', 'vendor/beta-model'], + suggestedModels: ['vendor/alpha-model'], + suggestionSource: 'fuzzy' as const, + }; + + await handler.handleIngestMessage( + ws, + makeStreamMessage('error', { + fatal: true, + error: 'Rate limit exceeded for provider request', + errorSource: 'assistant', + modelNotFoundRuntimeDiagnostics: diagnostics, + }) + ); + + expect(doContext.wrapperSupervisor.onTerminalEvent).toHaveBeenCalledWith({ + wrapperRunId: WRAPPER_RUN_ID, + status: 'failed', + error: 'Rate limit exceeded for provider request', + errorSource: 'assistant', + }); + const publicCalls = JSON.stringify({ + persisted: vi.mocked(eventQueries.insert).mock.calls, + broadcast: vi.mocked(broadcast).mock.calls, + }); + expect(publicCalls).not.toContain('retired-model'); + expect(publicCalls).not.toContain('vendor/alpha-model'); + }); + it('keeps unclassified fatal events as wrapper failures', async () => { const doContext = createNewPathDOContext(); const handler = createIngestHandler( diff --git a/services/cloud-agent-next/src/websocket/ingest.ts b/services/cloud-agent-next/src/websocket/ingest.ts index 0e656ccb60..78b1e3e903 100644 --- a/services/cloud-agent-next/src/websocket/ingest.ts +++ b/services/cloud-agent-next/src/websocket/ingest.ts @@ -30,6 +30,7 @@ import { logger } from '../logger.js'; import type { WrapperSupervisor, WrapperTerminalEvent } from '../session/wrapper-supervisor.js'; import type { TerminalizeParams } from '../session/session-message-state.js'; import { classifyAssistantFailureMessage } from '../session/safe-failure-projection.js'; +import { parseModelNotFoundRuntimeDiagnostics } from '../shared/runtime-model-diagnostics.js'; // --------------------------------------------------------------------------- // Ingest Attachment @@ -63,8 +64,11 @@ const errorEventSchema = z.object({ error: z.string().optional(), message: z.string().optional(), errorSource: z.literal('assistant').optional(), + modelNotFoundRuntimeDiagnostics: z.unknown().optional(), }); +const MODEL_NOT_FOUND_SAFE_ERROR_MESSAGE = 'Assistant request failed: model not found'; + const cloudMessageCompletedEventSchema = z.object({ messageId: z.string(), assistantMessageId: z.string().optional(), @@ -824,6 +828,38 @@ export function createIngestHandler( errorData.errorSource === 'assistant' ? classifyAssistantFailureMessage(fatalMessage) : 'Agent wrapper failed'; + const shouldForwardModelNotFoundDiagnostics = + errorData.errorSource === 'assistant' && + safeFatalMessage === MODEL_NOT_FOUND_SAFE_ERROR_MESSAGE; + const parsedDiagnostics = + shouldForwardModelNotFoundDiagnostics && + errorData.modelNotFoundRuntimeDiagnostics !== undefined + ? parseModelNotFoundRuntimeDiagnostics(errorData.modelNotFoundRuntimeDiagnostics) + : undefined; + if ( + !shouldForwardModelNotFoundDiagnostics && + errorData.modelNotFoundRuntimeDiagnostics !== undefined + ) { + logger + .withFields({ + sessionId, + wrapperRunId, + wrapperGeneration, + wrapperConnectionId, + }) + .warn('Ignoring runtime model diagnostics for non-model-not-found fatal error'); + } else if (parsedDiagnostics && !parsedDiagnostics.success) { + logger + .withFields({ + sessionId, + wrapperRunId, + wrapperGeneration, + wrapperConnectionId, + reason: parsedDiagnostics.reason, + serializedByteLength: parsedDiagnostics.serializedByteLength, + }) + .warn('Ignoring invalid runtime model diagnostics'); + } broadcastFn({ id: 0 as EventId, execution_id: eventSourceId, @@ -839,6 +875,9 @@ export function createIngestHandler( status: 'failed', error: fatalMessage, errorSource: errorData.errorSource, + ...(parsedDiagnostics?.success + ? { modelNotFoundRuntimeDiagnostics: parsedDiagnostics.data } + : {}), }); logger .withFields({ diff --git a/services/cloud-agent-next/test/unit/wrapper/kilo-api.test.ts b/services/cloud-agent-next/test/unit/wrapper/kilo-api.test.ts index a77e05fc31..b20f106b30 100644 --- a/services/cloud-agent-next/test/unit/wrapper/kilo-api.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/kilo-api.test.ts @@ -104,6 +104,50 @@ describe('createWrapperKiloClient prompt handoff', () => { }); }); + it('lists exact deduplicated effective model IDs for the requested provider', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response( + JSON.stringify([ + { + id: 'kilo', + models: { + 'openai/gpt-5.1': {}, + 'anthropic/claude-sonnet-4-20250514': {}, + }, + }, + { + id: 'openai', + models: { + 'gpt-5.1': {}, + }, + }, + { + id: 'kilo', + models: { + 'openai/gpt-5.1': {}, + 'google/gemini-3-pro': {}, + }, + }, + ]), + { status: 200, headers: { 'content-type': 'application/json' } } + ) + ); + vi.stubGlobal('fetch', fetchMock); + const client = createWrapperKiloClient(createSdkClient(), 'http://127.0.0.1:0', workspacePath); + + await expect(client.listEffectiveModels('kilo')).resolves.toEqual([ + 'anthropic/claude-sonnet-4-20250514', + 'google/gemini-3-pro', + 'openai/gpt-5.1', + ]); + const request = fetchMock.mock.calls[0]?.[0]; + expect(request).toBeInstanceOf(Request); + const url = new URL((request as Request).url); + expect(url.pathname).toBe('/config/providers'); + expect(url.searchParams.get('directory')).toBe(workspacePath); + expect(url.searchParams.get('workspace')).toBe(workspacePath); + }); + it('passes snapshot wait policy through command requests', async () => { const fetchMock = vi.fn().mockResolvedValue( new Response(JSON.stringify({}), { diff --git a/services/cloud-agent-next/test/unit/wrapper/model-diagnostics.test.ts b/services/cloud-agent-next/test/unit/wrapper/model-diagnostics.test.ts new file mode 100644 index 0000000000..c8cdbb5d4e --- /dev/null +++ b/services/cloud-agent-next/test/unit/wrapper/model-diagnostics.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { buildModelNotFoundRuntimeDiagnostics } from '../../../wrapper/src/model-diagnostics.js'; + +describe('buildModelNotFoundRuntimeDiagnostics', () => { + it('ranks broad fuzzy matches across provider prefixes and slash suffixes', () => { + const diagnostics = buildModelNotFoundRuntimeDiagnostics('kilo/claude-sonnet-4', [ + 'openai/gpt-5.1', + 'anthropic/claude-sonnet-4-20250514', + 'google/gemini-3-pro', + ]); + + expect(diagnostics).toMatchObject({ + requestedModel: 'kilo/claude-sonnet-4', + availableModelCount: 3, + suggestionSource: 'fuzzy', + }); + expect(diagnostics.suggestedModels[0]).toBe('anthropic/claude-sonnet-4-20250514'); + }); + + it('falls back to the first five lexicographic model IDs when fuzzy search has no result', () => { + const diagnostics = buildModelNotFoundRuntimeDiagnostics('kilo/zzzzzz', [ + 'vendor/theta', + 'vendor/beta', + 'vendor/epsilon', + 'vendor/delta', + 'vendor/alpha', + 'vendor/gamma', + ]); + + expect(diagnostics.suggestionSource).toBe('first-five'); + expect(diagnostics.suggestedModels).toEqual([ + 'vendor/alpha', + 'vendor/beta', + 'vendor/delta', + 'vendor/epsilon', + 'vendor/gamma', + ]); + }); + + it('reports no suggestions when the runtime exposes no models', () => { + const diagnostics = buildModelNotFoundRuntimeDiagnostics('kilo/missing', []); + + expect(diagnostics).toEqual({ + requestedModel: 'kilo/missing', + availableModelCount: 0, + availableModels: [], + suggestedModels: [], + suggestionSource: 'none', + }); + }); +}); diff --git a/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts b/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts index a21f5ce652..18c0666642 100644 --- a/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts @@ -167,6 +167,7 @@ const createMockKiloClient = (overrides: Partial = {}): Wrapp getPermissions: vi.fn().mockResolvedValue([]), getNetworkWaits: vi.fn().mockResolvedValue([]), resumeNetworkWait: vi.fn().mockResolvedValue(true), + listEffectiveModels: vi.fn().mockResolvedValue([]), // Return a stream that never yields — keeps event subscription alive subscribeEvents: vi.fn().mockResolvedValue({ stream: (async function* () { @@ -1091,8 +1092,26 @@ describe('ingest WS reconnection', () => { expect(callbacks2.onReconnecting).toHaveBeenCalledWith(1); }); - it('surfaces model-not-found session errors as terminal wrapper errors', async () => { + it('surfaces code-review model-not-found session errors with runtime diagnostics', async () => { + state = new WrapperState(); + state.bindSession(createCodeReviewSessionContext()); + state.acceptMessage('msg_0123456789abMODELNOTFOUND', { + autoCommit: false, + condenseOnComplete: false, + model: 'kilo/zzzzzzzz', + }); + const listEffectiveModels = vi + .fn() + .mockResolvedValue([ + 'vendor/theta', + 'vendor/beta', + 'vendor/epsilon', + 'vendor/delta', + 'vendor/alpha', + 'vendor/gamma', + ]); const kiloClient = createMockKiloClient({ + listEffectiveModels, subscribeEvents: vi.fn().mockResolvedValue({ stream: createEventStream([ { @@ -1101,7 +1120,7 @@ describe('ingest WS reconnection', () => { sessionID: 'kilo_sess_456', error: { name: 'UnknownError', - data: { message: 'Model not found: kilo/does-not-exist.' }, + data: { message: 'Model not found: kilo/zzzzzzzz.' }, }, }, }, @@ -1117,10 +1136,160 @@ describe('ingest WS reconnection', () => { event => event.streamEventType === 'kilocode' && event.data.event === 'session.error' ); expect(sessionErrors).toHaveLength(1); + expect(listEffectiveModels).toHaveBeenCalledWith('kilo'); expect(callbacks.onTerminalError).toHaveBeenCalledWith({ - error: 'Model not found: kilo/does-not-exist.', + error: 'Model not found: kilo/zzzzzzzz.', errorSource: 'assistant', + modelNotFoundRuntimeDiagnostics: { + requestedModel: 'kilo/zzzzzzzz', + availableModelCount: 6, + availableModels: [ + 'vendor/alpha', + 'vendor/beta', + 'vendor/delta', + 'vendor/epsilon', + 'vendor/gamma', + 'vendor/theta', + ], + suggestedModels: [ + 'vendor/alpha', + 'vendor/beta', + 'vendor/delta', + 'vendor/epsilon', + 'vendor/gamma', + ], + suggestionSource: 'first-five', + }, + }); + }); + + it('preserves the terminal model-not-found error when runtime diagnostics lookup fails', async () => { + state = new WrapperState(); + state.bindSession(createCodeReviewSessionContext()); + state.acceptMessage('msg_0123456789abLOOKUPFAILS', { + autoCommit: false, + condenseOnComplete: false, + model: 'kilo/retired-model', + }); + const listEffectiveModels = vi.fn().mockRejectedValue(new Error('config unavailable')); + const kiloClient = createMockKiloClient({ + listEffectiveModels, + subscribeEvents: vi.fn().mockResolvedValue({ + stream: createEventStream([ + { + type: 'session.error', + properties: { + sessionID: 'kilo_sess_456', + error: { data: { message: 'Model not found: kilo/retired-model.' } }, + }, + }, + ]), + }), }); + + const manager = createManagerWithClient(kiloClient); + await openConnection(manager); + await vi.advanceTimersByTimeAsync(0); + + expect(listEffectiveModels).toHaveBeenCalledWith('kilo'); + expect(callbacks.onTerminalError).toHaveBeenCalledWith({ + error: 'Model not found: kilo/retired-model.', + errorSource: 'assistant', + }); + }); + + it('preserves the terminal model-not-found error when runtime diagnostics lookup hangs', async () => { + state = new WrapperState(); + state.bindSession(createCodeReviewSessionContext()); + state.acceptMessage('msg_0123456789abLOOKUPHANGS', { + autoCommit: false, + condenseOnComplete: false, + model: 'kilo/retired-model', + }); + const listEffectiveModels = vi.fn().mockReturnValue(new Promise(() => {})); + const kiloClient = createMockKiloClient({ + listEffectiveModels, + subscribeEvents: vi.fn().mockResolvedValue({ + stream: createEventStream([ + { + type: 'session.error', + properties: { + sessionID: 'kilo_sess_456', + error: { data: { message: 'Model not found: kilo/retired-model.' } }, + }, + }, + ]), + }), + }); + + const manager = createManagerWithClient(kiloClient); + await openConnection(manager); + await vi.advanceTimersByTimeAsync(0); + + expect(listEffectiveModels).toHaveBeenCalledWith('kilo'); + expect(callbacks.onTerminalError).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1_000); + + expect(callbacks.onTerminalError).toHaveBeenCalledWith({ + error: 'Model not found: kilo/retired-model.', + errorSource: 'assistant', + }); + }); + + it('does not fetch model diagnostics for non-code-review model-not-found errors', async () => { + const listEffectiveModels = vi.fn().mockResolvedValue(['vendor/model']); + const kiloClient = createMockKiloClient({ + listEffectiveModels, + subscribeEvents: vi.fn().mockResolvedValue({ + stream: createEventStream([ + { + type: 'session.error', + properties: { + sessionID: 'kilo_sess_456', + error: { data: { message: 'Model not found: kilo/retired-model.' } }, + }, + }, + ]), + }), + }); + + const manager = createManagerWithClient(kiloClient); + await openConnection(manager); + await vi.advanceTimersByTimeAsync(0); + + expect(listEffectiveModels).not.toHaveBeenCalled(); + expect(callbacks.onTerminalError).toHaveBeenCalledWith({ + error: 'Model not found: kilo/retired-model.', + errorSource: 'assistant', + }); + }); + + it('does not fetch model diagnostics for child-session model-not-found errors', async () => { + state = new WrapperState(); + state.bindSession(createCodeReviewSessionContext()); + const listEffectiveModels = vi.fn().mockResolvedValue(['vendor/model']); + const kiloClient = createMockKiloClient({ + listEffectiveModels, + subscribeEvents: vi.fn().mockResolvedValue({ + stream: createEventStream([ + { + type: 'session.error', + properties: { + sessionID: 'kilo_child_789', + error: { data: { message: 'Model not found: kilo/retired-model.' } }, + }, + }, + ]), + }), + }); + + const manager = createManagerWithClient(kiloClient); + await openConnection(manager); + await vi.advanceTimersByTimeAsync(0); + + expect(listEffectiveModels).not.toHaveBeenCalled(); + expect(callbacks.onTerminalError).not.toHaveBeenCalled(); }); it.each(['usage_limit_exceeded', 'Too Many Requests'])( diff --git a/services/cloud-agent-next/test/unit/wrapper/server.test.ts b/services/cloud-agent-next/test/unit/wrapper/server.test.ts index a81a1f0991..c6edb59aae 100644 --- a/services/cloud-agent-next/test/unit/wrapper/server.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/server.test.ts @@ -47,6 +47,9 @@ function createMockKiloClient(): WrapperKiloClient { getSessionStatuses: vi.fn().mockResolvedValue({}), getQuestions: vi.fn().mockResolvedValue([]), getPermissions: vi.fn().mockResolvedValue([]), + getNetworkWaits: vi.fn().mockResolvedValue([]), + resumeNetworkWait: vi.fn().mockResolvedValue(true), + listEffectiveModels: vi.fn().mockResolvedValue([]), subscribeEvents: vi.fn().mockResolvedValue({ stream: undefined }), serverUrl: 'http://127.0.0.1:0', }; diff --git a/services/cloud-agent-next/test/unit/wrapper/snapshot.test.ts b/services/cloud-agent-next/test/unit/wrapper/snapshot.test.ts index 0b5edb162f..6114491e9f 100644 --- a/services/cloud-agent-next/test/unit/wrapper/snapshot.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/snapshot.test.ts @@ -140,6 +140,7 @@ function createMockKiloClient(overrides?: Partial): WrapperKi getPermissions: vi.fn().mockResolvedValue([]), getNetworkWaits: vi.fn().mockResolvedValue([]), resumeNetworkWait: vi.fn().mockResolvedValue(true), + listEffectiveModels: vi.fn().mockResolvedValue([]), subscribeEvents: vi.fn().mockResolvedValue({ stream: (async function* () { await new Promise(() => {}); diff --git a/services/cloud-agent-next/wrapper/package.json b/services/cloud-agent-next/wrapper/package.json index c8a1e68276..726b16e1d8 100644 --- a/services/cloud-agent-next/wrapper/package.json +++ b/services/cloud-agent-next/wrapper/package.json @@ -8,7 +8,8 @@ "typecheck": "tsgo --noEmit" }, "dependencies": { - "@kilocode/sdk": "7.3.21" + "@kilocode/sdk": "7.3.21", + "fuzzysort": "3.1.0" }, "devDependencies": { "@types/bun": "1.3.14", diff --git a/services/cloud-agent-next/wrapper/src/connection.ts b/services/cloud-agent-next/wrapper/src/connection.ts index 12f20e444b..ba7c48713c 100644 --- a/services/cloud-agent-next/wrapper/src/connection.ts +++ b/services/cloud-agent-next/wrapper/src/connection.ts @@ -14,6 +14,8 @@ import type { IngestEvent, WrapperCommand } from '../../src/shared/protocol.js'; import { trimPayload } from '../../src/shared/trim-payload.js'; import { logToFile } from './utils.js'; import type { KiloEvent, WrapperKiloClient } from './kilo-api.js'; +import type { ModelNotFoundRuntimeDiagnostics } from '../../src/shared/runtime-model-diagnostics.js'; +import { buildModelNotFoundRuntimeDiagnostics } from './model-diagnostics.js'; function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null; @@ -165,6 +167,7 @@ export type ConnectionConfig = { export type AssistantTerminalError = { error: string; errorSource: 'assistant'; + modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics; }; export type ConnectionCallbacks = { @@ -232,6 +235,7 @@ const RECONNECT_BASE_DELAY_MS = 1_000; * a silently stuck HTTP stream. */ const SUBSCRIBE_HANDSHAKE_TIMEOUT_MS = 5_000; const INGEST_INITIAL_CONNECT_TIMEOUT_MS = 10_000; +const MODEL_NOT_FOUND_DIAGNOSTICS_TIMEOUT_MS = 1_000; function buildIngestWebSocketUrl(session: NonNullable): string { const url = new URL(session.ingestUrl); @@ -803,6 +807,82 @@ export function createConnectionManager( return `Insufficient credits: ${eventType}`; } + function isModelNotFoundMessage(message: string): boolean { + return /\bmodel\s+(?:was\s+)?not\s+found\b/i.test(message); + } + + function shouldFetchModelNotFoundDiagnostics( + eventType: string, + properties: Record, + terminalErrorText: string + ): boolean { + if (eventType !== 'session.error') return false; + if (!isCodeReviewJob(state)) return false; + if (properties.sessionID !== state.currentSession?.kiloSessionId) return false; + return isModelNotFoundMessage(terminalErrorText); + } + + function logModelDiagnosticUnavailable(reason: string, requestedModel?: string): void { + logToFile( + JSON.stringify({ + message: 'model_not_found_runtime_diagnostics_unavailable', + reason, + agentSessionId: state.currentSession?.agentSessionId, + kiloSessionId: state.currentSession?.kiloSessionId, + requestedModel, + }) + ); + } + + async function listEffectiveModelsForDiagnostics( + requestedModel: string + ): Promise { + let timeoutId: ReturnType | undefined; + const timeout = new Promise(resolve => { + timeoutId = setTimeout(() => { + logModelDiagnosticUnavailable( + `timed out after ${MODEL_NOT_FOUND_DIAGNOSTICS_TIMEOUT_MS}ms`, + requestedModel + ); + resolve(undefined); + }, MODEL_NOT_FOUND_DIAGNOSTICS_TIMEOUT_MS); + }); + + try { + return await Promise.race([config.kiloClient.listEffectiveModels('kilo'), timeout]); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } + } + + async function maybeBuildModelNotFoundDiagnostics( + eventType: string, + properties: Record, + terminalErrorText: string + ): Promise { + if (!shouldFetchModelNotFoundDiagnostics(eventType, properties, terminalErrorText)) { + return undefined; + } + + const requestedModel = state.batchFinalizationConfig?.model?.trim(); + if (!requestedModel) { + logModelDiagnosticUnavailable('missing_requested_model'); + return undefined; + } + + try { + const availableModels = await listEffectiveModelsForDiagnostics(requestedModel); + if (!availableModels) return undefined; + return buildModelNotFoundRuntimeDiagnostics(requestedModel, availableModels); + } catch (error) { + logModelDiagnosticUnavailable( + error instanceof Error ? error.message : String(error), + requestedModel + ); + return undefined; + } + } + function maybeResumeNetworkWait(eventType: string, properties: Record): void { if (eventType !== 'session.network.restored') return; @@ -1025,9 +1105,16 @@ export function createConnectionManager( // Terminal error detection if (isTerminalError(eventType, properties)) { + const terminalErrorText = getTerminalErrorText(eventType, properties); + const modelNotFoundRuntimeDiagnostics = await maybeBuildModelNotFoundDiagnostics( + eventType, + properties, + terminalErrorText + ); callbacks.onTerminalError({ - error: getTerminalErrorText(eventType, properties), + error: terminalErrorText, errorSource: 'assistant', + ...(modelNotFoundRuntimeDiagnostics ? { modelNotFoundRuntimeDiagnostics } : {}), }); return; } diff --git a/services/cloud-agent-next/wrapper/src/kilo-api.ts b/services/cloud-agent-next/wrapper/src/kilo-api.ts index 0c002b1ff7..20ce869270 100644 --- a/services/cloud-agent-next/wrapper/src/kilo-api.ts +++ b/services/cloud-agent-next/wrapper/src/kilo-api.ts @@ -20,6 +20,63 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null; } +function providerIdFromRecord(provider: Record): string | undefined { + const id = provider.id ?? provider.providerID ?? provider.providerId; + return typeof id === 'string' ? id : undefined; +} + +function modelIdFromRecord(model: Record): string | undefined { + const id = model.id ?? model.modelID ?? model.modelId; + return typeof id === 'string' ? id : undefined; +} + +function modelKeysFromModels(models: unknown): string[] { + if (isRecord(models)) return Object.keys(models); + if (!Array.isArray(models)) return []; + return models.flatMap(model => { + if (typeof model === 'string') return [model]; + if (isRecord(model)) { + const modelID = modelIdFromRecord(model); + return modelID ? [modelID] : []; + } + return []; + }); +} + +function modelKeysFromProvider(provider: unknown): string[] { + if (!isRecord(provider)) return []; + return modelKeysFromModels(provider.models); +} + +function findProviderEntries(data: unknown, providerID: string): unknown[] { + if (Array.isArray(data)) { + return data.filter( + provider => isRecord(provider) && providerIdFromRecord(provider) === providerID + ); + } + + if (!isRecord(data)) return []; + + const providers = data.providers; + if (Array.isArray(providers)) { + const matchingProviders = providers.filter( + entry => isRecord(entry) && providerIdFromRecord(entry) === providerID + ); + if (matchingProviders.length > 0) return matchingProviders; + } + + const directProvider = data[providerID]; + if (directProvider !== undefined) return [directProvider]; + + return providerIdFromRecord(data) === providerID ? [data] : []; +} + +function exactDedupedModelKeys(data: unknown, providerID: string): string[] { + return [...new Set(findProviderEntries(data, providerID).flatMap(modelKeysFromProvider))].sort( + (left, right) => left.localeCompare(right) + ); +} + function formatSdkError(error: unknown): string { if (error instanceof Error) return error.message; @@ -154,6 +211,7 @@ export type WrapperKiloClient = { >; getNetworkWaits: () => Promise; resumeNetworkWait: (requestID: string) => Promise; + listEffectiveModels: (providerID: string) => Promise; generateCommitMessage: (opts: { path: string }) => Promise<{ message: string }>; createPty: (opts: { cwd: string; @@ -362,6 +420,15 @@ export function createWrapperKiloClient( return requireSdkData(result, `Network reply ${requestID}`); }, + listEffectiveModels: async providerID => { + const result = await v2Client.config.providers({ + directory: workspacePath, + workspace: workspacePath, + }); + const data = requireSdkData(result, 'Config providers'); + return exactDedupedModelKeys(data, providerID); + }, + generateCommitMessage: async opts => { const result = await v2Client.commitMessage.generate({ path: opts.path }); return result.data ?? { message: '' }; diff --git a/services/cloud-agent-next/wrapper/src/model-diagnostics.ts b/services/cloud-agent-next/wrapper/src/model-diagnostics.ts new file mode 100644 index 0000000000..2123f136b3 --- /dev/null +++ b/services/cloud-agent-next/wrapper/src/model-diagnostics.ts @@ -0,0 +1,92 @@ +import fuzzysort from 'fuzzysort'; +import { + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS, + type ModelNotFoundRuntimeDiagnostics, + type ModelNotFoundSuggestionSource, +} from '../../src/shared/runtime-model-diagnostics.js'; + +const FUZZY_THRESHOLD = -10_000; +const KILO_PROVIDER_PREFIX = 'kilo/'; + +type CandidateVariant = { + modelID: string; + key: string; +}; + +function unique(values: string[]): string[] { + return [...new Set(values)]; +} + +function sortModelIds(modelIDs: string[]): string[] { + return [...modelIDs].sort((left, right) => left.localeCompare(right)); +} + +function comparisonKeys(modelID: string): string[] { + const lower = modelID.toLocaleLowerCase(); + const withoutKiloPrefix = lower.startsWith(KILO_PROVIDER_PREFIX) + ? lower.slice(KILO_PROVIDER_PREFIX.length) + : lower; + const suffix = withoutKiloPrefix.split('/').at(-1) ?? withoutKiloPrefix; + return unique([lower, withoutKiloPrefix, suffix]); +} + +function buildCandidateVariants(modelIDs: string[]): CandidateVariant[] { + return sortModelIds(modelIDs).flatMap(modelID => + comparisonKeys(modelID).map(key => ({ modelID, key })) + ); +} + +function rankFuzzySuggestions(requestedModel: string, availableModels: string[]): string[] { + const candidateVariants = buildCandidateVariants(availableModels); + const bestScores = new Map(); + + for (const query of comparisonKeys(requestedModel)) { + const results = fuzzysort.go(query, candidateVariants, { + key: 'key', + threshold: FUZZY_THRESHOLD, + limit: candidateVariants.length, + }); + for (const result of results) { + const previousScore = bestScores.get(result.obj.modelID); + if (previousScore === undefined || result.score > previousScore) { + bestScores.set(result.obj.modelID, result.score); + } + } + } + + return [...bestScores.entries()] + .sort( + ([leftModel, leftScore], [rightModel, rightScore]) => + rightScore - leftScore || leftModel.localeCompare(rightModel) + ) + .slice(0, MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS) + .map(([modelID]) => modelID); +} + +export function buildModelNotFoundRuntimeDiagnostics( + requestedModel: string, + availableModels: string[] +): ModelNotFoundRuntimeDiagnostics { + const sortedAvailableModels = sortModelIds(unique(availableModels)); + let suggestedModels: string[] = []; + let suggestionSource: ModelNotFoundSuggestionSource = 'none'; + + if (sortedAvailableModels.length > 0) { + suggestedModels = rankFuzzySuggestions(requestedModel, sortedAvailableModels); + suggestionSource = suggestedModels.length > 0 ? 'fuzzy' : 'first-five'; + if (suggestionSource === 'first-five') { + suggestedModels = sortedAvailableModels.slice( + 0, + MODEL_NOT_FOUND_RUNTIME_DIAGNOSTIC_MAX_SUGGESTIONS + ); + } + } + + return { + requestedModel, + availableModelCount: sortedAvailableModels.length, + availableModels: sortedAvailableModels, + suggestedModels, + suggestionSource, + }; +} From 34f5d91033e1e269844ac1b494732b913a9ad055 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:04:53 +0000 Subject: [PATCH 2/3] fix(code-review): expand isModelNotFoundMessage to match unknown/invalid model patterns --- services/cloud-agent-next/wrapper/src/connection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cloud-agent-next/wrapper/src/connection.ts b/services/cloud-agent-next/wrapper/src/connection.ts index ba7c48713c..bad9511b00 100644 --- a/services/cloud-agent-next/wrapper/src/connection.ts +++ b/services/cloud-agent-next/wrapper/src/connection.ts @@ -808,7 +808,7 @@ export function createConnectionManager( } function isModelNotFoundMessage(message: string): boolean { - return /\bmodel\s+(?:was\s+)?not\s+found\b/i.test(message); + return /\b(model\s+(?:was\s+)?not\s+found|unknown\s+model|invalid\s+model)\b/i.test(message); } function shouldFetchModelNotFoundDiagnostics( From 957f429b2d45397d6bc70def145c2038620f9752 Mon Sep 17 00:00:00 2001 From: Alex Alecu Date: Fri, 12 Jun 2026 19:35:02 +0300 Subject: [PATCH 3/3] fix(cloud-agent): inspect full billing errors --- .../test/unit/wrapper/reconnection.test.ts | 29 +++++++++++++++++++ .../wrapper/src/connection.ts | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts b/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts index 8e593a46bf..237bdaad6c 100644 --- a/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts +++ b/services/cloud-agent-next/test/unit/wrapper/reconnection.test.ts @@ -1296,6 +1296,35 @@ describe('ingest WS reconnection', () => { expect(callbacks.onTerminalError).not.toHaveBeenCalled(); }); + it('detects billing markers in full session error objects', async () => { + const kiloClient = createMockKiloClient({ + subscribeEvents: vi.fn().mockResolvedValue({ + stream: createEventStream([ + { + type: 'session.error', + properties: { + sessionID: 'kilo_sess_456', + error: { + name: 'PaymentRequiredError', + data: { message: 'Request failed' }, + }, + }, + }, + ]), + }), + }); + + const manager = createManagerWithClient(kiloClient); + await openConnection(manager); + await vi.advanceTimersByTimeAsync(0); + + expect(callbacks.onTerminalError).toHaveBeenCalledWith({ + code: 'payment_required', + message: 'Request failed', + errorSource: 'assistant', + }); + }); + it.each(['usage_limit_exceeded', 'Too Many Requests'])( 'surfaces explicit assistant request failures as terminal errors: %s', async errorMessage => { diff --git a/services/cloud-agent-next/wrapper/src/connection.ts b/services/cloud-agent-next/wrapper/src/connection.ts index 4884030223..28e2dacb5f 100644 --- a/services/cloud-agent-next/wrapper/src/connection.ts +++ b/services/cloud-agent-next/wrapper/src/connection.ts @@ -773,7 +773,7 @@ export function createConnectionManager( } else if (eventType !== 'usage_limit_exceeded') { const error = properties.error; if (error) { - const normalizedError = terminalErrorText.toLowerCase(); + const normalizedError = JSON.stringify(error).toLowerCase(); if (eventType === 'session.error' && isModelNotFoundMessage(terminalErrorText)) { code = 'model_missing'; } else if (