fix(code-review): add model diagnostics

apps/web/src/app/api/internal/code-review-status/[reviewId]/route.test.ts

@@ -876,14 +876,25 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       );
     });
 
-    it('reclassifies failed model-not-found callbacks as cancelled while preserving the error message', async () => {
+    it('reclassifies failed model-not-found callbacks as cancelled while preserving dashboard diagnostics', async () => {
       mockGetCodeReviewById.mockResolvedValue(makeReview());
       mockFindKiloReviewComment.mockResolvedValue(null);
+      const diagnostics = {
+        requestedModel: 'kilo/retired-model',
+        availableModelCount: 3,
+        availableModels: ['vendor/alpha', 'vendor/beta', 'vendor/gamma'],
+        suggestedModels: ['vendor/alpha', 'vendor/beta'],
+        suggestionSource: 'fuzzy',
+      };
+      const detailedErrorMessage =
+        'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.';
 
       const response = await POST(
         makeRequest({
           status: 'failed',
-          errorMessage: 'Model not found: kilo/retired-model',
+          cloudAgentSessionId: 'agent_runtime_model_diagnostics',
+          errorMessage: detailedErrorMessage,
+          modelNotFoundRuntimeDiagnostics: diagnostics,
         }),
         makeParams(REVIEW_ID)
       );
@@ -892,18 +903,46 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
       expect(mockUpdateCodeReviewAttemptForCallback).toHaveBeenCalledWith(
         expect.objectContaining({
           status: 'cancelled',
-          errorMessage: 'Model not found: kilo/retired-model',
+          errorMessage: detailedErrorMessage,
           terminalReason: 'model_not_found',
         })
       );
       expect(mockUpdateCodeReviewStatusIfNonTerminal).toHaveBeenCalledWith(
         REVIEW_ID,
         'cancelled',
         expect.objectContaining({
-          errorMessage: 'Model not found: kilo/retired-model',
+          errorMessage: detailedErrorMessage,
           terminalReason: 'model_not_found',
         })
       );
+      expect(mockCaptureMessage).toHaveBeenCalledTimes(1);
+      expect(mockCaptureMessage).toHaveBeenCalledWith(
+        'Code review runtime model not found',
+        expect.objectContaining({
+          level: 'warning',
+          tags: expect.objectContaining({
+            source: 'code-review-runtime-model-not-found',
+            review_id: REVIEW_ID,
+            cloud_agent_session_id: 'agent_runtime_model_diagnostics',
+          }),
+          extra: expect.objectContaining({
+            requestedModel: 'kilo/retired-model',
+            availableModelCount: 3,
+            availableModels: ['vendor/alpha', 'vendor/beta', 'vendor/gamma'],
+            suggestedModels: ['vendor/alpha', 'vendor/beta'],
+            suggestionSource: 'fuzzy',
+          }),
+        })
+      );
+      const publicOutputs = JSON.stringify({
+        githubCheck: mockUpdateCheckRun.mock.calls,
+        githubSummary: mockCreatePRComment.mock.calls,
+        gitlabStatus: mockSetCommitStatus.mock.calls,
+        gitlabSummary: mockCreateMRNote.mock.calls,
+      });
+      expect(publicOutputs).not.toContain('vendor/alpha');
+      expect(publicOutputs).not.toContain('Available runtime models');
+      expect(publicOutputs).not.toContain('retired-model');
       expect(mockCreateInfraRetryAttemptIfMissing).not.toHaveBeenCalled();
       expect(mockRetryReviewFresh).not.toHaveBeenCalled();
     });
@@ -2148,9 +2187,11 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
     it('updates GitHub check runs with actionable cancelled copy', async () => {
       mockGetCodeReviewById.mockResolvedValue(makeReview());
       mockFindKiloReviewComment.mockResolvedValue(null);
+      const detailedErrorMessage =
+        'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.';
 
       await POST(
-        makeRequest({ status: 'failed', errorMessage: 'Model not found: kilo/retired-model' }),
+        makeRequest({ status: 'failed', errorMessage: detailedErrorMessage }),
         makeParams(REVIEW_ID)
       );
 
@@ -2169,16 +2210,25 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
         }),
         'standard'
       );
+      const publicOutputs = JSON.stringify({
+        githubCheck: mockUpdateCheckRun.mock.calls,
+        githubSummary: mockCreatePRComment.mock.calls,
+      });
+      expect(publicOutputs).not.toContain('retired-model');
+      expect(publicOutputs).not.toContain('vendor/alpha');
+      expect(publicOutputs).not.toContain('Available runtime models');
     });
 
     it('updates GitLab commit status with actionable cancelled copy', async () => {
       mockGetCodeReviewById.mockResolvedValue(
         makeReview({ platform: 'gitlab', platform_project_id: 42, check_run_id: null })
       );
       mockFindKiloReviewNote.mockResolvedValue(null);
+      const detailedErrorMessage =
+        'Model not found: kilo/retired-model. Available runtime models: 3. Closest matches: vendor/alpha, vendor/beta.';
 
       await POST(
-        makeRequest({ status: 'failed', errorMessage: 'Model not found: kilo/retired-model' }),
+        makeRequest({ status: 'failed', errorMessage: detailedErrorMessage }),
         makeParams(REVIEW_ID)
       );
 
@@ -2192,6 +2242,13 @@ describe('POST /api/internal/code-review-status/[reviewId]', () => {
         }),
         'https://gitlab.com'
       );
+      const publicOutputs = JSON.stringify({
+        gitlabStatus: mockSetCommitStatus.mock.calls,
+        gitlabSummary: mockCreateMRNote.mock.calls,
+      });
+      expect(publicOutputs).not.toContain('retired-model');
+      expect(publicOutputs).not.toContain('vendor/alpha');
+      expect(publicOutputs).not.toContain('Available runtime models');
     });
 
     it('creates the canonical GitHub summary when absent', async () => {

apps/web/src/app/api/internal/code-review-status/[reviewId]/route.ts

@@ -104,6 +104,7 @@ type CloudAgentNextCallbackPayload = {
   status: 'completed' | 'failed' | 'interrupted';
   errorMessage?: string;
   terminalReason?: CodeReviewTerminalReason;
+  modelNotFoundRuntimeDiagnostics?: unknown;
   lastSeenBranch?: string;
   gateResult?: 'pass' | 'fail';
 };
@@ -115,6 +116,135 @@ type TerminalOwnerResolution = {
   canDispatch: boolean;
 };
 
+type ModelNotFoundRuntimeDiagnostics = {
+  requestedModel: string;
+  availableModelCount: number;
+  availableModels: string[];
+  suggestedModels: string[];
+  suggestionSource: ModelNotFoundSuggestionSource;
+};
+
+type ModelNotFoundSuggestionSource = 'fuzzy' | 'first-five' | 'none';
+
+const MODEL_DIAGNOSTIC_MAX_MODEL_ID_LENGTH = 512;
+const MODEL_DIAGNOSTIC_MAX_SUGGESTIONS = 5;
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isValidDiagnosticModelId(value: unknown): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= MODEL_DIAGNOSTIC_MAX_MODEL_ID_LENGTH
+  );
+}
+
+function hasUniqueEntries(values: string[]): boolean {
+  return new Set(values).size === values.length;
+}
+
+function isModelDiagnosticSuggestionSource(value: unknown): value is ModelNotFoundSuggestionSource {
+  return value === 'fuzzy' || value === 'first-five' || value === 'none';
+}
+
+function parseModelNotFoundRuntimeDiagnostics(
+  value: unknown
+): ModelNotFoundRuntimeDiagnostics | undefined {
+  if (!isRecord(value)) return undefined;
+  const requestedModel = value.requestedModel;
+  const availableModelCount = value.availableModelCount;
+  const availableModels = value.availableModels;
+  const suggestedModels = value.suggestedModels;
+  const suggestionSource = value.suggestionSource;
+
+  if (!isValidDiagnosticModelId(requestedModel)) return undefined;
+  if (
+    typeof availableModelCount !== 'number' ||
+    !Number.isInteger(availableModelCount) ||
+    availableModelCount < 0
+  ) {
+    return undefined;
+  }
+  if (!Array.isArray(availableModels) || !availableModels.every(isValidDiagnosticModelId)) {
+    return undefined;
+  }
+  if (availableModels.length !== availableModelCount || !hasUniqueEntries(availableModels)) {
+    return undefined;
+  }
+  if (
+    !Array.isArray(suggestedModels) ||
+    suggestedModels.length > MODEL_DIAGNOSTIC_MAX_SUGGESTIONS ||
+    !suggestedModels.every(isValidDiagnosticModelId) ||
+    !hasUniqueEntries(suggestedModels)
+  ) {
+    return undefined;
+  }
+  if (!isModelDiagnosticSuggestionSource(suggestionSource)) {
+    return undefined;
+  }
+  if (suggestionSource === 'none' && suggestedModels.length > 0) return undefined;
+  if (availableModelCount === 0 && (availableModels.length > 0 || suggestedModels.length > 0)) {
+    return undefined;
+  }
+
+  return {
+    requestedModel,
+    availableModelCount,
+    availableModels,
+    suggestedModels,
+    suggestionSource,
+  };
+}
+
+function getModelNotFoundRuntimeDiagnostics(
+  payload: StatusUpdatePayload,
+  terminalReason?: CodeReviewTerminalReason
+): ModelNotFoundRuntimeDiagnostics | undefined {
+  if (terminalReason !== 'model_not_found') return undefined;
+  if (!('modelNotFoundRuntimeDiagnostics' in payload)) return undefined;
+  return parseModelNotFoundRuntimeDiagnostics(payload.modelNotFoundRuntimeDiagnostics);
+}
+
+function getLoggableStatusErrorMessage(
+  errorMessage: string | undefined,
+  terminalReason: CodeReviewTerminalReason | undefined
+): string | undefined {
+  if (!errorMessage) return undefined;
+  if (terminalReason === 'model_not_found') return 'Model not found';
+  return errorMessage;
+}
+
+function captureRuntimeModelNotFoundDiagnostics(params: {
+  reviewId: string;
+  sessionId?: string;
+  diagnostics: ModelNotFoundRuntimeDiagnostics;
+}): void {
+  const { reviewId, sessionId, diagnostics } = params;
+  const tags = {
+    source: 'code-review-runtime-model-not-found',
+    review_id: reviewId,
+    cloud_agent_session_id: sessionId ?? '',
+  };
+  const extra = {
+    requestedModel: diagnostics.requestedModel,
+    availableModelCount: diagnostics.availableModelCount,
+    availableModels: diagnostics.availableModels,
+    suggestedModels: diagnostics.suggestedModels,
+    suggestionSource: diagnostics.suggestionSource,
+  };
+  captureMessage('Code review runtime model not found', {
+    level: 'warning',
+    tags,
+    extra,
+  });
+  logExceptInTest('[code-review-status] Code review runtime model not found', {
+    reviewId,
+    sessionId,
+    ...extra,
+  });
+}
+
 /**
  * Normalize a payload from either the orchestrator or cloud-agent-next callback
  * into the common format expected by the update logic.
@@ -846,14 +976,15 @@ export async function POST(
       });
     }
 
+    const loggableErrorMessage = getLoggableStatusErrorMessage(errorMessage, terminalReason);
     logExceptInTest('[code-review-status] Received status update', {
       reviewId,
       attemptId,
       sessionId,
       cliSessionId,
       status,
       hasError: !!errorMessage,
-      ...(errorMessage ? { errorMessage } : {}),
+      ...(loggableErrorMessage ? { errorMessage: loggableErrorMessage } : {}),
     });
 
     // Get current review to check if update is needed
@@ -943,6 +1074,11 @@ export async function POST(
       });
     }
 
+    const modelNotFoundRuntimeDiagnostics = getModelNotFoundRuntimeDiagnostics(
+      rawPayload,
+      terminalReason
+    );
+
     let terminalOwnerResolution: TerminalOwnerResolution | undefined;
     const getTerminalOwnerResolution = async () => {
       terminalOwnerResolution ??= await resolveTerminalOwner(review, reviewId);
@@ -1118,6 +1254,13 @@ export async function POST(
           message: 'Review already in terminal state',
         });
       }
+      if (modelNotFoundRuntimeDiagnostics) {
+        captureRuntimeModelNotFoundDiagnostics({
+          reviewId,
+          sessionId,
+          diagnostics: modelNotFoundRuntimeDiagnostics,
+        });
+      }
     } else {
       await updateCodeReviewStatus(reviewId, status, parentStatusUpdates);
     }

services/cloud-agent-next/src/callbacks/types.ts

@@ -1,6 +1,7 @@
 import type { CloudAgentFailureStage } from '@kilocode/worker-utils/cloud-agent-failure';
 import type { ClientError } from '@kilocode/worker-utils/client-error';
 import type { SafeFailureProjection } from '../session/safe-failure-projection.js';
+import type { ModelNotFoundRuntimeDiagnostics } from '../shared/runtime-model-diagnostics.js';
 
 export type CallbackTarget = {
   url: string;
@@ -22,6 +23,7 @@ export type ExecutionCallbackPayload = {
   status: 'completed' | 'failed' | 'interrupted';
   errorMessage?: string;
   failure?: SafeFailureProjection;
+  modelNotFoundRuntimeDiagnostics?: ModelNotFoundRuntimeDiagnostics;
   failureStage?: CloudAgentFailureStage;
   clientError?: ClientError;
   /** Present when errorMessage was shortened to fit the callback queue. */