feat: export eval artifacts as json

.gitignore

@@ -41,6 +41,8 @@ packages/react-console/gen
 /app/.generated/
 /app/assets/generated
 /app/test/browser/.generated/
+/app/test/evals/.generated/
+/app/test/evals/eval-output/*
 /app/test/browser/test-output/*
 
 # Local checkout symlink

app/package.json

@@ -8,6 +8,8 @@
     "sync:codex-wasm": "node tools/sync_codex_wasm_assets.mjs",
     "typecheck": "tsc -p tsconfig.app.json --noEmit",
     "build": "vite build",
+    "evals:build": "tsc --target es2020 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/evals/.generated test/evals/runMinimalEvals.ts",
+    "evals:run": "pnpm run evals:build && node test/evals/.generated/runMinimalEvals.js",
     "cuj:build": "tsc --target es2020 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/browser/.generated test/browser/run-cuj-scenarios.ts",
     "cuj:run": "pnpm run cuj:build && node test/browser/.generated/run-cuj-scenarios.js",
     "cuj:upload": "tsc --target es2020 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/browser/.generated test/browser/upload-cuj-artifacts.ts && node test/browser/.generated/upload-cuj-artifacts.js",

app/src/App.tsx

@@ -48,6 +48,7 @@ import { SidePanelProvider } from "./contexts/SidePanelContext";
 import { appState } from "./lib/runtime/AppState";
 import GlobalToast from "./components/Toast";
 import DriveLinkCoordinatorHost from "./components/DriveLinkCoordinatorHost";
+import { RunmeEvalHost } from "./components/Evals/RunmeEvalHost";
 import { appLogger } from "./lib/logging/runtime";
 import {
   getConfiguredAgentEndpoint,
@@ -167,6 +168,7 @@ function App({ branding }: AppProps) {
                       <OutputProvider>
                         <NotebookProvider>
                           <SidePanelProvider>
+                            <RunmeEvalHost />
                             <GlobalToast />
                             <AppRouter />
                           </SidePanelProvider>

app/src/components/ChatKit/ChatKitPanel.test.tsx

@@ -375,7 +375,6 @@ describe("ChatKitPanel codex harness routing", () => {
       expect(proxyMock.connectWasm).toHaveBeenCalledWith({
         apiKey: "sk-test",
         sessionOptions: expect.objectContaining({
-          cwd: "/workspace",
           instructions: expect.objectContaining({
             developer: expect.stringContaining(
               "Executed JavaScript runs inside the Runme AppKernel runtime.",

app/src/components/ChatKit/ChatKitPanel.tsx

@@ -11,6 +11,7 @@ import {
   parser_pb,
 } from '../../contexts/CellContext'
 import { useNotebookContext } from '../../contexts/NotebookContext'
+import { useOutput } from '../../contexts/OutputContext'
 import { useCurrentDoc } from '../../contexts/CurrentDocContext'
 import {
   useHarness,
@@ -21,8 +22,8 @@ import {
 import {
   buildCodexChatKitFetchOptions,
 } from '../../lib/runtime/codexChatKitAdapter'
-import { createCodeModeExecutor } from '../../lib/runtime/codeModeExecutor'
 import { createChatKitFetchFromAdapter } from '../../lib/runtime/createChatKitFetchFromAdapter'
+import { buildPageCodeModeExecutor } from '../../lib/runtime/pageCodeModeExecutor'
 import { useCodexConversationSnapshot } from '../../lib/runtime/codexConversationController'
 import type {
   HarnessChatKitAdapter,
@@ -204,6 +205,7 @@ function ChatKitPanelInner({ defaultHarness }: ChatKitPanelInnerProps) {
   const harnessRuntimeManager = useMemo(() => getHarnessRuntimeManager(), [])
   const { getNotebookData, useNotebookList } =
     useNotebookContext()
+  const { getAllRenderers } = useOutput()
   const { getCurrentDoc } = useCurrentDoc()
   const responsesDirectConfig = useResponsesDirectConfigSnapshot()
   const codexProjects = useCodexProjects()
@@ -217,75 +219,19 @@ function ChatKitPanelInner({ defaultHarness }: ChatKitPanelInnerProps) {
   openNotebookListRef.current = openNotebookList
   const currentDocUriRef = useRef(currentDocUri)
   currentDocUriRef.current = currentDocUri
-
-  const resolveCodeModeNotebook = useCallback(
-    (target?: unknown) => {
-      const targetUri =
-        typeof target === 'string'
-          ? target
-          : typeof target === 'object' && target && 'uri' in target
-            ? (target as { uri?: string }).uri
-            : typeof target === 'object' &&
-                target &&
-                'handle' in target &&
-                (target as { handle?: { uri?: string } }).handle?.uri
-              ? (target as { handle?: { uri?: string } }).handle?.uri
-              : currentDocUriRef.current
-      if (!targetUri) {
-        return null
-      }
-      const data = getNotebookDataRef.current(targetUri)
-      if (!data) {
-        return null
-      }
-
-      return {
-        getUri: () => data.getUri(),
-        getName: () => data.getName(),
-        getNotebook: () => data.getNotebook(),
-        updateCell: (cell: parser_pb.Cell) => {
-          for (const renderer of getAllRenderersRef.current().values()) {
-            renderer.onCellUpdate(cell)
-          }
-          data.updateCell(cell)
-        },
-        getCell: (refId: string) => data.getCell(refId),
-        appendCodeCell: data.appendCodeCell?.bind(data),
-        addCodeCellAfter: data.addCodeCellAfter?.bind(data),
-        addCodeCellBefore: data.addCodeCellBefore?.bind(data),
-        removeCell: data.removeCell?.bind(data),
-      }
-    },
-    []
-  )
+  const getAllRenderersRef = useRef(getAllRenderers)
+  getAllRenderersRef.current = getAllRenderers
 
   const codeModeExecutor = useMemo(
     () =>
-      createCodeModeExecutor({
-        mode: 'sandbox',
-        resolveNotebook: resolveCodeModeNotebook,
-        listNotebooks: () => {
-          const uris = new Set<string>()
-          for (const notebook of openNotebookListRef.current) {
-            if (typeof notebook?.uri === 'string' && notebook.uri.trim()) {
-              uris.add(notebook.uri)
-            }
-          }
-          if (currentDocUriRef.current) {
-            uris.add(currentDocUriRef.current)
-          }
-          return Array.from(uris)
-            .map((uri) => resolveCodeModeNotebook(uri))
-            .filter(
-              (
-                notebook
-              ): notebook is NonNullable<
-                ReturnType<typeof resolveCodeModeNotebook>
-              > => Boolean(notebook)
-            )
-        },
+      buildPageCodeModeExecutor({
+        getNotebookData: (uri) => getNotebookDataRef.current(uri),
+        getOpenNotebookUris: () =>
+          openNotebookListRef.current.map((notebook) => notebook.uri),
+        getCurrentDocUri: () => currentDocUriRef.current,
+        getRenderers: () => getAllRenderersRef.current().values(),
       }),
-    [resolveCodeModeNotebook]
+    []
   )
 
   const handleCodexBridgeToolCall = useMemo(