Merge branch 'main' into derekx/new-checkpointing

Author: xzrderek

eval_protocol/dataset_logger/__init__.py

@@ -3,16 +3,31 @@
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
 from eval_protocol.dataset_logger.sqlite_dataset_logger_adapter import SqliteDatasetLoggerAdapter
 
+
 # Allow disabling sqlite logger to avoid environment-specific constraints in simple CLI runs.
-if os.getenv("DISABLE_EP_SQLITE_LOG", "0").strip() != "1":
-    default_logger = SqliteDatasetLoggerAdapter()
-else:
+def _get_default_logger():
+    if os.getenv("DISABLE_EP_SQLITE_LOG", "0").strip() != "1":
+        return SqliteDatasetLoggerAdapter()
+    else:
+
+        class _NoOpLogger(DatasetLogger):
+            def log(self, row):
+                return None
+
+            def read(self, rollout_id=None):
+                return []
+
+        return _NoOpLogger()
+
+
+# Lazy property that creates the logger only when accessed
+class _LazyLogger(DatasetLogger):
+
+    def log(self, row):
+        return _get_default_logger().log(row)
 
-    class _NoOpLogger(DatasetLogger):
-        def log(self, row):
-            return None
+    def read(self, rollout_id=None):
+        return _get_default_logger().read(rollout_id)
 
-        def read(self, rollout_id=None):
-            return []
 
-    default_logger = _NoOpLogger()
+default_logger: DatasetLogger = _LazyLogger()

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -59,7 +59,7 @@ async def call_agent(self) -> str:
         self.append_message_and_log(message)
         if message.tool_calls:
             # Create tasks for all tool calls to run them in parallel
-            tool_tasks = []
+            tool_tasks: List[asyncio.Task[tuple[str, List[TextContent]]]] = []
             for tool_call in message.tool_calls:
                 tool_call_id = tool_call.id
                 tool_name = tool_call.function.name
@@ -71,7 +71,7 @@ async def call_agent(self) -> str:
                 tool_tasks.append(task)
 
             # Execute all tool calls in parallel
-            tool_results: List[tuple[str, List[TextContent]]] = await asyncio.gather(*tool_tasks)
+            tool_results = await asyncio.gather(*tool_tasks)
 
             # Add all tool results to messages (they will be in the same order as tool_calls)
             for tool_call, (tool_call_id, content) in zip(message.tool_calls, tool_results):

tests/pytest/test_pytest_ensure_logging.py

@@ -1,4 +1,4 @@
-from typing import List
+import os
 from unittest.mock import Mock, patch
 
 import eval_protocol.dataset_logger as dataset_logger
@@ -13,28 +13,21 @@ async def test_ensure_logging(monkeypatch):
     """
     Ensure that default SQLITE logger gets called by mocking the storage and checking that the storage is called.
     """
-    from eval_protocol.pytest.evaluation_test import evaluation_test
-
     # Mock the SqliteEvaluationRowStore to track calls
-    mock_store = Mock(spec=SqliteEvaluationRowStore)
+    mock_store = Mock()
     mock_store.upsert_row = Mock()
     mock_store.read_rows = Mock(return_value=[])
     mock_store.db_path = "/tmp/test.db"
 
-    # Create a custom logger that uses our mocked store
-    class MockSqliteLogger(DatasetLogger):
-        def __init__(self, store: SqliteEvaluationRowStore):
-            self._store = store
-
-        def log(self, row: EvaluationRow) -> None:
-            data = row.model_dump(exclude_none=True, mode="json")
-            self._store.upsert_row(data=data)
-
-        def read(self, rollout_id=None) -> List[EvaluationRow]:
-            results = self._store.read_rows(rollout_id=rollout_id)
-            return [EvaluationRow(**data) for data in results]
-
-    mock_logger = MockSqliteLogger(mock_store)
+    # Mock the SqliteEvaluationRowStore constructor so that when SqliteDatasetLoggerAdapter
+    # creates its store, it gets our mock instead
+    with patch(
+        "eval_protocol.dataset_logger.sqlite_dataset_logger_adapter.SqliteEvaluationRowStore", return_value=mock_store
+    ):
+        from eval_protocol.models import EvaluationRow
+        from eval_protocol.pytest.default_no_op_rollout_process import default_no_op_rollout_processor
+        from eval_protocol.pytest.evaluation_test import evaluation_test
+        from tests.pytest.test_markdown_highlighting import markdown_dataset_to_evaluation_row
 
     @evaluation_test(
         input_dataset=[
@@ -46,7 +39,7 @@ def read(self, rollout_id=None) -> List[EvaluationRow]:
         mode="pointwise",
         combine_datasets=False,
         num_runs=2,
-        logger=mock_logger,  # Use our mocked logger
+        # Don't pass logger parameter - let it use the default_logger (which we've replaced)
     )
     def eval_fn(row: EvaluationRow) -> EvaluationRow:
         return row

vite-app/dist/assets/index-CpScNe1P.css

@@ -5,8 +5,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>EP | Log Viewer</title>
     <link rel="icon" href="/assets/favicon-BkAAWQga.png" />
-    <script type="module" crossorigin src="/assets/index-dHlKwEPE.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-CpScNe1P.css">
+    <script type="module" crossorigin src="/assets/index-D1ErODUS.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-D5KxcfFQ.css">
   </head>
   <body>
     <div id="root"></div>