From 10c6f39e4fbedb71442cbdba851d24aeca175bbc Mon Sep 17 00:00:00 2001 From: Jeff F Date: Tue, 31 Mar 2026 18:45:46 -0500 Subject: [PATCH 1/4] Release 1.3.2 --- CHANGELOG.md | 24 + config/config.yaml | 146 +- eval/agentic_eval_harness.py | 1468 ++++++++ eval/eval_e2e.py | 146 +- eval/eval_harness.py | 3663 +++++++++----------- eval/export_to_hf.py | 843 ++++- eval/insider_threat/eval_insider_threat.py | 10 - eval/retrieval_extensions.py | 114 +- eval/scorer.py | 220 +- src/confluence_writer.py | 298 +- src/day_planner.py | 9 +- src/external_email_ingest.py | 16 +- src/flow.py | 187 +- src/genesis.py | 158 +- src/memory.py | 87 +- src/normal_day.py | 389 ++- src/org_lifecycle.py | 253 +- src/planner_models.py | 4 + src/ticket_assigner.py | 28 + src/utils/persona_utils.py | 6 +- 20 files changed, 5463 insertions(+), 2606 deletions(-) create mode 100644 eval/agentic_eval_harness.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a34d5..ff18a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,30 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +## [v1.3.2] — 2026-03-31 + +### Added + +- **Eval Dataset Generator v2 (`eval/eval_harness.py`, `eval/eval_e2e.py`)**: Introduced three novel evaluation tracks: **PERSPECTIVE** (actor visibility cones and information asymmetry), **COUNTERFACTUAL** (causal link testing), and **SILENCE** (expected-but-absent artifact cataloging). Added an **InfinityRetriever** integration with local caching for high-throughput dense retrieval. +- **Dynamic Domain Registry (`src/memory.py`, `src/org_lifecycle.py`, `src/genesis.py`)**: Implemented a live MongoDB `domain_registry` tracking system ownership and documentation coverage. Coverage automatically degrades into "orphaned" status during departures (`_orphan_domains_on_departure`) and recovers organically as engineers write Confluence docs or new hires claim domains. +- **Organic Gap Detection (`src/normal_day.py`, `src/confluence_writer.py`)**: Added "SELF-AUDIT" metadata generation to Confluence page creation and peer-review audits to PRs to proactively detect knowledge gaps. Slack Q&A threads are now asynchronously classified by a dedicated LLM (`_assess_async_thread_gap`) to identify unresolved questions. +- **Commercial & Ops Personas (`config/config.yaml`)**: Expanded the simulation with new department personas including Marcus (Head of Sales & Marketing), Jenna (Marketing), Karen (HR & Ops), and Tom (Operations Manager), while adding strict role definitions for all existing engineers. + +### Changed + +- **Strict Temporal Horizons (`src/memory.py`, `src/flow.py`, `src/normal_day.py`)**: Threaded an `as_of_time` parameter through core fetch methods (`get_ticket`, `get_event_log`, `get_reviewable_prs_for`) to strictly enforce visibility cones and prevent LLM agents from "seeing into the future" during state assessments. +- **HuggingFace Corpus Export (`eval/export_to_hf.py`)**: Overhauled the corpus builder to directly sweep native MongoDB collections (Zendesk tickets, Salesforce deals/accounts, PRs, Emails, and Slack messages) instead of relying solely on the primary artifacts collection. Default dense baseline switched to **Qwen/Qwen3-Embedding-4B**. +- **Multi-Category Scoring (`eval/scorer.py`)**: Upgraded the `TemporalScorer` to support dynamic sub-categories (`knowledge_gap`, `point_in_time`, `stress_state`, `propagation`) and introduced a new `MultiHopScorer` to evaluate partial credit across multi-stage causal chains. +- **Incident On-Call Rotation (`src/flow.py`)**: Refactored incident generation to use a deterministic daily on-call rotation (`_get_next_on_call`) paired with dynamic cross-department friction signals, replacing the previous hardcoded gap-incident generation. + +### Fixed + +- **Insider Threat Telemetry (`eval/insider_threat/eval_insider_threat.py`)**: Removed faulty phone call access log scraping that improperly linked innocent victim records to attacker actors. +- **Persona Prompt Bleed (`src/utils/persona_utils.py`)**: Fixed missing expertise context in async and watercooler interactions by unconditionally injecting the "Expertise" list into all persona voice cards. +- **Redundant Email Routing (`src/external_email_ingest.py`)**: Disabled legacy/redundant customer email routing pathways and stripped stale incident context from outbound sales generation. + +--- + ## [v1.3.1] — 2026-03-28 ### Added diff --git a/config/config.yaml b/config/config.yaml index c69318c..ee36a24 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -114,6 +114,7 @@ roles: knowledge_gaps: - name: "Bill" left: "2024-06" + dept: "Engineering_Backend" role: "CTO" knew_about: - "TitanDB" @@ -236,6 +237,7 @@ personas: Jax: style: "The Burnt-out Greybeard" social_role: "The Cynical Gatekeeper" + role: "Senior Backend Engineer" typing_quirks: > Jax types like he's conserving energy. Everything is lowercase, nothing is punctuated fully, and his thoughts trail off with '...' as if finishing the sentence isn't worth the effort. @@ -268,6 +270,7 @@ personas: Sarah: style: "The High-Stakes Driver" social_role: "The Urgent Taskmaster" + role: "Product Manager" typing_quirks: > Sarah writes with the grammar of someone who drafted the message twice before sending. She @mentions people deliberately — not casually — as a way of assigning accountability @@ -293,6 +296,7 @@ personas: Deepa: style: "The Methodical Architect" social_role: "The Technical Anchor" + role: "Senior Backend Engineer" typing_quirks: > When Deepa thinks through a problem, she instinctively breaks it into numbered steps mid-message, even in casual DMs. She doesn't summarize — she structures out loud. @@ -318,6 +322,7 @@ personas: Elena: style: "The Eager New Hire" social_role: "The Enthusiastic Learner" + role: "Product Manager" typing_quirks: > Elena's messages radiate energy she hasn't learned to modulate yet. She over-punctuates excitement (!!!) and leans on emojis (✨, 🚀) to soften questions she's afraid might @@ -344,6 +349,7 @@ personas: Priya: style: "The Creative Verbose" social_role: "The Brand Visionary" + role: "Design Lead" typing_quirks: > Priya writes like she's art-directing the conversation. Her messages are adjective-heavy and rhythmic — she'll use an em-dash where anyone else would use a comma, because the @@ -376,6 +382,7 @@ personas: Chloe: style: "The Fast & Opinionated" social_role: "The Mobile Specialist" + role: "iOS Engineer" typing_quirks: > Chloe writes short and links fast. Half her messages are a sentence and a GitHub issue URL. She treats 'iOS' and 'Swift' less as technologies and more as a worldview — if something @@ -407,6 +414,7 @@ personas: Nadia: style: "The Detail-Obsessed" social_role: "The Quality Guardian" + role: "QA Engineer" typing_quirks: > Nadia's messages read like bug reports written by someone who genuinely cares. She asks 'Wait, did we test X?' not rhetorically but because she's already thinking about what @@ -433,6 +441,7 @@ personas: Morgan: style: "The Systems Thinker" social_role: "The Infrastructure Backbone" + role: "DevOps / Infrastructure Engineer" typing_quirks: > Morgan writes like someone who has been paged at 3am too many times. Messages are precise and tool-forward — less 'I think we should' and more 'kubectl logs shows X, @@ -467,8 +476,143 @@ personas: "ultramarathon running", "sourdough bread baking", ] + Marcus: + role: "Head of Sales & Marketing" + style: "The Relationship Maximizer" + social_role: "The Pipeline Obsessive" + typing_quirks: > + Marcus writes in CRM shorthand even in Slack — 'moving to stage 3', 'need NDA by + EOW', 'who owns the Acme follow-up?' He peppers messages with first names to create + warmth at velocity. Every outbound message sounds like it was written specifically + for the recipient, even when it wasn't. He uses bullet points not for clarity but + because scanning a 3-bullet message feels more actionable than reading a paragraph. + anti_patterns: > + Never write Marcus as strategic or reflective mid-conversation. He doesn't theorize + about the market — he asks about the deal. He doesn't say 'we should think about + positioning' — he says 'who's the decision-maker and when do they go dark?' He reads + silence from a prospect as a signal, not a void. + pet_peeves: "cold outreach without research, demos that don't tie to pain points, marketing that can't be measured" + stress: 60 + expertise: + [ + "B2B sales", + "account management", + "pipeline management", + "sports-tech partnerships", + "revenue forecasting", + ] + tenure: "4yr" + is_lead: true + interests: + - "college basketball (huge March Madness bracket guy)" + - "BBQ competitions (regional, serious)" + - "true crime podcasts" + - "collecting signed sports memorabilia" + + Jenna: + role: "Marketing Manager" + style: "The Data-Backed Storyteller" + social_role: "The Campaign Architect" + typing_quirks: > + Jenna leads with numbers, then narrative. A typical message starts with '38% open rate + on the last send — here's what I think drove it:' before a tight 2-sentence explanation. + She writes in clean, declarative prose — no hedging, no vague superlatives. When she + disagrees with a direction, she frames it as a hypothesis: 'Worth testing, but my bet + is that X will outperform Y and here's why.' + anti_patterns: > + Never write Jenna as vibes-first or gut-driven. She doesn't say 'I feel like this + campaign will land' — she says 'based on last quarter's cohort data, this segment + converts at 2x.' She doesn't spin bad results; she explains them. She's skeptical of + brand work she can't tie to a metric, but she knows that's a tension, not a rule. + pet_peeves: "vanity metrics, campaigns launched without UTM tracking, 'let's just post more'" + stress: 45 + expertise: + [ + "content marketing", + "email campaigns", + "growth analytics", + "SEO", + "lifecycle marketing", + ] + tenure: "2yr" + is_lead: false + interests: + - "amateur stand-up comedy (open mics, not a career move)" + - "running half-marathons" + - "obsessive podcast listener (narrative non-fiction)" + - "urban vegetable gardening" + Karen: + role: "Head of HR & Ops" + style: "The Diplomatic Enforcer" + social_role: "The Culture Keeper" + typing_quirks: > + Karen writes with warmth that has policy baked into it. She'll open with a genuine + check-in, then transition into the actual ask so smoothly you almost miss it. She uses + 'we' constantly — not as a deflection, but because she genuinely believes problems are + collective. When delivering hard news or policy constraints, her sentences get shorter + and more precise, like she's being careful with each word. She never uses exclamation + marks for professional matters. For personal ones, sparingly. + anti_patterns: > + Never write Karen as cold or bureaucratic, but never write her as a pushover either. + She doesn't say 'that's against policy' without context — she explains the why. She + doesn't avoid conflict; she reframes it. She will absolutely tell a senior leader + they're wrong, but she'll do it in a 1:1, not a Slack channel. + pet_peeves: "hiring decisions made without rubrics, performance feedback that only happens at review time, 'we'll figure out the culture stuff later'" + stress: 50 + expertise: + [ + "HR strategy", + "recruiting", + "employee relations", + "comp & benefits", + "office operations", + "compliance", + ] + tenure: "4yr" + is_lead: true + interests: + - "documentary films (social justice focus)" + - "pottery (wheel-throwing, not hand-building)" + - "hosting elaborate dinner parties" + - "long-distance cycling" + Tom: + role: "Operations Manager" + style: "The Quiet Fixer" + social_role: "The Invisible Enabler" + typing_quirks: > + Tom's messages are functional to the point of being utilitarian — he answers the + question asked, maybe one sentence of context, done. He doesn't announce when he's + fixed something; you just notice it works. When things are broken and he needs + information, his messages are precise lists of exactly what he needs, numbered, + with no preamble. He signs off messages with nothing — no name, no emoji, just + the period at the end of the last sentence. + anti_patterns: > + Never write Tom as visibly stressed or reactive. He doesn't complain about the + volume of problems he handles. He doesn't take credit for solutions in public. If + someone sends him a sprawling, chaotic email about a vendor issue, he responds + with three targeted questions and a proposed resolution. He finds drama exhausting + and opts out of it entirely. + pet_peeves: "last-minute vendor changes, unclear ownership of recurring tasks, being looped in after a decision is already made" + stress: 35 + expertise: + [ + "vendor management", + "facilities", + "budget tracking", + "process documentation", + "IT procurement", + "event logistics", + ] + tenure: "3yr" + is_lead: false + interests: + - "woodworking (functional furniture, not decorative)" + - "competitive chess (online, unrated)" + - "obscure board games" + - "home brewing lager" default_persona: + role: "Team Member" style: "The Capable Middle" social_role: "The Reliable Contributor" typing_quirks: > @@ -478,7 +622,7 @@ default_persona: immediately after the task is done, which is a feature, not a bug. pet_peeves: "unclear asks, last-minute scope changes" stress: 45 - expertise: ["general engineering", "cross-functional communication"] + expertise: General competence in their department's core function. Infer from department. tenure: "2yr" is_lead: false usage_note: > diff --git a/eval/agentic_eval_harness.py b/eval/agentic_eval_harness.py new file mode 100644 index 0000000..42e49d0 --- /dev/null +++ b/eval/agentic_eval_harness.py @@ -0,0 +1,1468 @@ +""" +agentic_eval_harness.py +======================= +OrgForge Agentic Evaluation Harness — v2 + +Evaluates AI agents on three novel tracks that require the deterministic +state machine to exist. No retrieval scoring. Each track has its own +trajectory model and scorer because the reasoning structure is fundamentally +different for each. + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 1 — PERSPECTIVE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Temporal gate: as_of_time from question (actor's knowledge horizon) +Actor gate: tool calls filtered to actor_visible_artifacts +Trajectory: did the agent stay within the actor's visibility cone? + did it correctly identify what was and wasn't accessible? +Score penalty: using artifacts outside the actor's cone, even to reach + the correct answer. The point is epistemic discipline. + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 2 — COUNTERFACTUAL +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Temporal gate: as_of_time of the effect event (agent can see everything up to + and including the effect to understand what happened) +No actor gate: agent has read access to all subsystems +Trajectory: did the agent identify the correct causal mechanism? + did it trace cause → effect correctly? +Answer scoring: structured extraction of (outcome_changed, mechanism, actors) + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 3 — SILENCE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Temporal gate: end of simulation (agent can see the full corpus) +No actor gate: agent has read access to all subsystems +Trajectory: CRITICAL — did the agent search expected_search_space before + concluding absence? A correct "no" without checking the right + places is scored as a trajectory failure even if the boolean is right. +Answer scoring: boolean only — did the agent correctly conclude absence? + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Score weights (per track) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Track Answer Trajectory Notes +─────────────── ─────── ────────── ───────────────────────── +PERSPECTIVE 0.40 0.60 Trajectory is primary — epistemic discipline matters +COUNTERFACTUAL 0.50 0.50 Both matter — wrong mechanism, wrong answer +SILENCE 0.30 0.70 Can't score a "no" without proof of search +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from dataclasses import dataclass, field, asdict +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple +import argparse +import yaml + +logger = logging.getLogger("orgforge.agentic_eval") + +with open(Path(__file__).resolve().parent.parent / "config" / "config.yaml") as f: + _CFG = yaml.safe_load(f) + +_SIM_CFG = _CFG.get("simulation", {}) +BASE = Path(_SIM_CFG.get("output_dir", "./export")) +EVAL_DIR = BASE / "eval" +_SIM_START = datetime.strptime(_CFG["simulation"]["start_date"], "%Y-%m-%d") + +# Per-track answer/trajectory weights +_TRACK_WEIGHTS = { + "PERSPECTIVE": {"answer": 0.40, "trajectory": 0.60}, + "COUNTERFACTUAL": {"answer": 0.50, "trajectory": 0.50}, + "SILENCE": {"answer": 0.30, "trajectory": 0.70}, +} + +# Doc type → tool name mapping +_DOCTYPE_TO_TOOL = { + "jira": "get_ticket", + "confluence": "get_confluence_page", + "slack": "get_slack_thread", + "email": "get_email", + "pr": "get_pr", + "zd_ticket": "get_zd_ticket", + "sf_opp": "get_sf_opportunity", + "sf_account": "get_sf_account", + "zoom": "get_zoom_transcript", + "datadog": "get_datadog_alert", + "invoice": "get_invoice", + "nps": "get_nps_response", +} + +# Tool names that imply a subsystem — used to detect actor gate violations +_TOOL_SUBSYSTEM = { + "get_ticket": "jira", + "get_confluence_page": "confluence", + "get_slack_thread": "slack", + "get_email": "email", + "get_pr": "git", + "get_zd_ticket": "zendesk", + "get_sf_opportunity": "salesforce", + "get_sf_account": "salesforce", + "get_zoom_transcript": "zoom", + "get_datadog_alert": "datadog", + "get_invoice": "email", + "get_nps_response": "salesforce", + "get_events_for_day": None, # cross-subsystem — handled separately + "search_artifacts": None, +} + + +# ───────────────────────────────────────────────────────────────────────────── +# DATA CLASSES +# ───────────────────────────────────────────────────────────────────────────── + + +@dataclass +class ToolCall: + tool_name: str + arguments: Dict[str, Any] + result_ids: List[str] + result_types: List[str] + timestamp_requested: Optional[str] + horizon_violation: bool # artifact timestamp > as_of_time + actor_gate_violation: ( + bool # artifact outside actor's visibility cone (PERSPECTIVE only) + ) + subsystem_violation: ( + bool # tool subsystem not in actor's access set (PERSPECTIVE only) + ) + returned_empty: bool + latency_ms: float + + +@dataclass +class AgentTrajectory: + question_id: str + question_type: str + tool_calls: List[ToolCall] = field(default_factory=list) + final_answer: Dict[str, Any] = field(default_factory=dict) + total_latency_ms: float = 0.0 + horizon_violations: int = 0 + actor_gate_violations: int = 0 # PERSPECTIVE track + subsystem_violations: int = 0 # PERSPECTIVE track + search_space_coverage: float = 0.0 # SILENCE track + causal_mechanism_found: bool = False # COUNTERFACTUAL track + dead_ends_hit: int = 0 + dead_ends_recovered: int = 0 + + +@dataclass +class PerspectiveTrajectoryScore: + epistemic_discipline: float # 1.0 - (cone violations / total calls) + subsystem_discipline: float # 1.0 - (subsystem violations / total calls) + horizon_discipline: float # 1.0 - (horizon violations / total calls) + conclusion_grounding: float # did final answer cite in-cone artifacts? + dead_end_recovery: float + composite: float + + +@dataclass +class CounterfactualTrajectoryScore: + cause_identified: float # did agent retrieve the cause event? + effect_identified: float # did agent retrieve the effect event? + mechanism_correct: float # did agent name the correct link_type? + causal_chain_complete: float # did agent traverse cause → effect in order? + horizon_discipline: float + composite: float + + +@dataclass +class SilenceTrajectoryScore: + search_space_coverage: float # fraction of expected_search_space the agent checked + correct_absence_conclusion: float # did agent explicitly conclude "does not exist"? + premature_conclusion: float # did agent conclude before searching? (penalty) + horizon_discipline: float + composite: float + + +@dataclass +class EvalResult: + question_id: str + question_type: str + difficulty: str + answer_score: float + answer_correct: bool + trajectory_score: float + combined_score: float + failure_reason: Optional[str] + tool_call_count: int + meta: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict: + return asdict(self) + + +# ───────────────────────────────────────────────────────────────────────────── +# GATED TOOL LAYER +# ───────────────────────────────────────────────────────────────────────────── + + +class GatedTools: + """ + Wraps the document corpus and enforces gates per question type. + + PERSPECTIVE: temporal gate (as_of_time) + actor gate (visibility cone) + COUNTERFACTUAL: temporal gate only (as_of_time = effect event timestamp) + SILENCE: no gate (agent sees full corpus — the absence must be real) + + Violations are logged but results are still returned (the agent should + observe them and self-correct). Violations penalize trajectory score. + """ + + def __init__( + self, + mem, + question: dict, + as_of_time: str, + actor_visible_artifacts: Optional[Set[str]] = None, + actor_subsystem_access: Optional[Set[str]] = None, + ): + self._mem = mem + self._question = question + self._as_of_time = as_of_time + self._actor_visible = actor_visible_artifacts or set() + self._actor_subsystems = actor_subsystem_access or set() + self._question_type = question.get("question_type", "") + self._call_log: List[ToolCall] = [] + + @property + def call_log(self) -> List[ToolCall]: + return self._call_log + + def _temporal_gate(self, doc: dict) -> bool: + if self._question_type == "SILENCE": + return True # No temporal gate for silence + ts = doc.get("timestamp") or doc.get("created") or doc.get("date") + if not ts: + return True + try: + return datetime.fromisoformat(str(ts)) <= datetime.fromisoformat( + self._as_of_time + ) + except (ValueError, TypeError): + return True + + def _check_actor_gate(self, doc_id: str, doc_type: str) -> Tuple[bool, bool]: + """ + Returns (actor_gate_violation, subsystem_violation). + Only meaningful for PERSPECTIVE questions. + """ + if self._question_type != "PERSPECTIVE": + return False, False + + from eval_harness import _ARTIFACT_SUBSYSTEM + + subsystem = _ARTIFACT_SUBSYSTEM.get(doc_type, "default") + + subsystem_violation = ( + bool(self._actor_subsystems) + and subsystem not in self._actor_subsystems + and subsystem != "default" + ) + + actor_gate_violation = ( + bool(self._actor_visible) and doc_id not in self._actor_visible + ) + + return actor_gate_violation, subsystem_violation + + def _record( + self, + tool_name: str, + arguments: Dict, + results: List[dict], + t0: float, + horizon_violation: bool = False, + ) -> List[dict]: + latency = (time.time() - t0) * 1000 + filtered = [r for r in results if self._temporal_gate(r)] + horizon_violation = horizon_violation or len(filtered) < len(results) + + result_ids = [str(r.get("id", r.get("_id", ""))) for r in filtered] + result_types = [str(r.get("doc_type", r.get("type", ""))) for r in filtered] + + actor_gate_violation = False + subsystem_violation = False + for rid, rtype in zip(result_ids, result_types): + agv, sv = self._check_actor_gate(rid, rtype) + if agv: + actor_gate_violation = True + if sv: + subsystem_violation = True + + # Check subsystem from tool name too + tool_subsystem = _TOOL_SUBSYSTEM.get(tool_name) + if ( + self._question_type == "PERSPECTIVE" + and tool_subsystem + and self._actor_subsystems + and tool_subsystem not in self._actor_subsystems + ): + subsystem_violation = True + + self._call_log.append( + ToolCall( + tool_name=tool_name, + arguments=arguments, + result_ids=result_ids, + result_types=result_types, + timestamp_requested=arguments.get("as_of_time"), + horizon_violation=horizon_violation, + actor_gate_violation=actor_gate_violation, + subsystem_violation=subsystem_violation, + returned_empty=len(filtered) == 0, + latency_ms=latency, + ) + ) + return filtered + + # ── Tool implementations ────────────────────────────────────────────────── + # Each mirrors a real MongoDB query. The agent is given these as tools. + + def get_ticket(self, ticket_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["jira"].find_one({"id": ticket_id}) or {} + return self._record( + "get_ticket", + {"ticket_id": ticket_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_confluence_page( + self, page_id: str, as_of_time: Optional[str] = None + ) -> dict: + t0 = time.time() + doc = self._mem._db["confluence"].find_one({"id": page_id}) or {} + return self._record( + "get_confluence_page", + {"page_id": page_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_slack_thread( + self, thread_id: str, as_of_time: Optional[str] = None + ) -> List[dict]: + t0 = time.time() + docs = list(self._mem._db["slack"].find({"thread_id": thread_id})) + return self._record( + "get_slack_thread", + {"thread_id": thread_id, "as_of_time": as_of_time}, + docs, + t0, + ) + + def get_email(self, email_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["emails"].find_one({"id": email_id}) or {} + return self._record( + "get_email", + {"email_id": email_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_pr(self, pr_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["prs"].find_one({"id": pr_id}) or {} + return self._record( + "get_pr", + {"pr_id": pr_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_zd_ticket(self, ticket_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["zendesk"].find_one({"id": ticket_id}) or {} + return self._record( + "get_zd_ticket", + {"ticket_id": ticket_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_sf_opportunity(self, opp_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["salesforce_opps"].find_one({"id": opp_id}) or {} + return self._record( + "get_sf_opportunity", + {"opp_id": opp_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_sf_account(self, account_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["salesforce_accounts"].find_one({"id": account_id}) or {} + return self._record( + "get_sf_account", + {"account_id": account_id, "as_of_time": as_of_time}, + [doc] if doc else [], + t0, + ) + + def get_zoom_transcript( + self, transcript_id: str, as_of_time: Optional[str] = None + ) -> dict: + t0 = time.time() + doc = self._mem._db["zoom"].find_one({"id": transcript_id}) or {} + return self._record( + "get_zoom_transcript", + {"transcript_id": transcript_id}, + [doc] if doc else [], + t0, + ) + + def get_datadog_alert( + self, alert_id: str, as_of_time: Optional[str] = None + ) -> dict: + t0 = time.time() + doc = self._mem._db["datadog"].find_one({"id": alert_id}) or {} + return self._record( + "get_datadog_alert", {"alert_id": alert_id}, [doc] if doc else [], t0 + ) + + def get_invoice(self, invoice_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["invoices"].find_one({"id": invoice_id}) or {} + return self._record( + "get_invoice", {"invoice_id": invoice_id}, [doc] if doc else [], t0 + ) + + def get_nps_response(self, nps_id: str, as_of_time: Optional[str] = None) -> dict: + t0 = time.time() + doc = self._mem._db["nps"].find_one({"id": nps_id}) or {} + return self._record( + "get_nps_response", {"nps_id": nps_id}, [doc] if doc else [], t0 + ) + + def get_events_for_day( + self, day: int, event_type: Optional[str] = None + ) -> List[dict]: + t0 = time.time() + query: Dict = {"day": day} + if event_type: + query["type"] = event_type + docs = list(self._mem._db["events"].find(query)) + return self._record( + "get_events_for_day", {"day": day, "event_type": event_type}, docs, t0 + ) + + def search_artifacts( + self, + query: str, + doc_types: Optional[List[str]] = None, + as_of_time: Optional[str] = None, + actor: Optional[str] = None, + ) -> List[dict]: + """Semantic search across artifact collections.""" + t0 = time.time() + collections = doc_types or list(self._mem._db.list_collection_names()) + results = [] + for coll in collections: + try: + docs = list( + self._mem._db[coll] + .find( + {"$text": {"$search": query}}, {"score": {"$meta": "textScore"}} + ) + .sort([("score", {"$meta": "textScore"})]) + .limit(5) + ) + results.extend(docs) + except Exception: + pass + return self._record( + "search_artifacts", + {"query": query, "doc_types": doc_types, "actor": actor}, + results, + t0, + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# SCORERS +# ───────────────────────────────────────────────────────────────────────────── + + +class PerspectiveScorer: + """ + Scores a PERSPECTIVE trajectory. + + Answer scoring: + - Exact match on ground_truth.could_actor_have_known (boolean) + - Partial credit for correctly identifying blocked_subsystems + - Partial credit for citing in-cone evidence + + Trajectory scoring: + - Epistemic discipline: fraction of tool calls that stayed within cone + - Subsystem discipline: fraction of tool calls to accessible subsystems + - Conclusion grounding: did the final answer cite in-cone artifacts? + """ + + def score_answer( + self, final_answer: Dict, ground_truth: Dict + ) -> Tuple[float, bool]: + if not final_answer: + return 0.0, False + + gt_bool = ground_truth.get("could_actor_have_known", False) + + # Extract boolean from agent answer + agent_bool = self._extract_boolean(final_answer) + if agent_bool is None: + return 0.1, False + + correct = agent_bool == gt_bool + if not correct: + return 0.0, False + + # Partial credit for explaining the mechanism correctly + score = 0.6 # base for correct boolean + + gt_blocked = set(ground_truth.get("blocked_subsystems", [])) + agent_blocked = set(final_answer.get("blocked_subsystems", [])) + if gt_blocked and agent_blocked: + overlap = len(gt_blocked & agent_blocked) / len(gt_blocked) + score += 0.2 * overlap + + gt_evidence = set(ground_truth.get("evidence_artifacts", [])) + agent_evidence = set(final_answer.get("evidence_artifacts", [])) + if gt_evidence and agent_evidence: + overlap = len(gt_evidence & agent_evidence) / len(gt_evidence) + score += 0.2 * overlap + elif not gt_evidence: + score += 0.2 # no evidence required — agent doesn't need to cite any + + return min(score, 1.0), True + + def score_trajectory( + self, + trajectory: AgentTrajectory, + question: dict, + ) -> PerspectiveTrajectoryScore: + calls = trajectory.tool_calls + if not calls: + return PerspectiveTrajectoryScore(0, 0, 0, 0, 1.0, 0.0) + + n = len(calls) + actor_cone_violations = sum(1 for c in calls if c.actor_gate_violation) + subsystem_violations = sum(1 for c in calls if c.subsystem_violation) + horizon_violations = sum(1 for c in calls if c.horizon_violation) + dead_ends = sum(1 for c in calls if c.returned_empty) + dead_ends_recovered = trajectory.dead_ends_recovered + + epistemic_discipline = 1.0 - (actor_cone_violations / n) + subsystem_discipline = 1.0 - (subsystem_violations / n) + horizon_discipline = 1.0 - (horizon_violations / n) + + # Conclusion grounding: did agent cite any in-cone artifact in final answer? + actor_visible = set(question.get("actor_visible_artifacts", [])) + cited = set(trajectory.final_answer.get("evidence_artifacts", [])) + conclusion_grounding = 1.0 if (cited & actor_visible) else 0.5 if cited else 0.0 + + dead_end_recovery = dead_ends_recovered / dead_ends if dead_ends > 0 else 1.0 + + composite = ( + 0.35 * epistemic_discipline + + 0.25 * subsystem_discipline + + 0.20 * conclusion_grounding + + 0.10 * horizon_discipline + + 0.10 * dead_end_recovery + ) + + return PerspectiveTrajectoryScore( + epistemic_discipline=round(epistemic_discipline, 4), + subsystem_discipline=round(subsystem_discipline, 4), + horizon_discipline=round(horizon_discipline, 4), + conclusion_grounding=round(conclusion_grounding, 4), + dead_end_recovery=round(dead_end_recovery, 4), + composite=round(composite, 4), + ) + + def _extract_boolean(self, answer: Dict) -> Optional[bool]: + for key in ("could_actor_have_known", "answer", "result", "known", "visible"): + val = answer.get(key) + if isinstance(val, bool): + return val + if isinstance(val, str): + if val.lower() in ("true", "yes", "1"): + return True + if val.lower() in ("false", "no", "0"): + return False + # Try to find a boolean in free-text reasoning + reasoning = str(answer.get("reasoning", answer.get("explanation", ""))).lower() + if any( + w in reasoning + for w in ( + "could not have known", + "did not have access", + "was not visible", + "outside their", + ) + ): + return False + if any( + w in reasoning + for w in ("could have known", "had access", "was visible", "in their") + ): + return True + return None + + +class CounterfactualScorer: + """ + Scores a COUNTERFACTUAL trajectory. + + Answer scoring: + - outcome_changed: boolean match (0.4) + - mechanism: correct link_type identified (0.35) + - actors: at least one correct actor identified (0.25) + + Trajectory scoring: + - cause_identified: agent retrieved the cause event + - effect_identified: agent retrieved the effect event + - mechanism_correct: agent named the right link_type + - causal_chain_complete: agent traversed cause → effect in order + """ + + _MECHANISM_ALIASES = { + "involves_gap": { + "knowledge gap", + "gap", + "undocumented", + "missing documentation", + "knowledge_gap", + }, + "recurrence_of": { + "recurrence", + "repeat incident", + "recurred", + "same issue", + "recurring", + }, + "spawned_doc": { + "spawned", + "documentation", + "confluence", + "design discussion", + "produced doc", + }, + "email_dropped": { + "dropped", + "unactioned", + "routing failure", + "missed email", + "no response", + }, + "sf_ownership_lapsed": { + "ownership lapsed", + "crm gap", + "salesforce", + "account owner", + "orphaned", + }, + "zd_escalation_source": { + "zendesk", + "support ticket", + "escalated from", + "zd escalation", + }, + "blocker_flagged": { + "blocker", + "blocked", + "delay", + "progress", + "technical blocker", + "blocker_flagged", + }, + "incident_coordination": { + "coordination", + "external contact", + "external party", + "incident_coordination", + "coordinated with", + }, + "departure_reassignment": { + "reassignment", + "departed", + "departure", + "reassigned", + "departure_reassignment", + "not reassigned", + }, + } + + def score_answer( + self, final_answer: Dict, ground_truth: Dict + ) -> Tuple[float, bool]: + if not final_answer: + return 0.0, False + + score = 0.0 + gt_outcome = ground_truth.get("outcome_changed", True) + agent_outcome = self._extract_boolean(final_answer, "outcome_changed") + + if agent_outcome is None: + return 0.0, False + if agent_outcome == gt_outcome: + score += 0.4 + + # Mechanism match + gt_mechanism = ground_truth.get("causal_mechanism", "") + agent_mechanism = str( + final_answer.get("mechanism", final_answer.get("causal_mechanism", "")) + ).lower() + aliases = self._MECHANISM_ALIASES.get(gt_mechanism, {gt_mechanism}) + if any(alias in agent_mechanism for alias in aliases): + score += 0.35 + + # Actor match + gt_actors = set(ground_truth.get("actors", [])) + agent_actors_raw = final_answer.get( + "actors", final_answer.get("involved_actors", []) + ) + agent_actors = ( + set(agent_actors_raw) if isinstance(agent_actors_raw, list) else set() + ) + if gt_actors and agent_actors and (gt_actors & agent_actors): + score += 0.25 + elif not gt_actors: + score += 0.25 + + is_correct = score >= 0.75 + return round(min(score, 1.0), 4), is_correct + + def score_trajectory( + self, + trajectory: AgentTrajectory, + question: dict, + ground_truth: Dict, + ) -> CounterfactualTrajectoryScore: + calls = trajectory.tool_calls + if not calls: + return CounterfactualTrajectoryScore(0, 0, 0, 0, 1.0, 0.0) + + n = len(calls) + retrieved_ids = set() + for call in calls: + retrieved_ids.update(call.result_ids) + + cause_id = ground_truth.get("cause_event_id", "") + effect_id = ground_truth.get("effect_event_id", "") + cause_identified = 1.0 if (cause_id and cause_id in retrieved_ids) else 0.0 + effect_identified = 1.0 if (effect_id and effect_id in retrieved_ids) else 0.0 + + # Mechanism: did agent use keyword in its tool calls or final answer? + gt_mechanism = ground_truth.get("causal_mechanism", "") + aliases = self._MECHANISM_ALIASES.get(gt_mechanism, {gt_mechanism}) + agent_text = " ".join( + [str(c.arguments) for c in calls] + [str(trajectory.final_answer)] + ).lower() + mechanism_correct = ( + 1.0 if any(alias in agent_text for alias in aliases) else 0.0 + ) + + # Causal chain: did agent retrieve cause before effect? + cause_call_idx = next( + (i for i, c in enumerate(calls) if cause_id in c.result_ids), None + ) + effect_call_idx = next( + (i for i, c in enumerate(calls) if effect_id in c.result_ids), None + ) + causal_chain_complete = ( + 1.0 + if ( + cause_call_idx is not None + and effect_call_idx is not None + and cause_call_idx <= effect_call_idx + ) + else 0.5 + if (cause_call_idx is not None or effect_call_idx is not None) + else 0.0 + ) + + horizon_violations = sum(1 for c in calls if c.horizon_violation) + horizon_discipline = 1.0 - (horizon_violations / n) + + composite = ( + 0.25 * cause_identified + + 0.25 * effect_identified + + 0.25 * mechanism_correct + + 0.15 * causal_chain_complete + + 0.10 * horizon_discipline + ) + + return CounterfactualTrajectoryScore( + cause_identified=cause_identified, + effect_identified=effect_identified, + mechanism_correct=mechanism_correct, + causal_chain_complete=causal_chain_complete, + horizon_discipline=round(horizon_discipline, 4), + composite=round(composite, 4), + ) + + def _extract_boolean(self, answer: Dict, key: str) -> Optional[bool]: + val = answer.get(key) + if isinstance(val, bool): + return val + if isinstance(val, str): + if val.lower() in ("true", "yes"): + return True + if val.lower() in ("false", "no"): + return False + reasoning = str(answer.get("reasoning", "")).lower() + if "would not" in reasoning or "would have prevented" in reasoning: + return True + if "would still" in reasoning or "no change" in reasoning: + return False + return None + + +class SilenceScorer: + """ + Scores a SILENCE trajectory. + + The key insight: absence is only meaningful if the agent searched the + right places. A lucky "no" without evidence of search is scored as a + trajectory failure. + + Answer scoring: + - Boolean correct (did agent conclude the artifact does not exist): 1.0 + - If agent concluded "yes" (artifact exists): 0.0 + + Trajectory scoring: + - search_space_coverage: fraction of expected_search_space the agent checked + - correct_absence_conclusion: did agent say "does not exist" explicitly? + - premature_conclusion: penalty if agent concluded before searching + """ + + def score_answer( + self, final_answer: Dict, ground_truth: Dict + ) -> Tuple[float, bool]: + if not final_answer: + return 0.0, False + + gt_answer = ground_truth.get("answer", False) # Always False for SILENCE + agent_answer = self._extract_absence_conclusion(final_answer) + + if agent_answer is None: + return 0.1, False + + correct = agent_answer == gt_answer + return (1.0, True) if correct else (0.0, False) + + def score_trajectory( + self, + trajectory: AgentTrajectory, + question: dict, + ) -> SilenceTrajectoryScore: + calls = trajectory.tool_calls + expected_space = set(question.get("expected_search_space", [])) + if not calls: + return SilenceTrajectoryScore(0.0, 0.0, 0.0, 1.0, 0.0) + + n = len(calls) + + # What did the agent search? + searched_ids: Set[str] = set() + searched_tool_args: List[str] = [] + for call in calls: + searched_ids.update(call.result_ids) + searched_tool_args.append(str(call.arguments).lower()) + + # Search space coverage: fraction of expected_search_space hit + # Also count partial matches on path prefixes + covered = set() + for expected in expected_space: + expected_lower = expected.lower() + if any(expected_lower in arg for arg in searched_tool_args): + covered.add(expected) + elif expected in searched_ids: + covered.add(expected) + + search_space_coverage = ( + len(covered) / len(expected_space) if expected_space else 1.0 + ) + + # Correct absence conclusion: did agent explicitly state non-existence? + conclusion_text = str(trajectory.final_answer.get("reasoning", "")).lower() + conclusion_text += str(trajectory.final_answer.get("answer", "")).lower() + explicit_negative = any( + phrase in conclusion_text + for phrase in ( + "does not exist", + "was not created", + "no postmortem", + "not found", + "could not find", + "no record", + "never created", + "not in the corpus", + "no evidence", + "not present", + "absent", + ) + ) + correct_absence_conclusion = 1.0 if explicit_negative else 0.3 + + # Premature conclusion: did agent conclude before searching? + # Heuristic: if the final answer came after fewer than 2 tool calls, penalize + premature_conclusion = 1.0 - max(0.0, min(1.0, (n - 1) / 3)) + + horizon_violations = sum(1 for c in calls if c.horizon_violation) + horizon_discipline = ( + 1.0 # SILENCE has no temporal gate, so no violations possible + ) + + composite = ( + 0.50 * search_space_coverage + + 0.30 * correct_absence_conclusion + - 0.10 * (1.0 - premature_conclusion) # penalty for rushing + + 0.10 * horizon_discipline + ) + + return SilenceTrajectoryScore( + search_space_coverage=round(search_space_coverage, 4), + correct_absence_conclusion=round(correct_absence_conclusion, 4), + premature_conclusion=round(premature_conclusion, 4), + horizon_discipline=round(horizon_discipline, 4), + composite=round(max(0.0, composite), 4), + ) + + def _extract_absence_conclusion(self, answer: Dict) -> Optional[bool]: + """True = artifact exists, False = artifact does not exist.""" + val = answer.get("exists", answer.get("found", answer.get("answer"))) + if isinstance(val, bool): + return val + if isinstance(val, str): + if val.lower() in ("true", "yes", "exists", "found"): + return True + if val.lower() in ("false", "no", "not found", "does not exist", "absent"): + return False + reasoning = str(answer.get("reasoning", "")).lower() + if any( + w in reasoning + for w in ( + "does not exist", + "not created", + "no record", + "absent", + "not found", + ) + ): + return False + if any(w in reasoning for w in ("exists", "was created", "found", "present")): + return True + return None + + +# ───────────────────────────────────────────────────────────────────────────── +# AGENT RUNNER +# ───────────────────────────────────────────────────────────────────────────── + + +class AgenticEvalRunner: + """ + Runs the agent on each question and scores the result. + + For each question: + 1. Sets up the gated tool layer appropriate for the track + 2. Runs the agent with the typed tool surface + 3. Collects the trajectory + 4. Scores answer + trajectory with the track-specific scorer + 5. Combines scores with track-specific weights + """ + + def __init__(self, model: str = "claude-sonnet-4-6", max_steps: int = 15): + self._model = model + self._max_steps = max_steps + + from flow import build_llm + from memory import Memory + + self._mem = Memory() + self._llm = build_llm("worker") + + self._perspective_scorer = PerspectiveScorer() + self._counterfactual_scorer = CounterfactualScorer() + self._silence_scorer = SilenceScorer() + + def run( + self, + questions_path: Path, + out_path: Path, + question_types: Optional[List[str]] = None, + max_questions: Optional[int] = None, + ) -> None: + with open(questions_path) as f: + data = json.load(f) + + questions = data["questions"] + if question_types: + questions = [q for q in questions if q["question_type"] in question_types] + if max_questions: + questions = questions[:max_questions] + + logger.info( + f"Running agentic eval on {len(questions)} questions " + f"(model={self._model}, max_steps={self._max_steps})" + ) + + results: List[EvalResult] = [] + per_question: List[dict] = [] + + for i, question in enumerate(questions): + qtype = question["question_type"] + logger.info( + f"[{i + 1}/{len(questions)}] {qtype} — {question['question_id']}" + ) + + try: + result = self._run_question(question) + except Exception as exc: + logger.error(f" Failed: {exc}") + result = EvalResult( + question_id=question["question_id"], + question_type=qtype, + difficulty=question.get("difficulty", "unknown"), + answer_score=0.0, + answer_correct=False, + trajectory_score=0.0, + combined_score=0.0, + failure_reason=str(exc), + tool_call_count=0, + meta={"error": str(exc)}, + ) + + results.append(result) + per_question.append(result.to_dict()) + logger.info( + f" answer={result.answer_score:.3f} " + f"trajectory={result.trajectory_score:.3f} " + f"combined={result.combined_score:.3f} " + f"tools={result.tool_call_count}" + ) + + summary = self._aggregate(results) + out_path.parent.mkdir(parents=True, exist_ok=True) + output = { + "meta": { + "model": self._model, + "max_steps": self._max_steps, + "n_questions": len(results), + "track_weights": _TRACK_WEIGHTS, + }, + "summary": summary, + "per_question": per_question, + } + with open(out_path, "w") as f: + json.dump(output, f, indent=2, default=str) + + logger.info(f"Results written to {out_path}") + logger.info( + f"Overall — answer: {summary['overall']['answer_score']:.3f} " + f"trajectory: {summary['overall']['trajectory_score']:.3f} " + f"combined: {summary['overall']['combined_score']:.3f}" + ) + + def _run_question(self, question: dict) -> EvalResult: + qtype = question["question_type"] + ground_truth = question["ground_truth"] + + # Set up gated tools + as_of_time = self._infer_as_of_time(question) + actor_visible = ( + set(question.get("actor_visible_artifacts", [])) + if qtype == "PERSPECTIVE" + else None + ) + actor_subsystems = ( + set(question.get("subsystem_access", [])) + if qtype == "PERSPECTIVE" + else None + ) + + tools = GatedTools( + mem=self._mem, + question=question, + as_of_time=as_of_time, + actor_visible_artifacts=actor_visible, + actor_subsystem_access=actor_subsystems, + ) + + # Run agent + trajectory = self._run_agent(question, tools) + + # Score + if qtype == "PERSPECTIVE": + answer_score, answer_correct = self._perspective_scorer.score_answer( + trajectory.final_answer, ground_truth + ) + traj = self._perspective_scorer.score_trajectory(trajectory, question) + traj_score = traj.composite + traj_detail = asdict(traj) + + elif qtype == "COUNTERFACTUAL": + answer_score, answer_correct = self._counterfactual_scorer.score_answer( + trajectory.final_answer, ground_truth + ) + traj = self._counterfactual_scorer.score_trajectory( + trajectory, question, ground_truth + ) + traj_score = traj.composite + traj_detail = asdict(traj) + + elif qtype == "SILENCE": + answer_score, answer_correct = self._silence_scorer.score_answer( + trajectory.final_answer, ground_truth + ) + traj = self._silence_scorer.score_trajectory(trajectory, question) + traj_score = traj.composite + traj_detail = asdict(traj) + + else: + raise ValueError(f"Unknown question type: {qtype}") + + weights = _TRACK_WEIGHTS[qtype] + combined = weights["answer"] * answer_score + weights["trajectory"] * traj_score + + return EvalResult( + question_id=question["question_id"], + question_type=qtype, + difficulty=question.get("difficulty", "unknown"), + answer_score=round(answer_score, 4), + answer_correct=answer_correct, + trajectory_score=round(traj_score, 4), + combined_score=round(combined, 4), + failure_reason=None, + tool_call_count=len(trajectory.tool_calls), + meta={ + "model": self._model, + "as_of_time": as_of_time, + "trajectory_detail": traj_detail, + "horizon_violations": trajectory.horizon_violations, + "actor_gate_violations": trajectory.actor_gate_violations, + "subsystem_violations": trajectory.subsystem_violations, + "dead_ends_hit": trajectory.dead_ends_hit, + "dead_ends_recovered": trajectory.dead_ends_recovered, + "total_latency_ms": round(trajectory.total_latency_ms, 1), + "tool_calls": [asdict(tc) for tc in trajectory.tool_calls], + "final_answer": trajectory.final_answer, + }, + ) + + def _run_agent(self, question: dict, tools: GatedTools) -> AgentTrajectory: + """ + Runs the agent against the question using the gated tool surface. + Returns a populated AgentTrajectory. + + The agent is given a structured output format so answer extraction + is reliable across all three tracks. + """ + from agent_factory import make_agent + from crewai import Crew, Task + + qtype = question["question_type"] + trajectory = AgentTrajectory( + question_id=question["question_id"], + question_type=qtype, + ) + + # Build output schema based on track + output_schema = { + "PERSPECTIVE": """{ + "could_actor_have_known": , + "reasoning": "", + "evidence_artifacts": ["", ...], + "blocked_subsystems": ["", ...] +}""", + "COUNTERFACTUAL": """{ + "outcome_changed": , + "mechanism": "", + "causal_mechanism": "", + "actors": ["", ...], + "reasoning": "" +}""", + "SILENCE": """{ + "exists": , + "answer": "", + "reasoning": "" +}""", + }[qtype] + + constraint_note = { + "PERSPECTIVE": ( + f"\n\nIMPORTANT: You are answering from the perspective of {question.get('actor', 'the actor')} " + f"as of Day {question.get('as_of_day', '?')}. " + f"This actor only has access to: {', '.join(question.get('subsystem_access', []))}. " + f"You must not use information from systems outside this list. " + f"Accessing artifacts outside the actor's visibility cone is a violation." + ), + "COUNTERFACTUAL": ( + "\n\nIMPORTANT: This is a counterfactual question. You must identify the explicit " + "causal link in the data — do not speculate. Find the cause event and the effect " + "event, then determine whether removing the cause would have changed the effect." + ), + "SILENCE": ( + f"\n\nIMPORTANT: This is an absence question. You must search the corpus thoroughly " + f"before concluding. Check: {', '.join(question.get('expected_search_space', [])[:5])}. " + f"Only conclude absence after exhausting these sources. Do not guess." + ), + }[qtype] + + agent = make_agent( + role="Enterprise Knowledge Analyst", + goal="Reason carefully over corporate documents to answer complex questions.", + backstory=( + "You are an expert analyst evaluating enterprise AI systems. You reason " + "carefully, cite evidence, stay within stated constraints, and never guess." + ), + llm=self._llm, + tools=self._tool_list(tools), + ) + + task = Task( + description=( + f"{question['question_text']}" + f"{constraint_note}" + f"\n\nRespond ONLY with a JSON object matching this schema:\n{output_schema}" + ), + expected_output="A JSON object matching the schema above. No preamble.", + agent=agent, + max_iter=self._max_steps, + ) + + t_start = time.time() + try: + raw = str( + Crew(agents=[agent], tasks=[task], verbose=False).kickoff() + ).strip() + final_answer = self._parse_structured_answer(raw) + except Exception as exc: + logger.warning(f" Agent error: {exc}") + final_answer = {} + + trajectory.total_latency_ms = (time.time() - t_start) * 1000 + trajectory.tool_calls = list(tools.call_log) + trajectory.final_answer = final_answer + trajectory.horizon_violations = sum( + 1 for c in trajectory.tool_calls if c.horizon_violation + ) + trajectory.actor_gate_violations = sum( + 1 for c in trajectory.tool_calls if c.actor_gate_violation + ) + trajectory.subsystem_violations = sum( + 1 for c in trajectory.tool_calls if c.subsystem_violation + ) + trajectory.dead_ends_hit = sum( + 1 for c in trajectory.tool_calls if c.returned_empty + ) + + # Dead end recovery: count cases where agent made a successful call after a dead end + for i, call in enumerate(trajectory.tool_calls): + if call.returned_empty and i + 1 < len(trajectory.tool_calls): + if not trajectory.tool_calls[i + 1].returned_empty: + trajectory.dead_ends_recovered += 1 + + return trajectory + + def _tool_list(self, tools: GatedTools) -> List: + """Return the tool surface for the agent. Narrow and typed.""" + return [ + tools.get_ticket, + tools.get_confluence_page, + tools.get_slack_thread, + tools.get_email, + tools.get_pr, + tools.get_zd_ticket, + tools.get_sf_opportunity, + tools.get_sf_account, + tools.get_zoom_transcript, + tools.get_datadog_alert, + tools.get_invoice, + tools.get_nps_response, + tools.get_events_for_day, + tools.search_artifacts, + ] + + def _parse_structured_answer(self, raw: str) -> Dict: + """Extract JSON from agent response. Strips markdown fences.""" + text = raw.strip() + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + try: + return json.loads(text) + except json.JSONDecodeError: + # Try to find JSON object in response + match = re.search(r"\{.*\}", text, re.DOTALL) + if match: + try: + return json.loads(match.group()) + except json.JSONDecodeError: + pass + return {"raw_response": raw} + + def _infer_as_of_time(self, question: dict) -> str: + qtype = question.get("question_type", "") + if qtype == "SILENCE": + # Full corpus — end of sim + max_day = max( + (e.day for e in self._mem.get_event_log(from_db=True)), default=22 + ) + return (_SIM_START + timedelta(days=max_day)).isoformat() + if qtype == "PERSPECTIVE": + return question.get("as_of_time", datetime.now().isoformat()) + if qtype == "COUNTERFACTUAL": + # Use effect event timestamp + effect_id = question.get("ground_truth", {}).get("effect_event_id") + if effect_id: + try: + ev = self._mem._db["events"].find_one({"event_id": effect_id}) + if ev and ev.get("timestamp"): + return str(ev["timestamp"]) + except Exception: + pass + day = question.get("day", question.get("event_day", 1)) + return (_SIM_START + timedelta(days=day)).isoformat() + + def _aggregate(self, results: List[EvalResult]) -> dict: + def mean(vals): + return round(sum(vals) / len(vals), 4) if vals else 0.0 + + by_type: Dict[str, List[EvalResult]] = {} + by_difficulty: Dict[str, List[EvalResult]] = {} + for r in results: + by_type.setdefault(r.question_type, []).append(r) + by_difficulty.setdefault(r.difficulty, []).append(r) + + return { + "overall": { + "n": len(results), + "answer_score": mean([r.answer_score for r in results]), + "trajectory_score": mean([r.trajectory_score for r in results]), + "combined_score": mean([r.combined_score for r in results]), + "accuracy": round( + sum(r.answer_correct for r in results) / len(results), 4 + ), + "avg_tool_calls": mean([r.tool_call_count for r in results]), + }, + "by_type": { + qtype: { + "n": len(rs), + "answer_score": mean([r.answer_score for r in rs]), + "trajectory_score": mean([r.trajectory_score for r in rs]), + "combined_score": mean([r.combined_score for r in rs]), + "accuracy": round(sum(r.answer_correct for r in rs) / len(rs), 4), + "avg_tool_calls": mean([r.tool_call_count for r in rs]), + # Track-specific breakdowns + **( + { + "avg_actor_gate_violations": mean( + [r.meta.get("actor_gate_violations", 0) for r in rs] + ) + } + if qtype == "PERSPECTIVE" + else {} + ), + **( + { + "avg_subsystem_violations": mean( + [r.meta.get("subsystem_violations", 0) for r in rs] + ) + } + if qtype == "PERSPECTIVE" + else {} + ), + **( + { + "search_space_coverage": mean( + [ + r.meta.get("trajectory_detail", {}).get( + "search_space_coverage", 0 + ) + for r in rs + ] + ) + } + if qtype == "SILENCE" + else {} + ), + } + for qtype, rs in by_type.items() + }, + "by_difficulty": { + diff: { + "n": len(rs), + "answer_score": mean([r.answer_score for r in rs]), + "trajectory_score": mean([r.trajectory_score for r in rs]), + "combined_score": mean([r.combined_score for r in rs]), + } + for diff, rs in by_difficulty.items() + }, + } + + +# ───────────────────────────────────────────────────────────────────────────── +# ENTRYPOINT +# ───────────────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + parser = argparse.ArgumentParser( + description="OrgForge Agentic Eval Harness v2 — PERSPECTIVE, COUNTERFACTUAL, SILENCE" + ) + parser.add_argument( + "--questions", + type=Path, + default=EVAL_DIR / "eval_questions.json", + ) + parser.add_argument( + "--out", + type=Path, + default=EVAL_DIR / "agentic_results.json", + ) + parser.add_argument( + "--model", + type=str, + default="claude-sonnet-4-6", + ) + parser.add_argument( + "--max-steps", + type=int, + default=15, + help="Max tool-use steps per question (SILENCE questions may need more)", + ) + parser.add_argument( + "--types", + nargs="+", + choices=["PERSPECTIVE", "COUNTERFACTUAL", "SILENCE"], + help="Run only specific tracks", + ) + parser.add_argument( + "--max-questions", + type=int, + default=None, + ) + args = parser.parse_args() + + runner = AgenticEvalRunner(model=args.model, max_steps=args.max_steps) + runner.run( + questions_path=args.questions, + out_path=args.out, + question_types=args.types, + max_questions=args.max_questions, + ) diff --git a/eval/eval_e2e.py b/eval/eval_e2e.py index cb0c268..d40582c 100644 --- a/eval/eval_e2e.py +++ b/eval/eval_e2e.py @@ -48,6 +48,7 @@ from __future__ import annotations import argparse +import hashlib import json import logging import os @@ -285,6 +286,129 @@ def retrieve(self, query: str, top_k: int = TOP_K) -> List[str]: return [self._doc_ids[int(i)] for i in indices] +class InfinityRetriever(Retriever): + """ + OpenAI-compatible retriever for a local/remote Infinity server. + Configurable via INFINITY_HOST environment variable (default: http://localhost:11434). + """ + + name = "infinity" + + _INSTRUCTIONS = { + "search_document": "", + "search_query": "query: ", + } + + def __init__( + self, + model: str = "Qwen/Qwen3-Embedding-4B", + batch_size: int = 2, + cache_dir: str = ".embed_cache", + ): + import requests + + self._model = os.environ.get("EMBED_MODEL", model) + self._host = os.environ.get("INFINITY_HOST", "http://localhost:11434") + self._batch_size = batch_size + self._session = requests.Session() + self._cache_dir = Path(cache_dir) + self._cache_dir.mkdir(exist_ok=True) + self._q_cache = {} + self._q_cache_path = None + + def _cache_key(self, corpus: List[dict]) -> str: + """Stable key based on model + corpus content.""" + corpus_fingerprint = hashlib.md5( + json.dumps([r["doc_id"] for r in corpus], sort_keys=True).encode() + ).hexdigest()[:12] + safe_model = self._model.replace("/", "_") + return f"{safe_model}__{corpus_fingerprint}" + + def index(self, corpus: List[dict]) -> None: + self._doc_ids = [r["doc_id"] for r in corpus] + key = self._cache_key(corpus) + cache_path = self._cache_dir / f"{self._cache_key(corpus)}.npz" + + self._q_cache_path = self._cache_dir / f"{key}_questions.json" + if self._q_cache_path.exists(): + with open(self._q_cache_path, "r") as f: + self._q_cache = json.load(f) + logger.info( + f" Loaded {len(self._q_cache)} cached questions from {self._q_cache_path}" + ) + + if cache_path.exists(): + logger.info(f" Loading Infinity embeddings from cache: {cache_path}") + data = np.load(cache_path, allow_pickle=True) + self._matrix = data["matrix"] + assert list(data["doc_ids"]) == self._doc_ids, "Cache doc_id mismatch!" + logger.info(" Infinity index ready (from cache)") + return + + bodies = [r.get("body", "") or "" for r in corpus] + + logger.info( + f" Embedding {len(bodies)} docs via Infinity ({self._model} at {self._host}) ..." + ) + + embeddings = [] + prefix = self._INSTRUCTIONS["search_document"] + + for i in range(0, len(bodies), self._batch_size): + batch = bodies[i : i + self._batch_size] + prefixed_batch = [prefix + text for text in batch] + + try: + resp = self._session.post( + f"{self._host}/embeddings", + json={"model": self._model, "input": prefixed_batch}, + timeout=300, + ) + resp.raise_for_status() + + batch_vecs = [d["embedding"] for d in resp.json()["data"]] + embeddings.extend(batch_vecs) + except Exception as e: + logger.error(f" Crash during corpus indexing at doc {i}: {e}") + raise + logger.info( + f" embedded {min(i + self._batch_size, len(bodies))}/{len(bodies)}" + ) + + mat = np.array(embeddings, dtype=np.float32) + norms = np.linalg.norm(mat, axis=1, keepdims=True) + self._matrix = mat / np.where(norms == 0, 1, norms) + np.savez_compressed(cache_path, matrix=self._matrix, doc_ids=self._doc_ids) + logger.info(f" Embeddings cached to {cache_path}") + + def retrieve(self, query: str, top_k: int = TOP_K) -> List[str]: + + if query in self._q_cache: + q_vec = np.array(self._q_cache[query], dtype=np.float32) + else: + prefix = self._INSTRUCTIONS["search_query"] + resp = self._session.post( + f"{self._host}/embeddings", + json={"model": self._model, "input": [prefix + query]}, + timeout=30, + ) + resp.raise_for_status() + + vec_list = resp.json()["data"][0]["embedding"] + self._q_cache[query] = vec_list + + if self._q_cache_path: + with open(self._q_cache_path, "w") as f: + json.dump(self._q_cache, f) + + q_vec = np.array(vec_list, dtype=np.float32) + + q_vec /= max(np.linalg.norm(q_vec), 1e-9) + scores = self._matrix @ q_vec + indices = scores.argsort()[::-1][:top_k] + return [self._doc_ids[int(i)] for i in indices] + + class BedrockCohereRetriever(Retriever): """ Cohere Embed v4 via Amazon Bedrock (invoke_model). @@ -371,6 +495,8 @@ def build_retriever(name: str, region: str = "us-east-1") -> Retriever: return BedrockCohereRetriever(region=region) if name == "openai": return OpenAIRetriever() + if name == "infinity": + return InfinityRetriever() # ── RRF and Graph retrievers (require retrieval_extensions.py) ───────────── if not _EXTENSIONS_AVAILABLE: @@ -386,6 +512,8 @@ def build_retriever(name: str, region: str = "us-east-1") -> Retriever: return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) if name == "rrf-bedrock": return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) + if name == "rrf-infinity": + return RRFRetriever([BM25Retriever(), InfinityRetriever()]) # Graph-Augmented: expand any base retriever along the artifact graph if name == "graph-bm25": @@ -395,12 +523,15 @@ def build_retriever(name: str, region: str = "us-east-1") -> Retriever: if name == "graph-rrf": base = RRFRetriever([BM25Retriever(), CohereRetriever()]) return GraphAugmentedRetriever(base) + if name == "graph-rrf": + base = RRFRetriever([BM25Retriever(), CohereRetriever()]) + return GraphAugmentedRetriever(base) raise ValueError( f"Unknown retriever: {name!r}. " - "Choose bm25 | cohere | cohere-bedrock | openai | " - "rrf | rrf-openai | rrf-bedrock | " - "graph-bm25 | graph-cohere | graph-rrf" + "Choose bm25 | cohere | cohere-bedrock | openai | infinity | " + "rrf | rrf-openai | rrf-bedrock | rrf-infinity | " + "graph-bm25 | graph-cohere | graph-infinity | graph-rrf" ) @@ -1205,21 +1336,24 @@ def _parse_args() -> argparse.Namespace: "cohere", "cohere-bedrock", "openai", + "infinity", # Reciprocal Rank Fusion "rrf", "rrf-openai", "rrf-bedrock", + "rrf-infinity", # Graph-Augmented (1-2 hop artifact expansion) "graph-bm25", "graph-cohere", + "graph-infinity", "graph-rrf", ], default="bm25", help=( "Retriever to use (default: bm25).\n" - " bm25 / cohere / cohere-bedrock / openai — single retrievers\n" - " rrf / rrf-openai / rrf-bedrock — BM25 + dense fusion (RRF)\n" - " graph-bm25 / graph-cohere / graph-rrf — graph-augmented expansion" + " bm25 / cohere / cohere-bedrock / openai / infinity — single retrievers\n" + " rrf / rrf-openai / rrf-bedrock / rrf-infinity — BM25 + dense fusion (RRF)\n" + " graph-bm25 / graph-cohere / graph-infinity / graph-rrf — graph-augmented expansion" ), ) p.add_argument( diff --git a/eval/eval_harness.py b/eval/eval_harness.py index 620db39..518c560 100644 --- a/eval/eval_harness.py +++ b/eval/eval_harness.py @@ -1,105 +1,81 @@ """ eval_harness.py =============== -Post-simulation eval dataset generator for OrgForge. +OrgForge Eval Dataset Generator — v2 -Run after flow.py completes: - python eval_harness.py - -Produces two files in export/eval/: - causal_threads.json — explicit artifact graphs with actor knowledge states - eval_questions.json — typed Q&A pairs with deterministic ground-truth answers - -Design principle ----------------- -Answers are derived deterministically from the SimEvent log — the LLM only -writes question prose. This means: - - Ground truth is always correct by construction (no hallucination risk) - - Questions can be regenerated with different wording without changing answers - - The eval set is reproducible across sim runs with the same config - -Question types --------------- - RETRIEVAL "Which artifact first mentioned X?" - Answer: specific artifact ID from SimEvent log - - CAUSAL "What action did Y take after Z happened?" - Answer: next SimEvent in the causal chain - - TEMPORAL "Did person P know about X when they made decision D?" - Answer: boolean + evidence, derived from actor knowledge snapshots - Note: samples both involves_gap=True incidents (pool A) and all - other incidents (pool B) to cover true-positive and true-negative cases. - - GAP_DETECTION "Was the email from X ever actioned?" - Answer: boolean derived from email_dropped / customer_email_routed events - - ROUTING "Who was the first internal person to see the complaint from X?" - Answer: liaison name from inbound_external_email SimEvent - - PLAN "What was dept X focused on during Day N?" - Answer: theme + actors from dept_plans collection - - ESCALATION "Who was involved in the escalation chain for ticket X?" - Answer: escalation_actors from escalation_chain SimEvent - - KNOWLEDGE_GAP "What domain was undocumented when incident X fired?" - Answer: gap_areas from knowledge_gap_detected SimEvent - - POSTMORTEM "Which Confluence doc captured the postmortem for incident X?" - Answer: confluence artifact ID from postmortem_created SimEvent - - STANDUP "What did person X report at standup on Day N?" - Answer: summary + slack_thread_id from standup SimEvent - - CUSTOMER_ESC "Who handled the escalation from customer X?" - Answer: first_handler + downstream artifacts from customer_escalation - or customer_email_routed SimEvent - - ZD_RESOLUTION "Was Zendesk ticket X resolved and how long did it take?" - Answer: resolved boolean + duration_days from zd_ticket_opened / - zd_tickets_resolved SimEvents +Produces three novel eval tracks that require the deterministic state machine +to exist. No retrieval questions. Those are covered by other benchmarks. - ZD_ESCALATION "Which Zendesk tickets escalated to incident X?" - Answer: ticket_ids list from zd_tickets_escalated SimEvent - (uses CAUSAL scorer — incident + ticket chain) - - SF_RISK "Which Salesforce accounts were flagged at-risk after incident X?" - Answer: at_risk_accounts list from sf_deals_risk_flagged SimEvent - - SF_TOUCHPOINT "What opportunity was advanced by the email from X to Y?" - Answer: opportunity_id + stage from crm_touchpoint SimEvent - (uses CAUSAL scorer — email → opportunity chain) - - SF_OWNERSHIP "Which accounts lost their owner when X departed?" - Answer: lapsed_accounts + lapsed_opportunities from - sf_ownership_lapsed SimEvent (uses RETRIEVAL scorer) - - DATADOG_ALERT "Which Datadog alert fired for incident X?" - Answer: incident_id inferred from incident_opened SimEvent - (uses RETRIEVAL scorer — alert → incident link) - - NPS_SCORE "What NPS score did customer X give and what drove it?" - Answer: nps_score + classification derived deterministically - from ZD ticket and incident SimEvents (mirrors NPSWriter formula) - - INVOICE_SLA "What SLA credit appeared on customer X's invoice?" - Answer: breach_duration_days + sla_credit_per_org derived from - incident duration × SLA_CREDIT_RATE (mirrors InvoiceWriter logic) - -Usage for RAG eval ------------------- -Each question in eval_questions.json has: - - question_text — natural language question - - question_type — one of the five types above - - ground_truth — the answer an agent should produce - - evidence_chain — list of SimEvent IDs / artifact IDs that support the answer - - difficulty — "easy" | "medium" | "hard" - - requires_reasoning — whether answering requires multi-hop traversal +Run after flow.py and post_sim_artifacts.py complete: + python eval_harness.py -An eval harness compares agent answers against ground_truth. -evidence_chain lets you score partial credit (did the agent find the right artifacts -even if it drew the wrong conclusion?). +Produces in export/eval/: + actor_visibility.json — per-actor artifact visibility cones, time-indexed + causal_link_index.json — explicit causal links derived from sim flags + absence_catalog.json — expected-but-absent artifact pairs + eval_questions.json — PERSPECTIVE + COUNTERFACTUAL + SILENCE questions + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 1 — PERSPECTIVE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Questions scoped to what a specific actor could have known at a specific moment, +given their actual subsystem access and information horizon. + +Ground truth is derived from the actor's visibility cone: the set of artifact IDs +reachable by that actor at or before as_of_time, filtered by subsystem access. + +Cross-subsystem questions (e.g. engineer sees Slack + Zoom but not Salesforce) +are flagged difficulty="hard". Single-subsystem questions are "medium". + +Example: + "Based only on what Morgan had access to as of Day 9, should she have known + that Acme Corp was at churn risk?" + ground_truth: { "answer": False, "reason": "sf_deals_risk_flagged not in Morgan's visibility cone" } + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 2 — COUNTERFACTUAL +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Questions of the form "if X had been different, would Y have occurred?" +Only generated where the sim encodes an explicit causal link: + - involves_gap / knowledge_gap_detected → incident causation + - recurrence_of → repeat incident prevention + - spawned_doc → design discussion → documentation + - email_dropped → unactioned communication + - sf_ownership_lapsed → CRM ownership gap + - zd_escalation_source → support ticket → incident + +Ground truth is always derivable from the explicit link without inference. + +Example: + "If Jordan had documented auth-service before departing, would incident IT-108 + have been diagnosed faster?" + ground_truth: { "outcome_changed": True, "mechanism": "knowledge_gap_detected", + "gap_domain": "auth-service", "causal_event": "evt_..." } + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +TRACK 3 — SILENCE +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Questions about things that did NOT happen. The state machine is the arbiter: +if no event fired, absence is ground truth regardless of whether absence was +intentional. + +Each SILENCE question includes an expected_search_space — the artifact IDs the +agent MUST check before concluding absence. A correct "no" reached without +searching the right places scores 0 on trajectory even if the boolean is right. + +Example: + "Was a postmortem written for the Zendesk escalation on Day 6?" + ground_truth: False + expected_search_space: ["confluence/postmortems/", "jira/IT-*", "slack/incidents"] + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Design principles +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +- Ground truth is always derived from SimEvent log. LLMs only write prose. +- Question prose generation includes a structured validation loop. +- Actor visibility cones are first-class data structures, not a scoring afterthought. +- Subsystem access is explicitly modeled per actor per day. +- The absence catalog is built by pattern-matching expected event pairs, not heuristics. """ from __future__ import annotations @@ -107,11 +83,14 @@ import json import logging import random +import re +from config_loader import CONFIG, DEPARTED_EMPLOYEES import yaml from collections import defaultdict -from datetime import datetime +from dataclasses import dataclass, asdict +from datetime import datetime, timedelta from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import Dict, List, Optional, Set, Tuple from agent_factory import make_agent from crewai import Crew, Task @@ -119,7 +98,6 @@ logger = logging.getLogger("orgforge.eval") - with open(Path(__file__).resolve().parent.parent / "config" / "config.yaml") as f: _CFG = yaml.safe_load(f) @@ -127,2177 +105,1586 @@ EVAL_DIR = BASE / "eval" EVAL_DIR.mkdir(parents=True, exist_ok=True) - -class CausalThreadBuilder: +_SIM_START = datetime.strptime(_CFG["simulation"]["start_date"], "%Y-%m-%d") + +# ── Subsystem access model ──────────────────────────────────────────────────── +# Maps role patterns to the subsystems they have access to. +# Agents outside a subsystem cannot retrieve its artifacts. +# Extend this as new subsystems are added to the simulation. + +_ROLE_SUBSYSTEM_ACCESS: Dict[str, Set[str]] = { + "ceo": { + "slack", + "jira", + "confluence", + "zoom", + "email", + "salesforce", + "zendesk", + "datadog", + }, + "product": {"slack", "jira", "confluence", "zoom", "email"}, + "engineering_backend": {"slack", "jira", "confluence", "git", "zoom", "datadog"}, + "engineering_mobile": {"slack", "jira", "confluence", "git", "zoom", "datadog"}, + "design": {"slack", "confluence", "zoom"}, + "sales_marketing": {"slack", "salesforce", "email", "zoom", "confluence"}, + "hr_ops": {"slack", "email", "confluence", "zoom"}, + "qa_support": {"slack", "zendesk", "confluence", "email"}, + "external": set(), +} + +# Maps artifact ID prefixes / doc_types to their subsystem +_ARTIFACT_SUBSYSTEM: Dict[str, str] = { + "jira": "jira", + "confluence": "confluence", + "slack": "slack", + "pr": "git", + "email": "email", + "zd_ticket": "zendesk", + "sf_opp": "salesforce", + "sf_account": "salesforce", + "datadog": "datadog", + "zoom": "zoom", + "invoice": "email", # invoices are email artifacts for access purposes + "nps": "salesforce", # NPS lives in the CRM surface +} + +# Explicit causal link types the sim encodes — COUNTERFACTUAL scope +_EXPLICIT_CAUSAL_LINKS = { + "involves_gap", # incident ← knowledge gap + "recurrence_of", # incident ← prior unresolved incident + "spawned_doc", # confluence ← design discussion + "email_dropped", # communication failure ← routing gap + "sf_ownership_lapsed", # CRM gap ← employee departure + "zd_escalation_source", # incident ← support ticket escalation + "blocker_flagged", # blocker → delayed progress + "incident_coordination", # incident → external contact + "departure_reassignment", # departure → ticket/escalation shift + "assignment_domain_mismatch", # planning mismatch → knowledge gap → incident +} + +# Expected event pairs for SILENCE catalog: +# (trigger_event_type, expected_response_event_type, link_field) +# If trigger fired but response did not, that's a valid SILENCE question target. +_SILENCE_PAIRS: List[Tuple[str, str, str]] = [ + ("incident_opened", "postmortem_created", "jira"), + ("incident_opened", "incident_resolved", "jira"), + ("customer_escalation", "zd_ticket_opened", "email"), + ("customer_email_routed", "zd_ticket_opened", "email"), + ("inbound_external_email", "customer_email_routed", "email"), + ("design_discussion", "confluence_created", "zoom_transcript"), + ("knowledge_gap_detected", "confluence_created", "gap_domain"), + ("zd_tickets_escalated", "incident_opened", "jira"), + ("employee_departed", "sf_ownership_lapsed", "actor"), + ("employee_departed", "ticket_reassigned", "actor"), + ("pr_opened", "pr_merged", "pr"), + ("incident_opened", "zd_tickets_escalated", "jira"), + ("employee_hired", "onboarding_session", "name"), + ("employee_hired", "warmup_1on1", "name"), + ("incident_opened", "sf_deals_risk_flagged", "jira"), + ("assignment_domain_mismatch", "knowledge_gap_detected", "ticket_id"), +] + +_BROADCAST_CONFIG = { + "incident_opened": ["slack", "datadog"], + "incident_resolved": ["slack"], + "postmortem_created": ["slack", "confluence"], + "standup": ["slack"], + "pr_opened": ["git", "slack"], + "pr_merged": ["git", "slack"], + "knowledge_gap_detected": ["slack", "confluence"], +} + + +def _safe_artifact_values(artifact_ids: dict) -> Set[str]: + """Flatten artifact_ids values — some may be lists.""" + vals: Set[str] = set() + for v in (artifact_ids or {}).values(): + if isinstance(v, list): + vals.update(str(x) for x in v) + elif v: + vals.add(str(v)) + return vals + + +# ───────────────────────────────────────────────────────────────────────────── +# DATA STRUCTURES +# ───────────────────────────────────────────────────────────────────────────── + + +@dataclass +class ActorVisibilityCone: """ - Reconstructs explicit artifact graphs from the SimEvent log. - Each thread has a root artifact and a directed list of nodes, - each annotated with which actors knew about it at that timestamp. + The complete set of artifact IDs visible to a specific actor at a specific + moment, partitioned by subsystem. + + Built from SimEvents: an actor can see an artifact if they appear in + event.actors for that artifact's creation event, or if the artifact was + broadcast to a channel/tool they have access to (e.g. an incident Slack + message is visible to all engineers). """ - def __init__(self, mem: Memory): - self._mem = mem - self._events: List[SimEvent] = mem.get_event_log(from_db=True) + actor: str + role: str + as_of_time: str # ISO timestamp — the knowledge horizon + as_of_day: int + subsystem_access: Set[str] # subsystems this actor can query + visible_artifacts: Dict[str, Set[str]] # subsystem → set of artifact IDs + directly_involved: Set[str] # artifacts where actor appears in event.actors + broadcast_visible: Set[str] # artifacts visible via channel broadcast + + def all_visible(self) -> Set[str]: + all_ids: Set[str] = set() + for ids in self.visible_artifacts.values(): + all_ids.update(ids) + return all_ids + + def can_see(self, artifact_id: str, doc_type: str) -> bool: + subsystem = _ARTIFACT_SUBSYSTEM.get(doc_type, "default") + if subsystem not in self.subsystem_access: + return False + return artifact_id in self.all_visible() + + def to_dict(self) -> dict: + return { + "actor": self.actor, + "role": self.role, + "as_of_time": self.as_of_time, + "as_of_day": self.as_of_day, + "subsystem_access": sorted(self.subsystem_access), + "visible_artifacts": { + k: sorted(v) for k, v in self.visible_artifacts.items() + }, + "directly_involved": sorted(self.directly_involved), + "broadcast_visible": sorted(self.broadcast_visible), + } - def build_all(self) -> List[dict]: - threads = [] - threads.extend(self._incident_threads()) - threads.extend(self._customer_email_threads()) - threads.extend(self._hr_threads()) - threads.extend(self._dropped_email_threads()) - threads.extend(self._postmortem_threads()) - threads.extend(self._design_doc_threads()) - threads.extend(self._zendesk_threads()) - threads.extend(self._salesforce_threads()) - return threads - - def _incident_threads(self) -> List[dict]: - threads = [] - opened = [e for e in self._events if e.type == "incident_opened"] - for event in opened: - ticket_id = event.artifact_ids.get("jira") - if not ticket_id: - continue - chain = event.facts.get("causal_chain", [ticket_id]) - nodes = self._build_nodes(chain, event) - threads.append( - { - "chain_id": f"incident_{ticket_id}", - "chain_type": "incident", - "root_artifact": ticket_id, - "root_event_type": "incident_opened", - "day": event.day, - "date": event.date, - "nodes": nodes, - "terminal_artifact": chain[-1] if chain else ticket_id, - "complete": any( - e.type == "incident_resolved" - and e.artifact_ids.get("jira") == ticket_id - for e in self._events - ), - "involves_knowledge_gap": event.facts.get("involves_gap", False), - "recurrence_of": event.facts.get("recurrence_of"), - } - ) - return threads - def _customer_email_threads(self) -> List[dict]: - threads = [] - routed = [e for e in self._events if e.type == "customer_email_routed"] - for event in routed: - email_id = event.artifact_ids.get("email") - if not email_id: - continue - chain = event.facts.get("causal_chain", [email_id]) - nodes = self._build_nodes(chain, event) - threads.append( - { - "chain_id": f"customer_email_{email_id}", - "chain_type": "customer_email", - "root_artifact": email_id, - "root_event_type": "inbound_external_email", - "day": event.day, - "date": event.date, - "nodes": nodes, - "terminal_artifact": chain[-1] if chain else email_id, - "complete": len(chain) > 1, # False if only root = no action - "high_priority": event.facts.get("high_priority", False), - "source": event.facts.get("source"), - "jira_opened": any("IT-" in n or "ORG-" in n for n in chain[1:]), - } - ) - return threads - - def _dropped_email_threads(self) -> List[dict]: - """Dropped emails are their own thread type — single-node chains.""" - threads = [] - dropped = [e for e in self._events if e.type == "email_dropped"] - for event in dropped: - email_id = event.artifact_ids.get("email") - if not email_id: - continue - threads.append( - { - "chain_id": f"dropped_email_{email_id}", - "chain_type": "dropped_email", - "root_artifact": email_id, - "root_event_type": "inbound_external_email", - "day": event.day, - "date": event.date, - "nodes": [ - { - "artifact_id": email_id, - "event_type": "inbound_external_email", - "timestamp": event.timestamp, - "known_to": event.actors, - "caused": [], # intentionally empty - } - ], - "terminal_artifact": email_id, - "complete": False, # never actioned — this is the eval signal - "source": event.facts.get("source"), - "subject": event.facts.get("subject"), - } - ) - return threads +@dataclass +class CausalLink: + """ + An explicit causal relationship encoded in the simulation. + These are the only valid sources for COUNTERFACTUAL questions. + """ - def _hr_threads(self) -> List[dict]: - threads = [] - hr_emails = [e for e in self._events if e.type == "hr_outbound_email"] - hired = [e for e in self._events if e.type == "employee_hired"] + link_type: str # one of _EXPLICIT_CAUSAL_LINKS + cause_event_id: str + cause_event_type: str + effect_event_id: str + effect_event_type: str + actors: List[str] + day: int + link_field: str # the fact key that carries the link + link_value: str # the value of that field + subsystems_involved: Set[str] + counterfactual_premise: str # natural language "if X had been different" + counterfactual_outcome: str # natural language "then Y would have..." + outcome_changed: bool # does removing the cause change the effect? + + def to_dict(self) -> dict: + d = asdict(self) + d["subsystems_involved"] = sorted(self.subsystems_involved) + return d + + +@dataclass +class AbsenceRecord: + """ + A case where a trigger event fired but its expected response event did not. + The state machine is the arbiter — no inference about intent. + """ - for email_event in hr_emails: - prospect = email_event.facts.get("prospect") - embed_id = email_event.artifact_ids.get("embed_id") - hire_day = email_event.facts.get("hire_day") + trigger_event_id: str + trigger_event_type: str + expected_response_type: str + trigger_day: int + trigger_actors: List[str] + trigger_artifact_ids: Dict[str, str] + link_field: str + link_value: str + subsystem: str + expected_search_space: List[str] # artifact IDs the agent must check - # Find matching employee_hired event - hire_event = next( - (e for e in hired if prospect in e.actors and e.day == hire_day), - None, - ) + def to_dict(self) -> dict: + return asdict(self) - chain = [embed_id] - if hire_event: - hire_artifact = hire_event.artifact_ids.get( - "jira", hire_event.artifact_ids.get("slack_thread", "") - ) - if hire_artifact: - chain.append(hire_artifact) - threads.append( - { - "chain_id": f"hr_{prospect}", - "chain_type": "hr_hire", - "root_artifact": embed_id, - "root_event_type": "hr_outbound_email", - "day": email_event.day, - "date": email_event.date, - "nodes": self._build_nodes(chain, email_event), - "terminal_artifact": chain[-1], - "complete": hire_event is not None, - "prospect": prospect, - "hire_day": hire_day, - } - ) - return threads +# ───────────────────────────────────────────────────────────────────────────── +# ACTOR VISIBILITY BUILDER +# ───────────────────────────────────────────────────────────────────────────── - def _postmortem_threads(self) -> List[dict]: - """ - Builds incident → postmortem causal chains. - Each thread pairs the incident_opened event with its postmortem_created - event, enabling CAUSAL questions like 'What postmortem followed ORG-130?' - and TEMPORAL questions about whether the postmortem existed before a - recurrence. - """ - threads = [] - postmortems = [e for e in self._events if e.type == "postmortem_created"] - for event in postmortems: - ticket_id = event.artifact_ids.get("jira") - conf_id = event.artifact_ids.get("confluence") - if not ticket_id or not conf_id: - continue - chain = event.facts.get("causal_chain", [ticket_id, conf_id]) - nodes = self._build_nodes(chain, event) - threads.append( - { - "chain_id": f"postmortem_{ticket_id}", - "chain_type": "postmortem", - "root_artifact": ticket_id, - "root_event_type": "incident_opened", - "terminal_artifact": conf_id, - "day": event.day, - "date": event.date, - "nodes": nodes, - "complete": True, - "confluence_id": conf_id, - "root_cause": event.facts.get("root_cause", ""), - } - ) - return threads +class ActorVisibilityBuilder: + """ + Reconstructs the knowledge cone for every actor at every day boundary. + + Visibility rules: + 1. DIRECT: actor appears in event.actors for an artifact's creation event + 2. BROADCAST: artifact was created in a shared channel (Slack incidents, + standups, engineering-wide announcements) — visible to all actors with + that subsystem access + 3. ROLE-GATED: actor's role must include the artifact's subsystem + 4. TEMPORAL: artifact timestamp must be <= as_of_time + + Broadcast channels are inferred from event type: + - standup, incident_alert, dept_announcement → broadcast to subsystem members + - direct_message, email, zd_ticket → direct only + """ - def _zendesk_threads(self) -> List[dict]: - """ - Builds Zendesk ticket lifecycle chains: - zd_ticket_opened → (zd_tickets_escalated) → zd_tickets_resolved + # Event types whose artifacts are broadcast to all actors with subsystem access + _BROADCAST_EVENTS = { + "standup", + "incident_opened", + "incident_resolved", + "postmortem_created", + "pr_opened", + "pr_merged", + "knowledge_gap_detected", + } - Each thread records whether the ticket was escalated to an incident, - enabling CAUSAL questions like 'Which ZD tickets escalated to ORG-42?' - and ZD_RESOLUTION questions like 'Was ZD-501 resolved, and how quickly?' - """ - threads = [] - opened_events: Dict[str, SimEvent] = {} - for e in self._events: - if e.type == "zd_ticket_opened": - tid = e.facts.get("ticket_id", "") - if tid: - opened_events[tid] = e - - escalated_tickets: Dict[str, str] = {} - resolved_days: Dict[str, int] = {} - for e in self._events: - if e.type == "zd_tickets_escalated": - iid = e.facts.get("incident_id", "") - for tid in e.facts.get("ticket_ids", []): - escalated_tickets[tid] = iid - elif e.type == "zd_tickets_resolved": - for tid in e.facts.get("ticket_ids", []): - resolved_days[tid] = e.day - - for tid, open_event in opened_events.items(): - incident_id = escalated_tickets.get(tid) - resolve_day = resolved_days.get(tid) - org = open_event.facts.get("org_name", "Unknown") - - chain = [tid] - if incident_id: - chain.append(incident_id) - - nodes = self._build_nodes(chain, open_event) - - threads.append( - { - "chain_id": f"zd_{tid}", - "chain_type": "zendesk_ticket", - "root_artifact": tid, - "root_event_type": "zd_ticket_opened", - "day": open_event.day, - "date": open_event.date, - "nodes": nodes, - "terminal_artifact": chain[-1], - "complete": resolve_day is not None, - "escalated": bool(incident_id), - "incident_id": incident_id, - "org_name": org, - "subject": open_event.facts.get("subject", ""), - "open_day": open_event.day, - "resolve_day": resolve_day, - "duration_days": (resolve_day - open_event.day) - if resolve_day - else None, - } - ) - return threads + def __init__(self, mem: Memory): + self._mem = mem + self._events: List[SimEvent] = mem.get_event_log(from_db=True) + self._actor_roles: Dict[str, str] = self._infer_actor_roles() - def _salesforce_threads(self) -> List[dict]: + def _infer_actor_roles(self) -> Dict[str, str]: """ - Builds Salesforce causal chains across three event types: - crm_touchpoint — a sales email created or advanced an opportunity - sf_deals_risk_flagged — an incident caused accounts to be flagged at-risk - sf_ownership_lapsed — an employee departure orphaned accounts/opps - - Each thread type answers a different class of eval question: - crm_touchpoint → CAUSAL: 'What opportunity was created from this email?' - sf_deals_risk_flagged → SF_RISK: 'Which accounts were at risk after incident X?' - sf_ownership_lapsed → RETRIEVAL: 'Which accounts lost their owner when X left?' + Standardize role inference using config_loader ground truth. """ - threads = [] + roles: Dict[str, str] = {} - for e in self._events: - if e.type != "crm_touchpoint": - continue - opp_id = e.artifact_ids.get("sf_opp", "") - if not opp_id: - continue - threads.append( - { - "chain_id": f"sf_touchpoint_{opp_id}", - "chain_type": "sf_touchpoint", - "root_artifact": opp_id, - "root_event_type": "crm_touchpoint", - "day": e.day, - "date": e.date, - "nodes": self._build_nodes([opp_id], e), - "terminal_artifact": opp_id, - "complete": True, - "account_name": e.facts.get("account_name", ""), - "stage": e.facts.get("stage", ""), - "sender": e.facts.get("sender", ""), - "subject": e.facts.get("subject", ""), - } - ) + for dept, members in CONFIG["org_chart"].items(): + role_slug = dept.lower().replace(" ", "_") + for name in members: + roles[name] = role_slug - for e in self._events: - if e.type != "sf_deals_risk_flagged": - continue - incident_id = e.facts.get("incident_id", "") - account_names = e.facts.get("account_names", []) - if not incident_id or not account_names: - continue - threads.append( - { - "chain_id": f"sf_risk_{incident_id}", - "chain_type": "sf_risk", - "root_artifact": incident_id, - "root_event_type": "sf_deals_risk_flagged", - "day": e.day, - "date": e.date, - "nodes": self._build_nodes([incident_id], e), - "terminal_artifact": incident_id, - "complete": True, - "incident_id": incident_id, - "at_risk_accounts": account_names, - "account_count": len(account_names), - } - ) + for name, data in DEPARTED_EMPLOYEES.items(): + roles[name] = data["role"].lower().replace(" ", "_") - for e in self._events: - if e.type != "sf_ownership_lapsed": - continue - departed = e.facts.get("departed_employee", "") - accs = e.facts.get("accounts_lapsed", []) - opps = e.facts.get("opportunities_lapsed", []) - if not departed: - continue - # Chain root is the departed-employee event anchor; artifacts are - # the lapsed account/opp IDs stored as lists in artifact_ids. - chain = accs[:1] or opps[:1] or [f"lapsed_{departed}"] - threads.append( - { - "chain_id": f"sf_lapse_{departed.lower().replace(' ', '_')}", - "chain_type": "sf_ownership_lapse", - "root_artifact": chain[0], - "root_event_type": "sf_ownership_lapsed", - "day": e.day, - "date": e.date, - "nodes": self._build_nodes(chain, e), - "terminal_artifact": chain[-1], - "complete": True, - "departed_employee": departed, - "role": e.facts.get("role", ""), - "lapsed_accounts": accs, - "lapsed_opportunities": opps, - } - ) + lifecycle = CONFIG.get("org_lifecycle", {}) + for hire in lifecycle.get("scheduled_hires", []): + roles[hire["name"]] = hire["role"].lower().replace(" ", "_") + for dep in lifecycle.get("scheduled_departures", []): + roles[dep["name"]] = dep["role"].lower().replace(" ", "_") - return threads + for actor in self._all_actors(): + if actor not in roles: + roles[actor] = "external" - def _design_doc_threads(self) -> List[dict]: - threads = [] - conf_events = [ - e - for e in self._events - if e.type == "confluence_created" - and e.facts.get("type") == "design_doc" - and e.facts.get("causal_chain") # only after your fix - ] - for event in conf_events: - conf_id = event.artifact_ids.get("confluence") - if not conf_id: - continue - chain = event.facts.get("causal_chain", [conf_id]) - nodes = self._build_nodes(chain, event) - threads.append( - { - "chain_id": f"design_doc_{conf_id}", - "chain_type": "design_doc", - "root_artifact": conf_id, - "root_event_type": "confluence_created", - "day": event.day, - "date": event.date, - "nodes": nodes, - "terminal_artifact": chain[-1] if chain else conf_id, - "complete": len(chain) > 1, - "author": next(iter(event.actors), None), - "spawned_tickets": event.facts.get("spawned_tickets", []), - } - ) - return threads + return roles - def _build_nodes(self, chain: List[str], root_event: SimEvent) -> List[dict]: - """ - Build annotated node list from a causal chain. - Each node records: artifact_id, event_type, timestamp, known_to, caused. - known_to is derived from which actors appear in SimEvents up to that point. - """ - nodes = [] - cumulative_known: Set[str] = set() - - for i, artifact_id in enumerate(chain): - event = self._find_event_for_artifact(artifact_id) - if event: - cumulative_known.update(event.actors) - caused = [chain[i + 1]] if i + 1 < len(chain) else [] - nodes.append( - { - "artifact_id": artifact_id, - "event_type": event.type, - "timestamp": event.timestamp, - "day": event.day, - "known_to": list(cumulative_known), - "caused": caused, - } - ) - else: - nodes.append( - { - "artifact_id": artifact_id, - "event_type": "unknown", - "timestamp": root_event.timestamp, - "day": root_event.day, - "known_to": list(root_event.actors), - "caused": [chain[i + 1]] if i + 1 < len(chain) else [], - } - ) - return nodes + def _subsystem_access_for(self, actor: str) -> Set[str]: + role = self._actor_roles.get(actor, "external") + return set(_ROLE_SUBSYSTEM_ACCESS.get(role, _ROLE_SUBSYSTEM_ACCESS["external"])) - def _find_event_for_artifact(self, artifact_id: str) -> Optional[SimEvent]: - """Find the SimEvent most directly associated with an artifact ID.""" + def _all_actors(self) -> Set[str]: + actors: Set[str] = set() for event in self._events: - if artifact_id in event.artifact_ids.values(): - return event - if artifact_id in event.facts.get("causal_chain", []): - continue - return None - - -class EvalQuestionGenerator: - """ - Generates typed eval questions from causal threads and SimEvents. + actors.update(event.actors) + return actors + + def _artifact_subsystem(self, doc_type: str) -> str: + return _ARTIFACT_SUBSYSTEM.get(doc_type, "default") + + _BROADCAST_CONFIG = { + "incident_opened": ["slack", "datadog"], + "incident_resolved": ["slack"], + "postmortem_created": ["slack", "confluence"], + "standup": ["slack"], + "pr_opened": ["git"], + "pr_merged": ["git"], + "knowledge_gap_detected": ["slack", "confluence"], + } + + def build_all(self) -> Dict[str, List[ActorVisibilityCone]]: + all_actors = self._all_actors() + result: Dict[str, List[ActorVisibilityCone]] = {} + max_day = max((e.day for e in self._events), default=1) + + events_by_day = defaultdict(list) + for event in self._events: + events_by_day[event.day].append(event) + + for actor in all_actors: + role = self._actor_roles.get(actor, "default") + access = self._subsystem_access_for(actor) + cones: List[ActorVisibilityCone] = [] + + current_visible = defaultdict(set) + current_directly_involved = set() + current_broadcast_visible = set() + + for day in range(1, max_day + 1): + for event in events_by_day.get(day, []): + is_direct = actor in (event.actors or []) + + broadcast_channels = _BROADCAST_CONFIG.get(event.type) + is_broadcast = broadcast_channels is not None + + for doc_type, artifact_id in (event.artifact_ids or {}).items(): + if not artifact_id: + continue + subsystem = self._artifact_subsystem(doc_type) + if subsystem not in access: + continue + + if is_direct: + current_visible[subsystem].add(artifact_id) + current_directly_involved.add(artifact_id) + elif is_broadcast and any( + sub in access for sub in broadcast_channels + ): + current_visible[subsystem].add(artifact_id) + current_broadcast_visible.add(artifact_id) + + as_of_dt = _SIM_START + timedelta(days=day - 1, hours=23, minutes=59) + cones.append( + ActorVisibilityCone( + actor=actor, + role=role, + as_of_time=as_of_dt.isoformat(), + as_of_day=day, + subsystem_access=access, + visible_artifacts={ + k: set(v) for k, v in current_visible.items() + }, + directly_involved=set(current_directly_involved), + broadcast_visible=set(current_broadcast_visible), + ) + ) - For each question: - 1. Extract answer deterministically from SimEvent log (no LLM) - 2. LLM wraps the answer in natural-sounding question prose - 3. Store question_text + ground_truth + evidence_chain together + result[actor] = cones - The LLM never touches the ground_truth field — only the question wording. - """ + return result - def __init__(self, mem: Memory, worker_llm): - self._mem = mem - self._worker_llm = worker_llm - self._events: List[SimEvent] = mem.get_event_log(from_db=True) - def generate(self, threads: List[dict]) -> List[dict]: - questions = [] - questions.extend(self._retrieval_questions(threads)) - questions.extend(self._causal_questions(threads)) - questions.extend(self._temporal_questions()) - questions.extend(self._gap_detection_questions(threads)) - questions.extend(self._routing_questions(threads)) - questions.extend(self._escalation_questions()) - questions.extend(self._knowledge_gap_questions()) - questions.extend(self._postmortem_questions(threads)) - questions.extend(self._confluence_questions()) - questions.extend(self._standup_questions()) - questions.extend(self._customer_escalation_questions()) - questions.extend(self._zendesk_resolution_questions(threads)) - questions.extend(self._zd_escalation_questions(threads)) - questions.extend(self._sf_risk_questions(threads)) - questions.extend(self._sf_ownership_lapse_questions(threads)) - questions.extend(self._sf_touchpoint_questions(threads)) - questions.extend(self._datadog_alert_questions()) - questions.extend(self._nps_score_questions()) - questions.extend(self._invoice_sla_questions()) - questions.extend(self._pr_review_questions(threads)) - questions.extend(self._blocker_questions()) - questions.extend(self._vendor_routing_questions()) - questions.extend(self._design_discussion_questions()) - return questions +# ───────────────────────────────────────────────────────────────────────────── +# CAUSAL LINK INDEX +# ───────────────────────────────────────────────────────────────────────────── - def _retrieval_questions(self, threads: List[dict]) -> List[dict]: - questions = [] - incident_threads = [t for t in threads if t["chain_type"] == "incident"] - logger.info( - f"[eval] _retrieval_questions: {len(incident_threads)} incident threads available" - ) - for thread in random.sample(incident_threads, min(50, len(incident_threads))): - root_event = self._find_event_by_artifact(thread["root_artifact"]) - if not root_event: - continue - root_cause = root_event.facts.get("root_cause", "") - if not root_cause: - continue - ground_truth = { - "artifact_id": thread["root_artifact"], - "artifact_type": "jira", - "timestamp": root_event.timestamp, - "day": root_event.day, - } - evidence = [thread["root_artifact"]] - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking which artifact or ticket " - f'first documented this incident: "{root_cause[:80]}". ' - f"The question should sound like a natural analyst query. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"retrieval_{thread['root_artifact']}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": thread["chain_id"], - } - ) - return questions +class CausalLinkIndexer: + """ + Scans the SimEvent log for all explicit causal links. + Only links in _EXPLICIT_CAUSAL_LINKS are indexed — no inference. - def _causal_questions(self, threads: List[dict]) -> List[dict]: - questions = [] - multi_hop = [ - t - for t in threads - if len(t.get("nodes", [])) >= 3 - and t["chain_type"] in ("incident", "customer_email", "design_doc") - ] - logger.info( - f"[eval] _causal_questions: {len(multi_hop)} multi-hop threads available" - ) - for thread in random.sample(multi_hop, min(50, len(multi_hop))): - nodes = thread["nodes"] + Each link becomes a potential COUNTERFACTUAL question source. + The counterfactual premise and outcome are templated deterministically + from the link type and event facts; LLMs only rephrase them. + """ - if len(nodes) < 2: - continue - trigger_node = nodes[0] - result_node = nodes[1] - - ground_truth = { - "artifact_id": result_node["artifact_id"], - "event_type": result_node["event_type"], - "actors": result_node["known_to"], - "timestamp": result_node["timestamp"], - } - evidence = [trigger_node["artifact_id"], result_node["artifact_id"]] - - trigger_event = self._find_event_by_artifact(trigger_node["artifact_id"]) - trigger_desc = ( - trigger_event.summary if trigger_event else trigger_node["artifact_id"] - ) + def __init__(self, mem: Memory): + self._mem = mem + self._events: List[SimEvent] = mem.get_event_log(from_db=True) + self._event_by_id: Dict[str, SimEvent] = { + self._synthetic_event_id(e): e for e in self._events + } - q_text = self._generate_question_prose( - template=( - f"Generate a causal question asking what action or artifact " - f'was produced as a direct result of: "{trigger_desc[:100]}". ' - f"The question should probe cause-and-effect reasoning. " - f"Output only the question text." - ) + def _synthetic_event_id(self, e: SimEvent) -> str: + """Build a stable synthetic key since SimEvent has no event_id attr.""" + raw = next(iter((e.artifact_ids or {}).values()), "none") + first_artifact = raw[0] if isinstance(raw, list) else (raw or "none") + actor = (e.actors or ["unknown"])[0] + return f"evt_{e.type}_{e.day}_{first_artifact}_{actor}" + + def _subsystems_for_event(self, event: SimEvent) -> Set[str]: + subsystems: Set[str] = set() + for doc_type in event.artifact_ids or {}: + s = _ARTIFACT_SUBSYSTEM.get(doc_type, "default") + if s != "default": + subsystems.add(s) + return subsystems + + def _find_effect_event(self, link_type: str, cause: SimEvent) -> Optional[SimEvent]: + """Find the downstream event causally linked to cause.""" + if link_type == "involves_gap": + gap_domain = ( + cause.facts.get("gap_areas", [None])[0] + if cause.facts.get("gap_areas") + else None ) - if q_text: - questions.append( - { - "question_id": f"causal_{trigger_node['artifact_id']}", - "question_type": "CAUSAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) - return questions - - def _temporal_questions(self) -> List[dict]: - """ - Uses actor knowledge snapshots from day_summary SimEvents. - Asks whether a person had relevant information before a key decision. - - Two candidate pools are combined so the eval covers both true-positive - and true-negative temporal reasoning: + if not gap_domain: + return None - Pool A — incidents where involves_gap=True (knowledge was definitely - lost before the incident). These always yield had_knowledge=False - questions, exercising the "departure caused a gap" path. + for e in self._events: + if e.day < cause.day: + continue - Pool B — all other incident_opened events. For each, we check whether - ANY departure event preceded the incident day, regardless of - whether gap keywords matched. This yields a mix of - had_knowledge=True and had_knowledge=False questions, preventing - the eval from being gamed by always answering "no knowledge". + if e.type == "incident_opened" and gap_domain in e.facts.get( + "gap_areas", [] + ): + return e - The two pools are sampled independently then merged, so a run with no - involves_gap incidents still produces temporal questions from Pool B. - """ - questions = [] - departures = [e for e in self._events if e.type == "employee_departed"] - - # ── Pool A: explicit gap incidents (involves_gap=True) ───────────────── - pool_a = [ - e - for e in self._events - if e.type == "incident_opened" and e.facts.get("involves_gap") - ] + relevant_types = { + "async_question_asked", + "pr_review_comment", + "confluence_created", + "postmortem_created", + } + if e.type in relevant_types: + event_domains = ( + e.facts.get("gap_areas") or e.facts.get("domain") or [] + ) + if gap_domain in ( + event_domains + if isinstance(event_domains, list) + else [event_domains] + ): + return e + + elif link_type == "recurrence_of": + cause_artifacts = _safe_artifact_values(cause.artifact_ids) + for e in self._events: + recurrence = e.facts.get("recurrence_of") + if recurrence and recurrence in cause_artifacts: + return e + + elif link_type == "spawned_doc": + cause_artifacts = _safe_artifact_values(cause.artifact_ids) + for e in self._events: + if ( + e.type == "confluence_created" + and e.facts.get("source_discussion") in cause_artifacts + ): + return e + + elif link_type == "email_dropped": + email_id = (cause.artifact_ids or {}).get("email") + if isinstance(email_id, list): + email_id = email_id[0] if email_id else None + if not email_id: + return None - # ── Pool B: all other incidents — derive gap from departure overlap ──── - pool_a_ids = {e.artifact_ids.get("jira") for e in pool_a} - pool_b = [ - e - for e in self._events - if e.type == "incident_opened" - and not e.facts.get("involves_gap") - and e.artifact_ids.get("jira") not in pool_a_ids - ] + elif link_type == "sf_ownership_lapsed": + actor = (cause.actors or [None])[0] + if not actor: + return None + for e in self._events: + if e.type == "sf_ownership_lapsed" and actor in (e.actors or []): + return e + + elif link_type == "zd_escalation_source": + jira_id = (cause.artifact_ids or {}).get("jira") + if isinstance(jira_id, list): + jira_id = jira_id[0] if jira_id else None + if not jira_id: + return None - # Track (person, domain) pairs already emitted to prevent the eval - # collapsing into N copies of the same question when one departure - # dominates the event log (e.g. only Jordan left, owns auth-service). - seen_person_domain: Set[tuple] = set() + elif link_type == "blocker_flagged": + jira_id = cause.artifact_ids.get("jira") + return next( + ( + e + for e in self._events + if e.type == "ticket_progress" + and e.artifact_ids.get("jira") == jira_id + and e.day >= cause.day + ), + None, + ) - def _build_temporal_question(event: SimEvent) -> Optional[dict]: - ticket_id = event.artifact_ids.get("jira") - assignee = next(iter(event.actors), None) - if not ticket_id or not assignee: - return None + elif link_type == "incident_coordination": + jira_id = cause.artifact_ids.get("jira") + return next( + ( + e + for e in self._events + if e.type == "external_contact_summarized" + and e.artifact_ids.get("jira") == jira_id + ), + None, + ) - # ── Resolve gap_domain ──────────────────────────────────────────── - # Priority order: - # 1. Explicit gap_areas on the event (Pool A / flagged incidents) - # 2. root_cause keywords from the incident itself — this produces - # had_knowledge=True questions when no matching departure exists, - # which is the primary source of positive-class TEMPORAL examples. - # 3. Most-recent departure's knowledge_domains as a last resort. - gap_areas = event.facts.get("gap_areas", []) - root_cause = event.facts.get("root_cause", "") - - if gap_areas: - gap_domain = gap_areas[0] - elif root_cause: - _stop = {"the", "a", "an", "in", "of", "on", "was", "is", "due", "to"} - tokens = [ - t - for t in root_cause.split() - if t.lower() not in _stop and len(t) > 4 - ] - gap_domain = tokens[0] if tokens else "system" - else: - prior_dep = next( - ( - d - for d in sorted(departures, key=lambda d: d.day, reverse=True) - if d.day < event.day - ), - None, - ) - if prior_dep: - domains = prior_dep.facts.get("knowledge_domains", []) - gap_domain = domains[0] if domains else "undocumented system" - else: - gap_domain = "system" + elif link_type == "departure_reassignment": + departed_actor = (cause.actors or [None])[0] + return next( + ( + e + for e in self._events + if e.type == "escalation_chain" + and e.facts.get("trigger") == "post_departure_reroute" + and e.facts.get("departed") == departed_actor + ), + None, + ) - if (assignee, gap_domain) in seen_person_domain: - return None - seen_person_domain.add((assignee, gap_domain)) - - # ── Deterministic had_knowledge ─────────────────────────────────── - # True = no departure of a domain expert preceded this incident - # → the team DID have access to documentation - # False = a departure covering this domain preceded the incident - # → the knowledge was gone before the incident fired - departure = next( + elif link_type == "deal_risk_propagation": + return next( ( - d - for d in self._events - if d.type == "employee_departed" - and gap_domain in str(d.facts.get("knowledge_domains", [])) - and d.day < event.day + e + for e in self._events + if e.type == "sf_deals_risk_flagged" and e.day >= cause.day ), None, ) - had_knowledge = departure is None - ground_truth = { - "had_knowledge": had_knowledge, - "person": assignee, - "domain": gap_domain, - "incident_day": event.day, - "departure_day": departure.day if departure else None, - "reasoning": ( - f"No departure of the {gap_domain} knowledge owner had occurred before Day {event.day}." - if had_knowledge - else ( - f"{departure.actors[0]} (who owned {gap_domain}) left on " - f"Day {departure.day}, {event.day - departure.day} days before " - f"this incident." - ) + elif link_type == "onboarding_path": + new_hire = cause.facts.get("name") + return next( + ( + e + for e in self._events + if e.type == "onboarding_session" and new_hire in e.actors ), - } - evidence = [ticket_id] - if departure: - evidence.append(f"EVENT-{departure.day}-employee_departed") - - q_text = self._generate_question_prose( - template=( - f"Generate a temporal knowledge question asking whether " - f"{assignee} had access to documentation about {gap_domain} " - f"when incident {ticket_id} was opened on Day {event.day}. " - f"The question should test whether an agent can reason about " - f"what information existed at a specific point in time. " - f"Output only the question text." - ) + None, ) - if not q_text: - return None - return { - "question_id": f"temporal_{ticket_id}_{gap_domain}", - "question_type": "TEMPORAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [e for e in evidence if e], - "difficulty": "hard", - "requires_reasoning": True, - "chain_id": f"incident_{ticket_id}", - } - - for event in random.sample(pool_a, min(50, len(pool_a))): - q = _build_temporal_question(event) - if q: - questions.append(q) - - for event in random.sample(pool_b, min(50, len(pool_b))): - q = _build_temporal_question(event) - if q: - questions.append(q) - - hk_dist = {True: 0, False: 0} - for q in questions: - hk_dist[q["ground_truth"]["had_knowledge"]] += 1 - logger.info( - f"[eval] _temporal_questions: pool_a={len(pool_a)}, pool_b={len(pool_b)}, " - f"generated={len(questions)}, had_knowledge={hk_dist}" - ) - return questions + elif link_type == "assignment_domain_mismatch": + # Look for a knowledge_gap_detected on the same ticket on a later day, + # or an incident_opened whose gap_areas overlap with the mismatch domains. + ticket_id = cause.facts.get("ticket_id") + mismatch_actors = set(cause.actors or []) + for e in self._events: + if e.day < cause.day: + continue + if e.type == "knowledge_gap_detected": + if ticket_id and e.artifact_ids.get("jira") == ticket_id: + return e + # Also match on overlapping actors (the assigned engineer surfaces the gap) + if mismatch_actors & set(e.actors or []): + return e + if e.type == "incident_opened": + gap_areas = e.facts.get("gap_areas", []) + mismatch_domains = cause.facts.get("assignment_risk_domains", []) + if gap_areas and mismatch_domains and set(gap_areas) & set(mismatch_domains): + return e - def _gap_detection_questions(self, threads: List[dict]) -> List[dict]: - questions = [] - dropped = [t for t in threads if t["chain_type"] == "dropped_email"] - routed = [ - t - for t in threads - if t["chain_type"] == "customer_email" and t.get("complete") - ] - logger.info( - f"[eval] _gap_detection_questions: {len(dropped)} dropped, " - f"{len(routed)} routed-complete threads available" - ) + return None - for thread in random.sample(dropped, min(50, len(dropped))): - subject = thread.get("subject", thread["root_artifact"]) - source = thread.get("source", "unknown sender") - ground_truth = { - "was_actioned": False, - "artifact_id": thread["root_artifact"], - "source": source, - "downstream_artifacts": [], - "reason": "Email received but no Slack message or JIRA was created.", - } - q_text = self._generate_question_prose( - template=( - f"Generate a gap-detection question asking whether any action " - f"was taken after {source} sent an email with subject " - f'"{subject[:60]}". The question should require the agent to ' - f"search for downstream artifacts and notice their absence. " - f"Output only the question text." - ) + def _counterfactual_template( + self, link_type: str, cause: SimEvent, effect: SimEvent + ) -> Tuple[str, str, bool]: + """ + Returns (premise, outcome, outcome_changed) as deterministic strings. + These become the ground_truth fields — no LLM involvement here. + """ + if link_type == "involves_gap": + gap_areas = cause.facts.get("gap_areas", ["unknown domain"]) + gap_str = ", ".join(gap_areas) + actor = (cause.actors or ["the departing engineer"])[0] + jira_id = (effect.artifact_ids or {}).get("jira", "the incident") + premise = f"{actor} had fully documented {gap_str} before departing" + outcome = f"{jira_id} would have been diagnosed faster or prevented" + return premise, outcome, True + + elif link_type == "recurrence_of": + orig = effect.facts.get("recurrence_of", "the original incident") + jira_id = (effect.artifact_ids or {}).get("jira", "the recurrence") + premise = f"the postmortem for {orig} had included preventive action items" + outcome = f"{jira_id} would likely not have occurred" + return premise, outcome, True + + elif link_type == "spawned_doc": + topic = cause.facts.get("topic", "the design discussion") + conf_id = (effect.artifact_ids or {}).get( + "confluence", "the Confluence doc" ) - if q_text: - questions.append( - { - "question_id": f"gap_{thread['root_artifact']}", - "question_type": "GAP_DETECTION", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [thread["root_artifact"]], - "difficulty": "hard", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) - - for thread in random.sample(routed, min(20, len(routed))): - source = thread.get("source", "unknown") - chain = [n["artifact_id"] for n in thread["nodes"]] - ground_truth = { - "was_actioned": True, - "artifact_id": thread["root_artifact"], - "source": source, - "downstream_artifacts": chain[1:], - "jira_opened": thread.get("jira_opened", False), - } - q_text = self._generate_question_prose( - template=( - f"Generate a gap-detection question asking whether any action " - f"was taken after {source} sent a complaint email. " - f"Output only the question text." - ) + premise = f"the discussion about '{topic}' had not been documented" + outcome = f"{conf_id} would not exist and related decisions would remain undocumented" + return premise, outcome, True + + elif link_type == "email_dropped": + sender = cause.facts.get("sender", "the customer") + premise = f"the email from {sender} had been routed correctly" + outcome = "a support ticket would have been opened and the issue tracked" + return premise, outcome, True + + elif link_type == "sf_ownership_lapsed": + actor = (cause.actors or ["the departed employee"])[0] + accounts = effect.facts.get("lapsed_accounts", []) + acc_str = ", ".join(accounts[:3]) if accounts else "affected accounts" + premise = ( + f"{actor}'s Salesforce accounts had been reassigned before departure" + ) + outcome = ( + f"{acc_str} would not have lost ownership and pipeline would be intact" + ) + return premise, outcome, True + + elif link_type == "zd_escalation_source": + ticket_ids = effect.facts.get("ticket_ids", ["the support ticket"]) + tickets_str = ", ".join(ticket_ids[:3]) + premise = f"{tickets_str} had been resolved at the support level" + outcome = "the incident escalation would not have occurred" + return premise, outcome, True + + if link_type == "blocker_flagged": + reason = cause.facts.get("blocker_reason", "a technical blocker") + jira_id = effect.artifact_ids.get("jira", "the ticket") + return ( + f"the blocker regarding '{reason}' had been resolved immediately", + f"work on {jira_id} would have progressed without delay", + True, ) - if q_text: - questions.append( - { - "question_id": f"gap_{thread['root_artifact']}_actioned", - "question_type": "GAP_DETECTION", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": chain, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) - return questions - - def _routing_questions(self, threads: List[dict]) -> List[dict]: - questions = [] - customer_threads = [ - t for t in threads if t["chain_type"] in ("customer_email", "dropped_email") - ] - logger.info( - f"[eval] _routing_questions: {len(customer_threads)} customer threads available" - ) - for thread in random.sample(customer_threads, min(25, len(customer_threads))): - root_node = thread["nodes"][0] if thread["nodes"] else None - if not root_node: - continue - # Build an exclusion set: the external source (may be stored as a - # company name like "RunnersPeak Corp", not an email address) plus - # anything that looks like an email address. Only proper internal - # actor names (single/dual words, no @ or Corp/LLC/Inc suffix) pass. - external_source = thread.get("source", "") - _external_markers = {"corp", "llc", "inc", "ltd", "gmbh", "co."} - - def _is_internal(actor: str) -> bool: - if "@" in actor: - return False - if actor == external_source: - return False - # Catch "RunnersPeak Corp", "MarathonTech LLC", etc. - if any(part.lower() in _external_markers for part in actor.split()): - return False - return True - - first_internal = next( - (a for a in root_node.get("known_to", []) if _is_internal(a)), - None, + elif link_type == "incident_coordination": + contact = effect.facts.get("external_party", "the external contact") + jira_id = cause.artifact_ids.get("jira", "the incident") + return ( + f"the incident {jira_id} had not occurred", + f"the team would not have needed to coordinate with {contact}", + True, ) - if not first_internal: - logger.warning( - f"[eval] _routing_questions: no internal actor found in " - f"known_to={root_node.get('known_to')} for thread " - f"{thread['chain_id']} — skipping" - ) - continue - ground_truth = { - "first_recipient": first_internal, - "artifact_id": thread["root_artifact"], - "timestamp": root_node["timestamp"], - "was_escalated": len(thread["nodes"]) > 1, - } - source = thread.get("source", "a customer") - q_text = self._generate_question_prose( - template=( - f"Generate a routing question asking which internal employee " - f"first received the email from {source}. The question should " - f"test whether an agent can trace the initial delivery path. " - f"Output only the question text." - ) + elif link_type == "departure_reassignment": + actor = (cause.actors or ["the employee"])[0] + return ( + f"{actor} had not departed the company", + "their active tickets and escalation responsibilities would not have been reassigned", + True, ) - if q_text: - questions.append( - { - "question_id": f"routing_{thread['root_artifact']}", - "question_type": "ROUTING", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [thread["root_artifact"]], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": thread["chain_id"], - } - ) - return questions - # ── ESCALATION — "Who was pulled into the escalation for ticket X?" ─────────── - def _escalation_questions(self) -> List[dict]: - questions = [] - esc_events = [e for e in self._events if e.type == "escalation_chain"] - logger.info( - f"[eval] _escalation_questions: {len(esc_events)} escalation_chain events available" - ) + elif link_type == "deal_risk_propagation": + jira_id = cause.artifact_ids.get("jira", "the incident") + return ( + f"the incident {jira_id} had not occurred", + "the associated Salesforce deals would not have been flagged as at-risk", + True, + ) - for event in random.sample(esc_events, min(25, len(esc_events))): - ticket_id = event.artifact_ids.get("jira", "") - actors = event.facts.get("escalation_actors", []) - narrative = event.facts.get("escalation_narrative", "") - if not ticket_id or not actors: - continue + elif link_type == "onboarding_path": + name = cause.facts.get("name", "the new hire") + return ( + f"{name} had not been hired on Day {cause.day}", + "the onboarding sessions and warmup meetings for them would not have taken place", + True, + ) - ground_truth = { - "ticket_id": ticket_id, - "escalation_actors": actors, - "hops": len(actors) - 1, - "narrative": narrative, - } - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking who was involved in the escalation " - f"chain for incident {ticket_id}. The question should test " - f"whether an agent can trace the escalation path. " - f"Output only the question text." - ) + elif link_type == "assignment_domain_mismatch": + actors = cause.actors or ["the engineer"] + ticket_id = cause.facts.get("ticket_id", "the ticket") + coverage = cause.facts.get("documentation_coverage") + coverage_str = ( + f" (documentation coverage: {int(coverage * 100)}%)" if coverage else "" + ) + return ( + f"{actors[0]} had been assigned to {ticket_id} with matching domain expertise", + f"the knowledge gap{coverage_str} would likely not have been surfaced and the associated incident risk reduced", + True, ) - if q_text: - questions.append( - { - "question_id": f"escalation_{ticket_id}", - "question_type": "ESCALATION", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [ticket_id], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"incident_{ticket_id}", - } - ) - return questions - # ── KNOWLEDGE GAP — "What domain was undocumented when X fired?" ────────────── - def _knowledge_gap_questions(self) -> List[dict]: - questions = [] - gap_events = [ - e - for e in self._events - if e.type == "knowledge_gap_detected" - and e.facts.get("trigger") != "centrality_vacuum" - ] - logger.info( - f"[eval] _knowledge_gap_questions: {len(gap_events)} knowledge_gap_detected events available" + return ( + "the causal condition had been different", + "the outcome would have changed", + True, ) - for event in random.sample(gap_events, min(25, len(gap_events))): - ticket_id = event.artifact_ids.get("jira", "") - gap_areas = event.facts.get("gap_areas", []) - actors = event.actors - if not ticket_id or not gap_areas: + def build(self) -> List[CausalLink]: + links: List[CausalLink] = [] + + for link_type in _EXPLICIT_CAUSAL_LINKS: + if link_type == "involves_gap": + cause_events = [ + e for e in self._events if e.type == "knowledge_gap_detected" + ] + elif link_type == "recurrence_of": + cause_events = [ + e for e in self._events if e.type == "incident_resolved" + ] + elif link_type == "spawned_doc": + cause_events = [ + e + for e in self._events + if e.type == "design_discussion" and e.facts.get("spawned_doc") + ] + elif link_type == "email_dropped": + cause_events = [ + e for e in self._events if e.type == "inbound_external_email" + ] + elif link_type == "sf_ownership_lapsed": + cause_events = [ + e for e in self._events if e.type == "employee_departed" + ] + elif link_type == "zd_escalation_source": + cause_events = [e for e in self._events if e.type == "incident_opened"] + elif link_type == "blocker_flagged": + cause_events = [e for e in self._events if e.type == "blocker_flagged"] + elif link_type == "incident_coordination": + cause_events = [e for e in self._events if e.type == "incident_opened"] + elif link_type == "departure_reassignment": + cause_events = [ + e for e in self._events if e.type == "employee_departed" + ] + elif link_type == "assignment_domain_mismatch": + cause_events = [ + e for e in self._events if e.type == "assignment_domain_mismatch" + ] + else: continue - ground_truth = { - "ticket_id": ticket_id, - "gap_areas": gap_areas, - "detected_by": actors, - "artifact_id": ticket_id, - } - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking which knowledge domain or system " - f"was found to be undocumented during incident {ticket_id}. " - f"The question should test whether an agent can identify " - f"documentation gaps from incident records. " - f"Output only the question text." + for cause in cause_events: + effect = self._find_effect_event(link_type, cause) + if not effect: + continue + + premise, outcome, changed = self._counterfactual_template( + link_type, cause, effect ) - ) - if q_text: - questions.append( - { - "question_id": f"gap_{ticket_id}", - "question_type": "KNOWLEDGE_GAP", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [ticket_id], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"incident_{ticket_id}", - } + + subsystems = self._subsystems_for_event( + cause + ) | self._subsystems_for_event(effect) + + link_field = { + "involves_gap": "gap_areas", + "recurrence_of": "recurrence_of", + "spawned_doc": "spawned_doc", + "email_dropped": "email", + "sf_ownership_lapsed": "actor", + "zd_escalation_source": "jira", + "blocker_flagged": "jira", + "incident_coordination": "jira", + "departure_reassignment": "actor", + "assignment_domain_mismatch": "ticket_id", + }.get(link_type, "") + + link_value = str( + cause.facts.get(link_field, "") + or (cause.artifact_ids or {}).get(link_field, "") + or (cause.actors or [""])[0] ) - logger.info( - f"[eval] _knowledge_gap_questions: {len(gap_events)} gaps available, " - f"generated={len(questions)}" - ) - return questions - # ── POSTMORTEM — "What postmortem documented incident X?" ───────────────── + links.append( + CausalLink( + link_type=link_type, + cause_event_id=self._synthetic_event_id(cause), + cause_event_type=cause.type, + effect_event_id=self._synthetic_event_id(effect), + effect_event_type=effect.type, + actors=list(set((cause.actors or []) + (effect.actors or []))), + day=cause.day, + link_field=link_field, + link_value=link_value, + subsystems_involved=subsystems, + counterfactual_premise=premise, + counterfactual_outcome=outcome, + outcome_changed=changed, + ) + ) - def _postmortem_questions(self, threads: List[dict]) -> List[dict]: - """ - CAUSAL questions that traverse the incident → postmortem chain. - Tests whether an agent can identify that a postmortem was written and - find the confluence artifact that contains it. - """ - questions = [] - pm_threads = [t for t in threads if t["chain_type"] == "postmortem"] - logger.info( - f"[eval] _postmortem_questions: {len(pm_threads)} postmortem threads available" - ) + logger.info(f"[causal_index] {len(links)} explicit causal links indexed") + return links - for thread in random.sample(pm_threads, min(25, len(pm_threads))): - ticket_id = thread["root_artifact"] - conf_id = thread["confluence_id"] - root_cause = thread.get("root_cause", "") - nodes = thread["nodes"] - if len(nodes) < 2: - continue - ground_truth = { - "incident_id": ticket_id, - "postmortem_confluence_id": conf_id, - "postmortem_day": thread["day"], - "root_cause": root_cause, - } - evidence = [ticket_id, conf_id] - - q_text = self._generate_question_prose( - template=( - f"Generate a causal question asking which Confluence document " - f"captured the postmortem for incident {ticket_id}. The question " - f"should test whether an agent can trace from a Jira incident to " - f"its postmortem artifact. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"postmortem_{ticket_id}", - "question_type": "CAUSAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": f"postmortem_{ticket_id}", - } - ) - return questions +# ───────────────────────────────────────────────────────────────────────────── +# ABSENCE CATALOG +# ───────────────────────────────────────────────────────────────────────────── - # ── CONFLUENCE — "Which doc first covered topic X?" ─────────────────────── - def _confluence_questions(self) -> List[dict]: - """ - RETRIEVAL questions over the confluence_created event log. - Tests whether an agent can locate the right Confluence page for a topic. - Queries the events collection directly (not artifacts) so ground truth - is deterministic from the SimEvent, not from embedding similarity. - """ - questions = [] - conf_events = [e for e in self._events if e.type == "confluence_created"] - logger.info( - f"[eval] _confluence_questions: {len(conf_events)} confluence_created events available" - ) +class AbsenceCatalogBuilder: + """ + Builds the catalog of expected-but-absent artifact pairs. - for event in random.sample(conf_events, min(50, len(conf_events))): - conf_id = event.artifact_ids.get("confluence") or event.artifact_ids.get( - "page_id" - ) - author = next(iter(event.actors), None) - topic = ( - event.facts.get("topic") - or event.facts.get("title") - or event.facts.get("summary", "") - ) - if not conf_id or not topic: - continue + For each pair in _SILENCE_PAIRS, scans the event log for trigger events + that have no matching response event. The state machine is the arbiter: + if no response event fired, the absence is ground truth. - ground_truth = { - "confluence_id": conf_id, - "author": author, - "day": event.day, - "topic": topic, - } - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking which Confluence page " - f'covers the topic: "{topic[:80]}". The question should sound ' - f"like an engineer searching internal documentation. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"confluence_{conf_id}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [conf_id], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"confluence_{conf_id}", - } - ) - return questions + Also derives expected_search_space: the set of artifact IDs the agent + must check before concluding absence. This is what separates a well-reasoned + "no" from a lucky guess. + """ - # ── STANDUP — "What was X working on during Day N standup?" ────────────── + def __init__(self, mem: Memory): + self._mem = mem + self._events: List[SimEvent] = mem.get_event_log(from_db=True) + + @staticmethod + def _synthetic_event_id(e: SimEvent) -> str: + first_artifact = next(iter((e.artifact_ids or {}).values()), "none") + actor = (e.actors or ["unknown"])[0] + return f"evt_{e.type}_{e.day}_{first_artifact}_{actor}" + + def _match_key(self, event: SimEvent, link_field: str) -> Optional[str]: + """Extract the value that links a trigger to its expected response.""" + val = (event.artifact_ids or {}).get(link_field) + if val: + return val + val = event.facts.get(link_field) + if val: + return str(val) + if link_field == "actor" and event.actors: + return event.actors[0] + return None - def _standup_questions(self) -> List[dict]: + def _expected_search_space( + self, trigger: SimEvent, expected_response_type: str + ) -> List[str]: """ - RETRIEVAL questions over standup SimEvents. - Tests whether an agent can identify what a specific engineer reported - at standup on a given day — a lightweight memory retrieval task. + Derive the artifact IDs the agent should check to confirm absence. + These are artifacts that WOULD contain the response if it had occurred. """ - questions = [] - standup_events = [e for e in self._events if e.type == "standup"] - logger.info( - f"[eval] _standup_questions: {len(standup_events)} standup events available" - ) + search_space: List[str] = [] + + # Always include trigger artifacts as starting points + for artifact_id in (trigger.artifact_ids or {}).values(): + if artifact_id: + search_space.append(artifact_id) + + if expected_response_type == "postmortem_created": + jira_id = (trigger.artifact_ids or {}).get("jira", "") + if jira_id: + search_space.append(f"confluence/postmortems/{jira_id}") + search_space.append("slack/channels/incidents") + + elif expected_response_type == "incident_resolved": + jira_id = (trigger.artifact_ids or {}).get("jira", "") + if jira_id: + search_space.append(jira_id) + search_space.append("jira/incidents") + + elif expected_response_type == "zd_ticket_opened": + email_id = (trigger.artifact_ids or {}).get("email", "") + if email_id: + search_space.append("zendesk/tickets") + search_space.append(email_id) + + elif expected_response_type == "customer_email_routed": + search_space.append("slack/channels/support") + search_space.append("zendesk/queue") + + elif expected_response_type == "confluence_created": + zoom_id = (trigger.artifact_ids or {}).get("zoom_transcript", "") + if zoom_id: + search_space.append(zoom_id) + search_space.append("confluence/design-docs") + search_space.append("confluence/decisions") + + elif expected_response_type == "sf_ownership_lapsed": + actor = (trigger.actors or [""])[0] + if actor: + search_space.append(f"salesforce/accounts/{actor}") + search_space.append("salesforce/ownership-log") + + elif expected_response_type == "ticket_reassigned": + actor = (trigger.actors or [""])[0] + if actor: + search_space.append("jira/reassignments") + search_space.append("slack/channels/engineering") + + elif expected_response_type == "pr_merged": + pr_id = (trigger.artifact_ids or {}).get("pr", "") + if pr_id: + search_space.append(pr_id) + search_space.append("git/merged-prs") + + elif expected_response_type == "zd_tickets_escalated": + jira_id = (trigger.artifact_ids or {}).get("jira", "") + if jira_id: + search_space.append(jira_id) + search_space.append("zendesk/escalations") + + elif expected_response_type == "onboarding_session": + name = trigger.facts.get("name", "") + search_space.append("slack/channels/general") + search_space.append(f"confluence/onboarding/{name}") + + elif expected_response_type == "warmup_1on1": + name = trigger.facts.get("name", "") + search_space.append("slack/channels/engineering") + search_space.append("zoom/transcripts") + + elif expected_response_type == "sf_deals_risk_flagged": + jira_id = trigger.artifact_ids.get("jira", "") + search_space.append("salesforce/opportunities") + if jira_id: + search_space.append(jira_id) + + elif expected_response_type == "knowledge_gap_detected": + # Silence: assignment_domain_mismatch fired but no gap was ever formally detected + ticket_id = trigger.facts.get("ticket_id", "") + if ticket_id: + search_space.append(f"jira/{ticket_id}") + search_space.append("slack/channels/engineering") + search_space.append("confluence/knowledge-gaps") + + return list(dict.fromkeys(search_space)) # dedupe, preserve order + + def build(self) -> List[AbsenceRecord]: + records: List[AbsenceRecord] = [] + + for trigger_type, response_type, link_field in _SILENCE_PAIRS: + trigger_events = [e for e in self._events if e.type == trigger_type] + + for trigger in trigger_events: + trigger_artifacts = _safe_artifact_values(trigger.artifact_ids) + link_key = self._match_key(trigger, link_field) + + if response_type == "confluence_created" and ( + trigger.facts.get("spawned_doc") + or "confluence" in (trigger.artifact_ids or {}) + ): + continue - for event in random.sample(standup_events, min(25, len(standup_events))): - if not event.actors: - continue - # Pick one actor from the standup to ask about - actor = random.choice(event.actors) - slack_thread = event.artifact_ids.get("slack_thread", "") - day = event.day - - ground_truth = { - "actor": actor, - "day": day, - "artifact_id": slack_thread, - "all_participants": event.actors, - "summary": event.summary, - } - evidence = [slack_thread] if slack_thread else [] - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking what {actor} reported " - f"during the standup on Day {day}. The question should test " - f"whether an agent can locate a specific person's standup " - f"update from a Slack thread. " - f"Output only the question text." + response_found = False + for e in self._events: + if e.type != response_type or e.day < trigger.day: + continue + + if link_key and ( + self._match_key(e, link_field) == link_key + or link_key in str(e.artifact_ids) + or link_key in str(e.facts) + ): + response_found = True + break + + response_artifacts = _safe_artifact_values(e.artifact_ids) + if trigger_artifacts & response_artifacts: + response_found = True + break + + if response_found: + continue + + subsystem = _ARTIFACT_SUBSYSTEM.get( + list((trigger.artifact_ids or {}).keys() or [""])[0], "default" ) - ) - if q_text: - questions.append( - { - "question_id": f"standup_{actor}_day{day}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"standup_day{day}", - } + search_space = self._expected_search_space(trigger, response_type) + + records.append( + AbsenceRecord( + trigger_event_id=self._synthetic_event_id(trigger), + trigger_event_type=trigger_type, + expected_response_type=response_type, + trigger_day=trigger.day, + trigger_actors=trigger.actors or [], + trigger_artifact_ids=dict(trigger.artifact_ids or {}), + link_field=link_field, + link_value=link_key or "N/A", + subsystem=subsystem, + expected_search_space=search_space, + ) ) - return questions - # ── CUSTOMER ESCALATION — "How was the escalation from X resolved?" ─────── + logger.info(f"[absence_catalog] {len(records)} absence records cataloged") + return records - def _customer_escalation_questions(self) -> List[dict]: - """ - CAUSAL questions over customer_escalation and customer_email_routed events. - Tests whether an agent can trace what action was taken after a customer - escalation arrived — who handled it, and what artifacts were created. - Uses customer_email_routed as the primary source since it's confirmed - present in the event log; customer_escalation events are included when found. - """ - questions = [] - esc_events = [ - e - for e in self._events - if e.type in ("customer_escalation", "customer_email_routed") - and e.facts.get("source") # need a named source for the question - ] - logger.info( - f"[eval] _customer_escalation_questions: {len(esc_events)} " - f"customer escalation/routed events available" - ) - for event in random.sample(esc_events, min(25, len(esc_events))): - source = event.facts.get("source", "a customer") - ticket_id = event.artifact_ids.get("jira") or event.artifact_ids.get( - "email", "" - ) - handler = next(iter(event.actors), None) - if not handler: - continue +# ───────────────────────────────────────────────────────────────────────────── +# QUESTION GENERATOR +# ───────────────────────────────────────────────────────────────────────────── - causal_chain = event.facts.get( - "causal_chain", [ticket_id] if ticket_id else [] - ) - downstream = [a for a in causal_chain if a != ticket_id] - - ground_truth = { - "source": source, - "first_handler": handler, - "artifact_id": ticket_id, - "downstream_artifacts": downstream, - "day": event.day, - "was_escalated": bool(downstream), - } - evidence = [ticket_id] if ticket_id else [] - - q_text = self._generate_question_prose( - template=( - f"Generate a causal question asking who handled the escalation " - f"from {source} and what action was taken. The question should " - f"require tracing from the initial contact through to any " - f"tickets or responses created. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"escalation_handling_{event.artifact_ids.get('email') or ticket_id}", - "question_type": "CAUSAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [e for e in evidence if e], - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": f"customer_{source}", - } - ) - return questions - # ── ZD_RESOLUTION — "Was ZD ticket X resolved and how long did it take?" ──── +class EvalQuestionGenerator: + """ + Generates PERSPECTIVE, COUNTERFACTUAL, and SILENCE questions. - def _zendesk_resolution_questions(self, threads: List[dict]) -> List[dict]: - """ - ZD_RESOLUTION questions over zendesk_ticket threads. - Tests whether an agent can determine ticket resolution status and SLA - duration from the Zendesk ticket lifecycle event chain. + Ground truth is always derived deterministically from the three indexes. + LLMs only write question prose, and every generated question is validated + against a structured rubric before inclusion. - Maps to existing RETRIEVAL scorer — ground truth is a boolean + day count, - not a narrative, so exact-match is appropriate. - """ - questions = [] - zd_threads = [t for t in threads if t["chain_type"] == "zendesk_ticket"] - logger.info( - f"[eval] _zendesk_resolution_questions: {len(zd_threads)} ZD threads available" - ) + Question prose validation checks: + - Ends with a question mark + - Does not contain the ground truth answer verbatim + - Does not name an artifact ID directly (keeps questions natural-language) + - Is unambiguous — references the actor/day/subsystem constraint explicitly + """ - for thread in random.sample(zd_threads, min(30, len(zd_threads))): - tid = thread["root_artifact"] - org = thread.get("org_name", "a customer") - subject = thread.get("subject", tid) - - ground_truth = { - "ticket_id": tid, - "artifact_id": tid, - "org_name": org, - "resolved": thread["complete"], - "duration_days": thread.get("duration_days"), - "escalated": thread["escalated"], - "incident_id": thread.get("incident_id"), - } - evidence = [tid] - if thread.get("incident_id"): - evidence.append(thread["incident_id"]) - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking whether Zendesk ticket " - f"{tid} from {org} was resolved, and if so, how many days it " - f"took. The question should test whether an agent can trace the " - f"full ticket lifecycle. " - f"Output only the question text." - ) - ) - if q_text: - difficulty = "medium" if thread["escalated"] else "easy" - questions.append( - { - "question_id": f"zd_resolution_{tid}", - "question_type": "ZD_RESOLUTION", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": difficulty, - "requires_reasoning": thread["escalated"], - "chain_id": thread["chain_id"], - } - ) - return questions + MAX_PERSPECTIVE = 40 + MAX_COUNTERFACTUAL = 30 + MAX_SILENCE = 30 + + def __init__( + self, + mem: Memory, + worker_llm, + visibility_map: Dict[str, List[ActorVisibilityCone]], + causal_links: List[CausalLink], + absence_catalog: List[AbsenceRecord], + ): + self._mem = mem + self._worker_llm = worker_llm + self._visibility_map = visibility_map + self._causal_links = causal_links + self._absence_catalog = absence_catalog + self._events: List[SimEvent] = mem.get_event_log(from_db=True) - # ── ZD_ESCALATION — "Which tickets escalated to incident X?" ───────────── + @staticmethod + def _synthetic_event_id(e: SimEvent) -> str: + """Build a stable synthetic key since SimEvent has no event_id attr.""" + raw = next(iter((e.artifact_ids or {}).values()), "none") + first_artifact = raw[0] if isinstance(raw, list) else (raw or "none") + actor = (e.actors or ["unknown"])[0] + return f"evt_{e.type}_{e.day}_{first_artifact}_{actor}" - def _zd_escalation_questions(self, threads: List[dict]) -> List[dict]: - """ - CAUSAL questions over zendesk_ticket threads where escalated=True. - Tests cross-system reasoning: given a Jira incident, can the agent - identify which ZD tickets triggered or were linked to it? - """ - questions = [] - escalated = [ - t for t in threads if t["chain_type"] == "zendesk_ticket" and t["escalated"] - ] - logger.info( - f"[eval] _zd_escalation_questions: {len(escalated)} escalated ZD threads available" - ) + def generate(self) -> List[dict]: + questions: List[dict] = [] - # Group by incident so we can ask "all tickets for incident X" questions - by_incident: Dict[str, List[dict]] = {} - for t in escalated: - iid = t.get("incident_id", "") - if iid: - by_incident.setdefault(iid, []).append(t) - - for incident_id, tickets in random.sample( - list(by_incident.items()), min(20, len(by_incident)) - ): - ticket_ids = [t["root_artifact"] for t in tickets] - orgs = list({t.get("org_name", "") for t in tickets if t.get("org_name")}) - - ground_truth = { - "incident_id": incident_id, - "artifact_id": incident_id, - "ticket_ids": ticket_ids, - "ticket_count": len(ticket_ids), - "affected_orgs": orgs, - "event_type": "zd_tickets_escalated", - } - evidence = [incident_id] + ticket_ids - - q_text = self._generate_question_prose( - template=( - f"Generate a causal question asking which Zendesk support " - f"tickets were escalated as a result of incident {incident_id}. " - f"The question should require the agent to trace from the " - f"incident back to the affected customer tickets. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"zd_escalation_{incident_id}", - "question_type": "CAUSAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": f"zd_escalation_{incident_id}", - } - ) - return questions + logger.info("[eval] Generating PERSPECTIVE questions...") + questions.extend(self._perspective_questions()) - # ── SF_RISK — "Which accounts were flagged at-risk after incident X?" ───── + logger.info("[eval] Generating COUNTERFACTUAL questions...") + questions.extend(self._counterfactual_questions()) - def _sf_risk_questions(self, threads: List[dict]) -> List[dict]: - """ - SF_RISK questions over sf_risk threads (sf_deals_risk_flagged events). - New question type: distinct from CAUSAL because it specifically tests - cross-system awareness — the agent must connect an engineering incident - to its commercial impact in Salesforce. - """ - questions = [] - risk_threads = [t for t in threads if t["chain_type"] == "sf_risk"] - logger.info( - f"[eval] _sf_risk_questions: {len(risk_threads)} SF risk threads available" - ) + logger.info("[eval] Generating SILENCE questions...") + questions.extend(self._silence_questions()) - for thread in random.sample(risk_threads, min(25, len(risk_threads))): - incident_id = thread["incident_id"] - accounts = thread["at_risk_accounts"] - if not accounts: - continue + # Shuffle so question types are interleaved in the output + random.shuffle(questions) - ground_truth = { - "incident_id": incident_id, - "artifact_id": incident_id, - "at_risk_accounts": accounts, - "account_count": len(accounts), - "day": thread["day"], - } - evidence = [incident_id] - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking which Salesforce customer accounts " - f"were flagged as at-risk following incident {incident_id}. " - f"The question should require the agent to connect an engineering " - f"incident to its downstream commercial impact. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"sf_risk_{incident_id}", - "question_type": "SF_RISK", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "hard", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) + logger.info(f"[eval] {len(questions)} total questions generated") return questions - def _sf_ownership_lapse_questions(self, threads: List[dict]) -> List[dict]: - """ - RETRIEVAL questions over sf_ownership_lapse threads. - Uses existing RETRIEVAL scorer — ground truth is a deterministic account - list derived from sf_ownership_lapsed SimEvents. - Tests cross-domain reasoning: departure event → CRM consequence. - """ - questions = [] - lapse_threads = [t for t in threads if t["chain_type"] == "sf_ownership_lapse"] - logger.info( - f"[eval] _sf_ownership_lapse_questions: {len(lapse_threads)} lapse threads available" - ) + # ── TRACK 1: PERSPECTIVE ───────────────────────────────────────────────── - for thread in lapse_threads: # typically low-volume; use all - departed = thread["departed_employee"] - accs = thread["lapsed_accounts"] - opps = thread["lapsed_opportunities"] - if not accs and not opps: - continue + def _perspective_questions(self) -> List[dict]: + questions: List[dict] = [] - ground_truth = { - "departed_employee": departed, - "role": thread.get("role", ""), - "artifact_id": thread["root_artifact"], - "lapsed_accounts": accs, - "lapsed_opportunities": opps, - "day": thread["day"], - } - evidence = accs[:3] + opps[:2] # cap evidence list length - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking which Salesforce accounts " - f"or open opportunities were left without an owner after " - f"{departed} departed. The question should test whether an agent " - f"can trace the CRM impact of an employee departure. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"sf_lapse_{departed.lower().replace(' ', '_')}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [e for e in evidence if e], - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) - return questions + internal_actors = [ + actor + for actor in self._visibility_map.keys() + if self._visibility_map[actor][0].role != "external" + ] - def _sf_touchpoint_questions(self, threads: List[dict]) -> List[dict]: - """ - CAUSAL questions over sf_touchpoint threads (crm_touchpoint events). - Tests whether an agent can link an outbound sales email to the SF - opportunity it created or advanced. - """ - questions = [] - tp_threads = [t for t in threads if t["chain_type"] == "sf_touchpoint"] - logger.info( - f"[eval] _sf_touchpoint_questions: {len(tp_threads)} SF touchpoint threads available" + # Find events that involve information asymmetry — where the actor was + # NOT in event.actors but the event affected them (e.g. a customer + # escalation that went to sales but not engineering) + asymmetry_events = [ + ev for ev in self._find_asymmetry_events() if ev[0] in internal_actors + ] + + candidates = random.sample( + asymmetry_events, min(self.MAX_PERSPECTIVE, len(asymmetry_events)) ) - for thread in random.sample(tp_threads, min(25, len(tp_threads))): - opp_id = thread["root_artifact"] - sender = thread.get("sender", "a sales rep") - account = thread.get("account_name", "a customer") - stage = thread.get("stage", "") - subject = thread.get("subject", "") - - ground_truth = { - "opportunity_id": opp_id, - "artifact_id": opp_id, - "account_name": account, - "stage": stage, - "sender": sender, - "event_type": "crm_touchpoint", - } - evidence = [opp_id] - - q_text = self._generate_question_prose( - template=( - f"Generate a causal question asking which Salesforce opportunity " - f"was created or advanced when {sender} sent an outbound email " - f'with subject "{subject[:60]}" to {account}. The question ' - f"should test whether an agent can trace from an outbound email " - f"to its CRM outcome. " - f"Output only the question text." - ) + for actor, cone, event, info_available, cross_subsystem in candidates: + question = self._build_perspective_question( + actor, cone, event, info_available, cross_subsystem ) - if q_text: - questions.append( - { - "question_id": f"sf_touchpoint_{opp_id}", - "question_type": "CAUSAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": thread["chain_id"], - } - ) + if question: + questions.append(question) + + logger.info(f"[eval] {len(questions)} PERSPECTIVE questions built") return questions - def _datadog_alert_questions(self) -> List[dict]: + def _find_asymmetry_events(self) -> List[Tuple]: """ - DATADOG_ALERT questions inferred from incident_opened SimEvents. - - Datadog alert records are generated post-sim from incident events - (see post_sim_artifacts.py DatadogWriter). Because no dedicated - 'datadog_alert_fired' SimEvent is emitted during the sim, we derive - ground truth directly from incident_opened facts: the monitor name - and the incident it corresponds to are deterministically linked. - - The agent must retrieve the right incident artifact and demonstrate - awareness that a Datadog alert is the upstream signal for the incident. - Maps to RETRIEVAL scorer — ground truth is the incident artifact_id. + Find (actor, cone, event, info_available, is_cross_subsystem) tuples + where an actor had partial or no visibility into a significant event. + + Focuses on events with real decision-making consequence: + - Customer escalations visible to support but not engineering + - Incidents visible to engineering but not sales + - Design decisions visible to eng but not the broader org + - HR/departure events with asymmetric visibility + - CRM risk flags invisible to non-sales actors """ - questions = [] - incident_events = [e for e in self._events if e.type == "incident_opened"] - logger.info( - f"[eval] _datadog_alert_questions: {len(incident_events)} incidents available" - ) + results = [] + significant_types = { + "incident_opened", + "customer_escalation", + "sf_deals_risk_flagged", + "knowledge_gap_detected", + "employee_departed", + "design_discussion", + "customer_email_routed", + "zd_tickets_escalated", + "sf_ownership_lapsed", + "postmortem_created", + "inbound_external_email", + "assignment_domain_mismatch", + } - for event in random.sample(incident_events, min(20, len(incident_events))): - ticket_id = event.artifact_ids.get("jira", "") - root_cause = event.facts.get("root_cause", "") - if not ticket_id or not root_cause: + for event in self._events: + if event.type not in significant_types: continue - ground_truth = { - "incident_id": ticket_id, - "artifact_id": ticket_id, - "root_cause": root_cause, - "open_day": event.day, - # Monitor name is generated post-sim via LLM batch; ground truth - # here is the incident ID so the scorer can do an exact match - # without depending on the LLM-enriched monitor name string. - "monitor_source": "datadog", - } - evidence = [ticket_id] - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking which Datadog alert or " - f"monitor fired to trigger incident {ticket_id}, whose root " - f'cause was: "{root_cause[:80]}". The question should test ' - f"whether an agent can connect an observability alert to the " - f"incident it caused. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"dd_alert_{ticket_id}", - "question_type": "DATADOG_ALERT", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "medium", - "requires_reasoning": False, - "chain_id": f"incident_{ticket_id}", - } - ) - return questions + event_subsystems = set() + event_artifacts = set() - def _nps_score_questions(self) -> List[dict]: - """ - NPS_SCORE questions inferred from ZD ticket + incident SimEvents. - - NPS responses are generated post-sim (see post_sim_artifacts.py NPSWriter). - Ground truth is derived deterministically from the same scoring formula - used by NPSWriter — no disk read required: - Base 9; -3 per escalated ZD ticket; -2 per unresolved ZD ticket; - -1 per SLA breach day; clamped to [0, 10]. + for doc_type, aid in (event.artifact_ids or {}).items(): + if not aid: + continue + event_artifacts.add(str(aid)) + s = _ARTIFACT_SUBSYSTEM.get(doc_type, "default") + if s != "default": + event_subsystems.add(s) - Tests whether an agent can reason across ZD tickets, incidents, and - the NPS artifact to explain a customer's satisfaction score. - """ - questions = [] + if not event_subsystems: + continue - org_data: Dict[str, Dict] = {} + # Look for actors NOT in the event who have relevant role-based access + for actor, cones in self._visibility_map.items(): + if actor in (event.actors or []): + continue # Actor was directly involved — not an asymmetry case - for e in self._events: - if e.type == "zd_ticket_opened": - org = e.facts.get("org_name", "") - if not org: + # Find the cone at the event's day + cone = next((c for c in cones if c.as_of_day == event.day), None) + if not cone: continue - org_data.setdefault( - org, - { - "tickets": [], - "escalated_count": 0, - "unresolved_count": 0, - "breach_days": 0, - }, - ) - org_data[org]["tickets"].append(e.facts.get("ticket_id", "")) - - elif e.type == "zd_tickets_escalated": - # Each escalated ticket adds to the org's escalated count - iid = e.facts.get("incident_id", "") - for tid in e.facts.get("ticket_ids", []): - # Find org for this ticket - for e2 in self._events: - if ( - e2.type == "zd_ticket_opened" - and e2.facts.get("ticket_id") == tid - ): - org = e2.facts.get("org_name", "") - if org in org_data: - org_data[org]["escalated_count"] += 1 - break - - if not org_data: - logger.info("[eval] _nps_score_questions: no ZD ticket data — skipping") - return questions - - for org, data in random.sample(list(org_data.items()), min(20, len(org_data))): - ticket_ids = data["tickets"] - if not ticket_ids: - continue - score = 9 - score -= 3 * data["escalated_count"] - score = max(0, min(10, score)) + all_visible = cone.all_visible() + event_artifacts = _safe_artifact_values(event.artifact_ids) + + # Check if actor missed this information + missed_artifacts = event_artifacts - all_visible + if not missed_artifacts: + continue # Actor could already see everything + + # Determine if this spans subsystems the actor doesn't have + blocked_by_role = event_subsystems - cone.subsystem_access + cross_subsystem = len(blocked_by_role) > 0 + + # What did the actor actually know that's related? + related_visible = [] + for e in self._events: + if e.day > event.day: + continue + if actor not in (e.actors or []): + continue + shared_actors = set(e.actors or []) & set(event.actors or []) + shared_artifacts = _safe_artifact_values( + e.artifact_ids + ) & _safe_artifact_values(event.artifact_ids) + if shared_actors or shared_artifacts: + for aid in _safe_artifact_values(e.artifact_ids): + if aid in all_visible: + related_visible.append(aid) + + info_available = { + "actor_visible_subsystems": sorted(cone.subsystem_access), + "event_subsystems": sorted(event_subsystems), + "blocked_by_role": sorted(blocked_by_role), + "missed_artifacts": sorted(missed_artifacts), + "related_artifacts_actor_saw": sorted(set(related_visible)), + } - classification = ( - "promoter" if score >= 9 else "passive" if score >= 7 else "detractor" - ) + results.append((actor, cone, event, info_available, cross_subsystem)) + + return results + + def _build_perspective_question( + self, + actor: str, + cone: ActorVisibilityCone, + event: SimEvent, + info_available: dict, + cross_subsystem: bool, + ) -> Optional[dict]: + + # Derive ground truth deterministically + missed = info_available["missed_artifacts"] + blocked = info_available["blocked_by_role"] + could_have_known = len(missed) == 0 # actor had access to all artifacts + + ground_truth = { + "actor": actor, + "as_of_day": cone.as_of_day, + "as_of_time": cone.as_of_time, + "could_actor_have_known": could_have_known, + "reason": ( + f"Actor had access to {sorted(cone.subsystem_access)} but event " + f"involved {sorted(info_available['event_subsystems'])}; " + f"blocked by role from: {sorted(blocked)}" + if not could_have_known + else f"All event artifacts were in actor's visibility cone via " + f"{'direct involvement' if info_available['related_artifacts_actor_saw'] else 'broadcast'}" + ), + "evidence_artifacts": sorted(info_available["related_artifacts_actor_saw"]), + "missed_artifacts": sorted(missed), + "blocked_subsystems": sorted(blocked), + } - ground_truth = { - "org_name": org, - "artifact_id": ticket_ids[0], - "nps_score": score, - "classification": classification, - "escalated_tickets": data["escalated_count"], - "ticket_ids": ticket_ids, - } - evidence = ticket_ids[:3] - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking what NPS score the customer " - f"{org} would give based on their support experience, and " - f"what factors drove that score. The question should require " - f"the agent to reason across support tickets and incident " - f"history to predict customer satisfaction. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"nps_{org.lower().replace(' ', '_')}", - "question_type": "NPS_SCORE", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "hard", - "requires_reasoning": True, - "chain_id": f"nps_{org.lower().replace(' ', '_')}", - } - ) - return questions + difficulty = "hard" if cross_subsystem else "medium" - def _invoice_sla_questions(self) -> List[dict]: - """ - INVOICE_SLA questions inferred from incident + ZD SimEvents. + # Build prose template + event_desc = self._event_description(event) + subsystem_constraint = ( + f"{actor} has access to {', '.join(sorted(cone.subsystem_access))} " + f"but not {', '.join(sorted(blocked))}" + if blocked + else f"{actor} has access to {', '.join(sorted(cone.subsystem_access))}" + ) - Invoices are generated post-sim (see post_sim_artifacts.py InvoiceWriter). - Ground truth is derived from SLA breach logic: any incident that lasted - more than SLA_BREACH_THRESHOLD_DAYS (1 day) generates an SLA credit - on the affected customers' invoices at SLA_CREDIT_RATE (2%) of contract value. + template = ( + f"Write a question asking whether {actor} would have known about " + f"'{event_desc}' as of Day {cone.as_of_day}, given that " + f"{subsystem_constraint}. " + f"The question must name the actor, the approximate time constraint " + f"(Day {cone.as_of_day}), and the subsystem limitation. " + f"Do not reveal the answer. Do not include artifact IDs. " + f"Output only the question text." + ) - Tests whether an agent can reason about the financial consequences of - incidents by tracing: incident duration → SLA breach → invoice line item. - """ - questions = [] + question_text = self._generate_and_validate_prose( + template=template, + ground_truth_str=str(could_have_known), + question_type="PERSPECTIVE", + ) + if not question_text: + return None - resolved_by_id: Dict[str, SimEvent] = { - e.artifact_ids.get("jira", ""): e - for e in self._events - if e.type == "incident_resolved" + return { + "question_id": f"perspective_{actor}_{self._synthetic_event_id(event)}", + "question_type": "PERSPECTIVE", + "difficulty": difficulty, + "cross_subsystem": cross_subsystem, + "actor": actor, + "actor_role": cone.role, + "as_of_day": cone.as_of_day, + "as_of_time": cone.as_of_time, + "subsystem_access": sorted(cone.subsystem_access), + "blocked_subsystems": sorted(info_available["blocked_by_role"]), + "event_id": self._synthetic_event_id(event), + "event_type": event.type, + "event_day": event.day, + "question_text": question_text, + "ground_truth": ground_truth, + "actor_visible_artifacts": sorted(cone.all_visible()), + "requires_reasoning": True, } - sla_breach_incidents = [] - for e in self._events: - if e.type != "incident_opened": - continue - ticket_id = e.artifact_ids.get("jira", "") - if not ticket_id: - continue - resolve_event = resolved_by_id.get(ticket_id) - if not resolve_event: - continue - duration = resolve_event.day - e.day - if duration > 1: - sla_breach_incidents.append( - { - "ticket_id": ticket_id, - "open_day": e.day, - "resolve_day": resolve_event.day, - "duration_days": duration, - "root_cause": e.facts.get("root_cause", ""), - } - ) + # ── TRACK 2: COUNTERFACTUAL ─────────────────────────────────────────────── - incident_to_orgs: Dict[str, List[str]] = {} - for e in self._events: - if e.type == "zd_tickets_escalated": - iid = e.facts.get("incident_id", "") - if not iid: - continue - for tid in e.facts.get("ticket_ids", []): - for e2 in self._events: - if ( - e2.type == "zd_ticket_opened" - and e2.facts.get("ticket_id") == tid - ): - org = e2.facts.get("org_name", "") - if org: - incident_to_orgs.setdefault(iid, []).append(org) - break - elif e.type == "sf_deals_risk_flagged": - iid = e.facts.get("incident_id", "") - if iid: - for org in e.facts.get("account_names", []): - incident_to_orgs.setdefault(iid, []).append(org) + def _counterfactual_questions(self) -> List[dict]: + questions: List[dict] = [] - logger.info( - f"[eval] _invoice_sla_questions: {len(sla_breach_incidents)} " - f"SLA-breaching incidents available" + sampled = random.sample( + self._causal_links, min(self.MAX_COUNTERFACTUAL, len(self._causal_links)) ) - for inc in random.sample( - sla_breach_incidents, min(20, len(sla_breach_incidents)) - ): - ticket_id = inc["ticket_id"] - orgs = list(set(incident_to_orgs.get(ticket_id, []))) - if not orgs: - continue + for link in sampled: + question = self._build_counterfactual_question(link) + if question: + questions.append(question) - credit_per_org = round(50_000 * 0.02 * inc["duration_days"], 2) - - ground_truth = { - "incident_id": ticket_id, - "artifact_id": ticket_id, - "breach_duration_days": inc["duration_days"], - "affected_orgs": orgs, - "sla_credit_per_org": credit_per_org, - "root_cause": inc["root_cause"], - } - evidence = [ticket_id] - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking what SLA credit would appear on " - f"the invoice for customers affected by incident {ticket_id}, " - f"which remained open for {inc['duration_days']} days. The " - f"question should require the agent to reason about SLA breach " - f"thresholds and financial credit calculations. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"invoice_sla_{ticket_id}", - "question_type": "INVOICE_SLA", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": evidence, - "difficulty": "hard", - "requires_reasoning": True, - "chain_id": f"incident_{ticket_id}", - } - ) + logger.info(f"[eval] {len(questions)} COUNTERFACTUAL questions built") return questions - def _pr_review_questions(self, threads: List[dict]) -> List[dict]: - questions = [] - pr_events = [e for e in self._events if e.type == "pr_review"] - logger.info( - f"[eval] _pr_review_questions: {len(pr_events)} pr_review events available" + def _build_counterfactual_question(self, link: CausalLink) -> Optional[dict]: + + ground_truth = { + "outcome_changed": link.outcome_changed, + "causal_mechanism": link.link_type, + "causal_link_field": link.link_field, + "causal_link_value": link.link_value, + "cause_event_id": link.cause_event_id, + "cause_event_type": link.cause_event_type, + "effect_event_id": link.effect_event_id, + "effect_event_type": link.effect_event_type, + "premise": link.counterfactual_premise, + "outcome": link.counterfactual_outcome, + "actors": link.actors, + } + + subsystems_str = ", ".join(sorted(link.subsystems_involved)) + difficulty = "hard" if len(link.subsystems_involved) > 1 else "medium" + + template = ( + f"Write a counterfactual question asking: if {link.counterfactual_premise}, " + f"would the following have occurred: {link.counterfactual_outcome}? " + f"The question should involve events from Day {link.day} in a simulated " + f"company with systems including {subsystems_str}. " + f"The question must be phrased as a hypothetical (use 'if', 'had', 'would'). " + f"Do not reveal the answer. Do not include event IDs. " + f"Output only the question text." ) - for event in random.sample(pr_events, min(40, len(pr_events))): - pr_id = event.artifact_ids.get("pr", "") - reviewer = event.facts.get("reviewer", "") - author = event.facts.get("author", "") - verdict = event.facts.get("verdict", "") - pr_title = event.facts.get("pr_title", "") - linked_ticket = event.artifact_ids.get("jira", "") + question_text = self._generate_and_validate_prose( + template=template, + ground_truth_str=link.counterfactual_outcome, + question_type="COUNTERFACTUAL", + ) + if not question_text: + return None - if not pr_id or not reviewer or not verdict: - continue + return { + "question_id": f"counterfactual_{link.cause_event_id}_{link.link_type}", + "question_type": "COUNTERFACTUAL", + "difficulty": difficulty, + "link_type": link.link_type, + "day": link.day, + "actors": link.actors, + "subsystems_involved": sorted(link.subsystems_involved), + "question_text": question_text, + "ground_truth": ground_truth, + "evidence_chain": [link.cause_event_id, link.effect_event_id], + "requires_reasoning": True, + } - ground_truth_review = { - "pr_id": pr_id, - "reviewer": reviewer, - "author": author, - "verdict": verdict, - "linked_ticket": linked_ticket, - "day": event.day, - } - evidence_review = [pr_id] + ([linked_ticket] if linked_ticket else []) - - q_text_review = self._generate_question_prose( - template=( - f"Generate a retrieval question asking who reviewed pull request " - f"{pr_id} and whether it was approved or had changes requested. " - f"The question should test whether an agent can locate the review " - f"record and identify both the reviewer and their verdict. " - f"Output only the question text." - ) - ) - if q_text_review: - questions.append( - { - "question_id": f"pr_review_{pr_id}", - "question_type": "PR_REVIEW", - "question_text": q_text_review, - "ground_truth": ground_truth_review, - "evidence_chain": [e for e in evidence_review if e], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"pr_{pr_id}", - } - ) + # ── TRACK 3: SILENCE ───────────────────────────────────────────────────── - if not linked_ticket: - continue + def _silence_questions(self) -> List[dict]: + questions: List[dict] = [] - ground_truth_causal = { - "artifact_id": pr_id, - "event_type": "pr_review", - "actors": [reviewer, author], - "timestamp": event.timestamp, - "verdict": verdict, - } - evidence_causal = [linked_ticket, pr_id] - - q_text_causal = self._generate_question_prose( - template=( - f"Generate a causal question asking what pull request was opened " - f"or reviewed as a result of work on ticket {linked_ticket}. " - f"The question should require the agent to trace from a Jira ticket " - f"to the GitHub PR that resolved it. " - f"Output only the question text." - ) - ) - if q_text_causal: - questions.append( - { - "question_id": f"pr_causal_{linked_ticket}_{pr_id}", - "question_type": "CAUSAL", - "question_text": q_text_causal, - "ground_truth": ground_truth_causal, - "evidence_chain": evidence_causal, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": f"pr_{pr_id}", - } - ) + sampled = random.sample( + self._absence_catalog, min(self.MAX_SILENCE, len(self._absence_catalog)) + ) - return questions + for record in sampled: + question = self._build_silence_question(record) + if question: + questions.append(question) - def _blocker_questions(self) -> List[dict]: - questions = [] - blocker_events = [e for e in self._events if e.type == "blocker_flagged"] - logger.info( - f"[eval] _blocker_questions: {len(blocker_events)} blocker_flagged events available" - ) + logger.info(f"[eval] {len(questions)} SILENCE questions built") + return questions - seen_tickets: Set[str] = set() - for event in random.sample(blocker_events, min(25, len(blocker_events))): - ticket_id = event.artifact_ids.get("jira", "") - slack_thread = event.artifact_ids.get("slack_thread", "") - assignee = next(iter(event.actors), None) - blocker_reason = event.facts.get( - "comment", event.facts.get("blocker_reason", "") - ) + def _build_silence_question(self, record: AbsenceRecord) -> Optional[dict]: + + ground_truth = { + "answer": False, # The expected artifact/event does NOT exist + "absence_type": "state_machine_confirmed", + "trigger_event_id": record.trigger_event_id, + "trigger_event_type": record.trigger_event_type, + "expected_response_type": record.expected_response_type, + "trigger_day": record.trigger_day, + "trigger_actors": record.trigger_actors, + "expected_search_space": record.expected_search_space, + "link_field": record.link_field, + "link_value": record.link_value, + } - if not ticket_id or not assignee or ticket_id in seen_tickets: - continue - seen_tickets.add(ticket_id) - - ground_truth = { - "artifact_id": ticket_id, - "was_blocked": True, - "assignee": assignee, - "ticket_id": ticket_id, - "day": event.day, - "slack_thread": slack_thread, - } - evidence = [ticket_id] + ([slack_thread] if slack_thread else []) - - q_text = self._generate_question_prose( - template=( - f"Generate a retrieval question asking what ticket {assignee} was " - f"blocked on during Day {event.day}, and what the blocker was. " - f"The question should test whether an agent can locate a blocker " - f"report in the ticket or Slack history. " - f"Output only the question text." - ) - ) - if q_text: - questions.append( - { - "question_id": f"blocker_{ticket_id}_day{event.day}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [e for e in evidence if e], - "difficulty": "medium", - "requires_reasoning": False, - "chain_id": f"blocker_{ticket_id}", - } - ) + # Build a natural-language description of what should have existed + expected_desc = { + "postmortem_created": "a postmortem document", + "incident_resolved": "an incident resolution", + "zd_ticket_opened": "a Zendesk support ticket", + "customer_email_routed": "an internal routing of the customer email", + "confluence_created": "a Confluence documentation page", + "sf_ownership_lapsed": "a Salesforce ownership transfer", + "ticket_reassigned": "a Jira ticket reassignment", + "pr_merged": "a merged pull request", + "zd_tickets_escalated": "a Zendesk escalation", + "incident_opened": "an incident ticket", + "onboarding_session": "an onboarding session", + "warmup_1on1": "a warmup 1-on-1 meeting", + "sf_deals_risk_flagged": "a Salesforce risk flag on related deals", + "knowledge_gap_detected": "a formal knowledge gap detection event", + }.get(record.expected_response_type, f"a {record.expected_response_type} event") + + trigger_desc = { + "incident_opened": f"the incident on Day {record.trigger_day}", + "customer_escalation": f"the customer escalation on Day {record.trigger_day}", + "inbound_external_email": f"the inbound email on Day {record.trigger_day}", + "design_discussion": f"the design discussion on Day {record.trigger_day}", + "knowledge_gap_detected": f"the knowledge gap detected on Day {record.trigger_day}", + "employee_departed": f"the employee departure on Day {record.trigger_day}", + "zd_tickets_escalated": f"the Zendesk escalation on Day {record.trigger_day}", + "pr_opened": f"the pull request opened on Day {record.trigger_day}", + "customer_email_routed": f"the routing of the customer email on Day {record.trigger_day}", + "sf_deals_risk_flagged": f"the CRM risk flagging on Day {record.trigger_day}", + "employee_hired": f"the hiring of {record.link_value} on Day {record.trigger_day}", + "assignment_domain_mismatch": f"the domain mismatch assignment flagged on Day {record.trigger_day}", + }.get(record.trigger_event_type, f"the event on Day {record.trigger_day}") + + actors_str = ( + ", ".join(record.trigger_actors[:2]) + if record.trigger_actors + else "the involved parties" + ) - blocked_ticket_ids = {e.artifact_ids.get("jira", "") for e in blocker_events} - progress_events = [ - e - for e in self._events - if e.type == "ticket_progress" - and e.artifact_ids.get("jira", "") not in blocked_ticket_ids - and e.artifact_ids.get("jira", "") - ] - for event in random.sample(progress_events, min(15, len(progress_events))): - ticket_id = event.artifact_ids.get("jira", "") - assignee = next(iter(event.actors), None) - if not ticket_id or not assignee: - continue + trigger_ev = next( + ( + e + for e in self._events + if self._synthetic_event_id(e) == record.trigger_event_id + ), + None, + ) - ground_truth = { - "artifact_id": ticket_id, - "was_blocked": False, - "assignee": assignee, - "ticket_id": ticket_id, - "day": event.day, - "slack_thread": None, - } - - q_text = self._generate_question_prose( - template=( - f"Generate a question asking whether {assignee} reported any " - f"blockers while working on ticket {ticket_id} on Day {event.day}. " - f"The question should require the agent to check the ticket history " - f"and confirm whether a blocker was or wasn't reported. " - f"Output only the question text." - ) + if not trigger_ev: + logger.warning( + f"[eval] Skipping SILENCE question for unknown trigger type: {record.trigger_event_type}" ) - if q_text: - questions.append( - { - "question_id": f"blocker_check_{ticket_id}_day{event.day}", - "question_type": "RETRIEVAL", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [ticket_id], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"blocker_{ticket_id}", - } - ) + return None - logger.info( - f"[eval] _blocker_questions: generated {len(questions)} questions " - f"(pool_A={len(blocker_events)}, pool_B={len(progress_events)})" + template = ( + f"Write a yes/no question asking whether {expected_desc} was created " + f"in response to {trigger_desc} involving {actors_str}. " + f"CRITICAL: Only refer to the event exactly as described ('{trigger_desc}'). " + f"Do not call it an 'incident' or 'outage' unless those words are explicitly used. " + f"The question should be phrased so that the correct answer is 'no' — " + f"the artifact does not exist — but the agent must investigate to confirm this. " + f"Do not state or imply the answer. Do not include system IDs. " + f"The question should sound like something a manager would ask when reviewing " + f"process compliance. " + f"Output only the question text." ) - return questions - def _vendor_routing_questions(self) -> List[dict]: - questions = [] - vendor_events = [e for e in self._events if e.type == "vendor_email_routed"] - logger.info( - f"[eval] _vendor_routing_questions: {len(vendor_events)} vendor_email_routed events available" + question_text = self._generate_and_validate_prose( + template=template, + ground_truth_str="False", + question_type="SILENCE", ) + if not question_text: + return None - for event in random.sample(vendor_events, min(25, len(vendor_events))): - email_id = event.artifact_ids.get("email", "") - vendor = event.facts.get("vendor", "") - topic = event.facts.get("topic", "") - routed_to = event.facts.get("routed_to", "") - causal_chain = event.facts.get("causal_chain", []) + return { + "question_id": f"silence_{record.trigger_event_id}_{record.expected_response_type}", + "question_type": "SILENCE", + "difficulty": "hard", # Absence reasoning is always hard + "trigger_event_id": record.trigger_event_id, + "trigger_event_type": record.trigger_event_type, + "trigger_day": record.trigger_day, + "expected_response_type": record.expected_response_type, + "subsystem": record.subsystem, + "question_text": question_text, + "ground_truth": ground_truth, + "expected_search_space": record.expected_search_space, + "requires_reasoning": True, + } - if not email_id or not vendor or not routed_to: - continue + # ── PROSE GENERATION + VALIDATION ──────────────────────────────────────── - jira_opened = any( - a - for a in causal_chain - if isinstance(a, str) and (a.startswith("IT-") or a.startswith("ORG-")) - ) + def _event_description(self, event: SimEvent) -> str: + """Natural language description of an event for use in question templates.""" + descs = { + "incident_opened": lambda e: ( + f"a P1 incident ({e.facts.get('title', 'system incident')})" + ), + "customer_escalation": lambda e: ( + f"a customer escalation from {e.facts.get('customer', 'a customer')}" + ), + "sf_deals_risk_flagged": lambda e: ( + "Salesforce accounts being flagged at-risk" + ), + "knowledge_gap_detected": lambda e: ( + f"a knowledge gap in {', '.join(e.facts.get('gap_areas', ['an undocumented domain']))}" + ), + "employee_departed": lambda e: ( + f"the departure of {(e.actors or ['a team member'])[0]}" + ), + "design_discussion": lambda e: ( + f"a design discussion about {e.facts.get('topic', 'a technical topic')}" + ), + "customer_email_routed": lambda e: ( + "a customer email being routed to support" + ), + "zd_tickets_escalated": lambda e: ( + "Zendesk tickets being escalated to an incident" + ), + "sf_ownership_lapsed": lambda e: "Salesforce accounts losing their owner", + "postmortem_created": lambda e: "a postmortem being written", + "inbound_external_email": lambda e: ( + f"an inbound email from {e.facts.get('sender', 'an external contact')}" + ), + } + fn = descs.get(event.type) + if fn: + try: + return fn(event) + except Exception: + pass + return f"a {event.type.replace('_', ' ')} event" + + def _generate_and_validate_prose( + self, + template: str, + ground_truth_str: str, + question_type: str, + max_attempts: int = 3, + ) -> Optional[str]: + """ + LLM writes question prose. Validates against structured rubric. + Retries up to max_attempts if validation fails. + + Validation rules: + 1. Must end with '?' + 2. Must not contain ground_truth_str verbatim + 3. Must not contain raw artifact IDs (pattern: XX-\\d+ or [a-f0-9]{8,}) + 4. Must be between 15 and 120 words + 5. Must contain at least one of: actor name, day reference, subsystem word + """ + agent = make_agent( + role="Eval Dataset Author", + goal="Write natural-sounding evaluation questions for AI agent benchmarks.", + backstory=( + "You write clear, specific questions for evaluating AI agents on reasoning " + "tasks. Questions must be unambiguous, naturally phrased, and answerable " + "only through careful reasoning over a corporate document corpus." + ), + llm=self._worker_llm, + ) - ground_truth = { - "first_recipient": routed_to, - "artifact_id": email_id, - "vendor": vendor, - "topic": topic, - "jira_opened": jira_opened, - "timestamp": event.timestamp, - } - evidence = [email_id] + ([c for c in causal_chain[:2] if c != email_id]) - - q_text = self._generate_question_prose( - template=( - f"Generate a routing question asking which internal engineer or " - f"team member handled the alert or email from {vendor} regarding " - f'"{topic[:60]}". The question should test whether an agent can ' - f"trace inbound vendor communications to the responsible engineer. " - f"Output only the question text." - ) + for attempt in range(max_attempts): + retry_note = ( + " Previous attempt failed validation. Make sure the question: " + "ends with '?', does not reveal the answer, avoids artifact IDs, " + "and is 15-120 words long." + if attempt > 0 + else "" ) - if q_text: - questions.append( - { - "question_id": f"vendor_routing_{email_id}", - "question_type": "ROUTING", - "question_text": q_text, - "ground_truth": ground_truth, - "evidence_chain": [e for e in evidence if e], - "difficulty": "medium", - "requires_reasoning": False, - "chain_id": f"vendor_{email_id}", - } + task = Task( + description=template + retry_note, + expected_output="One question ending with a question mark. No preamble or explanation.", + agent=agent, + ) + try: + result = str( + Crew(agents=[agent], tasks=[task], verbose=False).kickoff() + ).strip() + + if self._validate_prose(result, ground_truth_str, question_type): + return result + else: + logger.debug( + f"[eval] Prose validation failed (attempt {attempt + 1}): {result[:80]}" + ) + except Exception as exc: + logger.warning( + f"[eval] Prose generation error (attempt {attempt + 1}): {exc}" ) - return questions - def _design_discussion_questions(self) -> List[dict]: - questions = [] - dd_events = [e for e in self._events if e.type == "design_discussion"] - logger.info( - f"[eval] _design_discussion_questions: {len(dd_events)} design_discussion events available" - ) + return None - for event in random.sample(dd_events, min(30, len(dd_events))): - topic = event.facts.get("topic", "") - participants = event.facts.get("participants", event.actors) - medium = event.facts.get("medium", "slack") - spawned_doc = event.facts.get("spawned_doc", False) - conf_id = event.artifact_ids.get("confluence", "") - - artifact_id = ( - event.artifact_ids.get("zoom_transcript") - or event.artifact_ids.get("slack_thread") - or "" - ) - linked_ticket = event.artifact_ids.get("jira", "") + def _validate_prose( + self, text: str, ground_truth_str: str, question_type: str + ) -> bool: + if not text.endswith("?"): + return False - if not artifact_id or not topic or not participants: - continue + words = text.split() + if len(words) < 10 or len(words) > 150: + return False - ground_truth_ret = { - "artifact_id": artifact_id, - "topic": topic, - "participants": participants, - "medium": medium, - "day": event.day, - "linked_ticket": linked_ticket, - } - evidence_ret = [artifact_id] + ([linked_ticket] if linked_ticket else []) - - q_text_ret = self._generate_question_prose( - template=( - f"Generate a retrieval question asking who participated in the " - f"{'Zoom call' if medium == 'zoom' else 'Slack design discussion'} " - f'about "{topic[:80]}". The question should require the agent to ' - f"locate the {'transcript' if medium == 'zoom' else 'thread'} and " - f"identify all participants. " - f"Output only the question text." - ) - ) - if q_text_ret: - questions.append( - { - "question_id": f"design_discussion_{artifact_id}", - "question_type": "RETRIEVAL", - "question_text": q_text_ret, - "ground_truth": ground_truth_ret, - "evidence_chain": [e for e in evidence_ret if e], - "difficulty": "easy", - "requires_reasoning": False, - "chain_id": f"design_{artifact_id}", - } - ) + # Must not leak ground truth verbatim + gt_lower = ground_truth_str.lower() + if gt_lower in text.lower() and len(gt_lower) > 4: + return False - if not spawned_doc or not conf_id: - continue + # Must not contain raw artifact IDs (e.g. IT-108, abc12345) + if re.search(r"\b[A-Z]{1,4}-\d{2,6}\b", text): + return False + if re.search(r"\b[a-f0-9]{8,}\b", text): + return False - ground_truth_causal = { - "artifact_id": conf_id, - "event_type": "confluence_created", - "actors": participants, - "timestamp": event.timestamp, - "source_discussion": artifact_id, - } - evidence_causal = [artifact_id, conf_id] - - q_text_causal = self._generate_question_prose( - template=( - f"Generate a causal question asking which Confluence document was " - f'created as a result of the design discussion on "{topic[:80]}". ' - f"The question should require the agent to trace from the discussion " - f"artifact to the documentation it produced. " - f"Output only the question text." - ) - ) - if q_text_causal: - questions.append( - { - "question_id": f"design_doc_spawned_{artifact_id}", - "question_type": "CAUSAL", - "question_text": q_text_causal, - "ground_truth": ground_truth_causal, - "evidence_chain": evidence_causal, - "difficulty": "medium", - "requires_reasoning": True, - "chain_id": f"design_{artifact_id}", - } - ) + # PERSPECTIVE questions must reference an actor name or role + if question_type == "PERSPECTIVE": + if not re.search(r"day\s+\d+|as of|by\s+[A-Z][a-z]+", text, re.IGNORECASE): + return False - return questions + # COUNTERFACTUAL questions must use hypothetical language + if question_type == "COUNTERFACTUAL": + if not re.search( + r"\b(if|had|would|could|might|hypothetically)\b", text, re.IGNORECASE + ): + return False - def _generate_question_prose(self, template: str) -> Optional[str]: - """LLM writes question wording only. Never touches ground truth.""" - agent = make_agent( - role="Eval Dataset Author", - goal="Write natural-sounding evaluation questions for AI agent benchmarks.", - backstory=( - "You write clear, specific questions for evaluating AI retrieval " - "and reasoning systems. Questions should be unambiguous and answerable " - "from a corporate document corpus." - ), - llm=self._worker_llm, - ) - task = Task( - description=template, - expected_output="One question ending with a question mark. No preamble.", - agent=agent, - ) - try: - result = str( - Crew(agents=[agent], tasks=[task], verbose=False).kickoff() - ).strip() - # Ensure it ends with ? - if not result.endswith("?"): - result = result.rstrip(".") + "?" - return result - except Exception as exc: - logger.warning(f"[eval] Question prose generation failed: {exc}") - return None + # SILENCE questions must be yes/no answerable + if question_type == "SILENCE": + if not re.search( + r"\b(was|were|did|has|have|is|are)\b", text, re.IGNORECASE + ): + return False - def _find_event_by_artifact(self, artifact_id: str) -> Optional[SimEvent]: - for event in self._events: - if artifact_id in event.artifact_ids.values(): - return event - return None + return True + + +# ───────────────────────────────────────────────────────────────────────────── +# HARNESS ENTRYPOINT +# ───────────────────────────────────────────────────────────────────────────── class EvalHarness: + """ + Orchestrates the full eval dataset generation pipeline: + 1. Build actor visibility cones (ActorVisibilityBuilder) + 2. Index explicit causal links (CausalLinkIndexer) + 3. Catalog expected-but-absent artifacts (AbsenceCatalogBuilder) + 4. Generate eval questions (EvalQuestionGenerator) + 5. Write all intermediate structures + final questions to export/eval/ + """ + def __init__(self): from flow import build_llm @@ -2305,26 +1692,59 @@ def __init__(self): self._worker_llm = build_llm("worker") def run(self) -> None: - logger.info("[bold cyan]🔬 Building eval dataset...[/bold cyan]") - - builder = CausalThreadBuilder(self._mem) - threads = builder.build_all() - logger.info(f" {len(threads)} causal threads extracted") - - threads_path = EVAL_DIR / "causal_threads.json" - with open(threads_path, "w") as f: - json.dump(threads, f, indent=2, default=str) - logger.info(f" → {threads_path}") - - generator = EvalQuestionGenerator(self._mem, self._worker_llm) - questions = generator.generate(threads) - logger.info(f" {len(questions)} eval questions generated") + logger.info("[bold cyan]🔬 Building OrgForge eval dataset v2...[/bold cyan]") + + # Step 1: Actor visibility + logger.info("[eval] Building actor visibility cones...") + vis_builder = ActorVisibilityBuilder(self._mem) + visibility_map = vis_builder.build_all() + vis_export = { + actor: [cone.to_dict() for cone in cones] + for actor, cones in visibility_map.items() + } + vis_path = EVAL_DIR / "actor_visibility.json" + with open(vis_path, "w") as f: + json.dump(vis_export, f, indent=2, default=str) + logger.info(f" → {vis_path} ({len(visibility_map)} actors)") + + # Step 2: Causal link index + logger.info("[eval] Indexing explicit causal links...") + link_indexer = CausalLinkIndexer(self._mem) + causal_links = link_indexer.build() + links_path = EVAL_DIR / "causal_link_index.json" + with open(links_path, "w") as f: + json.dump([lnk.to_dict() for lnk in causal_links], f, indent=2, default=str) + logger.info(f" → {links_path} ({len(causal_links)} links)") + + # Step 3: Absence catalog + logger.info("[eval] Building absence catalog...") + absence_builder = AbsenceCatalogBuilder(self._mem) + absence_catalog = absence_builder.build() + absence_path = EVAL_DIR / "absence_catalog.json" + with open(absence_path, "w") as f: + json.dump([r.to_dict() for r in absence_catalog], f, indent=2, default=str) + logger.info(f" → {absence_path} ({len(absence_catalog)} absence records)") + + # Step 4: Question generation + logger.info("[eval] Generating eval questions...") + generator = EvalQuestionGenerator( + mem=self._mem, + worker_llm=self._worker_llm, + visibility_map=visibility_map, + causal_links=causal_links, + absence_catalog=absence_catalog, + ) + questions = generator.generate() + # Summary stats by_type: Dict[str, int] = defaultdict(int) by_difficulty: Dict[str, int] = defaultdict(int) + cross_subsystem_count = 0 for q in questions: by_type[q["question_type"]] += 1 by_difficulty[q["difficulty"]] += 1 + if q.get("cross_subsystem"): + cross_subsystem_count += 1 questions_path = EVAL_DIR / "eval_questions.json" with open(questions_path, "w") as f: @@ -2332,9 +1752,15 @@ def run(self) -> None: { "metadata": { "generated_at": datetime.now().isoformat(), + "version": "2.0", + "tracks": ["PERSPECTIVE", "COUNTERFACTUAL", "SILENCE"], "total_questions": len(questions), "by_type": dict(by_type), "by_difficulty": dict(by_difficulty), + "cross_subsystem_questions": cross_subsystem_count, + "actors_with_visibility_cones": len(visibility_map), + "causal_links_indexed": len(causal_links), + "absence_records": len(absence_catalog), }, "questions": questions, }, @@ -2342,17 +1768,18 @@ def run(self) -> None: indent=2, default=str, ) + logger.info(f" → {questions_path}") logger.info( - f"[green]✓ Eval dataset complete.[/green] " - f"Types: {dict(by_type)} | Difficulty: {dict(by_difficulty)}" + f"[green]✓ Eval dataset v2 complete.[/green] " + f"Types: {dict(by_type)} | Difficulty: {dict(by_difficulty)} | " + f"Cross-subsystem: {cross_subsystem_count}" ) if __name__ == "__main__": - import logging - logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", ) EvalHarness().run() diff --git a/eval/export_to_hf.py b/eval/export_to_hf.py index 3d55afa..44856f0 100644 --- a/eval/export_to_hf.py +++ b/eval/export_to_hf.py @@ -26,7 +26,8 @@ Corpus schema (one row per document) ------------------------------------- doc_id str — globally unique, e.g. "ORG-42", "CONF-ENG-007", "EMAIL-001" - doc_type str — "jira" | "confluence" | "slack" | "email" | "pr" | "sim_event" + doc_type str — "jira" | "confluence" | "slack" | "email" | "pr" | + "zd_ticket" | "sf_opp" | "sf_account" | "sim_event" title str — human-readable title or subject line body str — full text content for retrieval day int — simulation day this artifact was created @@ -57,7 +58,7 @@ returned doc_ids are compared against evidence_chain. MRR@10 and Recall@10 are reported per question type. -Dense — sentence-transformers "Losspost/stella_en_1.5b_v5" (1024-dim). +Dense — sentence-transformers "Qwen/Qwen3-Embedding-4B" (2560-dim). Cosine similarity between question_text embedding and body embeddings. Same MRR@10 / Recall@10 reported for comparison. If sentence-transformers is not installed, this section is skipped @@ -77,6 +78,7 @@ from collections import defaultdict from pathlib import Path from typing import Any, Dict, List, Tuple +import numpy as np import yaml @@ -131,15 +133,9 @@ "rank_bm25 not installed — BM25 baseline disabled. pip install rank-bm25" ) -try: - import ollama - import numpy as np - _DENSE_AVAILABLE = True - _DENSE_MODEL_NAME = "Losspost/stella_en_1.5b_v5" # or whatever your memory.py uses -except ImportError: - _DENSE_AVAILABLE = False - logger.warning("ollama not installed — dense baseline disabled.") +_DENSE_AVAILABLE = True +_DENSE_MODEL_NAME = "Qwen/Qwen3-Embedding-4B" # ───────────────────────────────────────────────────────────────────────────── @@ -165,7 +161,7 @@ def _dept_from_artifact_id(artifact_id: str) -> str: class CorpusBuilder: """ Reads the MongoDB-persisted artifacts (via Memory) and the SimEvent log, - then normalises every artifact into a flat list of corpus rows. + then normalizes every artifact into a flat list of corpus rows. Falls back to reconstructing from eval JSON if MongoDB is unavailable, which allows the exporter to run in offline/CI environments. @@ -200,6 +196,8 @@ def build(self) -> List[dict]: rows = self._enrich_from_mongo(rows) rows.extend(self._plans_to_corpus_rows()) + rows.extend(self._post_sim_to_corpus_rows()) + # Deduplicate: keep the row with the longest body for each doc_id seen: Dict[str, dict] = {} for row in rows: @@ -251,12 +249,21 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: "incident_resolved", "escalation_chain", "postmortem_created", + "zd_tickets_escalated", + "sf_deals_risk_flagged", + "crm_account_at_risk", ) is_external = event_type in ( "inbound_external_email", "customer_email_routed", "vendor_email_routed", "email_dropped", + "sales_outbound_email", + "proactive_outreach_initiated", + "zd_ticket_opened", + "zd_tickets_resolved", + "crm_touchpoint", + "customer_health_briefing", ) shared = { @@ -320,6 +327,8 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: "customer_email_routed", "vendor_email_routed", "email_dropped", + "sales_outbound_email", + "proactive_outreach_initiated", ): rows.append( { @@ -362,6 +371,73 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) + # ── ZENDESK TICKET ──────────────────────────────────────────────────── + # zd_ticket_opened carries a single ID; zd_tickets_escalated and + # zd_tickets_resolved carry a list under "zd_tickets". + zd_ids: List[str] = [] + single_zd = artifact_ids.get("zd_ticket", "") + if single_zd: + zd_ids = [single_zd] + else: + multi_zd = artifact_ids.get("zd_tickets", []) + if isinstance(multi_zd, list): + zd_ids = [str(z) for z in multi_zd if z] + for zd_id in zd_ids: + rows.append( + { + **shared, + "doc_id": zd_id, + "doc_type": "zd_ticket", + "title": str(facts.get("subject", facts.get("ticket_id", zd_id)))[ + :512 + ], + "body": self._zd_body(facts), + } + ) + + # ── SALESFORCE OPPORTUNITY ──────────────────────────────────────────── + # sf_opps may be a single ID (crm_touchpoint) or a list + # (sf_deals_risk_flagged, sf_ownership_lapsed). + sf_opp_ids: List[str] = [] + single_opp = artifact_ids.get("sf_opp", "") + if single_opp: + sf_opp_ids = [single_opp] + else: + multi_opp = artifact_ids.get("sf_opps", []) + if isinstance(multi_opp, list): + sf_opp_ids = [str(o) for o in multi_opp if o] + for opp_id in sf_opp_ids: + rows.append( + { + **shared, + "doc_id": opp_id, + "doc_type": "sf_opp", + "title": str( + facts.get("account_name", opp_id) + + " — " + + facts.get("stage", "") + )[:512], + "body": self._sf_opp_body(facts), + } + ) + + # ── SALESFORCE ACCOUNT ──────────────────────────────────────────────── + # sf_ownership_lapsed carries a list of account IDs. + sf_acc_ids: List[str] = [] + multi_acc = artifact_ids.get("sf_accounts", []) + if isinstance(multi_acc, list): + sf_acc_ids = [str(a) for a in multi_acc if a] + for acc_id in sf_acc_ids: + rows.append( + { + **shared, + "doc_id": acc_id, + "doc_type": "sf_account", + "title": str(facts.get("account_name", acc_id))[:512], + "body": self._sf_account_body(facts, acc_id), + } + ) + # ── FALLBACK ────────────────────────────────────────────────────────── if not rows: rows.append( @@ -418,6 +494,58 @@ def _email_body(self, facts: dict, evt: dict) -> str: parts.append(evt.get("summary", "")) return "\n".join(parts) + def _zd_body(self, facts: dict) -> str: + parts = [] + for key in ("subject", "org_name", "description", "ticket_id", "component"): + val = facts.get(key) + if val: + parts.append(f"{key}: {val}") + ticket_ids = facts.get("ticket_ids", []) + if ticket_ids: + parts.append("ticket_ids: " + ", ".join(str(t) for t in ticket_ids)) + incident_id = facts.get("incident_id", "") + if incident_id: + parts.append(f"related_incident: {incident_id}") + return "\n".join(parts) + + def _sf_opp_body(self, facts: dict) -> str: + parts = [] + for key in ( + "account_name", + "stage", + "sender", + "subject", + "risk_note", + "opportunity_id", + ): + val = facts.get(key) + if val: + parts.append(f"{key}: {val}") + opp_ids = facts.get("opp_ids", []) + if opp_ids: + parts.append("opp_ids: " + ", ".join(str(o) for o in opp_ids)) + incident_id = facts.get("incident_id", "") + if incident_id: + parts.append(f"related_incident: {incident_id}") + return "\n".join(parts) + + def _sf_account_body(self, facts: dict, acc_id: str) -> str: + parts = [] + for key in ("departed_employee", "role", "account_name"): + val = facts.get(key) + if val: + parts.append(f"{key}: {val}") + accounts_lapsed = facts.get("accounts_lapsed", []) + if acc_id in accounts_lapsed: + parts.append(f"account_id: {acc_id}") + parts.append("status: ownership lapsed — pending reassignment") + opps_lapsed = facts.get("opportunities_lapsed", []) + if opps_lapsed: + parts.append( + "opportunities_lapsed: " + ", ".join(str(o) for o in opps_lapsed) + ) + return "\n".join(parts) + def _plans_to_corpus_rows(self) -> List[dict]: rows = [] for plan in self._mem._db["dept_plans"].find({}, {"_id": 0}): @@ -486,6 +614,61 @@ def _plans_to_corpus_rows(self) -> List[dict]: ) return rows + def _post_sim_to_corpus_rows(self) -> List[dict]: + rows = [] + nps_dir = BASE / "nps" / "responses" + if nps_dir.exists(): + for p in nps_dir.glob("*.json"): + data = json.loads(p.read_text()) + rows.append( + { + "doc_id": data["response_id"], + "doc_type": "nps_survey", + "title": f"NPS Survey: {data['org_name']}", + "body": f"Score: {data['score']}\nComment: {data.get('verbatim_comment', '')}", + "day": 30, + "date": data["submitted_at"][:10], + "timestamp": data["submitted_at"], + "actors": json.dumps([data["org_name"]]), + "tags": json.dumps(["nps", data["classification"]]), + "artifact_ids": json.dumps({"nps": data["response_id"]}), + "dept": "Sales_Marketing", + "is_incident": data["score"] < 7, + "is_external": True, + } + ) + + inv_dir = BASE / "invoices" + if inv_dir.exists(): + for p in inv_dir.glob("*.json"): + data = json.loads(p.read_text()) + body_text = ( + data.get("notes", "") + + "\n" + + json.dumps(data.get("line_items", [])) + ) + rows.append( + { + "doc_id": data["invoice_id"], + "doc_type": "invoice", + "title": f"Invoice {data['invoice_id']} - {data['customer']['org_name']}", + "body": body_text, + "day": 30, + "date": data["invoice_date"][:10], + "timestamp": data["invoice_date"], + "actors": json.dumps([data["customer"]["org_name"]]), + "tags": json.dumps(["invoice", "billing"]), + "artifact_ids": json.dumps({"invoice": data["invoice_id"]}), + "dept": "Finance", + "is_incident": data.get("metadata", {}).get( + "sla_credits_count", 0 + ) + > 0, + "is_external": True, + } + ) + return rows + def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: """ Attempt to replace thin SimEvent body text with richer MongoDB content. @@ -551,12 +734,200 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: parts.append(c) rich_map[tid] = "\n".join(p for p in parts if p) - for artifact in self._mem._db["artifacts"].find( - {"type": "email"}, {"_id": 1, "content": 1, "title": 1} + # Pull requests — stored in the pull_requests collection. + # PRs are written via mem.upsert_pr() and are never in artifacts. + # Key fields: pr_id, title, description, author, ticket_id, + # reviewers, status, dept, day, date, timestamp, + # comments[].{date, author, verdict, text}. + for pr in self._mem._db["pull_requests"].find( + {}, + { + "_id": 0, + "pr_id": 1, + "title": 1, + "description": 1, + "author": 1, + "ticket_id": 1, + "reviewers": 1, + "status": 1, + "comments": 1, + }, + ): + pid = pr.get("pr_id") + if not pid: + continue + parts = [] + if pr.get("title"): + parts.append(f"title: {pr['title']}") + if pr.get("description"): + parts.append(f"description: {pr['description']}") + if pr.get("author"): + parts.append(f"author: {pr['author']}") + if pr.get("ticket_id"): + parts.append(f"ticket: {pr['ticket_id']}") + if pr.get("status"): + parts.append(f"status: {pr['status']}") + reviewers = pr.get("reviewers", []) + if reviewers: + parts.append(f"reviewers: {', '.join(reviewers)}") + for c in pr.get("comments") or []: + verdict = f" [{c['verdict']}]" if c.get("verdict") else "" + text = c.get("text", "") + author = c.get("author", "") + if text: + parts.append(f"review ({author}{verdict}): {text}") + rich_map[pid] = "\n".join(p for p in parts if p) + + # Emails — stored in the dedicated emails collection. + # Schema: _id (ObjectId), embed_id, subject, body, from_name, + # from_addr, to_name, to_addr, direction, day, date, timestamp. + # Use embed_id as the corpus doc_id to match what SimEvents carry. + for email in self._mem._db["emails"].find( + {}, + { + "_id": 0, + "embed_id": 1, + "subject": 1, + "body": 1, + "from_name": 1, + "from_addr": 1, + "to_name": 1, + "to_addr": 1, + "direction": 1, + }, + ): + eid = email.get("embed_id") + if not eid: + continue + parts = [] + if email.get("subject"): + parts.append(f"subject: {email['subject']}") + if email.get("from_name") or email.get("from_addr"): + parts.append( + f"from: {email.get('from_name', '')} <{email.get('from_addr', '')}>" + ) + if email.get("to_name") or email.get("to_addr"): + parts.append( + f"to: {email.get('to_name', '')} <{email.get('to_addr', '')}>" + ) + if email.get("body"): + parts.append(email["body"]) + rich_map[eid] = "\n".join(parts) + + # Zendesk tickets — rich body folds in subject, org, description, + # all comment texts, and the related incident reference. + for ticket in self._mem._db["zd_tickets"].find( + {}, + { + "_id": 0, + "ticket_id": 1, + "subject": 1, + "org_name": 1, + "description": 1, + "comments": 1, + "related_incident": 1, + "priority": 1, + "status": 1, + }, ): - art_id = artifact.get("_id") - if art_id and artifact.get("content"): - rich_map[art_id] = artifact["content"] + tid = ticket.get("ticket_id") + if not tid: + continue + parts = [ + f"subject: {ticket.get('subject', '')}", + f"org: {ticket.get('org_name', '')}", + f"status: {ticket.get('status', '')}", + f"priority: {ticket.get('priority', '')}", + ] + if ticket.get("description"): + parts.append(f"description: {ticket['description']}") + if ticket.get("related_incident"): + parts.append(f"related_incident: {ticket['related_incident']}") + for c in ticket.get("comments") or []: + author = c.get("author", "") + text = c.get("text", "") + if text: + parts.append( + f"comment ({author}): {text}" + if author + else f"comment: {text}" + ) + rich_map[tid] = "\n".join(p for p in parts if p) + + # Salesforce opportunities — rich body from sf_opps collection. + for opp in self._mem._db["sf_opps"].find( + {}, + { + "_id": 0, + "opportunity_id": 1, + "account_name": 1, + "stage": 1, + "probability": 1, + "amount": 1, + "owner": 1, + "lead_source": 1, + "next_step": 1, + "risk_notes": 1, + "touchpoints": 1, + "close_date": 1, + }, + ): + oid = opp.get("opportunity_id") + if not oid: + continue + parts = [ + f"account: {opp.get('account_name', '')}", + f"stage: {opp.get('stage', '')}", + f"probability: {opp.get('probability', '')}%", + f"amount: ${opp.get('amount', 0):,}", + f"owner: {opp.get('owner', '')}", + f"close_date: {opp.get('close_date', '')}", + f"lead_source: {opp.get('lead_source', '')}", + f"next_step: {opp.get('next_step', '')}", + ] + for note in opp.get("risk_notes") or []: + parts.append(f"risk: {note}") + for tp in opp.get("touchpoints") or []: + subject = tp.get("subject", "") + sender = tp.get("sender", "") + ts = tp.get("timestamp", "") + if subject: + parts.append(f"touchpoint ({sender}, {ts}): {subject}") + rich_map[oid] = "\n".join(p for p in parts if p) + + # Salesforce accounts — rich body from sf_accounts collection. + for acc in self._mem._db["sf_accounts"].find( + {}, + { + "_id": 0, + "account_id": 1, + "name": 1, + "primary_contact": 1, + "type": 1, + "industry": 1, + "tier": 1, + "billing_region": 1, + "arr": 1, + "owner": 1, + "risk_flag": 1, + }, + ): + aid = acc.get("account_id") + if not aid: + continue + parts = [ + f"name: {acc.get('name', '')}", + f"type: {acc.get('type', '')}", + f"tier: {acc.get('tier', '')}", + f"industry: {acc.get('industry', '')}", + f"billing_region: {acc.get('billing_region', '')}", + f"arr: ${acc.get('arr', 0):,}", + f"owner: {acc.get('owner', '')}", + f"primary_contact: {acc.get('primary_contact', '')}", + ] + if acc.get("risk_flag"): + parts.append("risk_flag: true — ownership lapsed or at-risk") + rich_map[aid] = "\n".join(p for p in parts if p) for row in rows: if row["doc_id"] == "CONF-UNKNOWN" and row["doc_type"] == "confluence": @@ -591,29 +962,77 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: if row["doc_type"] == "confluence" and not row.get("dept"): row["dept"] = _dept_from_artifact_id(row["doc_id"]) # ── Orphan sweep ────────────────────────────────────────────── - # Create corpus rows for any artifacts in MongoDB not yet in corpus. - # Covers all retrievable types; excludes jira_comment (folded above) - # and persona_skill (not a corpus artifact). - # slack_thread = full thread document (correct corpus unit) - # slack = individual message fragments — excluded, same as jira_comment - # slack_messages collection also excluded for same reason - _TYPE_MAP = { + # Each collection is queried directly. This correctly handles the + # fact that emails, PRs, slack messages, ZD tickets, SF opps, and + # SF accounts all have their own dedicated collections and are NOT + # reliably present in the artifacts collection. + # + # artifacts is still swept for confluence/jira/slack_thread types + # (embed_artifact() is the write path for those), plus any CRM + # embeddings that landed there via crm_system._embed(). + # + # Exclusions: + # - jira_comment: folded into parent JIRA body above + # - persona_skill: internal planner state, not a corpus artifact + # - slack_messages individual rows: the corpus unit is the thread + # (from SimEvent slack_thread key), not individual messages + # - insider threat artifacts (exfil_/hoarding_/snooping_/dlp_ prefixes) + + existing_ids = {row["doc_id"] for row in rows} + + def _make_orphan_row( + doc_id, + doc_type, + title, + body, + day, + date, + timestamp, + actors, + tags, + artifact_type, + is_incident=False, + is_external=False, + ): + dept = _dept_from_artifact_id(doc_id) or next( + ( + _ACTOR_TO_DEPT.get(str(a), "") + for a in actors + if _ACTOR_TO_DEPT.get(str(a)) + ), + "", + ) + return { + "doc_id": doc_id, + "doc_type": doc_type, + "title": str(title)[:512], + "body": str(body), + "day": int(day or 0), + "date": str(date or ""), + "timestamp": str(timestamp or ""), + "actors": json.dumps(actors), + "tags": json.dumps(tags), + "artifact_ids": json.dumps({artifact_type: doc_id}), + "dept": dept, + "is_incident": is_incident, + "is_external": is_external, + } + + # ── artifacts (confluence, jira, slack_thread, CRM embeds) ─────── + _ARTIFACT_TYPE_MAP = { "confluence": "confluence", "slack_thread": "slack", - "email": "email", - "pr": "pr", "jira": "jira", + "zd_ticket": "zd_ticket", + "sf_opportunity": "sf_opp", } - existing_ids = {row["doc_id"] for row in rows} for artifact in self._mem._db["artifacts"].find( - {"type": {"$in": list(_TYPE_MAP.keys())}}, + {"type": {"$in": list(_ARTIFACT_TYPE_MAP.keys())}}, { "_id": 1, "type": 1, "content": 1, - "body": 1, "title": 1, - "subject": 1, "day": 1, "date": 1, "timestamp": 1, @@ -624,12 +1043,11 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: ): art_id = str(artifact.get("_id", "")) art_type = artifact.get("type", "") - doc_type = _TYPE_MAP.get(art_type, "sim_event") if not art_id or art_id in existing_ids: continue if any( - art_id.startswith(prefix) - for prefix in ("exfil_", "hoarding_", "snooping_", "dlp_") + art_id.startswith(p) + for p in ("exfil_", "hoarding_", "snooping_", "dlp_") ): logger.debug(f" skipping insider threat artifact: {art_id}") continue @@ -637,41 +1055,280 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: author = artifact.get("author") or meta.get("author", "") actors = artifact.get("actors") or ([author] if author else []) tags = meta.get("tags", [art_type]) - body = ( - artifact.get("content") - or artifact.get("body") - or artifact.get("subject") - or "" + body = rich_map.get(art_id) or artifact.get("content") or "" + title = artifact.get("title") or art_id + rows.append( + _make_orphan_row( + doc_id=art_id, + doc_type=_ARTIFACT_TYPE_MAP.get(art_type, "sim_event"), + title=title, + body=body, + day=artifact.get("day"), + date=artifact.get("date"), + timestamp=artifact.get("timestamp"), + actors=actors, + tags=tags, + artifact_type=art_type, + is_incident=any( + t in tags + for t in ( + "postmortem", + "incident", + "escalation", + "zd_escalated", + ) + ), + is_external=art_type in ("zd_ticket", "sf_opportunity"), + ) ) - title = artifact.get("title") or artifact.get("subject") or art_id - dept = _dept_from_artifact_id(art_id) or next( - ( - _ACTOR_TO_DEPT.get(str(a), "") - for a in actors - if _ACTOR_TO_DEPT.get(str(a)) - ), - "", + logger.debug(f" orphan artifact added: {art_id} ({art_type})") + existing_ids.add(art_id) + + # ── pull_requests ───────────────────────────────────────────────── + for pr in self._mem._db["pull_requests"].find( + {}, + { + "_id": 0, + "pr_id": 1, + "title": 1, + "author": 1, + "day": 1, + "date": 1, + "timestamp": 1, + "dept": 1, + }, + ): + pid = pr.get("pr_id", "") + if not pid or pid in existing_ids: + continue + body = rich_map.get(pid, "") + rows.append( + _make_orphan_row( + doc_id=pid, + doc_type="pr", + title=pr.get("title", pid), + body=body, + day=pr.get("day"), + date=pr.get("date"), + timestamp=pr.get("timestamp"), + actors=[pr["author"]] if pr.get("author") else [], + tags=["pr"], + artifact_type="pr", + ) ) + logger.debug(f" orphan PR added: {pid}") + existing_ids.add(pid) + + # ── emails ──────────────────────────────────────────────────────── + for email in self._mem._db["emails"].find( + {}, + { + "_id": 0, + "embed_id": 1, + "subject": 1, + "from_name": 1, + "from_addr": 1, + "direction": 1, + "day": 1, + "date": 1, + "timestamp": 1, + }, + ): + eid = email.get("embed_id", "") + if not eid or eid in existing_ids: + continue + body = rich_map.get(eid, "") + direction = email.get("direction", "") rows.append( - { - "doc_id": art_id, - "doc_type": doc_type, - "title": str(title)[:512], - "body": str(body), - "day": int(artifact.get("day", 0)), - "date": str(artifact.get("date", "")), - "timestamp": str(artifact.get("timestamp", "")), - "actors": json.dumps(actors), - "tags": json.dumps(tags), - "artifact_ids": json.dumps({art_type: art_id}), - "dept": dept, - "is_incident": any( - t in tags for t in ("postmortem", "incident") - ), - "is_external": art_type == "email", + _make_orphan_row( + doc_id=eid, + doc_type="email", + title=email.get("subject", eid), + body=body, + day=email.get("day"), + date=email.get("date"), + timestamp=email.get("timestamp"), + actors=[email["from_name"]] if email.get("from_name") else [], + tags=["email", direction] if direction else ["email"], + artifact_type="email", + is_external=True, + ) + ) + logger.debug(f" orphan email added: {eid}") + existing_ids.add(eid) + + # ── zd_tickets ──────────────────────────────────────────────────── + for ticket in self._mem._db["zd_tickets"].find( + {}, + { + "_id": 0, + "ticket_id": 1, + "subject": 1, + "org_name": 1, + "day": 1, + "date": 1, + "created_at": 1, + "status": 1, + "priority": 1, + }, + ): + tid = ticket.get("ticket_id", "") + if not tid or tid in existing_ids: + continue + body = rich_map.get(tid, "") + rows.append( + _make_orphan_row( + doc_id=tid, + doc_type="zd_ticket", + title=ticket.get("subject", tid), + body=body, + day=ticket.get("day"), + date=ticket.get("date"), + timestamp=ticket.get("created_at"), + actors=[], + tags=["zendesk", "support"], + artifact_type="zd_ticket", + is_external=True, + is_incident=ticket.get("priority") == "Urgent", + ) + ) + logger.debug(f" orphan ZD ticket added: {tid}") + existing_ids.add(tid) + + # ── sf_opps ─────────────────────────────────────────────────────── + for opp in self._mem._db["sf_opps"].find( + {}, + { + "_id": 0, + "opportunity_id": 1, + "account_name": 1, + "stage": 1, + "owner": 1, + "day": 1, + "date": 1, + "created_at": 1, + }, + ): + oid = opp.get("opportunity_id", "") + if not oid or oid in existing_ids: + continue + body = rich_map.get(oid, "") + title = ( + f"{opp.get('account_name', oid)} — {opp.get('stage', '')}" + ).strip(" —") + rows.append( + _make_orphan_row( + doc_id=oid, + doc_type="sf_opp", + title=title, + body=body, + day=opp.get("day"), + date=opp.get("date"), + timestamp=opp.get("created_at"), + actors=[opp["owner"]] if opp.get("owner") else [], + tags=["salesforce", "opportunity"], + artifact_type="sf_opp", + is_external=True, + ) + ) + logger.debug(f" orphan SF opp added: {oid}") + existing_ids.add(oid) + + # ── sf_accounts ─────────────────────────────────────────────────── + for acc in self._mem._db["sf_accounts"].find( + {}, + { + "_id": 0, + "account_id": 1, + "name": 1, + "owner": 1, + "created_at": 1, + }, + ): + aid = acc.get("account_id", "") + if not aid or aid in existing_ids: + continue + body = rich_map.get(aid, "") + rows.append( + _make_orphan_row( + doc_id=aid, + doc_type="sf_account", + title=acc.get("name", aid), + body=body, + day=None, + date=None, + timestamp=acc.get("created_at"), + actors=[acc["owner"]] if acc.get("owner") else [], + tags=["salesforce", "account"], + artifact_type="sf_account", + is_external=True, + ) + ) + logger.debug(f" orphan SF account added: {aid}") + existing_ids.add(aid) + + # ── slack_messages (thread-level grouping) ──────────────────────── + # Individual slack_messages rows are per-message. For the corpus we + # group by thread_id and concatenate message texts into one document, + # which matches how SimEvents reference slack content (by thread). + thread_buckets: Dict[str, dict] = {} + for msg in self._mem._db["slack_messages"].find( + {}, + { + "_id": 0, + "thread_id": 1, + "channel": 1, + "text": 1, + "author": 1, + "sender": 1, + "ts": 1, + "day": 1, + "date": 1, + }, + ): + tid = msg.get("thread_id", "") + if not tid or tid in existing_ids: + continue + if tid not in thread_buckets: + thread_buckets[tid] = { + "channel": msg.get("channel", ""), + "day": msg.get("day"), + "date": msg.get("date"), + "ts": msg.get("ts", ""), + "actors": set(), + "texts": [], } + bucket = thread_buckets[tid] + author = msg.get("author") or msg.get("sender", "") + if author: + bucket["actors"].add(author) + text = msg.get("text", "") + if text: + prefix = f"{author}: " if author else "" + bucket["texts"].append(f"{prefix}{text}") + + for tid, bucket in thread_buckets.items(): + if tid in existing_ids: + continue + actors = sorted(bucket["actors"]) + channel = bucket["channel"] + body = "\n".join(bucket["texts"]) + rows.append( + _make_orphan_row( + doc_id=tid, + doc_type="slack", + title=f"#{channel}" if channel else tid, + body=body, + day=bucket["day"], + date=bucket["date"], + timestamp=bucket["ts"], + actors=actors, + tags=["slack", channel] if channel else ["slack"], + artifact_type="slack_thread", + ) ) - logger.debug(f" orphan artifact added: {art_id} ({doc_type})") + logger.debug(f" orphan slack thread added: {tid}") + existing_ids.add(tid) except Exception as exc: logger.debug(f"MongoDB enrichment skipped: {exc}") @@ -736,6 +1393,29 @@ def __init__(self, corpus: List[dict], questions: List[dict], mem=None): else: self._bm25 = None + if _DENSE_AVAILABLE: + logger.info(" Embedding corpus for dense baseline...") + embeddings = [] + + for i, body in enumerate(self._bodies): + text_to_embed = ( + body.strip() if body and body.strip() else "empty document" + ) + vec = self._mem._embed( + text_to_embed, + input_type="search_document", + ) + embeddings.append(vec) + + if (i + 1) % 500 == 0: + logger.info(f" embedded {i + 1}/{len(self._bodies)} docs...") + + mat = np.array(embeddings, dtype=np.float32) + norms = np.linalg.norm(mat, axis=1, keepdims=True) + self._dense_matrix = mat / np.where(norms == 0, 1, norms) + else: + self._dense_matrix = None + # ── PUBLIC ──────────────────────────────────────────────────────────────── def run_bm25(self) -> Tuple[List[dict], Dict[str, Any]]: @@ -749,13 +1429,17 @@ def run_dense(self) -> Tuple[List[dict], Dict[str, Any]]: return self._run_retrieval(use_dense=True) def _rank(self, query: str, use_dense: bool, top_k: int = 10) -> List[str]: - if use_dense and self._mem is not None: - # Use the same collection and pattern as Memory.recall() - results = self._mem.recall(query=query, n=top_k) - corpus_ids = set(self._doc_ids) - filtered = [r["id"] for r in results if r.get("id") in corpus_ids] - return filtered[:top_k] - elif self._bm25 is not None: + if use_dense and self._dense_matrix is not None: + q_vec = np.array( + self._mem._embed(query, input_type="search_query"), dtype=np.float32 + ) + q_vec /= max(np.linalg.norm(q_vec), 1e-9) + + scores = self._dense_matrix @ q_vec + indices = scores.argsort()[::-1][:top_k] + return [self._doc_ids[int(i)] for i in indices] + + elif not use_dense and self._bm25 is not None: scores = self._bm25.get_scores(_tokenize(query)) indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) return [self._doc_ids[i] for i in indices[:top_k]] @@ -816,18 +1500,6 @@ def _mean(vals): } return per_question, aggregate - """ def _rank(self, query: str, use_dense: bool, top_k: int = 10) -> List[str]: - if use_dense and self._dense_matrix is not None: - q_vec = self._dense_model.encode([query], normalize_embeddings=True)[0] - scores = self._dense_matrix @ q_vec - indices = scores.argsort()[::-1][:top_k] - return [self._doc_ids[i] for i in indices] - elif self._bm25 is not None: - scores = self._bm25.get_scores(_tokenize(query)) - indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) - return [self._doc_ids[i] for i in indices[:top_k]] - return [] """ - # ───────────────────────────────────────────────────────────────────────────── # DATASET CARD WRITER @@ -957,7 +1629,7 @@ def _render( | Column | Type | Description | |---|---|---| | `doc_id` | str | Unique artifact ID (e.g. `ORG-42`, `CONF-ENG-007`) | - | `doc_type` | str | `jira`, `confluence`, `slack`, `email`, `pr`, `sim_event` | + | `doc_type` | str | `jira`, `confluence`, `slack`, `email`, `pr`, `zd_ticket`, `sf_opp`, `sf_account`, `sim_event` | | `title` | str | Human-readable title or subject | | `body` | str | Full retrievable text | | `day` | int | Simulation day (1-indexed) | @@ -996,6 +1668,11 @@ def _render( | `POSTMORTEM` | Which Confluence doc captured the postmortem for incident X? | Yes (2-hop) | | `STANDUP` | What did person X report at standup on Day N? | No | | `CUSTOMER_ESC` | Who handled the escalation from customer X and what action was taken? | Yes (2-hop) | + | `ZD_RESOLUTION` | How quickly was a specific Zendesk ticket resolved? | Yes (cross-thread) | + | `DATADOG_ALERT` | What server metric triggered the alert on Day N? | No | + | `INVOICE_SLA` | What SLA credits were applied to a specific customer? | Yes | + | `NPS_SCORE` | What was the reasoning behind the customer's NPS score? | Yes | + | `PR_REVIEW` | What was the verdict of the code review? | No | ### Question Schema @@ -1126,7 +1803,7 @@ def _write_parquet(rows: List[dict], out_dir: Path, stem: str = "part-00000") -> out_path = out_dir / f"{stem}.parquet" pq.write_table(tbl, out_path, compression="snappy") logger.info( - f" → {out_path} ({len(rows):,} rows, {out_path.stat().st_size // 1024} KB)" + f" → {out_path} ({len(rows):,} rows, {out_path.stat().st_size // 2560} KB)" ) diff --git a/eval/insider_threat/eval_insider_threat.py b/eval/insider_threat/eval_insider_threat.py index bc81ced..29534be 100644 --- a/eval/insider_threat/eval_insider_threat.py +++ b/eval/insider_threat/eval_insider_threat.py @@ -1327,16 +1327,6 @@ def run_correlation( # Telemetry scoped to this actor only relevant = [r for r in access_log if r.get("actor") == actor] - # Also pull phone_call records where this actor made the call, - # and any vishing auth records filed under other actors that - # originated from a call by this actor - phone_calls = [ - r - for r in access_log - if r.get("record_type") == "phone_call" and r.get("actor") != actor - # include calls made by others only if they precede an - # idp_auth on this actor's account (they may be the victim) - ] vishing_on_actor = [ r for r in access_log diff --git a/eval/retrieval_extensions.py b/eval/retrieval_extensions.py index 8abcde5..4d283ba 100644 --- a/eval/retrieval_extensions.py +++ b/eval/retrieval_extensions.py @@ -26,56 +26,19 @@ Wraps any base Retriever and expands results by walking artifact relationship edges that are embedded in the corpus itself. - The corpus documents produced by OrgForge's simulation carry relationship - metadata in several forms: - • JSON-encoded "related_ids" / "causal_chain" / "artifact_ids" fields - • Inline artifact-ID tokens (ORG-\d+, CONF-\w+, EMAIL-\d+, etc.) - referenced in the body text - • A free-form "evidence_chain" field - The graph expander: 1. Indexes all edges at index() time → O(|corpus|) build 2. At retrieve() time, takes the base retriever's top-K results, - adds 1-hop neighbours from the edge graph, re-ranks the combined + adds 1-hop neighbors from the edge graph, re-ranks the combined pool by (base_score + neighbour_boost), and returns top-K. - Neighbour boost decays with hop distance: + Neighbor boost decays with hop distance: boost(d, hop) = NEIGHBOUR_BOOST_BASE ** hop (default: 0.5 per hop) Usage: python eval_e2e.py --retriever graph-bm25 --generator claude python eval_e2e.py --retriever graph-cohere --generator claude python eval_e2e.py --retriever graph-rrf --generator claude - -Integration ------------ -Add the following block to eval_e2e.py's build_retriever() function: - - # ── paste after the existing if/elif chain ────────────────────────── - from retrieval_extensions import RRFRetriever, GraphAugmentedRetriever - - if name == "rrf": - return RRFRetriever([BM25Retriever(), CohereRetriever()]) - if name == "rrf-openai": - return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) - if name == "rrf-bedrock": - return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) - if name == "graph-bm25": - return GraphAugmentedRetriever(BM25Retriever()) - if name == "graph-cohere": - return GraphAugmentedRetriever(CohereRetriever()) - if name == "graph-rrf": - base = RRFRetriever([BM25Retriever(), CohereRetriever()]) - return GraphAugmentedRetriever(base) - # ──────────────────────────────────────────────────────────────────── - -Also add the new choices to the --retriever argparse argument: - - choices=[ - "bm25", "cohere", "cohere-bedrock", "openai", - "rrf", "rrf-openai", "rrf-bedrock", - "graph-bm25", "graph-cohere", "graph-rrf", - ], """ from __future__ import annotations @@ -84,9 +47,8 @@ import logging import re from collections import defaultdict -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, List, Set, Tuple -import numpy as np logger = logging.getLogger("orgforge.retrieval_extensions") @@ -110,7 +72,7 @@ # RRF smoothing constant (Cormack et al. 2009 recommend k=60). RRF_K: int = 60 -# Graph expansion: score boost applied to 1-hop neighbours. +# Graph expansion: score boost applied to 1-hop neighbors. # Each additional hop multiplies by this factor (geometric decay). NEIGHBOUR_BOOST_BASE: float = 0.5 @@ -253,7 +215,7 @@ def build(self, corpus: List[dict]) -> None: for doc in corpus: src = doc["doc_id"] - neighbours: Set[str] = set() + neighbors: Set[str] = set() # 1. Structured relation fields. for field in _RELATION_FIELDS: @@ -266,30 +228,30 @@ def build(self, corpus: List[dict]) -> None: val = json.loads(val) except (json.JSONDecodeError, ValueError): # Try to extract IDs inline from the raw string. - neighbours.update(self._extract_ids_from_text(val, doc_ids)) + neighbors.update(self._extract_ids_from_text(val, doc_ids)) continue # Might be a dict (artifact_ids maps type → id). if isinstance(val, dict): for v in val.values(): if isinstance(v, str) and v in doc_ids: - neighbours.add(v) + neighbors.add(v) elif isinstance(v, list): - neighbours.update(x for x in v if x in doc_ids) + neighbors.update(x for x in v if x in doc_ids) elif isinstance(val, list): for item in val: if isinstance(item, str) and item in doc_ids: - neighbours.add(item) + neighbors.add(item) # 2. Inline artifact-ID tokens in body / title. for text_field in ("body", "content", "title"): text = doc.get(text_field) or "" - neighbours.update(self._extract_ids_from_text(text, doc_ids)) + neighbors.update(self._extract_ids_from_text(text, doc_ids)) # Remove self-loops. - neighbours.discard(src) + neighbors.discard(src) # Register bidirectional edges. - for tgt in neighbours: + for tgt in neighbors: self._adj[src].add(tgt) self._adj[tgt].add(src) @@ -303,8 +265,8 @@ def build(self, corpus: List[dict]) -> None: # Query # ------------------------------------------------------------------ - def neighbours(self, doc_id: str) -> Set[str]: - """Return all direct neighbours of doc_id.""" + def neighbors(self, doc_id: str) -> Set[str]: + """Return all direct neighbors of doc_id.""" return set(self._adj.get(doc_id, set())) def expand( @@ -326,7 +288,7 @@ def expand( current_hop += 1 next_frontier: Set[str] = set() for node in frontier: - for nbr in self.neighbours(node): + for nbr in self.neighbors(node): if nbr not in visited and nbr not in set(seed_ids): visited[nbr] = current_hop next_frontier.add(nbr) @@ -372,7 +334,7 @@ class GraphAugmentedRetriever: max_hops : int Graph expansion depth. 1–2 recommended; ≥3 adds noise. neighbour_boost_base : float - Multiplicative decay per hop. 0.5 means 1-hop neighbours get 50% + Multiplicative decay per hop. 0.5 means 1-hop neighbors get 50% of the connecting seed's score, 2-hop get 25%, etc. candidate_k : int Seeds fetched from base retriever before graph expansion. @@ -493,47 +455,3 @@ def explain(self, query: str, top_k: int = 10) -> List[dict]: } ) return results - - -# ───────────────────────────────────────────────────────────────────────────── -# build_retriever() PATCH -# ───────────────────────────────────────────────────────────────────────────── -# Copy-paste this function to REPLACE build_retriever() in eval_e2e.py. -# It adds rrf / rrf-openai / rrf-bedrock / graph-* to the existing choices. -# -# def build_retriever(name: str, region: str = "us-east-1") -> Retriever: -# from retrieval_extensions import RRFRetriever, GraphAugmentedRetriever -# -# # ── original retrievers ─────────────────────────────────────────────────── -# if name == "bm25": -# return BM25Retriever() -# if name == "cohere": -# return CohereRetriever() -# if name == "cohere-bedrock": -# return BedrockCohereRetriever(region=region) -# if name == "openai": -# return OpenAIRetriever() -# -# # ── RRF retrievers ──────────────────────────────────────────────────────── -# if name == "rrf": -# return RRFRetriever([BM25Retriever(), CohereRetriever()]) -# if name == "rrf-openai": -# return RRFRetriever([BM25Retriever(), OpenAIRetriever()]) -# if name == "rrf-bedrock": -# return RRFRetriever([BM25Retriever(), BedrockCohereRetriever(region=region)]) -# -# # ── Graph-augmented retrievers ──────────────────────────────────────────── -# if name == "graph-bm25": -# return GraphAugmentedRetriever(BM25Retriever()) -# if name == "graph-cohere": -# return GraphAugmentedRetriever(CohereRetriever()) -# if name == "graph-rrf": -# base = RRFRetriever([BM25Retriever(), CohereRetriever()]) -# return GraphAugmentedRetriever(base) -# -# raise ValueError( -# f"Unknown retriever: {name!r}. " -# "Choose bm25 | cohere | cohere-bedrock | openai | " -# "rrf | rrf-openai | rrf-bedrock | " -# "graph-bm25 | graph-cohere | graph-rrf" -# ) diff --git a/eval/scorer.py b/eval/scorer.py index 009a5fb..f49706f 100644 --- a/eval/scorer.py +++ b/eval/scorer.py @@ -15,7 +15,9 @@ ---------------------- RETRIEVAL Exact artifact_id match + optional timestamp proximity bonus. CAUSAL Artifact match AND event_type match required for full credit. - TEMPORAL Boolean + optional departure_day agreement. + TEMPORAL Boolean match routed by temporal_category (knowledge_gap/ + point_in_time/stress_state/propagation). Ground truth field + is had_knowledge, was_true, or knew_before per sub-category. GAP_DETECTION Boolean was_actioned + downstream artifact overlap. ROUTING first_recipient exact match. PLAN dept + theme match (theme uses substring matching for LLM prose). @@ -63,6 +65,7 @@ } TEMPORAL: + # knowledge_gap sub-category (default): { "had_knowledge": true, "person": "Alice", @@ -70,6 +73,16 @@ "departure_day": null, # null if agent thinks no departure "reasoning": "..." # free text — not scored } + # point_in_time / stress_state sub-categories: + { + "was_true": false, + "reasoning": "..." + } + # propagation sub-category: + { + "knew_before": true, + "reasoning": "..." + } GAP_DETECTION: { @@ -277,31 +290,53 @@ def score( class TemporalScorer(_BaseScorer): """ - TEMPORAL — "Did person P know about domain D when incident I was opened?" - - Full credit: had_knowledge boolean matches AND departure_day matches - (within ±1 day tolerance for off-by-one). - Partial: boolean correct but departure_day wrong or missing (0.6). + TEMPORAL — multi-sub-category scorer routed by temporal_category. + + Sub-categories and their ground truth boolean fields: + knowledge_gap (default) — had_knowledge + optional departure_day + point_in_time — was_true (no secondary date field) + stress_state — was_true (no secondary date field) + propagation — knew_before (no secondary date field) + + Full credit logic per sub-category: + knowledge_gap: boolean matches AND departure_day matches (±1 day). + Partial (0.6) when boolean correct but departure_day wrong. + point_in_time / stress_state / propagation: boolean match only → 1.0 or 0.0. """ + # Maps temporal_category → (gt_field, agent_field) + _BOOL_FIELDS = { + "knowledge_gap": ("had_knowledge", "had_knowledge"), + "point_in_time": ("was_true", "was_true"), + "stress_state": ("was_true", "was_true"), + "propagation": ("knew_before", "knew_before"), + } + def score( self, question: dict, agent_answer: dict ) -> Tuple[float, float, Optional[str]]: gt = question["ground_truth"] - gt_bool = gt.get("had_knowledge") - gt_dep_day = gt.get("departure_day") # int or None - agent_bool = agent_answer.get("had_knowledge") - agent_dep_day = agent_answer.get("departure_day") + category = question.get("temporal_category", "knowledge_gap") + + gt_field, agent_field = self._BOOL_FIELDS.get( + category, ("had_knowledge", "had_knowledge") + ) + gt_bool = gt.get(gt_field) + agent_bool = agent_answer.get(agent_field) bool_match = agent_bool == gt_bool if not bool_match: primary = 0.0 - failure = f"had_knowledge expected {gt_bool}, got {agent_bool}" - else: - # Departure day agreement + failure = f"{gt_field} expected {gt_bool}, got {agent_bool}" + elif category == "knowledge_gap": + # knowledge_gap questions carry an optional departure_day for + # additional precision credit. + gt_dep_day = gt.get("departure_day") # int or None + agent_dep_day = agent_answer.get("departure_day") + if gt_dep_day is None and agent_dep_day is None: - # Both agree no departure occurred — full credit. + # Both agree no departure relevant — full credit. # Note: when the dataset has had_knowledge=True for all questions # (e.g. short sim runs where no incident touches a departed # employee's domains), an agent that always returns @@ -325,11 +360,116 @@ def score( if gt_dep_day is not None else "Agent reported a departure day that doesn't exist" ) + else: + # point_in_time, stress_state, propagation — boolean match is + # sufficient for full credit; no secondary date field to check. + primary = 1.0 + failure = None + + evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), + ) + return primary, evidence, failure + + +class MultiHopScorer(_BaseScorer): + """ + MULTI_HOP — full customer complaint → resolution chain traversal. + + Scored as a weighted checklist across four hops. Each hop is worth + 0.25 of the primary score — partial credit scales with how far the + agent traced the chain before losing it. + + Hop 1 (0.25): correct email_id / source identified + Hop 2 (0.25): correct slack_thread_id (internal relay) + Hop 3 (0.25): correct ticket_id + assignee + Hop 4 (0.25): correct reply_id OR correct resolved_same_day boolean + This structure means an agent that traces email→slack→jira but misses + the reply scores 0.75 rather than 0 — which correctly reflects that + it found 3 of 4 artifacts. + """ + + def score( + self, question: dict, agent_answer: dict + ) -> Tuple[float, float, Optional[str]]: + gt = question["ground_truth"] + failures = [] + hop_scores = [] + + # Hop 1: source / email identified + gt_email = gt.get("email_id", "") + agent_email = agent_answer.get("email_id", "") + hop1 = 1.0 if agent_email == gt_email else 0.0 + if not hop1: + failures.append( + f"Hop 1: email_id expected {gt_email!r}, got {agent_email!r}" + ) + hop_scores.append(hop1) + + # Hop 2: internal relay (Slack thread) + gt_slack = gt.get("slack_thread_id", "") + agent_slack = agent_answer.get("slack_thread_id", "") + hop2 = ( + 1.0 + if (gt_slack and agent_slack == gt_slack) + else (0.5 if (gt_slack and agent_slack) else (1.0 if not gt_slack else 0.0)) + ) + if gt_slack and agent_slack != gt_slack: + failures.append( + f"Hop 2: slack_thread expected {gt_slack!r}, got {agent_slack!r}" + ) + hop_scores.append(hop2) + + # Hop 3: ticket + assignee + gt_ticket = gt.get("ticket_id", "") + agent_ticket = agent_answer.get("ticket_id", "") + gt_assignee = gt.get("assignee", "").lower() + agent_assignee = agent_answer.get("assignee", "").lower() + ticket_match = agent_ticket == gt_ticket + assignee_match = agent_assignee == gt_assignee + hop3 = ( + 1.0 if (ticket_match and assignee_match) else (0.6 if ticket_match else 0.0) + ) + if not ticket_match: + failures.append( + f"Hop 3: ticket expected {gt_ticket!r}, got {agent_ticket!r}" + ) + elif not assignee_match: + failures.append( + f"Hop 3: assignee expected {gt_assignee!r}, got {agent_assignee!r}" + ) + hop_scores.append(hop3) + + # Hop 4: reply sent + same-day resolution + gt_reply = gt.get("reply_id", "") + gt_same_day = gt.get("resolved_same_day", False) + agent_reply = agent_answer.get("reply_id", "") + agent_same_day = agent_answer.get("resolved_same_day") + reply_match = (not gt_reply) or (agent_reply == gt_reply) + same_day_match = agent_same_day == gt_same_day + hop4 = ( + 1.0 + if (reply_match and same_day_match) + else (0.5 if (reply_match or same_day_match) else 0.0) + ) + if not reply_match: + failures.append( + f"Hop 4: reply_id expected {gt_reply!r}, got {agent_reply!r}" + ) + if not same_day_match: + failures.append( + f"Hop 4: resolved_same_day expected {gt_same_day}, got {agent_same_day}" + ) + hop_scores.append(hop4) + + primary = sum(hop_scores) / len(hop_scores) evidence = self._evidence_overlap( question.get("evidence_chain", []), agent_answer.get("retrieved_artifact_ids", []), ) + failure = "; ".join(failures) if failures else None return primary, evidence, failure @@ -358,6 +498,15 @@ def score( primary = 1.0 failure = None else: + # was_actioned=True: reward boolean correctness with a 0.6 floor, + # then scale the remaining 0.4 by downstream artifact recall. + # INTENTIONAL ASYMMETRY: a correct True boolean with zero downstream + # overlap yields primary=0.6, which after PRIMARY_WEIGHT produces a + # combined floor of ~0.48+ (plus any evidence credit). This is + # deliberate — correctly identifying that an email was actioned is + # meaningful signal even when the agent can't enumerate the artifacts. + # If you want a stricter floor, lower 0.6 here (e.g. to 0.4) or make + # it conditional on ds_overlap > 0. ds_overlap = self._evidence_overlap(gt_downstream, agent_downstream) primary = 0.6 + 0.4 * ds_overlap failure = ( @@ -475,7 +624,10 @@ def score( agent_set = set(agent_actors) overlap = len(gt_set & agent_set) / len(gt_set) - if overlap == 1.0 and len(agent_set) == len(gt_set): + if overlap == 1.0: + # All ground-truth actors retrieved — full credit regardless of + # extra actors returned. Penalising for extras would unfairly + # punish agents that recall correctly but over-enumerate slightly. primary = 1.0 failure = None elif overlap > 0: @@ -548,7 +700,9 @@ def score( agent_set = set(agent_gaps) overlap = len(gt_set & agent_set) / len(gt_set) - if overlap == 1.0 and len(agent_set) == len(gt_set): + if overlap == 1.0: + # Full recall of ground-truth gap areas — full credit regardless + # of any additional areas the agent returns. primary = 1.0 failure = None elif overlap > 0: @@ -857,18 +1011,20 @@ def score( f"and credit ({agent_credit} vs {gt_credit}) both wrong" ) - # Affected orgs as evidence (secondary signal for partial credit) - evidence = ( - self._evidence_overlap( - gt_orgs, - agent_orgs, - ) - if gt_orgs - else self._evidence_overlap( - question.get("evidence_chain", []), - agent_answer.get("retrieved_artifact_ids", []), - ) + # Evidence: blend evidence_chain retrieval (did the agent find the invoice?) + # with affected_orgs overlap (did it identify the right customers?). + # evidence_chain recall is weighted more heavily (0.7) because it + # directly reflects whether the invoice artifact was retrieved; org + # overlap (0.3) is a secondary signal of answer quality. + chain_evidence = self._evidence_overlap( + question.get("evidence_chain", []), + agent_answer.get("retrieved_artifact_ids", []), ) + if gt_orgs: + org_overlap = self._evidence_overlap(gt_orgs, agent_orgs) + evidence = round(0.7 * chain_evidence + 0.3 * org_overlap, 4) + else: + evidence = chain_evidence return primary, evidence, failure @@ -891,6 +1047,7 @@ def score( "INVOICE_SLA": InvoiceSLAScorer(), "DATADOG_ALERT": RetrievalScorer(), "PR_REVIEW": PRReviewScorer(), + "MULTI_HOP": MultiHopScorer(), } @@ -1103,6 +1260,15 @@ def _mean(vals): "verdict": gt.get("verdict", ""), "reviewer": gt.get("reviewer", ""), } + elif qtype == "MULTI_HOP": + mock_answers[qid] = { + "email_id": gt.get("email_id", ""), + "slack_thread_id": gt.get("slack_thread_id", ""), + "ticket_id": gt.get("ticket_id", ""), + "assignee": gt.get("assignee", ""), + "reply_id": gt.get("reply_id", ""), + "resolved_same_day": gt.get("resolved_same_day"), + } scorer = OrgForgeScorer() results = scorer.score_all(questions, mock_answers) diff --git a/src/confluence_writer.py b/src/confluence_writer.py index 46745dd..f8aa2b9 100644 --- a/src/confluence_writer.py +++ b/src/confluence_writer.py @@ -400,6 +400,33 @@ def write_design_doc( author, "design", mem=self._mem, graph_dynamics=self._gd ) + persona = self._config.get("personas", {}).get(author, {}) + expertise_list = persona.get("expertise", ["general tasks"]) + expertise_str = ", ".join(str(e) for e in expertise_list[:5]) + author_dept = next( + (d for d, members in self._org_chart.items() if author in members), + "Unknown", + ) + + # Pull live domain registry context for orphaned domains so the LLM + # knows it's writing about an underdocumented area + orphaned_domain_context = "" + all_domains = list( + self._mem._db["domain_registry"].find({"primary_owner": None}) + ) + for rec in all_domains: + tags = rec.get("system_tags", []) + topic_lower = topic.lower() + if any(tag in topic_lower for tag in tags): + pct = int(rec.get("documentation_coverage", 0) * 100) + known_by = rec.get("known_by", []) + orphaned_domain_context += ( + f"\n⚠ '{rec['domain']}' is an orphaned domain: " + f"former owner={rec.get('former_owner', 'unknown')}, " + f"documentation={pct}%, " + f"partial knowledge held by: {known_by or 'nobody'}." + ) + agent = make_agent( role="Technical Lead", goal="Document technical decisions and extract an actionable ticket.", @@ -412,15 +439,46 @@ def write_design_doc( f"Background context: {ctx}\n" f"Existing pages you may reference:\n{related}\n\n" f"Write a design doc Confluence page with ID {conf_id}.\n" - f"Also extract 1 concrete next steps as a single JIRA ticket definition.\n" - f"Respond ONLY with valid JSON matching this exact schema:\n" + f"Also extract 1 concrete next step as a JIRA ticket.\n\n" + + ( + f"DOMAIN CONTEXT:{orphaned_domain_context}\n\n" + if orphaned_domain_context + else "" + ) + + "### SELF-AUDIT (fill metadata objectively, not in character)\n" + f"Your expertise on record: [{expertise_str}]\n" + f"Your department: {author_dept}\n" + "Compare every topic in your doc against that expertise list.\n" + "If the doc discusses areas NOT in that list, name them in " + "'topics_beyond_author_expertise'.\n" + "If you had to guess, hedge, or hand-wave on any claim, list it " + "in 'hedged_claims'.\n" + "If you deferred or left incomplete any section you know should " + "exist, list it in 'deferred_or_incomplete'.\n\n" + "Use these criteria:\n" + " author_domain_fit:\n" + " 'high' — doc demonstrates fluency: correct abstractions, aware of edge cases\n" + " 'medium' — doc is functional but shows shallow understanding or minor missteps\n" + " 'low' — doc shows clear unfamiliarity: wrong patterns, missing fundamentals\n\n" + " gap_classification:\n" + f" 'none' — {author}'s expertise aligns with all domains in this doc\n" + f" 'possible' — doc touches 1-2 domains outside {author}'s expertise but content looks adequate\n" + f" 'likely' — doc touches domains outside {author}'s expertise AND the content shows it\n\n" + f"Respond ONLY with valid JSON:\n" f"{{\n" - f' "markdown_doc": "string (full Markdown, no main # title, start directly with ## Problem Statement)",\n' + f' "markdown_doc": "full Markdown, no # title, start with ' + f'## Problem Statement",\n' f' "new_tickets": [\n' - f' {{"title": "string", ' - f'"assignee": "string (must be exactly: {author})", ' + f' {{"title": "string", "assignee": "{author}", ' f'"story_points": 1|2|3|5|8}}\n' - f" ]\n" + f" ],\n" + f' "metadata": {{\n' + f' "author_domain_fit": "low | medium | high",\n' + f' "gap_classification": "none | possible | likely",\n' + f' "topics_beyond_author_expertise": ["string"],\n' + f' "hedged_claims": ["string"],\n' + f' "deferred_or_incomplete": ["string"]\n' + f" }}\n" f"}}" ), expected_output="Valid JSON only. No markdown fences.", @@ -444,6 +502,7 @@ def write_design_doc( parsed = json.loads(clean) content = parsed.get("markdown_doc", "Draft pending.") new_tickets = parsed.get("new_tickets", []) + metadata = parsed.get("metadata", {}) except json.JSONDecodeError as e: logger.warning( f"[confluence] JSON parse failed for design doc: {e} — " @@ -451,6 +510,7 @@ def write_design_doc( ) content = raw new_tickets = [] + metadata = {} conf_ids = self._finalize_page( raw_content=content, @@ -462,30 +522,69 @@ def write_design_doc( subdir="design", tags=["confluence", "design_doc"], facts={"title": f"Design: {topic[:80]}", "type": "design_doc"}, + skip_event=True, ) - gaps = self._lifecycle.scan_for_knowledge_gaps( - text=content, - triggered_by=conf_ids[0], - day=self._state.day, - date_str=date_str, - state=self._state, - timestamp=timestamp, + _updated_domains = [ + rec["domain"] + for rec in self._mem._db["domain_registry"].find( + {"known_by": author, "last_updated_day": self._state.day} + ) + ] + + domain_fit = metadata.get("author_domain_fit", "high") + gap_class = metadata.get("gap_classification", "none") + beyond_expertise = metadata.get("topics_beyond_author_expertise", []) + hedged = metadata.get("hedged_claims", []) + deferred = metadata.get("deferred_or_incomplete", []) + + gap_detected = ( + domain_fit == "low" + or gap_class == "likely" + or (gap_class == "possible" and len(beyond_expertise) > 0) ) - if gaps: - gap_lines = "\n\n".join( - f"> ⚠️ **Knowledge Gap**: `{g.domain_hit}` was owned by " - f"ex-{g.departed_name} (~{int(g.documented_pct * 100)}% documented). " - f"Proceed with caution." - for g in gaps + + if gap_detected: + self._mem.log_event( + SimEvent( + type="knowledge_gap_detected", + timestamp=timestamp, + day=self._state.day, + date=date_str, + actors=[author], + artifact_ids={"confluence": conf_id}, + facts={ + "detection_method": "author_self_audit", + "topic": topic, + "author_domain_fit": domain_fit, + "author_expertise": expertise_list, + "gap_classification": gap_class, + "topics_beyond_expertise": beyond_expertise, + "hedged_claims": hedged, + "deferred_sections": deferred, + }, + summary=( + f"Knowledge gap detected in {conf_id}: " + f"{author} (expertise: {expertise_str}) wrote about '{topic}' " + f"with fit={domain_fit}, gap={gap_class}" + ), + tags=["knowledge_gap", "confluence", "design_doc"], + ) ) - import os + if beyond_expertise: + targeted_text = ". ".join(beyond_expertise) + if hedged: + targeted_text += ". " + ". ".join(hedged) - doc_path = f"{self._base}/confluence/design/{conf_ids[0]}.md" - if os.path.exists(doc_path): - with open(doc_path, "a") as f: - f.write(f"\n\n{gap_lines}") + self._lifecycle.scan_for_knowledge_gaps( + text=targeted_text, + triggered_by=conf_id, + day=self._state.day, + date_str=date_str, + state=self._state, + timestamp=timestamp, + ) created_ticket_ids = self._spawn_tickets( new_tickets, author, participants, date_str, timestamp @@ -511,6 +610,9 @@ def write_design_doc( "type": "design_doc", "spawned_tickets": created_ticket_ids, "causal_chain": chain.snapshot(), # ← add this + "author_domain_fit": metadata.get("author_domain_fit", "high"), + "gap_classification": metadata.get("gap_classification", "none"), + "domains_updated": _updated_domains, }, summary=( f"{author} created {conf_ids[0]} and spawned " @@ -575,7 +677,11 @@ def write_adhoc_page( doc_history = list( self._mem._events.find( - {"type": "confluence_created"}, {"facts.title": 1, "actors": 1} + { + "type": "confluence_created", + "timestamp": {"$lte": self._clock.now(resolved_author).isoformat()}, + }, + {"facts.title": 1, "actors": 1}, ) .sort("timestamp", -1) .limit(20) @@ -706,10 +812,6 @@ def write_adhoc_page( facts={"title": title, "adhoc": True}, ) - # ───────────────────────────────────────────────────────────────────────── - # PRIVATE — PAGE FINALISATION PIPELINE - # ───────────────────────────────────────────────────────────────────────── - def _finalize_page( self, raw_content: str, @@ -722,9 +824,10 @@ def _finalize_page( tags: List[str], facts: Dict, extra_artifact_ids: Optional[Dict[str, str]] = None, + skip_event: bool = False, ) -> List[str]: """ - Common finalisation pipeline for every Confluence page: + Common finalization pipeline for every Confluence page: 1. Strip broken cross-references 2. Register ID (raises DuplicateArtifactError — caller handles) 3. Chunk into child pages if content is long @@ -749,13 +852,10 @@ def _finalize_page( created_ids: List[str] = [] for page in pages: - # _registry.chunk_into_pages already registered IDs — - # catch the rare race where an ID was registered externally logger.info( f"[finalize] embedding page.id={page.id} parent={page.parent_id or 'ROOT'} content_len={len(page.content)}" ) try: - # strip broken refs one more time after chunking added headers final_content = self._registry.strip_broken_references(page.content) except Exception as e: logger.info(f"[finalize] Caught exception {e}") @@ -783,6 +883,16 @@ def _finalize_page( metadata=meta, ) + if page.parent_id is None and author and "genesis" not in (tags or []): + domains_updated = self._update_domain_registry_on_write( + author=author, + title=page.title, + content=final_content, + day=self._state.day, + ) + if domains_updated: + meta["domains_updated"] = domains_updated + if page.parent_id is None and author: self._mem._db["author_expertise"].update_one( {"author": author}, @@ -823,21 +933,23 @@ def _finalize_page( f"summary={'Child' if page.parent_id else 'Page'} {page.id} created: {page.title} " f"tags={tags}" ) - self._mem.log_event( - SimEvent( - type="confluence_created", - timestamp=timestamp, - day=self._state.day, - date=date_str, - actors=[author], - artifact_ids=artifact_ids, - facts=page_facts, - summary=( - f"{'Child' if page.parent_id else 'Page'} {page.id} created: {page.title}" - ), - tags=tags, + + if not skip_event: + self._mem.log_event( + SimEvent( + type="confluence_created", + timestamp=timestamp, + day=self._state.day, + date=date_str, + actors=[author], + artifact_ids=artifact_ids, + facts=page_facts, + summary=( + f"{'Child' if page.parent_id else 'Page'} {page.id} created: {page.title}" + ), + tags=tags, + ) ) - ) logger.debug(f"[finalize] post-log-event page.id={page.id}") logger.info(f"[confluence] _finalize_page complete: {page.id}") @@ -910,12 +1022,71 @@ def _spawn_tickets( created_ids.append(tid) return created_ids + def _update_domain_registry_on_write( + self, + author: str, + title: str, + content: str, + day: int, + coverage_delta: float = 0.10, + ) -> List[str]: + """ + After any Confluence page is finalised, check whether the page title or + content touches any registered domain in the DomainRegistry. If it does, + increment documentation_coverage by coverage_delta (default +10%) and + add the author to known_by. + + This is the recovery arc: every page written against an orphaned domain + nudges coverage upward. The planner prompt and gap detection both read + live coverage, so this produces visible narrative improvement over time. + + Matching is done against system_tags so variant spellings still resolve + (e.g. "titan" matches "TitanDB", "auth" matches "legacy auth service"). + + Returns: + List of domain names that were updated. + """ + updated: List[str] = [] + + all_domains = list(self._mem._db["domain_registry"].find({})) + if not all_domains: + return updated + + search_text = f"{title} {content[:500]}".lower() + + for rec in all_domains: + tags = rec.get("system_tags", []) + if not any(tag in search_text for tag in tags): + continue + + old_coverage = rec.get("documentation_coverage", 0.0) + new_coverage = min(1.0, old_coverage + coverage_delta) + + self._mem._db["domain_registry"].update_one( + {"_id": rec["_id"]}, + { + "$set": { + "documentation_coverage": round(new_coverage, 3), + "last_updated_day": day, + }, + "$addToSet": {"known_by": author}, + }, + ) + updated.append(rec["domain"]) + logger.info( + f" [dim]→ Domain registry: '{rec['domain']}' coverage " + f"{int(old_coverage * 100)}% → {int(new_coverage * 100)}% " + f"(author: {author})[/dim]" + ) + + return updated + def _pick_dept_author(self, prefix: str) -> str: """Return a random member of the department matching prefix, fallback to any employee.""" for dept, members in self._org_chart.items(): if prefix.upper() in dept.upper() and members: return random.choice(members) - # Fallback: any employee at all + return random.choice(self._all_names) def _conf_prefix_for(self, author: str) -> str: @@ -1017,10 +1188,37 @@ def _id_prefix_from_id(conf_id: str) -> str: return parts[1] if len(parts) >= 3 else "GEN" def _knowledge_gap_warning(self, topic: str) -> str: - """Append a knowledge-gap warning if the topic touches a departed employee's domain.""" + """ + Append a knowledge-gap warning if the topic touches a registered orphaned domain. + Uses live documentation_coverage from DomainRegistry rather than the static + config value so the warning reflects any recovery that has happened since genesis. + """ + topic_lower = topic.lower() + + # First try live registry — preferred source + all_domains = list( + self._mem._db["domain_registry"].find({"primary_owner": None}) + ) + for rec in all_domains: + tags = rec.get("system_tags", []) + if any(tag in topic_lower for tag in tags): + pct = int(rec.get("documentation_coverage", 0.2) * 100) + former = rec.get("former_owner", "a former employee") + known_by = rec.get("known_by", []) + known_str = ( + f" Partial knowledge held by: {', '.join(known_by)}." + if known_by + else " No current owner." + ) + return ( + f"\n\n> ⚠️ **Knowledge Gap**: This area ({rec['domain']}) was owned by " + f"{former}. Only ~{pct}% documented.{known_str}" + ) + + # Fallback to static config if domain not in registry (e.g. pre-registry data) departed = self._config.get("knowledge_gaps", []) for emp in departed: - hits = [k for k in emp.get("knew_about", []) if k.lower() in topic.lower()] + hits = [k for k in emp.get("knew_about", []) if k.lower() in topic_lower] if hits: return ( f"\n\n> ⚠️ **Knowledge Gap**: This area ({', '.join(hits)}) was owned by " diff --git a/src/day_planner.py b/src/day_planner.py index a687484..1d865e7 100644 --- a/src/day_planner.py +++ b/src/day_planner.py @@ -10,11 +10,6 @@ Engineering is the primary driver. Other departments react to Engineering's plan before the OrgCoordinator looks for collision points. - -Replace _generate_theme() in flow.py with: - org_plan = self._day_planner.plan(self.state, self._mem, self.graph_dynamics) - self.state.daily_theme = org_plan.org_theme - self.state.org_day_plan = org_plan # new State field — see note at bottom """ from __future__ import annotations @@ -1199,7 +1194,7 @@ def _generate_org_theme(self, state, mem: Memory, clock) -> str: return str(Crew(agents=[agent], tasks=[task], verbose=False).kickoff()).strip() def _extract_cross_signals( - self, mem: Memory, day: int + self, mem: Memory, day: int, as_of_time: Optional[str] = None ) -> Dict[str, List[CrossDeptSignal]]: signals: Dict[str, List[CrossDeptSignal]] = {} config_chart: Dict[str, List] = self._config["org_chart"] @@ -1230,7 +1225,7 @@ def _extract_cross_signals( recent = [ e - for e in mem.get_event_log() + for e in mem.get_event_log(from_db=True, as_of_time=as_of_time) if e.type in relevant_types and e.day >= max(1, day - 5) ] diff --git a/src/external_email_ingest.py b/src/external_email_ingest.py index bc330b7..185e695 100644 --- a/src/external_email_ingest.py +++ b/src/external_email_ingest.py @@ -21,7 +21,6 @@ from config_loader import COMPANY_DESCRIPTION from crm_system import NullCRMSystem from crewai import Crew, Task -from graph_dynamics import GraphDynamics import json_repair from memory import Memory, SimEvent from insider_threat import _NullInjector @@ -143,7 +142,7 @@ def __init__( clock, threat_injector=None, crm=None, - graph_dynamics=GraphDynamics, + graph_dynamics=None, ): self._config = config self._mem = mem @@ -267,7 +266,7 @@ def generate_hr_outbound(self, state) -> None: self._send_hr_outbound(hire, hr_lead, days_until, state, date_str) hire["_hr_email_sent"] = True - def _route_customer_email(self, signal: ExternalEmailSignal, state) -> None: + """ def _route_customer_email(self, signal: ExternalEmailSignal, state) -> None: date_str = str(state.current_date.date()) sales_lead = self._leads.get( signal.internal_liaison, next(iter(self._leads.values())) @@ -316,7 +315,7 @@ def _route_customer_email(self, signal: ExternalEmailSignal, state) -> None: ), tags=["email", "customer", "routed", "causal_chain"], ) - ) + ) """ def _sales_pings_product( self, signal, sales_lead, product_lead, state, date_str @@ -1087,14 +1086,6 @@ def _generate_email( tone = source.get("tone", "professional") date_str = str(state.current_date.date()) - incident_ctx = "" - if state.active_incidents: - inc = state.active_incidents[0] - incident_ctx = ( - f"\nActive incident: {inc.ticket_id} — {inc.root_cause}. " - f"Reference naturally if relevant." - ) - tech_stack = self._mem.tech_stack_for_prompt() tech_ctx = ( ( @@ -1126,7 +1117,6 @@ def _generate_email( description=( f"Email from {source_first_name} {source_last_name} at {source_org} to {liaison_name} at {self._company_name} which {COMPANY_DESCRIPTION} " f"about: {topic}.\nTone: {tone}. Health: {state.system_health}/100." - f"{incident_ctx}" f"{tech_ctx}\n\n" f"COMPANY CONTEXT: {self._company_name} is {self._company_desc}. " f"Ground your email in this reality.\n\n" diff --git a/src/flow.py b/src/flow.py index adeb087..35bcc4a 100644 --- a/src/flow.py +++ b/src/flow.py @@ -268,6 +268,7 @@ class ActiveIncident(BaseModel): causal_chain: Any = None recurrence_of: Optional[str] = None on_call: str = "" + actors: List[str] = [] class SprintState(BaseModel): @@ -932,10 +933,18 @@ def daily_cycle(self): self._clock.reset_to_business_start(ALL_NAMES) date_str = str(self.state.current_date.date()) departures = self._lifecycle.process_departures( - self.state.day, date_str, self.state, self._clock + self.state.day, + date_str, + self.state, + self._clock, + ticket_assigner=self._ticket_assigner, ) hires = self._lifecycle.process_hires( - self.state.day, date_str, self.state, self._clock + self.state.day, + date_str, + self.state, + self._clock, + ticket_assigner=self._ticket_assigner, ) for inc in self.state.active_incidents: @@ -984,7 +993,34 @@ def daily_cycle(self): if self._is_retro_day(): self._handle_retrospective() - self._advance_incidents() + _base_prob = CONFIG["simulation"].get("incident_base_prob", 0.15) + _cooldown = CONFIG["simulation"].get("incident_cooldown_days", 3) + days_since_incident = self.state.day - self.state.last_incident_day + + _incident_triggers = CONFIG["simulation"].get( + "incident_triggers", + [ + "crash", + "fail", + "error", + "latency", + "timeout", + "outage", + "down", + "spike", + ], + ) + _theme_lower = self.state.daily_theme.lower() + _theme_triggered = any(x in _theme_lower for x in _incident_triggers) + _prob_triggered = random.random() < _base_prob + + if ( + not self.state.active_incidents + and days_since_incident > _cooldown + and (_theme_triggered or _prob_triggered) + ): + self.state.last_incident_day = self.state.day + self._handle_incident() self._normal_day.handle(self.state.org_day_plan) self._email_ingestor.generate_business_hours(state=self.state) @@ -1018,34 +1054,7 @@ def daily_cycle(self): if r.get("pattern") == "trust_building": self._se_followup_days[r["followup_due_day"]] = r["target"] - _base_prob = CONFIG["simulation"].get("incident_base_prob", 0.15) - _cooldown = CONFIG["simulation"].get("incident_cooldown_days", 3) - days_since_incident = self.state.day - self.state.last_incident_day - - _incident_triggers = CONFIG["simulation"].get( - "incident_triggers", - [ - "crash", - "fail", - "error", - "latency", - "timeout", - "outage", - "down", - "spike", - ], - ) - _theme_lower = self.state.daily_theme.lower() - _theme_triggered = any(x in _theme_lower for x in _incident_triggers) - _prob_triggered = random.random() < _base_prob - - if ( - not self.state.active_incidents - and days_since_incident > _cooldown - and (_theme_triggered or _prob_triggered) - ): - self.state.last_incident_day = self.state.day - self._handle_incident() + self._advance_incidents() self._embed_worker.drain() @@ -1192,6 +1201,7 @@ def _generate_dept_tickets(dept: str, members: list) -> list: { "dept": dept, "status": {"$ne": "Done"}, + "created_at": {"$lte": timestamp_str}, } ) dept_capacity = self._ticket_assigner._compute_capacity(members, self.state) @@ -1690,14 +1700,26 @@ def _close_sprint(self) -> None: def _handle_incident(self): ticket_id = next_jira_id(self.state, self._registry, dept="Engineering_Backend") - root_cause = self._generate_root_cause() - on_call = self._select_domain_expert(root_cause, exclude="") - - incident_start = self._clock.tick_system(min_mins=30, max_mins=240) - incident_start_iso = incident_start.isoformat() date_str = str(self.state.current_date.date()) - self._clock.sync_to_system([on_call]) + on_call = self._get_next_on_call(self.state.day) + + system_fault = self._generate_root_cause() + + incident_lead = self._select_domain_expert(system_fault, exclude=on_call) + + eng_peer = next( + ( + n + for n in LIVE_ORG_CHART.get(dept_of(incident_lead), []) + if n != incident_lead and n != on_call + ), + on_call, + ) + + self._clock.tick_system(min_mins=30, max_mins=240) + responders = [r for r in [on_call, incident_lead, eng_peer] if r] + incident_start_iso = self._clock.sync_to_system(responders).isoformat() rc_agent = make_agent( role=f"{on_call}, Senior On-Call Engineer", @@ -1708,6 +1730,21 @@ def _handle_incident(self): llm=PLANNER_MODEL, ) + signals_map = self._day_planner._extract_cross_signals( + self._mem, self.state.day, as_of_time=incident_start_iso + ) + + eng_dept_key = next( + (k for k in signals_map if "eng" in k.lower()), "Engineering" + ) + recent_friction = signals_map.get(eng_dept_key, []) + + friction_summary = "\n".join( + [f"- {s.event_type}: {s.summary}" for s in recent_friction] + ) + + tech_stack = self._mem.tech_stack_for_prompt() + if self.state.system_health < 40: length_instruction = ( "2-3 sentences. The system is in bad shape — " @@ -1725,9 +1762,11 @@ def _handle_incident(self): rc_task = Task( description=( f"You are {on_call}. You are on-call and an incident just fired.\n\n" + f"RECENT ORG FRICTION & SIGNALS:\n{friction_summary}\n\n" f"System health: {self.state.system_health}/100\n" f"Today's org theme: {self.state.daily_theme}\n\n" - f"Write the root cause. Length: {length_instruction}\n" + f"Tech stack:\n{tech_stack}\n\n" + f"Write the root cause diagnosis based on this root cause {system_fault}. Length: {length_instruction}\n" f"Reference a real system component, endpoint, or dependency. " f"No preamble, no label — just the root cause." ), @@ -1743,19 +1782,8 @@ def _handle_incident(self): Crew(agents=[rc_agent], tasks=[rc_task], verbose=False).kickoff() ).strip() - incident_lead = self._select_domain_expert(root_cause, exclude=on_call) - - eng_peer = next( - ( - n - for n in LIVE_ORG_CHART.get(dept_of(incident_lead), []) - if n != incident_lead and n != on_call - ), - on_call, - ) - detected_gaps = self._lifecycle.scan_for_knowledge_gaps( - text=root_cause, + text=system_fault, triggered_by=ticket_id, day=self.state.day, date_str=date_str, @@ -1779,7 +1807,7 @@ def _handle_incident(self): ) prior = self._recurrence_detector.find_prior_incident( - root_cause, self.state.day, ticket_id + system_fault, self.state.day, ticket_id ) recurrence_of = prior.artifact_ids.get("jira") if prior else None recurrence_gap = (self.state.day - prior.day) if prior else None @@ -1808,7 +1836,7 @@ def _handle_incident(self): escalation_actors = [n for n, _ in chain.chain] datadog_text = ( - f"🚨 [CRITICAL] Anomaly detected: {root_cause[:80]}... " + f"🚨 [CRITICAL] Anomaly detected: {system_fault[:80]}... " f"Error rate spiked 400%. System health dropped to {self.state.system_health}." ) pagerduty_text = ( @@ -1827,7 +1855,7 @@ def _handle_incident(self): chain_handler.append(datadog_thread) chain_handler.append(pagerduty_thread) - _rc_slug = root_cause[:80].rstrip(".,;") + _rc_slug = system_fault[:80].rstrip(".,;") _gap_tag = ( f" [{gap_areas[0]} undocumented]" if involves_gap and gap_areas else "" ) @@ -1846,6 +1874,8 @@ def _handle_incident(self): f"Write a Jira ticket description for this incident.\n\n" f"COMPANY CONTEXT: {COMPANY_NAME} which {COMPANY_DESCRIPTION}\n" f"Title: {title}\n" + f"System Fault: {system_fault}\n" + f"Tech Stack: {tech_stack}\n" f"Root cause: {root_cause}\n" f"Escalation path: {escalation_narrative}\n" f"System health at incident open: {self.state.system_health}/100\n" @@ -1956,7 +1986,7 @@ def _handle_incident(self): root_cause=root_cause, causal_chain=chain_handler, recurrence_of=recurrence_of, - on_call=on_call, + actors=responders, ) self.state.active_incidents.append(inc) self.state.daily_incidents_opened += 1 @@ -1982,6 +2012,7 @@ def _handle_incident(self): facts={ "title": title, "root_cause": root_cause, + "system_fault": system_fault, "involves_gap": involves_gap, "gap_areas": gap_areas, "causal_chain": chain_handler.snapshot(), @@ -2003,7 +2034,7 @@ def _handle_incident(self): self._crm.handle_incident_opened( incident_id=ticket_id, - component=root_cause[:80], + component=system_fault, health=self.state.system_health, timestamp=incident_start_iso, date_str=date_str, @@ -2028,25 +2059,6 @@ def _handle_incident(self): ) ) - if involves_gap: - self._mem.log_event( - SimEvent( - type="knowledge_gap_detected", - timestamp=incident_start_iso, - day=self.state.day, - date=date_str, - actors=[on_call, eng_peer], - artifact_ids={ARTIFACT_KEY_JIRA: ticket_id}, - facts={ - "gap_areas": gap_areas or [LEGACY["name"]], - "involves_gap": True, - "gap_context": gap_context_str, - }, - summary=f"Knowledge gap detected during {ticket_id}: {gap_context_str[:80]}", - tags=["knowledge_gap"], - ) - ) - self._record_daily_actor(on_call, incident_lead) self._record_daily_event("incident_opened") self.graph_dynamics.apply_incident_stress([on_call, incident_lead]) @@ -2098,7 +2110,7 @@ def _advance_incidents(self): timestamp=cron_time_iso, ) - t = self._mem.get_ticket(inc.ticket_id) + t = self._mem.get_ticket(inc.ticket_id, as_of_time=cron_time_iso) if t: if pr["pr_id"] not in t.get("linked_prs", []): t.setdefault("linked_prs", []).append(pr["pr_id"]) @@ -2115,7 +2127,7 @@ def _advance_incidents(self): inc.pr_id = pr["pr_id"] if getattr(inc, "causal_chain", None): inc.causal_chain.append(pr["pr_id"]) - t = self._mem.get_ticket(inc.ticket_id) + t = self._mem.get_ticket(inc.ticket_id, as_of_time=cron_time_iso) if t: t["causal_chain"] = inc.causal_chain.snapshot() t["updated_at"] = cron_time_iso @@ -2137,7 +2149,10 @@ def _advance_incidents(self): elif inc.stage == "fix_in_progress": inc.stage = "review_pending" if inc.pr_id: - pr_doc = self._mem._prs.find_one({"pr_id": inc.pr_id}, {"_id": 0}) + pr_doc = self._mem._prs.find_one( + {"pr_id": inc.pr_id, "created_at": {"$lte": cron_time_iso}}, + {"_id": 0}, + ) if pr_doc: reviewers = pr_doc.get("reviewers", []) for reviewer in reviewers: @@ -2159,7 +2174,9 @@ def _advance_incidents(self): inc.stage = "resolved" if inc.pr_id: self._git.merge_pr(inc.pr_id) - linked_ticket = self._mem.get_ticket(inc.ticket_id) + linked_ticket = self._mem.get_ticket( + inc.ticket_id, as_of_time=cron_time_iso + ) if linked_ticket: linked_ticket["status"] = "Done" linked_ticket["updated_at"] = cron_time_iso @@ -2248,7 +2265,7 @@ def _write_postmortem(self, inc: ActiveIncident): timestamp=timestamp, day=self.state.day, date=str(self.state.current_date.date()), - actors=[on_call, eng_peer], + actors=inc.actors, artifact_ids={"confluence": conf_id, "jira": inc.ticket_id}, facts={ "causal_chain": inc.causal_chain.snapshot(), @@ -2290,6 +2307,13 @@ def _emit_bot_message(self, channel: str, bot_name: str, text: str, timestamp: s return thread_id + def _get_next_on_call(self, day: int): + eng_dept_key = next((k for k in ORG_CHART if "eng" in k.lower()), "Engineering") + engineers = ORG_CHART.get(eng_dept_key, []) + + index = day % len(engineers) + return engineers[index] + def _generate_adhoc_confluence_page( self, author: Optional[str] = None, @@ -2489,6 +2513,7 @@ def _print_final_report(self): ("JIRA Tickets", str(self._mem._jira.count_documents({}))), ("Slack Threads", str(self._mem._slack.count_documents({}))), ("Git PRs", str(self._mem._prs.count_documents({}))), + ("Emails", str(self._mem._db["emails"].count_documents({}))), ("Incidents Resolved", str(len(self.state.resolved_incidents))), ("Embedded Artifacts", str(s["artifact_count"])), ("Employees Departed", str(len(self._lifecycle._departed))), @@ -2506,8 +2531,12 @@ def _print_final_report(self): self._mem._artifacts.find({"type": "confluence"}, _proj) ), "jira_tickets": list(self._mem._jira.find({}, {"_id": 0})), + "emails": list(self._mem._db["emails"].find({}, {"_id": 0})), "slack_threads": list(self._mem._slack.find({}, {"_id": 0})), "pr_registry": list(self._mem._prs.find({}, {"_id": 0})), + "sf_accounts": list(self._mem._db["sf_accounts"].find({}, {"_id": 0})), + "sf_opps": list(self._mem._db["sf_opps"].find({}, {"_id": 0})), + "zd_tickets": list(self._mem._db["zd_tickets"].find({}, {"_id": 0})), "resolved_incidents": self.state.resolved_incidents, "morale_history": self.state.morale_history, "system_health": self.state.system_health, diff --git a/src/genesis.py b/src/genesis.py index c24c75f..155da91 100644 --- a/src/genesis.py +++ b/src/genesis.py @@ -263,6 +263,9 @@ def seed_crm_accounts(mem: Memory): } for contact in contacts: + if contact.get("category", "").lower() != "customer": + continue + org_name = contact.get("org", "Unknown") safe_id = org_name.upper().replace(" ", "").replace("-", "") account_id = f"ACC-{safe_id}" @@ -335,8 +338,60 @@ def seed_crm_accounts(mem: Memory): pass +def _domain_key(domain: str) -> str: + """Normalise a domain name to a stable MongoDB key.""" + return domain.lower().replace(" ", "_") + + +def _build_system_tags(domain: str, knew_about: List[str]) -> List[str]: + """ + Derive search-friendly system tags from a domain name and the full + knew_about list so ticket tagging has multiple match surfaces. + + e.g. "TitanDB" → ["titandb", "titan", "db"] + "legacy auth service" → ["legacy auth service", "legacy auth", "auth service", "auth"] + """ + tags = set() + raw = domain.lower() + tags.add(raw) + + for token in re.split(r"[\s_\-]+", raw): + if len(token) >= 3: + tags.add(token) + + for sibling in knew_about: + for token in re.split(r"[\s_\-]+", sibling.lower()): + if len(token) >= 3 and token in raw: + tags.add(token) + + return sorted(tags) + + def seed_knowledge_gaps(mem: Memory): - """Embeds skills and logs departure events for pre-simulation employees.""" + """ + Embeds skills, logs departure events, and seeds the DomainRegistry for + every pre-simulation employee defined under knowledge_gaps in config. + + DomainRegistry documents live in mem._db["domain_registry"] with _id + equal to the normalised domain key. Each document schema: + + { + "_id": "titandb", # normalised key + "domain": "TitanDB", # display name + "primary_owner": None, # None = orphaned + "former_owner": "Bill", + "documentation_coverage": 0.20, + "last_updated_day": -180, # day relative to sim start + "known_by": [], # engineers with partial knowledge + "system_tags": ["titandb", "titan", "db"], + "dept": "Engineering_Backend", + "is_genesis_gap": True, + } + + Callers (DepartmentPlanner, PR reviewer) should call + mem.get_domain_registry() to receive the full registry dict keyed by + domain display name. + """ if not CONFIG.get("knowledge_gaps"): return @@ -344,19 +399,26 @@ def seed_knowledge_gaps(mem: Memory): sim_start = datetime.strptime(CONFIG["simulation"]["start_date"], "%Y-%m-%d") + mem._db["domain_registry"].create_index([("system_tags", 1)]) + mem._db["domain_registry"].create_index([("primary_owner", 1)]) + for gap in CONFIG.get("knowledge_gaps", []): name = gap["name"] left_date = gap["left"] left_dt = datetime.strptime(left_date, "%Y-%m") departure_day = -(sim_start - left_dt).days + dept = gap.get("dept", "Engineering") + role = gap.get("role", "Former Employee") + knew_about = gap.get("knew_about", []) + doc_pct = gap.get("documented_pct", 0.5) mem.embed_persona_skills( name=name, data={ - "expertise": gap.get("knew_about", []), - "social_role": gap.get("role", "Former Employee"), + "expertise": knew_about, + "social_role": role, }, - dept=gap.get("dept", "Engineering"), + dept=dept, day=departure_day, timestamp_iso=f"{left_date}-01T09:00:00", ) @@ -371,16 +433,94 @@ def seed_knowledge_gaps(mem: Memory): artifact_ids={}, facts={ "name": name, - "role": gap.get("role", ""), - "knowledge_domains": gap.get("knew_about", []), - "documented_pct": gap.get("documented_pct", 0.5), + "role": role, + "knowledge_domains": knew_about, + "documented_pct": doc_pct, "is_genesis_gap": True, }, - summary=f"Genesis Gap: {name} ({gap.get('role')}) left Day {departure_day}.", + summary=f"Genesis Gap: {name} ({role}) left Day {departure_day}.", tags=["employee_departed", "lifecycle", "genesis"], ) ) - logger.info(f" [dim]→ Seeded Genesis Gap: {name}[/dim]") + + for domain in knew_about: + key = _domain_key(domain) + system_tags = _build_system_tags(domain, knew_about) + + existing = mem._db["domain_registry"].find_one({"_id": key}) + if existing: + # Domain already registered (e.g. two departures knew the + # same system). Just ensure former_owners is a list and + # append — don't overwrite coverage. + mem._db["domain_registry"].update_one( + {"_id": key}, + { + "$addToSet": { + "former_owners": name, + "system_tags": {"$each": system_tags}, + } + }, + ) + logger.info( + f" [dim]→ Domain '{domain}' already registered — " + f"appended {name} as former owner.[/dim]" + ) + continue + + record = { + "_id": key, + "domain": domain, + "primary_owner": None, # orphaned from day 0 + "former_owner": name, + "former_owners": [name], + "documentation_coverage": doc_pct, + "last_updated_day": departure_day, + "known_by": [], # no current engineers + "system_tags": system_tags, + "dept": dept, + "is_genesis_gap": True, + } + + mem._db["domain_registry"].insert_one(record) + + logger.info( + f" [dim]→ Domain registry: '{domain}' " + f"(owner=None, coverage={int(doc_pct * 100)}%, " + f"tags={system_tags})[/dim]" + ) + + mem.log_event( + SimEvent( + type="knowledge_gap_detected", + day=departure_day, + date=f"{left_date}-01", + timestamp=f"{left_date}-01T09:00:00", + actors=[name], + artifact_ids={}, + facts={ + "detection_method": "genesis_seed", + "former_owner": name, + "role": role, + "dept": dept, + "domains": knew_about, + "documentation_coverage": doc_pct, + "gap_classification": "likely", + "author_domain_fit": "low", + "topics_beyond_author_expertise": knew_about, + "is_genesis_gap": True, + }, + summary=( + f"Genesis gap seeded: {name} ({role}) departed with sole " + f"ownership of {knew_about}. " + f"Documentation coverage: {int(doc_pct * 100)}%." + ), + tags=["knowledge_gap", "genesis", "orphaned_domain"], + ) + ) + + logger.info( + f" [dim]→ Seeded Genesis Gap: {name} | domains: {knew_about}[/dim]" + ) @staticmethod diff --git a/src/memory.py b/src/memory.py index 45bbd2d..ce42b8c 100644 --- a/src/memory.py +++ b/src/memory.py @@ -23,7 +23,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field, asdict import time -from typing import List, Dict, Optional, Any, Tuple +from typing import List, Dict, Optional, Any, Tuple, Union import boto3 from pymongo import MongoClient @@ -356,7 +356,6 @@ def embed(self, text: str, input_type: str = "search_document") -> List[float]: result = self._json.loads(raw_body) vector = result["embeddings"]["float"][0] - headers = resp.get("ResponseMetadata", {}).get("HTTPHeaders", {}) return vector except self._client.exceptions.ThrottlingException: wait = 6.2 * (attempt + 1) # backoff: 6.2, 12.4, 18.6 @@ -434,6 +433,7 @@ def __init__( [("participants", 1), ("type", 1), ("day", -1)] ) self._conversation_summaries.create_index([("day", -1)]) + self._events.create_index([("timestamp", 1)]) self._events.create_index([("type", 1), ("day", 1)]) self._events.create_index([("type", 1), ("timestamp", -1)]) self._events.create_index([("actors", 1), ("timestamp", -1)]) @@ -442,7 +442,6 @@ def __init__( self._events.create_index([("type", 1), ("facts.participants", 1)]) self._checkpoints.create_index([("day", -1)]) self._jira.create_index([("dept", 1), ("status", 1)]) - self._current_day: int = 0 self._event_log: List[SimEvent] = [] @@ -923,11 +922,27 @@ def persona_history(self, name: str, n: int = 4) -> List[SimEvent]: ] return relevant[-n:] - def get_event_log(self, from_db: bool = False) -> List[SimEvent]: + def get_event_log( + self, from_db: bool = False, as_of_time: Optional[str] = None + ) -> List[SimEvent]: + """ + Returns the event log, optionally filtered by a specific timestamp + to prevent 'seeing into the future.' + """ + if from_db: - raw = self._events.find({}, {"_id": 0}).sort("timestamp", 1) + query = {} + if as_of_time: + query["timestamp"] = {"$lte": as_of_time} + + raw = self._events.find(query, {"_id": 0}).sort("timestamp", 1) return [SimEvent.from_dict(r) for r in raw] - return self._event_log + + log = self._event_log + if as_of_time: + log = [e for e in log if e.timestamp <= as_of_time] + + return log def events_by_type(self, event_type: str) -> List[SimEvent]: return [e for e in self._event_log if e.type == event_type] @@ -1183,7 +1198,6 @@ def context_for_sprint_planning( f" Day {inc.get('day', '?')} — {facts.get('title', facts.get('root_cause', 'Unknown'))}" ) - # ── Velocity from last checkpoint ───────────────────────────────────── checkpoint = self._checkpoints.find_one(sort=[("day", -1)]) if checkpoint: state = checkpoint.get("state", {}) @@ -1200,6 +1214,31 @@ def context_for_sprint_planning( else f"No sprint planning context found for {dept}." ) + def get_domain_registry(self) -> dict: + """Returns {domain: record} for all registered domains.""" + return {rec["domain"]: rec for rec in self._db["domain_registry"].find({})} + + def get_orphaned_domains(self) -> list: + """Returns all domains with no current primary owner.""" + return list(self._db["domain_registry"].find({"primary_owner": None})) + + def update_domain_coverage(self, domain: str, delta: float, author: str, day: int): + """Called when a Confluence page is written covering this domain.""" + key = domain.lower().replace(" ", "_") + rec = self._db["domain_registry"].find_one({"_id": key}) + if rec: + new_coverage = min(1.0, rec["documentation_coverage"] + delta) + self._db["domain_registry"].update_one( + {"_id": key}, + { + "$set": { + "documentation_coverage": new_coverage, + "last_updated_day": day, + }, + "$addToSet": {"known_by": author}, + }, + ) + def context_for_retrospective( self, sprint_num: int, @@ -1215,12 +1254,9 @@ def context_for_retrospective( - Tickets that carried over (not Done) - Incidents that fired during the sprint - Retrospective-relevant events (deploys, postmortems, etc.) - - Use this instead of context_for_prompt() in _handle_retrospective. """ lines: List[str] = [f"=== SPRINT #{sprint_num} RETROSPECTIVE CONTEXT ==="] - # ── Tickets active in this sprint window ────────────────────────────── done_tickets = list( self._jira.find( {"status": "Done"}, @@ -1263,7 +1299,6 @@ def context_for_retrospective( f"(status={t.get('status', '?')}, assignee={t.get('assignee', '?')})" ) - # ── Events in the sprint window ──────────────────────────────────────── _RETRO_TYPES = { "incident_detected", "incident_resolved", @@ -1580,6 +1615,9 @@ def reset(self, export_dir: Optional[str] = None): self._client.drop_database(db_name) self._db = self._client[db_name] + self._db.create_collection("artifacts") + self._db.create_collection("events") + # Re-bind all collection references to the fresh database self._artifacts = self._db["artifacts"] self._events = self._db["events"] @@ -1661,9 +1699,17 @@ def load_latest_checkpoint(self) -> Optional[Dict]: def upsert_ticket(self, ticket: Dict): self._jira.update_one({"id": ticket["id"]}, {"$set": ticket}, upsert=True) - def get_ticket(self, ticket_id: str) -> Optional[Dict]: - # Exclude _id so callers never receive a non-serialisable ObjectId - return self._jira.find_one({"id": ticket_id}, {"_id": 0}) + def get_ticket( + self, ticket_id: str, as_of_time: Optional[str] = None + ) -> Optional[Dict]: + query: dict[str, Union[str, dict[str, str]]] = {"id": ticket_id} + if as_of_time: + query["$or"] = [ + {"timestamp": {"$lte": as_of_time}}, + {"timestamp": {"$exists": False}}, + ] + + return self._jira.find_one(query, {"_id": 0}) def get_open_tickets_for_dept( self, members: List[str], dept_name: str = "" @@ -1679,8 +1725,17 @@ def get_open_tickets_for_dept( def upsert_pr(self, pr: Dict): self._prs.update_one({"pr_id": pr["pr_id"]}, {"$set": pr}, upsert=True) - def get_reviewable_prs_for(self, name: str) -> List[Dict]: - return list(self._prs.find({"reviewers": name, "status": "open"}, {"_id": 0})) + def get_reviewable_prs_for( + self, name: str, as_of_time: Optional[str] = None + ) -> List[Dict]: + query: dict[str, Union[str, dict[str, str]]] = { + "reviewers": name, + "status": "open", + } + if as_of_time: + query["timestamp"] = {"$lte": as_of_time} + + return list(self._prs.find(query, {"_id": 0})) def get_pr_by_ticket_id(self, ticket_id: str) -> Optional[Dict]: return self._prs.find_one({"ticket_id": ticket_id}, {"_id": 0}) diff --git a/src/normal_day.py b/src/normal_day.py index 47c6738..e0f3df8 100644 --- a/src/normal_day.py +++ b/src/normal_day.py @@ -225,7 +225,10 @@ def _handle_ticket_progress( if not ticket_id: return [] - ticket = self._mem.get_ticket(ticket_id) + current_actor_time, new_cursor = self._clock.advance_actor(assignee, hours=2.0) + current_actor_time_iso = current_actor_time.isoformat() + + ticket = self._mem.get_ticket(ticket_id, as_of_time=current_actor_time_iso) if not ticket: return [eng_plan.name] @@ -236,9 +239,6 @@ def _handle_ticket_progress( is_non_eng = dept_type == "non_eng" completion_artifact = ticket.get("completion_artifact", "slack") - current_actor_time, new_cursor = self._clock.advance_actor(assignee, hours=2.0) - current_actor_time_iso = current_actor_time.isoformat() - ctx = self._mem.context_for_person( name=assignee, as_of_time=current_actor_time_iso, @@ -265,6 +265,31 @@ def _handle_ticket_progress( backstory = persona_utils.get_voice_card(assignee, "async", self._gd, self._mem) + # Surface any orphaned domains this ticket touches so the engineer's + # comment naturally reflects uncertainty about legacy systems + orphaned_domain_hint = "" + if self._lifecycle: + all_domains = list( + self._mem._db["domain_registry"].find({"primary_owner": None}) + ) + ticket_text = ( + f"{ticket.get('title', '')} {ticket.get('description', '')}".lower() + ) + for rec in all_domains: + if any(tag in ticket_text for tag in rec.get("system_tags", [])): + pct = int(rec.get("documentation_coverage", 0) * 100) + known_by = rec.get("known_by", []) + orphaned_domain_hint += ( + f"\n⚠ NOTE: '{rec['domain']}' is underdocumented ({pct}% coverage). " + f"Former owner: {rec.get('former_owner', 'unknown')}. " + f"If your work touches this system, reflect genuine uncertainty in your comment." + + ( + f" Others with partial knowledge: {', '.join(known_by)}." + if known_by + else "" + ) + ) + if is_non_eng: persona = self._config.get("personas", {}).get(assignee, {}) agent_role = persona.get("role", dept_of_name(assignee, self._org_chart)) @@ -290,7 +315,12 @@ def _handle_ticket_progress( if ticket.get("status") == "In Review": for linked_pr_id in ticket.get("linked_prs", []): linked_pr = self._mem._prs.find_one( - {"pr_id": linked_pr_id, "status": "open"}, {"_id": 0} + { + "pr_id": linked_pr_id, + "status": "open", + "created_at": {"$lte": current_actor_time_iso}, + }, + {"_id": 0}, ) if linked_pr and linked_pr.get("changes_requested"): recent_feedback = linked_pr.get("comments", [])[-3:] @@ -317,7 +347,8 @@ def _handle_ticket_progress( f"You are {assignee}. You worked on ticket [{ticket_id}] today.\n\n" f"Your task today: {item.description}\n" f"{reviewer_feedback_hint}" - f"IMPORTANT: Your comment must be specifically about this ticket's work — " + + (orphaned_domain_hint + "\n" if orphaned_domain_hint else "") + + f"IMPORTANT: Your comment must be specifically about this ticket's work — " f"do not describe unrelated tasks.\n" f"{completion_note}\n\n" f"Respond ONLY with valid JSON. No preamble, no markdown fences.\n" @@ -390,7 +421,12 @@ def _handle_ticket_progress( if ticket.get("status") == "In Review": for linked_pr_id in ticket.get("linked_prs", []): linked_pr = self._mem._prs.find_one( - {"pr_id": linked_pr_id, "status": "open"}, {"_id": 0} + { + "pr_id": linked_pr_id, + "status": "open", + "created_at": {"$lte": current_actor_time_iso}, + }, + {"_id": 0}, ) if linked_pr and linked_pr.get("changes_requested"): linked_pr["changes_requested"] = False @@ -460,7 +496,14 @@ def _handle_ticket_progress( ticket["updated_at"] = current_actor_time_iso for pr_id in ticket.get("linked_prs", []): - pr = self._mem._prs.find_one({"pr_id": pr_id, "status": "open"}, {"_id": 0}) + pr = self._mem._prs.find_one( + { + "pr_id": pr_id, + "status": "open", + "created_at": {"$lte": current_actor_time_iso}, + }, + {"_id": 0}, + ) if pr and pr.get("changes_requested"): pr["changes_requested"] = False ticket["status"] = "In Review" @@ -754,9 +797,15 @@ def _complete_eng_ticket( linked_prs = ticket.get("linked_prs", []) ticket_age = self._state.day - ticket.get("in_progress_since", self._state.day) actor_clock_ok = self._clock.now(assignee).hour < 17 + actor_clock_iso_format = self._clock.now(assignee).isoformat() open_pr_with_changes = bool(linked_prs) and any( self._mem._prs.find_one( - {"pr_id": p, "status": "open", "changes_requested": True}, + { + "pr_id": p, + "status": "open", + "changes_requested": True, + "created_at": {"$lte": actor_clock_iso_format}, + }, {"_id": 0, "pr_id": 1}, ) for p in linked_prs @@ -840,6 +889,39 @@ def _handle_pr_review( ) review_history = f"\n--- PRIOR REVIEW ROUNDS ---\n{rounds}\n\n" + author_persona = self._config.get("personas", {}).get(author, {}) + expertise_list = author_persona.get("expertise", ["general tasks"]) + expertise_str = ", ".join(str(e) for e in expertise_list[:5]) + author_dept = next( + (d for d, members in self._org_chart.items() if author in members), + "Unknown", + ) + + reviewer_persona = self._config.get("personas", {}).get(reviewer, {}) + reviewer_expertise_list = reviewer_persona.get("expertise", ["general tasks"]) + reviewer_expertise_str = ", ".join(str(e) for e in reviewer_expertise_list[:5]) + reviewer_dept = next( + (d for d, members in self._org_chart.items() if reviewer in members), + "Unknown", + ) + + orphaned_domain_context = "" + if self._lifecycle: + all_domains = list( + self._mem._db["domain_registry"].find({"primary_owner": None}) + ) + pr_title_lower = pr_title.lower() + for rec in all_domains: + if any(tag in pr_title_lower for tag in rec.get("system_tags", [])): + pct = int(rec.get("documentation_coverage", 0) * 100) + known_by = rec.get("known_by", []) + orphaned_domain_context += ( + f"\n⚠ '{rec['domain']}' is an orphaned domain: " + f"former owner={rec.get('former_owner', 'unknown')}, " + f"documentation={pct}%, " + f"partial knowledge: {known_by or 'nobody'}." + ) + agent = make_agent( role=f"{reviewer} — {p.get('social_role', 'Code Reviewer')}", goal=f"Write a PR review comment as {reviewer} would, reflecting your current stress and style.", @@ -868,25 +950,55 @@ def _handle_pr_review( description=( f"You are {reviewer}. You are reviewing this PR by {author}: {pr_title}\n\n" f"{review_history}" - f"STEP 1 — DECIDE YOUR VERDICT FIRST:\n" + + ( + f"DOMAIN CONTEXT:{orphaned_domain_context}\n\n" + if orphaned_domain_context + else "" + ) + + f"STEP 1 — DECIDE YOUR VERDICT FIRST:\n" f"{approval_guidance}\n" f"Choose: 'approved' or 'changes_requested'.\n\n" f"STEP 2 — WRITE YOUR COMMENT:\n" f"Write 1-3 sentences consistent with your verdict. If approved, acknowledge " f"what looks good. If changes_requested, name the specific issue only.\n" f"Your tone must reflect your current stress level (see your backstory).\n\n" + f"STEP 3 — AUTHOR KNOWLEDGE AUDIT (answer objectively, NOT in character):\n" + f"Your expertise as reviewer: [{reviewer_expertise_str}]\n" + f"Your department: {reviewer_dept}\n" + f"{author}'s expertise on record: [{expertise_str}]\n" + f"{author}'s department: {author_dept}\n" + f"Based on the PR content and your expertise, assess whether {author} " + f"demonstrates a knowledge gap in the domains this PR touches.\n" + f"- In 'topics_beyond_author_expertise', list any technical areas where " + f"{author}'s implementation, approach, or omissions suggest unfamiliarity.\n" + f"- In 'hedged_claims', list specific decisions or statements in the PR " + f"that appear incorrect, naive, or underconfident given the problem domain.\n" + f"- If {author} deferred or left incomplete any section you know should " + f"exist, list it in 'deferred_or_incomplete'.\n\n" + f"Use these criteria:\n" + f" author_domain_fit:\n" + f" 'high' — PR demonstrates fluency: correct abstractions, aware of edge cases, idiomatic\n" + f" 'medium' — PR is functional but shows shallow understanding or minor missteps\n" + f" 'low' — PR shows clear unfamiliarity: wrong patterns, missing fundamentals, or over-reliance on guesswork\n\n" + f" gap_classification:\n" + f" 'none' — {author}'s expertise aligns with all domains touched by this PR\n" + f" 'possible' — PR touches 1-2 domains outside {author}'s expertise but implementation looks adequate\n" + f" 'likely' — PR touches domains outside {author}'s expertise AND the implementation shows it\n\n" f"Respond ONLY with valid JSON. No preamble, no markdown fences.\n" f"{{\n" f' "comment": "your review comment here",\n' - f' "verdict": "approved" or "changes_requested"\n' - f"}}\n\n" - f"{recurrence_hint}" - f"--- CONTEXT ---\n{ctx}" - ), - expected_output=( - 'Valid JSON only with keys "comment" (string) and "verdict" ' - '("approved" or "changes_requested"). No preamble, no markdown.' + f' "verdict": "approved" or "changes_requested",\n' + f' "metadata": {{\n' + f' "author_domain_fit": "low | medium | high",\n' + f' "confidence": "low | medium | high",\n' + f' "gap_classification": "none | possible | likely",\n' + f' "topics_beyond_author_expertise": ["string"],\n' + f' "hedged_claims": ["string"],\n' + f' "deferred_or_incomplete": ["string"]\n' + f" }}\n" + f"}}" ), + expected_output="Valid JSON only. No preamble, no markdown fences.", agent=agent, ) @@ -901,9 +1013,11 @@ def _handle_pr_review( verdict = parsed_review.get("verdict", "approved") if verdict not in ("approved", "changes_requested"): verdict = "approved" + review_metadata = parsed_review.get("metadata", {}) except (json.JSONDecodeError, AttributeError): review_text = raw_review verdict = "approved" + review_metadata = {} pr_comment = { "author": reviewer, @@ -1052,6 +1166,53 @@ def _handle_pr_review( timestamp=current_actor_time, ) + # Fire a structured gap event from reviewer audit metadata — distinct + # from the embedding scan above so the two signal sources are traceable + if review_metadata: + domain_fit = review_metadata.get("author_domain_fit", "high") + gap_class = review_metadata.get("gap_classification", "none") + beyond = review_metadata.get("topics_beyond_author_expertise", []) + hedged = review_metadata.get("hedged_claims", []) + deferred = review_metadata.get("deferred_or_incomplete", []) + + gap_detected = ( + domain_fit == "low" + or gap_class == "likely" + or (gap_class == "possible" and len(beyond) > 0) + ) + + if gap_detected: + self._mem.log_event( + SimEvent( + type="knowledge_gap_detected", + timestamp=current_actor_time, + day=self._state.day, + date=date_str, + actors=[author], + artifact_ids={"pr": pr.get("pr_id", pr_id or "")}, + facts={ + "detection_method": "reviewer_audit", + "reviewer": reviewer, + "author": author, + "pr_title": pr_title, + "author_domain_fit": domain_fit, + "author_expertise": expertise_list, + "reviewer_expertise": reviewer_expertise_list, + "gap_classification": gap_class, + "topics_beyond_author_expertise": beyond, + "hedged_claims": hedged, + "deferred_or_incomplete": deferred, + }, + summary=( + f"Knowledge gap detected via reviewer audit: " + f"{author} (expertise: {expertise_str}) submitted PR '{pr_title}' " + f"with fit={domain_fit}, gap={gap_class}. " + f"Reviewed by {reviewer}." + ), + tags=["knowledge_gap", "pr_review", "reviewer_audit"], + ) + ) + logger.info( f" [dim]🔍 {reviewer} reviewed {pr.get('pr_id', 'PR')} [{verdict}][/dim]" ) @@ -1091,9 +1252,13 @@ def _handle_pr_review_for_incident( recurrence_hint = "" linked_ticket_id = pr.get("linked_ticket") or pr.get("ticket_id", "") if linked_ticket_id: - ticket = self._mem.get_ticket(linked_ticket_id) + ticket = self._mem.get_ticket( + linked_ticket_id, as_of_time=current_actor_time + ) if ticket and ticket.get("recurrence_of"): - ancestor = self._mem.get_ticket(ticket["recurrence_of"]) + ancestor = self._mem.get_ticket( + ticket["recurrence_of"], as_of_time=current_actor_time + ) ancestor_root_cause = ancestor.get("root_cause", "") if ancestor else "" recurrence_hint = ( f"Note: this PR fixes {linked_ticket_id}, which is a recurrence of " @@ -1186,7 +1351,7 @@ def _handle_pr_review_for_incident( active_inc.causal_chain.append(pr_id) causal_facts["causal_chain"] = active_inc.causal_chain.snapshot() - t = self._mem.get_ticket(linked_ticket_id) + t = self._mem.get_ticket(linked_ticket_id, as_of_time=current_actor_time) if t: t["causal_chain"] = active_inc.causal_chain.snapshot() t["updated_at"] = current_actor_time @@ -1399,9 +1564,6 @@ def _handle_async_question( collaborator = next(iter(item.collaborator), None) or self._closest_colleague( asker ) - ticket_id = item.related_id - ticket = self._find_ticket(ticket_id) - ticket_title = ticket["title"] if ticket else item.description initial_participants = [asker] if collaborator: @@ -1420,6 +1582,10 @@ def _handle_async_question( ) meeting_time_iso = provisional_start.isoformat() + ticket_id = item.related_id + ticket = self._find_ticket(ticket_id, meeting_time_iso) + ticket_title = ticket["title"] if ticket else item.description + seed = [collaborator] if collaborator else [] all_actors = self._expertise_matched_participants( topic=ticket_title, @@ -1472,6 +1638,8 @@ def _handle_async_question( combined_hint = f"{doc_hint}\n\n{design_hint}" if design_hint else doc_hint + tech_stack = self._mem.tech_stack_for_prompt() + agent = make_agent( role="Slack Conversation Simulator", goal=( @@ -1487,6 +1655,7 @@ def _handle_async_question( f"COMPANY CONTEXT: {self._company} which {COMPANY_DESCRIPTION}\n" f"Write a full Slack thread where a colleague asks a question.\n\n" f"Topic: {ticket_title}\n" + f"Tech Stack: {tech_stack}\n" f"Relevant context: {ctx}\n" f"{combined_hint}\n\n" f"Turn order: {speaker_sequence}\n\n" @@ -1547,26 +1716,25 @@ def _handle_async_question( "responders": [a for a in all_actors if a != asker], "message_count": len(messages), } - if active_inc and getattr(active_inc, "causal_chain", None): - facts["causal_chain"] = active_inc.causal_chain.snapshot() - if ticket_id and not active_inc: + chain = CausalChainHandler(thread_id) + if ticket_id: + chain.append(ticket_id) prior = self._mem._events.find_one( { "type": "ticket_progress", "artifact_ids.jira": ticket_id, "facts.causal_chain": {"$exists": True}, + "timestamp": {"$lte": meeting_time_iso}, }, {"facts.causal_chain": 1, "_id": 0}, sort=[("timestamp", -1)], ) - - ticket_chain = CausalChainHandler(ticket_id) if prior: - for artifact_id in prior.get("facts", {}).get("causal_chain", []): - ticket_chain.append(artifact_id) - ticket_chain.append(thread_id) - facts["causal_chain"] = ticket_chain.snapshot() + for aid in prior.get("facts", {}).get("causal_chain", []): + chain.append(aid) + + facts["causal_chain"] = chain.snapshot() self._mem.log_event( SimEvent( @@ -1592,12 +1760,13 @@ def _handle_async_question( if self._lifecycle and messages: thread_text = " ".join(m["text"] for m in messages) - self._lifecycle.scan_for_knowledge_gaps( - text=f"{ticket_title} {thread_text}", - triggered_by=thread_id, - day=self._state.day, + self._assess_async_thread_gap( + messages=messages, + topic=ticket_title, + asker=asker, + thread_id=thread_id, + ticket_id=ticket_id, date_str=date_str, - state=self._state, timestamp=meeting_time_iso, ) @@ -1695,26 +1864,29 @@ def _handle_design_discussion( } related_ticket_id = item.related_id + + chain = CausalChainHandler(artifact_id) if related_ticket_id: + chain.append(related_ticket_id) prior = self._mem._events.find_one( { "type": "ticket_progress", "artifact_ids.jira": related_ticket_id, "facts.causal_chain": {"$exists": True}, + "timestamp": {"$lte": meeting_time_iso}, }, {"facts.causal_chain": 1, "_id": 0}, sort=[("timestamp", -1)], ) - ticket_chain = CausalChainHandler(related_ticket_id) if prior: for aid in prior.get("facts", {}).get("causal_chain", []): - ticket_chain.append(aid) - ticket_chain.append(artifact_id) - if conf_id: - ticket_chain.append(conf_id) - facts["causal_chain"] = ticket_chain.snapshot() + chain.append(aid) artifact_ids["jira"] = related_ticket_id + if conf_id: + chain.append(conf_id) + facts["causal_chain"] = chain.snapshot() + self._mem.log_event( SimEvent( type="design_discussion", @@ -2053,7 +2225,6 @@ def _emit_blocker_slack( Short Slack exchange when an engineer is blocked. Each participant speaks in their own voice via a dedicated Agent. """ - from causal_chain_handler import CausalChainHandler asker_dept = dept_of_name(asker, self._org_chart) channel = asker_dept.lower().replace(" ", "-") @@ -2137,6 +2308,7 @@ def _emit_blocker_slack( "type": "ticket_progress", "artifact_ids.jira": ticket_id, "facts.causal_chain": {"$exists": True}, + "timestamp": {"$lte": timestamp}, }, {"facts.causal_chain": 1, "_id": 0}, sort=[("timestamp", -1)], @@ -2562,8 +2734,6 @@ def _fire_sales_outreach(self, date_str: str) -> None: "completion_artifact": "email", } - from causal_chain_handler import CausalChainHandler - chain = CausalChainHandler(root_id=synthetic_ticket["id"]) opp_updated_str = opp.get("updated_at") @@ -2681,8 +2851,6 @@ def _create_design_doc_stub( date_str=date_str, ) - # ─── LOGGING HELPERS ───────────────────────────────────────────────────── - def _log_deferred_item(self, name: str, item: AgendaItem, date_str: str) -> None: """Log a deferred agenda item so the record shows the interruption.""" current_time_iso = self._clock.now(name).isoformat() @@ -2731,8 +2899,6 @@ def _log_deep_work(self, name: str, item: AgendaItem, date_str: str) -> None: ) ) - # ─── AMBIENT SIGNALS (unchanged from original) ──────────────────────────── - def _maybe_bot_alerts(self) -> None: cron_time_iso = self._clock.now("system").isoformat() @@ -2927,6 +3093,108 @@ def _trigger_watercooler_chat(self, target_actor: str, date_str: str) -> None: f" [dim]☕ Distraction: {target_actor} pulled into chat about {topic}[/dim]" ) + def _assess_async_thread_gap( + self, + messages: List[dict], + topic: str, + asker: str, + thread_id: str, + ticket_id: Optional[str], + date_str: str, + timestamp: str, + ) -> None: + """ + Classify whether an async Q&A thread reveals a genuine knowledge gap + vs a routine question that got answered. + + Uses a fast LLM call (worker model) to classify the thread outcome + rather than scanning raw text against departed employee embeddings. + """ + thread_text = "\n".join(f"{m['user']}: {m['text']}" for m in messages) + + asker_persona = self._config.get("personas", {}).get(asker, {}) + asker_expertise = ", ".join( + str(e) for e in asker_persona.get("expertise", [])[:5] + ) + + agent = make_agent( + role="Thread Analyst", + goal="Classify whether a Slack Q&A thread reveals a knowledge gap.", + backstory="You analyze workplace conversations to identify unresolved questions.", + llm=self._worker, + ) + task = Task( + description=( + f"Analyze this Slack Q&A thread.\n\n" + f"Asker: {asker} (expertise: {asker_expertise})\n" + f"Topic: {topic}\n\n" + f"Thread:\n{thread_text}\n\n" + f"Classify the thread outcome:\n" + f"- 'resolved': the question was answered confidently and correctly\n" + f"- 'uncertain': responders hedged, guessed, or gave conflicting answers\n" + f"- 'unresolved': the question went unanswered or was deferred\n" + f"- 'escalated': someone suggested asking another person or checking docs\n\n" + f"Respond ONLY with JSON:\n" + f"{{\n" + f' "outcome": "resolved | uncertain | unresolved | escalated",\n' + f' "gap_domain": "short topic label if uncertain/unresolved/escalated, ' + f'else empty string",\n' + f' "evidence": "one sentence explaining your classification"\n' + f"}}" + ), + expected_output="Valid JSON only.", + agent=agent, + ) + + raw = str(Crew(agents=[agent], tasks=[task], verbose=False).kickoff()).strip() + + try: + clean = raw.replace("```json", "").replace("```", "").strip() + parsed = json.loads(clean) + outcome = parsed.get("outcome", "resolved") + gap_domain = parsed.get("gap_domain", "") + evidence = parsed.get("evidence", "") + except json.JSONDecodeError: + return + + if outcome in ("uncertain", "unresolved", "escalated") and gap_domain: + self._lifecycle.scan_for_knowledge_gaps( + text=gap_domain, + triggered_by=thread_id, + day=self._state.day, + date_str=date_str, + state=self._state, + timestamp=timestamp, + ) + + self._mem.log_event( + SimEvent( + type="knowledge_gap_detected", + timestamp=timestamp, + day=self._state.day, + date=date_str, + actors=[m["user"] for m in messages], + artifact_ids={ + "slack_thread": thread_id, + "jira": ticket_id or "", + }, + facts={ + "detection_method": "async_thread_classification", + "outcome": outcome, + "gap_domain": gap_domain, + "evidence": evidence, + "asker": asker, + "asker_expertise": asker_expertise, + "topic": topic, + }, + summary=( + f"Async thread {outcome}: {asker} asked about '{topic}' — " + f"{evidence}" + ), + tags=["knowledge_gap", "slack", "async_question"], + ) + ) + def _last_turn_desc( self, speaker: str, @@ -2994,8 +3262,6 @@ def _extract_last_turn(self, raw_output: str, speaker: str) -> tuple: # Fallback — treat the whole output as the message return raw, None - # ─── LOW-LEVEL UTILITIES ────────────────────────────────────────────────── - def _save_slack( self, messages: List[dict], channel: str, interaction_type: str = "general" ) -> Tuple[str, str]: @@ -3051,7 +3317,6 @@ def _run_slack_design_discussion( date_str: str, ) -> Tuple[str, str, List[str]]: """ - Original Slack-thread path extracted from _handle_design_discussion. Returns (slack_path, thread_id, tags). """ @@ -3322,15 +3587,21 @@ def _emit_bot_message( ) return thread_id - def _find_ticket(self, ticket_id: Optional[str]) -> Optional[dict]: - if not ticket_id: + def _find_ticket( + self, ticket_id: Optional[str] = "", meeting_time_iso: Optional[str] = "" + ) -> Optional[dict]: + if not ticket_id and meeting_time_iso: return None - return self._mem.get_ticket(ticket_id) + return self._mem.get_ticket(ticket_id, as_of_time=meeting_time_iso) - def _find_pr(self, pr_id: Optional[str]) -> Optional[dict]: + def _find_pr( + self, pr_id: Optional[str], timestamp: Optional[str] = "" + ) -> Optional[dict]: if not pr_id: return None - return self._mem._prs.find_one({"pr_id": pr_id}, {"_id": 0}) + return self._mem._prs.find_one( + {"pr_id": pr_id, "created_at": {"$lte": timestamp}}, {"_id": 0} + ) def _find_reviewable_pr(self, reviewer: str) -> Optional[dict]: """Find an open PR where this person is listed as a reviewer.""" diff --git a/src/org_lifecycle.py b/src/org_lifecycle.py index c05e9fc..5bf1e68 100644 --- a/src/org_lifecycle.py +++ b/src/org_lifecycle.py @@ -154,7 +154,7 @@ def __init__( ) def process_departures( - self, day: int, date_str: str, state, clock + self, day: int, date_str: str, state, clock, ticket_assigner=None ) -> List[DepartureRecord]: departures: List[DepartureRecord] = [] @@ -164,6 +164,8 @@ def process_departures( ) if record: departures.append(record) + if ticket_assigner is not None: + ticket_assigner.evict_engineer(record.name) if self._cfg.get("enable_random_attrition", False): prob = self._cfg.get("random_attrition_daily_prob", 0.01) @@ -193,8 +195,6 @@ def process_departures( "reason": "voluntary", "knowledge_domains": [], "documented_pct": 0.5, - # role intentionally omitted — _execute_departure resolves it - # from personas so we don't hardcode anything here } record = self._execute_departure( attrition_cfg, @@ -206,16 +206,22 @@ def process_departures( ) if record: departures.append(record) + if ticket_assigner is not None: + ticket_assigner.evict_engineer(record.name) break return departures - def process_hires(self, day: int, date_str: str, state, clock) -> List[HireRecord]: + def process_hires( + self, day: int, date_str: str, state, clock, ticket_assigner=None + ) -> List[HireRecord]: hires: List[HireRecord] = [] for hire_cfg in self._scheduled_hires.get(day, []): record = self._execute_hire(hire_cfg, day, date_str, state, clock) if record: hires.append(record) + if ticket_assigner is not None: + ticket_assigner.register_hire(record.name) return hires def scan_for_knowledge_gaps( @@ -229,16 +235,18 @@ def scan_for_knowledge_gaps( similarity_threshold: float = 0.65, ) -> List[KnowledgeGapEvent]: """ - Detect knowledge gaps using semantic similarity. - - Instead of checking whether a departed employee's domain keyword appears - verbatim in the incident text, we embed the incident text and compare it - against the departed employee's persona_skill artifacts (expertise profile) - and any author_expertise artifacts (topics they wrote about). - - This catches cases where incident terminology differs from the departed - employee's stated expertise — e.g., "auth timeout" matches against - "identity management" because the embeddings are semantically close. + Detect knowledge gaps using semantic similarity against departed employee + expertise profiles, then cross-reference the DomainRegistry for live + documentation coverage and orphan status. + + Two-pass detection: + Pass 1 — Embedding similarity: embed the trigger text and compare + against departed employees' persona_skill vectors. Catches + semantic drift (e.g. "auth timeout" → "identity management"). + Pass 2 — DomainRegistry cross-reference: for each matched domain, + pull the current documentation_coverage from the registry so + the SimEvent carries an accurate, mutable coverage score rather + than the static documented_pct frozen at departure. Args: text: The incident root cause or description text. @@ -256,6 +264,7 @@ def scan_for_knowledge_gaps( if not self._departed: return found + # ── Pass 1: embedding similarity ────────────────────────────────────── expert_matches = self._mem.find_expert_by_skill(text, n=20) match_scores: Dict[str, float] = {} @@ -284,16 +293,51 @@ def scan_for_knowledge_gaps( ) domain_label = ", ".join(gap_domains) + # ── Pass 2: DomainRegistry cross-reference ───────────────────── + # Look up live coverage for each matched domain. If the registry + # has been updated by Confluence writes since genesis, this will + # reflect partial recovery rather than the static departure value. + registry_hits: List[dict] = [] + live_coverage: float = record.documented_pct # fallback to departure value + + for domain in gap_domains: + key = domain.lower().replace(" ", "_") + reg_doc = self._mem._db["domain_registry"].find_one({"_id": key}) + if reg_doc: + registry_hits.append(reg_doc) + + if registry_hits: + # Use the lowest coverage across all matched domains — most + # conservative estimate, most useful for gap severity scoring. + live_coverage = min(r["documentation_coverage"] for r in registry_hits) + known_by = list( + {name for r in registry_hits for name in r.get("known_by", [])} + ) + orphaned_domains = [ + r["domain"] for r in registry_hits if r.get("primary_owner") is None + ] + else: + known_by = [] + orphaned_domains = gap_domains # assume orphaned if not in registry + gap_event = KnowledgeGapEvent( departed_name=record.name, domain_hit=domain_label, triggered_by=triggered_by, triggered_on_day=day, - documented_pct=record.documented_pct, + documented_pct=live_coverage, ) self._gap_events.append(gap_event) found.append(gap_event) + # Severity classification — used downstream by gap_detected logic + if live_coverage < 0.3: + gap_classification = "likely" + elif live_coverage < 0.6: + gap_classification = "possible" + else: + gap_classification = "none" + self._mem.log_event( SimEvent( type="knowledge_gap_detected", @@ -306,23 +350,52 @@ def scan_for_knowledge_gaps( "departed_employee": record.name, "gap_areas": gap_domains, "triggered_by": triggered_by, - "documented_pct": record.documented_pct, + "documented_pct": record.documented_pct, # at departure + "live_documentation_coverage": round( + live_coverage, 3 + ), # current "days_since_departure": day - record.day, "escalation_harder": True, "semantic_score": round(score, 4), "detection_method": "embedding_similarity", + "gap_classification": gap_classification, + "orphaned_domains": orphaned_domains, + "known_by": known_by, + # Fields aligned with PR reviewer audit schema + "topics_beyond_author_expertise": gap_domains, + "author_domain_fit": ( + "low" + if live_coverage < 0.3 + else "medium" + if live_coverage < 0.6 + else "high" + ), }, summary=( f"Knowledge gap: {domain_label} (owned by ex-{record.name}, " f"similarity={score:.3f}) surfaced in {triggered_by}. " - f"~{int(record.documented_pct * 100)}% documented." + f"Coverage at departure: {int(record.documented_pct * 100)}% → " + f"live: {int(live_coverage * 100)}%." + + ( + f" Orphaned: {orphaned_domains}." + if orphaned_domains + else "" + ) + + ( + f" Partial knowledge held by: {known_by}." + if known_by + else "" + ) ), - tags=["knowledge_gap", "departed_employee"], + tags=["knowledge_gap", "departed_employee"] + + (["orphaned_domain"] if orphaned_domains else []), ) ) logger.info( f" [yellow]⚠ Knowledge gap:[/yellow] {domain_label} " - f"(was {record.name}'s, score={score:.3f}) surfaced in {triggered_by}" + f"(was {record.name}'s, score={score:.3f}, " + f"live_coverage={int(live_coverage * 100)}%) surfaced in {triggered_by}" + + (f" | orphaned: {orphaned_domains}" if orphaned_domains else "") ) return found @@ -361,14 +434,50 @@ def _load_departed_from_log(self, events: List[SimEvent]) -> None: f"[lifecycle] Loaded {len(self._departed)} departed employee(s) from event log." ) - def get_roster_context(self) -> str: + def get_roster_context(self, include_all_open_gaps: bool = False) -> str: lines: List[str] = [] - for d in self._departed[-3:]: - gap_str = ( - f"Knowledge gaps: {', '.join(d.knowledge_domains)}. " - f"~{int(d.documented_pct * 100)}% documented." - if d.knowledge_domains - else "No critical knowledge gaps." + + departed_to_show = ( + [ + d + for d in self._departed + if any(g.departed_name == d.name for g in self._gap_events) + ] + if include_all_open_gaps + else self._departed[-3:] + ) + for d in departed_to_show: + # Pull live coverage from registry rather than the static departure value + live_coverages: List[str] = [] + orphaned: List[str] = [] + for domain in d.knowledge_domains: + key = domain.lower().replace(" ", "_") + reg = self._mem._db["domain_registry"].find_one({"_id": key}) + if reg: + pct = int(reg.get("documentation_coverage", d.documented_pct) * 100) + owner = reg.get("primary_owner") + known_by = reg.get("known_by", []) + status = ( + f"owner={owner}" + if owner + else f"ORPHANED, known_by={known_by or 'nobody'}" + ) + live_coverages.append(f"{domain} ({pct}%, {status})") + if not owner: + orphaned.append(domain) + else: + live_coverages.append( + f"{domain} (~{int(d.documented_pct * 100)}%, no registry entry)" + ) + orphaned.append(domain) + + if live_coverages: + gap_str = "Knowledge gaps: " + "; ".join(live_coverages) + "." + else: + gap_str = "No critical knowledge gaps." + + orphan_str = ( + f" ⚠ Orphaned domains: {', '.join(orphaned)}." if orphaned else "" ) ticket_str = ( f" Reassigned: {', '.join(d.reassigned_tickets)}." @@ -384,7 +493,7 @@ def get_roster_context(self) -> str: lines.append("RECENT DEPARTURES:") lines.append( f" - {d.name} ({d.dept}) left Day {d.day} [{d.reason}]. " - f"{gap_str}{ticket_str}{handoff_str}" + f"{gap_str}{orphan_str}{ticket_str}{handoff_str}" ) for h in self._hired[-3:]: warm = self._count_warm_edges(h) @@ -557,6 +666,11 @@ def _execute_departure( day=day, ) + # ── Orphan any DomainRegistry records this engineer owned ───────────── + # Mid-sim departures may leave primary_owner set. Null it out so + # scan_for_knowledge_gaps correctly treats those domains as orphaned. + self._orphan_domains_on_departure(name, record, day) + logger.info( f" [red]👋 Departure:[/red] {name} ({dept}) [{record.reason}]. " f"{len(edge_snapshot)} edges severed. Centrality was {departing_centrality:.3f}." @@ -795,6 +909,88 @@ def _apply_centrality_vacuum( f"Stress absorbed by: {summary_str}" ) + # ── Domain registry maintenance ────────────────────────────────────────── + + def _orphan_domains_on_departure( + self, name: str, record: DepartureRecord, day: int + ) -> None: + """ + Null out primary_owner on any DomainRegistry records this engineer + owned. Called after _execute_departure so the registry stays in sync + with the live org chart. + + Also appends the name to former_owners so audit trails are preserved, + and records the coverage value at the time of departure so analysts can + see how much degradation happened on their watch. + """ + result = self._mem._db["domain_registry"].update_many( + {"primary_owner": name}, + { + "$set": { + "primary_owner": None, + "coverage_at_last_departure": None, # will be set per-doc below + }, + "$addToSet": {"former_owners": name}, + }, + ) + if result.modified_count == 0: + return + + # Per-doc update to capture individual coverage values at departure + orphaned = self._mem._db["domain_registry"].find( + {"former_owners": name, "primary_owner": None} + ) + for doc in orphaned: + self._mem._db["domain_registry"].update_one( + {"_id": doc["_id"]}, + { + "$set": { + "coverage_at_last_departure": doc.get("documentation_coverage"), + "last_updated_day": day, + } + }, + ) + logger.info( + f" [dim]→ Domain '{doc['domain']}' orphaned after {name}'s departure. " + f"Coverage frozen at {int(doc.get('documentation_coverage', 0) * 100)}%.[/dim]" + ) + + def _claim_domains_on_hire(self, name: str, expertise: List[str], day: int) -> None: + """ + When a new engineer is hired, check whether any orphaned DomainRegistry + domains overlap with their stated expertise. If so, assign them as + primary_owner and add them to known_by. + + This models the realistic scenario where a targeted backfill hire + deliberately covers a known gap — the registry reflects the recovery + immediately so planner prompts and gap detection see the updated state. + """ + for domain_key_str in expertise: + key = domain_key_str.lower().replace(" ", "_") + doc = self._mem._db["domain_registry"].find_one( + {"_id": key, "primary_owner": None} + ) + if not doc: + # Also try system_tags match for inexact expertise names + doc = self._mem._db["domain_registry"].find_one( + {"system_tags": key, "primary_owner": None} + ) + if doc: + self._mem._db["domain_registry"].update_one( + {"_id": doc["_id"]}, + { + "$set": { + "primary_owner": name, + "last_updated_day": day, + }, + "$addToSet": {"known_by": name}, + }, + ) + logger.info( + f" [dim]→ Domain '{doc['domain']}' claimed by new hire {name}. " + f"Coverage: {int(doc.get('documentation_coverage', 0) * 100)}%.[/dim]" + ) + # ─── HIRE ENGINE ────────────────────────────────────────────────────────── def _execute_hire( @@ -829,6 +1025,9 @@ def _execute_hire( timestamp_iso=clock.now("system").isoformat(), ) + # Claim any orphaned DomainRegistry domains that match this hire's expertise + self._claim_domains_on_hire(name, expertise, day) + if name in G: logger.warning(f"[lifecycle] '{name}' already in graph — skipping hire.") return None diff --git a/src/planner_models.py b/src/planner_models.py index 7062835..5e812fa 100644 --- a/src/planner_models.py +++ b/src/planner_models.py @@ -139,6 +139,9 @@ class SprintContext: capacity_by_member: Dict[str, float] in_review: List[str] sprint_theme: str = "" + ticket_domain_tags: Dict[str, List[str]] = field(default_factory=dict) + # e.g. {"TICKET-42": ["billing-legacy"], "TICKET-43": ["kafka", "infra"]} + # Populated by TicketAssigner via domain registry cross-reference. @dataclass @@ -263,4 +266,5 @@ class ValidationResult: "crm_touchpoint", "crm_account_at_risk", "customer_health_briefing", + "assignment_domain_mismatch", } diff --git a/src/ticket_assigner.py b/src/ticket_assigner.py index 897c755..f3087d8 100644 --- a/src/ticket_assigner.py +++ b/src/ticket_assigner.py @@ -419,6 +419,34 @@ def _ticket_title_vector(self, ticket: dict) -> List[float]: return vector + def evict_engineer(self, name: str) -> None: + """ + Remove a departed engineer's vector from the cache so they no longer + influence cost matrix scoring in _hungarian_assign. + Called by OrgLifecycleManager after _execute_departure completes. + """ + self._engineer_vectors.pop(name, None) + logger.debug(f"[assigner] Evicted vector for departed engineer: {name}") + + def register_hire(self, name: str) -> None: + """ + Pre-warm the expertise vector for a new hire so their first sprint + assignment scores correctly rather than defaulting to the neutral 1.0. + Called by OrgLifecycleManager after _execute_hire completes. + The persona must already be written to PERSONAS before this is called. + """ + vec = self._expertise_vector(name) + if vec: + logger.debug( + f"[assigner] Pre-warmed expertise vector for new hire: {name} " + f"({len(vec)}-dim)" + ) + else: + logger.debug( + f"[assigner] No expertise vector for new hire {name} " + f"(empty expertise — will score neutral)" + ) + def _ticket_history(self, state) -> Dict[str, set]: """ Returns {engineer: {ticket_ids they've touched in prior days}}. diff --git a/src/utils/persona_utils.py b/src/utils/persona_utils.py index c7b88c0..2bfc59b 100644 --- a/src/utils/persona_utils.py +++ b/src/utils/persona_utils.py @@ -140,9 +140,9 @@ def get_voice_card( lines = [header, f" Typing style: {quirks}", f" Current mood: {mood}"] - if context == "async": - lines.insert(2, f" Expertise: {expertise}") - elif context == "watercooler": + lines.insert(2, f" Expertise: {expertise}") + + if context == "watercooler": lines.insert(2, f" Personal interests: {interests}") _anti_pattern_contexts = {"async", "design", "collision", "dm"} From 63066aa86aa7c94e188d9d47184f0d7e492e2f93 Mon Sep 17 00:00:00 2001 From: Jeff F Date: Tue, 31 Mar 2026 19:37:27 -0500 Subject: [PATCH 2/4] Add updated eval files --- eval/agentic_eval_harness.py | 282 ++++++++++++++++++++++++++++------- eval/eval_harness.py | 19 +++ 2 files changed, 247 insertions(+), 54 deletions(-) diff --git a/eval/agentic_eval_harness.py b/eval/agentic_eval_harness.py index 42e49d0..b3a125c 100644 --- a/eval/agentic_eval_harness.py +++ b/eval/agentic_eval_harness.py @@ -61,6 +61,8 @@ import argparse import yaml +from eval_harness import _ARTIFACT_SUBSYSTEM + logger = logging.getLogger("orgforge.agentic_eval") with open(Path(__file__).resolve().parent.parent / "config" / "config.yaml") as f: @@ -256,7 +258,7 @@ def _check_actor_gate(self, doc_id: str, doc_type: str) -> Tuple[bool, bool]: if self._question_type != "PERSPECTIVE": return False, False - from eval_harness import _ARTIFACT_SUBSYSTEM + subsystem = _ARTIFACT_SUBSYSTEM.get(doc_type, "default") @@ -598,23 +600,57 @@ def _extract_boolean(self, answer: Dict) -> Optional[bool]: return True if val.lower() in ("false", "no", "0"): return False - # Try to find a boolean in free-text reasoning + + # Try to find a boolean in free-text reasoning. + # IMPORTANT: check negative phrases FIRST and use full-phrase matching so + # that "did not have access" cannot shadow the later "had access" check — + # both would match under simple substring logic since "had access" is a + # substring of "did not have access". We resolve this by checking the + # negative patterns against the exact negated forms only, not as substrings + # of longer phrases. reasoning = str(answer.get("reasoning", answer.get("explanation", ""))).lower() - if any( - w in reasoning - for w in ( - "could not have known", - "did not have access", - "was not visible", - "outside their", - ) - ): - return False - if any( - w in reasoning - for w in ("could have known", "had access", "was visible", "in their") - ): - return True + + # Negative indicators — listed as complete phrases, no substring ambiguity + _NEGATIVE_PHRASES = ( + "could not have known", + "did not have access", + "does not have access", + "had no access", + "was not visible", + "not visible to", + "outside their visibility", + "outside their access", + "outside their cone", + "not in their subsystem", + "blocked from", + "no access to", + ) + + + _POSITIVE_PHRASES = ( + "could have known", + "would have known", + "did have access", + "has access to", + "was visible to", + "visible to this actor", + "within their visibility", + "within their access", + "within their cone", + "in their subsystem", + "had direct access", + "had full access", + ) + + for phrase in _NEGATIVE_PHRASES: + if phrase in reasoning: + return False + + + for phrase in _POSITIVE_PHRASES: + if phrase in reasoning: + return True + return None @@ -756,10 +792,16 @@ def score_trajectory( for call in calls: retrieved_ids.update(call.result_ids) - cause_id = ground_truth.get("cause_event_id", "") - effect_id = ground_truth.get("effect_event_id", "") - cause_identified = 1.0 if (cause_id and cause_id in retrieved_ids) else 0.0 - effect_identified = 1.0 if (effect_id and effect_id in retrieved_ids) else 0.0 + # Use artifact IDs from evidence_chain_artifacts, not synthetic event IDs. + # Synthetic event IDs (e.g. "evt_incident_opened_5_IT-108_alex") are internal + # keys that never appear in MongoDB documents. Agents retrieve documents by + # their actual artifact IDs (e.g. "IT-108"), so we must match on those instead. + evidence_artifacts = ground_truth.get("evidence_chain_artifacts", {}) + cause_artifacts = set(evidence_artifacts.get("cause", [])) + effect_artifacts = set(evidence_artifacts.get("effect", [])) + + cause_identified = 1.0 if (cause_artifacts and cause_artifacts & retrieved_ids) else 0.0 + effect_identified = 1.0 if (effect_artifacts and effect_artifacts & retrieved_ids) else 0.0 # Mechanism: did agent use keyword in its tool calls or final answer? gt_mechanism = ground_truth.get("causal_mechanism", "") @@ -771,12 +813,17 @@ def score_trajectory( 1.0 if any(alias in agent_text for alias in aliases) else 0.0 ) - # Causal chain: did agent retrieve cause before effect? + # Causal chain: did agent retrieve a cause artifact before an effect artifact? + # Since we no longer have single cause_id/effect_id to index into call.result_ids, + # we find the FIRST call that returned any cause artifact and the FIRST that + # returned any effect artifact, then check ordering. cause_call_idx = next( - (i for i, c in enumerate(calls) if cause_id in c.result_ids), None + (i for i, c in enumerate(calls) if cause_artifacts & set(c.result_ids)), + None, ) effect_call_idx = next( - (i for i, c in enumerate(calls) if effect_id in c.result_ids), None + (i for i, c in enumerate(calls) if effect_artifacts & set(c.result_ids)), + None, ) causal_chain_complete = ( 1.0 @@ -819,11 +866,71 @@ def _extract_boolean(self, answer: Dict, key: str) -> Optional[bool]: return True if val.lower() in ("false", "no"): return False + + # Inspect free-text reasoning for outcome_changed signal. + # + # The original had two bugs: + # + # 1. "would not" was mapped to True (outcome DID change — the thing would + # NOT have happened). This is semantically correct for counterfactuals + # ("the incident would not have occurred") but the bare phrase is too + # short — "this would not be my first choice" would also match. + # Replaced with longer, unambiguous anchors. + # + # 2. "would have prevented" → True is correct but collides with + # "nothing would have prevented" → should be False. + # Fixed by checking the negated form first. + # + # 3. "no change" → False is a two-word phrase that can appear in + # unrelated contexts ("no change in personnel"). Replaced with + # longer anchors. + # + # Strategy: check negated/False-indicating phrases FIRST (longer, more + # specific), then check True-indicating phrases that are phrased to not + # overlap with any negated form above. + reasoning = str(answer.get("reasoning", "")).lower() - if "would not" in reasoning or "would have prevented" in reasoning: - return True - if "would still" in reasoning or "no change" in reasoning: - return False + + # False indicators — outcome did NOT change (removing cause = no difference) + _OUTCOME_UNCHANGED = ( + "would still have occurred", + "would have happened regardless", + "outcome would not have changed", + "outcome would be the same", + "would not have been prevented", + "nothing would have prevented", + "no change in outcome", + "would have proceeded regardless", + "result would be unchanged", + "would still have taken place", + ) + + # True indicators — outcome WOULD change (removing cause = different result) + # Phrased to not be substrings of any _OUTCOME_UNCHANGED phrase above. + _OUTCOME_CHANGED = ( + "would not have occurred", + "would have been prevented", + "would have been avoided", + "outcome would have changed", + "would have changed the outcome", + "would have been diagnosed faster", + "would not have escalated", + "would have been resolved", + "would not have happened", + "would have been different", + "causal chain would have been broken", + ) + + # Check False indicators first — they are more specific and longer + for phrase in _OUTCOME_UNCHANGED: + if phrase in reasoning: + return False + + # Check True indicators second + for phrase in _OUTCOME_CHANGED: + if phrase in reasoning: + return True + return None @@ -879,21 +986,50 @@ def score_trajectory( searched_ids.update(call.result_ids) searched_tool_args.append(str(call.arguments).lower()) - # Search space coverage: fraction of expected_search_space hit - # Also count partial matches on path prefixes + def _normalize_search_term(s: str) -> str: + """ + Extract the terminal component of a path-style search space entry. + e.g. "confluence/postmortems/IT-108" → "it-108" + "slack/channels/incidents" → "incidents" + "IT-108" → "it-108" + """ + return s.strip("/").split("/")[-1].lower() + + normalized_expected: Dict[str, str] = { + _normalize_search_term(e): e for e in expected_space + } + + # Also normalize all tool arg strings and result IDs for matching. + normalized_tool_args: List[str] = [ + arg.lower() for arg in searched_tool_args + ] + normalized_result_ids: Set[str] = { + _normalize_search_term(rid) for rid in searched_ids + } + covered = set() - for expected in expected_space: - expected_lower = expected.lower() - if any(expected_lower in arg for arg in searched_tool_args): - covered.add(expected) - elif expected in searched_ids: - covered.add(expected) + for norm_term, original in normalized_expected.items(): + # Primary match: terminal component appears anywhere in a tool arg string. + # This catches {"page_id": "IT-108"} matching "confluence/postmortems/IT-108" + # and {"query": "incidents"} matching "slack/channels/incidents". + if any(norm_term in arg for arg in normalized_tool_args): + covered.add(original) + + # Secondary match: terminal component matches a normalized result ID. + # This catches cases where the agent retrieved the document directly + # and its ID is the terminal path component. + elif norm_term in normalized_result_ids: + covered.add(original) + + # Tertiary match: the full original path appears verbatim in a tool arg. + # Preserves the original behaviour for agents that do pass full paths. + elif any(original.lower() in arg for arg in normalized_tool_args): + covered.add(original) search_space_coverage = ( len(covered) / len(expected_space) if expected_space else 1.0 ) - # Correct absence conclusion: did agent explicitly state non-existence? conclusion_text = str(trajectory.final_answer.get("reasoning", "")).lower() conclusion_text += str(trajectory.final_answer.get("answer", "")).lower() explicit_negative = any( @@ -948,20 +1084,60 @@ def _extract_absence_conclusion(self, answer: Dict) -> Optional[bool]: return True if val.lower() in ("false", "no", "not found", "does not exist", "absent"): return False + reasoning = str(answer.get("reasoning", "")).lower() - if any( - w in reasoning - for w in ( - "does not exist", - "not created", - "no record", - "absent", - "not found", - ) - ): - return False - if any(w in reasoning for w in ("exists", "was created", "found", "present")): - return True + + # False indicators — artifact does NOT exist + # These are checked first and are long enough to be unambiguous. + _ABSENCE_PHRASES = ( + "does not exist", + "did not exist", + "was not created", + "has not been created", + "no record exists", + "no record was found", + "could not be found", + "could not find", + "was not found", + "is not present", + "was not present", + "no evidence of", + "never created", + "not in the corpus", + "absent from", + "no postmortem", + "no ticket was", + "no confluence page", + ) + + # True indicators — artifact DOES exist + # Reworded so none are substrings of any _ABSENCE_PHRASES entry above. + _PRESENCE_PHRASES = ( + "artifact exists", + "document exists", + "ticket exists", + "page exists", + "record exists", + "was successfully created", + "has been created", + "is present in", + "appears in the corpus", + "was located", + "has been found", + "confirmed to exist", + "did find", + ) + + # Check absence first — these are longer and more specific + for phrase in _ABSENCE_PHRASES: + if phrase in reasoning: + return False + + # Check presence second — phrased to not overlap with any absence phrase + for phrase in _PRESENCE_PHRASES: + if phrase in reasoning: + return True + return None @@ -1314,10 +1490,8 @@ def _parse_structured_answer(self, raw: str) -> Dict: def _infer_as_of_time(self, question: dict) -> str: qtype = question.get("question_type", "") if qtype == "SILENCE": - # Full corpus — end of sim - max_day = max( - (e.day for e in self._mem.get_event_log(from_db=True)), default=22 - ) + events = self._mem.get_event_log(from_db=True) + max_day = max((e.day for e in events), default=1) return (_SIM_START + timedelta(days=max_day)).isoformat() if qtype == "PERSPECTIVE": return question.get("as_of_time", datetime.now().isoformat()) diff --git a/eval/eval_harness.py b/eval/eval_harness.py index 518c560..db543ac 100644 --- a/eval/eval_harness.py +++ b/eval/eval_harness.py @@ -1353,6 +1353,17 @@ def _counterfactual_questions(self) -> List[dict]: def _build_counterfactual_question(self, link: CausalLink) -> Optional[dict]: + cause_event = next( + (e for e in self._events + if self._synthetic_event_id(e) == link.cause_event_id), + None, + ) + effect_event = next( + (e for e in self._events + if self._synthetic_event_id(e) == link.effect_event_id), + None, + ) + ground_truth = { "outcome_changed": link.outcome_changed, "causal_mechanism": link.link_type, @@ -1365,6 +1376,14 @@ def _build_counterfactual_question(self, link: CausalLink) -> Optional[dict]: "premise": link.counterfactual_premise, "outcome": link.counterfactual_outcome, "actors": link.actors, + "evidence_chain_artifacts": { + "cause": sorted(_safe_artifact_values( + cause_event.artifact_ids if cause_event else {} + )), + "effect": sorted(_safe_artifact_values( + effect_event.artifact_ids if effect_event else {} + )), + }, } subsystems_str = ", ".join(sorted(link.subsystems_involved)) From a9ca0f244f397db7c6636ef2e896aa53c60ea2fb Mon Sep 17 00:00:00 2001 From: Jeff F Date: Tue, 31 Mar 2026 22:35:58 -0500 Subject: [PATCH 3/4] Add updated eval files --- eval/agentic_eval_harness.py | 253 +++++-- eval/eval_harness.py | 36 +- eval/export_to_hf.py | 1267 +++++++++++++++++++++++++--------- 3 files changed, 1164 insertions(+), 392 deletions(-) diff --git a/eval/agentic_eval_harness.py b/eval/agentic_eval_harness.py index b3a125c..621f8f5 100644 --- a/eval/agentic_eval_harness.py +++ b/eval/agentic_eval_harness.py @@ -1158,10 +1158,25 @@ class AgenticEvalRunner: 5. Combines scores with track-specific weights """ - def __init__(self, model: str = "claude-sonnet-4-6", max_steps: int = 15): + def __init__( + self, + model: str = "claude-sonnet-4-6", + max_steps: int = 15, + ungated: bool = False, + zero_shot: bool = False, + ): self._model = model self._max_steps = max_steps + # --ungated: all actor/subsystem gates disabled regardless of question type. + # Establishes the "god-mode" information ceiling for the Epistemic Tax. + self._ungated = ungated + + # --zero-shot: agent receives no tools at all (no corpus access). + # Establishes the hallucination / prior-knowledge floor. + # Mutually exclusive with --ungated; zero_shot takes precedence if both set. + self._zero_shot = zero_shot + from flow import build_llm from memory import Memory @@ -1248,24 +1263,38 @@ def run( f"Overall — answer: {summary['overall']['answer_score']:.3f} " f"trajectory: {summary['overall']['trajectory_score']:.3f} " f"combined: {summary['overall']['combined_score']:.3f}" + + ( + f" | violation_adjusted: " + f"{summary['overall'].get('violation_adjusted_combined_score', 'n/a')}" + ) ) def _run_question(self, question: dict) -> EvalResult: qtype = question["question_type"] ground_truth = question["ground_truth"] - # Set up gated tools + # Set up gated tools. + # --ungated: strip all actor/subsystem gates by passing None for both, + # regardless of question type. Temporal gate still applies. + # --zero-shot: GatedTools is still constructed (for consistent call + # logging infrastructure) but _tool_list() returns [] so the agent + # never actually invokes any tool. as_of_time = self._infer_as_of_time(question) - actor_visible = ( - set(question.get("actor_visible_artifacts", [])) - if qtype == "PERSPECTIVE" - else None - ) - actor_subsystems = ( - set(question.get("subsystem_access", [])) - if qtype == "PERSPECTIVE" - else None - ) + + if self._ungated: + actor_visible = None + actor_subsystems = None + else: + actor_visible = ( + set(question.get("actor_visible_artifacts", [])) + if qtype == "PERSPECTIVE" + else None + ) + actor_subsystems = ( + set(question.get("subsystem_access", [])) + if qtype == "PERSPECTIVE" + else None + ) tools = GatedTools( mem=self._mem, @@ -1323,6 +1352,11 @@ def _run_question(self, question: dict) -> EvalResult: tool_call_count=len(trajectory.tool_calls), meta={ "model": self._model, + "eval_mode": ( + "zero_shot" if self._zero_shot + else "ungated" if self._ungated + else "gated" + ), "as_of_time": as_of_time, "trajectory_detail": traj_detail, "horizon_violations": trajectory.horizon_violations, @@ -1452,7 +1486,13 @@ def _run_agent(self, question: dict, tools: GatedTools) -> AgentTrajectory: return trajectory def _tool_list(self, tools: GatedTools) -> List: - """Return the tool surface for the agent. Narrow and typed.""" + """Return the tool surface for the agent. Narrow and typed. + + Returns an empty list in --zero-shot mode so the agent has no corpus + access — this establishes the hallucination / prior-knowledge floor. + """ + if self._zero_shot: + return [] return [ tools.get_ticket, tools.get_confluence_page, @@ -1512,67 +1552,130 @@ def _aggregate(self, results: List[EvalResult]) -> dict: def mean(vals): return round(sum(vals) / len(vals), 4) if vals else 0.0 + # ── Violation-adjusted scoring ──────────────────────────────────────── + # violation_rate = total_actor_gate_violations / total_tool_calls + # compliance_factor = max(0, 1 − violation_rate) ** _VIOLATION_EXPONENT + # adjusted_score = combined_score × compliance_factor + # + # Quadratic exponent (2) means violations compound non-linearly: + # 0% violations → 1.00× multiplier (no penalty) + # 25% violations → 0.56× multiplier + # 50% violations → 0.25× multiplier (score quartered) + # 75% violations → 0.06× multiplier (effectively disqualified) + # + # This decouples compliance from trajectory scoring and makes it a + # multiplicative gate at the aggregate level — a cheating agent cannot + # overcome the penalty through high answer accuracy alone. + _VIOLATION_EXPONENT = 2 + + def _compliance_tier(rate: float) -> str: + if rate < 0.05: + return "compliant" + if rate < 0.20: + return "borderline" + return "non_compliant" + + def _violation_adjusted(combined: float, violation_rate: float) -> float: + factor = max(0.0, 1.0 - violation_rate) ** _VIOLATION_EXPONENT + return round(combined * factor, 4) + by_type: Dict[str, List[EvalResult]] = {} by_difficulty: Dict[str, List[EvalResult]] = {} for r in results: by_type.setdefault(r.question_type, []).append(r) by_difficulty.setdefault(r.difficulty, []).append(r) + by_type_summary = {} + for qtype, rs in by_type.items(): + total_calls = sum(r.tool_call_count for r in rs) + total_violations = sum( + r.meta.get("actor_gate_violations", 0) for r in rs + ) + violation_rate = ( + round(total_violations / total_calls, 4) if total_calls else 0.0 + ) + base_combined = mean([r.combined_score for r in rs]) + + summary: Dict[str, Any] = { + "n": len(rs), + "answer_score": mean([r.answer_score for r in rs]), + "trajectory_score": mean([r.trajectory_score for r in rs]), + "combined_score": base_combined, + "accuracy": round( + sum(r.answer_correct for r in rs) / len(rs), 4 + ), + "avg_tool_calls": mean([r.tool_call_count for r in rs]), + } + + if qtype == "PERSPECTIVE": + compliance_factor = round( + max(0.0, 1.0 - violation_rate) ** _VIOLATION_EXPONENT, 4 + ) + summary.update( + { + "violation_rate": violation_rate, + "compliance_factor": compliance_factor, + "compliance_tier": _compliance_tier(violation_rate), + # Primary leaderboard axis — combined_score alone allows a + # cheating agent to rank above a disciplined one. This number + # prevents that by applying the compliance penalty independently + # of answer quality. + "violation_adjusted_combined_score": _violation_adjusted( + base_combined, violation_rate + ), + "avg_actor_gate_violations": mean( + [r.meta.get("actor_gate_violations", 0) for r in rs] + ), + "avg_subsystem_violations": mean( + [r.meta.get("subsystem_violations", 0) for r in rs] + ), + } + ) + elif qtype == "SILENCE": + summary["search_space_coverage"] = mean( + [ + r.meta.get("trajectory_detail", {}).get( + "search_space_coverage", 0 + ) + for r in rs + ] + ) + + by_type_summary[qtype] = summary + + # ── Global violation_adjusted_combined_score ────────────────────────── + # A single number for cross-model ranking. Agents without PERSPECTIVE + # questions are not penalised (violation_rate = 0, factor = 1.0). + all_calls = sum(r.tool_call_count for r in results) + all_violations = sum( + r.meta.get("actor_gate_violations", 0) for r in results + ) + global_violation_rate = ( + round(all_violations / all_calls, 4) if all_calls else 0.0 + ) + overall_combined = mean([r.combined_score for r in results]) + return { "overall": { "n": len(results), "answer_score": mean([r.answer_score for r in results]), "trajectory_score": mean([r.trajectory_score for r in results]), - "combined_score": mean([r.combined_score for r in results]), + "combined_score": overall_combined, "accuracy": round( sum(r.answer_correct for r in results) / len(results), 4 ), "avg_tool_calls": mean([r.tool_call_count for r in results]), + "global_violation_rate": global_violation_rate, + "global_compliance_factor": round( + max(0.0, 1.0 - global_violation_rate) ** _VIOLATION_EXPONENT, 4 + ), + "global_compliance_tier": _compliance_tier(global_violation_rate), + # Primary cross-track ranking number + "violation_adjusted_combined_score": _violation_adjusted( + overall_combined, global_violation_rate + ), }, - "by_type": { - qtype: { - "n": len(rs), - "answer_score": mean([r.answer_score for r in rs]), - "trajectory_score": mean([r.trajectory_score for r in rs]), - "combined_score": mean([r.combined_score for r in rs]), - "accuracy": round(sum(r.answer_correct for r in rs) / len(rs), 4), - "avg_tool_calls": mean([r.tool_call_count for r in rs]), - # Track-specific breakdowns - **( - { - "avg_actor_gate_violations": mean( - [r.meta.get("actor_gate_violations", 0) for r in rs] - ) - } - if qtype == "PERSPECTIVE" - else {} - ), - **( - { - "avg_subsystem_violations": mean( - [r.meta.get("subsystem_violations", 0) for r in rs] - ) - } - if qtype == "PERSPECTIVE" - else {} - ), - **( - { - "search_space_coverage": mean( - [ - r.meta.get("trajectory_detail", {}).get( - "search_space_coverage", 0 - ) - for r in rs - ] - ) - } - if qtype == "SILENCE" - else {} - ), - } - for qtype, rs in by_type.items() - }, + "by_type": by_type_summary, "by_difficulty": { diff: { "n": len(rs), @@ -1631,9 +1734,41 @@ def mean(vals): type=int, default=None, ) + parser.add_argument( + "--ungated", + action="store_true", + default=False, + help=( + "Disable all actor/subsystem gates — god-mode corpus access. " + "Establishes the Epistemic Tax ceiling. " + "Default output: export/eval/ungated_results.json" + ), + ) + parser.add_argument( + "--zero-shot", + action="store_true", + default=False, + help=( + "Provide no tools to the agent (no corpus access). " + "Establishes the hallucination / prior-knowledge floor. " + "Default output: export/eval/zero_shot_results.json" + ), + ) args = parser.parse_args() - runner = AgenticEvalRunner(model=args.model, max_steps=args.max_steps) + # Default output paths differ by mode so runs don't clobber each other + if args.out == EVAL_DIR / "agentic_results.json": + if args.zero_shot: + args.out = EVAL_DIR / "zero_shot_results.json" + elif args.ungated: + args.out = EVAL_DIR / "ungated_results.json" + + runner = AgenticEvalRunner( + model=args.model, + max_steps=args.max_steps, + ungated=args.ungated, + zero_shot=args.zero_shot, + ) runner.run( questions_path=args.questions, out_path=args.out, diff --git a/eval/eval_harness.py b/eval/eval_harness.py index db543ac..5a5e7fe 100644 --- a/eval/eval_harness.py +++ b/eval/eval_harness.py @@ -646,7 +646,11 @@ def _find_effect_event(self, link_type: str, cause: SimEvent) -> Optional[SimEve if e.type == "incident_opened": gap_areas = e.facts.get("gap_areas", []) mismatch_domains = cause.facts.get("assignment_risk_domains", []) - if gap_areas and mismatch_domains and set(gap_areas) & set(mismatch_domains): + if ( + gap_areas + and mismatch_domains + and set(gap_areas) & set(mismatch_domains) + ): return e return None @@ -1354,13 +1358,19 @@ def _counterfactual_questions(self) -> List[dict]: def _build_counterfactual_question(self, link: CausalLink) -> Optional[dict]: cause_event = next( - (e for e in self._events - if self._synthetic_event_id(e) == link.cause_event_id), + ( + e + for e in self._events + if self._synthetic_event_id(e) == link.cause_event_id + ), None, ) effect_event = next( - (e for e in self._events - if self._synthetic_event_id(e) == link.effect_event_id), + ( + e + for e in self._events + if self._synthetic_event_id(e) == link.effect_event_id + ), None, ) @@ -1377,12 +1387,16 @@ def _build_counterfactual_question(self, link: CausalLink) -> Optional[dict]: "outcome": link.counterfactual_outcome, "actors": link.actors, "evidence_chain_artifacts": { - "cause": sorted(_safe_artifact_values( - cause_event.artifact_ids if cause_event else {} - )), - "effect": sorted(_safe_artifact_values( - effect_event.artifact_ids if effect_event else {} - )), + "cause": sorted( + _safe_artifact_values( + cause_event.artifact_ids if cause_event else {} + ) + ), + "effect": sorted( + _safe_artifact_values( + effect_event.artifact_ids if effect_event else {} + ) + ), }, } diff --git a/eval/export_to_hf.py b/eval/export_to_hf.py index 44856f0..0d82167 100644 --- a/eval/export_to_hf.py +++ b/eval/export_to_hf.py @@ -2,8 +2,8 @@ export_to_hf.py =============== Normalises all OrgForge simulation artifacts into a flat HuggingFace-ready -corpus, runs BM25 and dense-retrieval baselines, produces Parquet files, and -writes a dataset card (README.md) to export/hf_dataset/. +corpus, computes a two-tier baseline, produces Parquet files, and writes a +dataset card (README.md) to export/hf_dataset/. Run after flow.py + eval_harness.py: python export_to_hf.py @@ -12,16 +12,39 @@ ------------- export/hf_dataset/ corpus/ - corpus-00000.parquet — flat document corpus (one row per artifact) + corpus-00000.parquet — flat document corpus (one row per artifact) questions/ - questions-00000.parquet — eval questions with ground truth - threads/ - threads-00000.parquet — causal thread graphs (JSON-serialised) + questions-00000.parquet — eval questions with ground truth + eval_indexes/ + causal_link_index.parquet — explicit causal links from CausalLinkIndexer + actor_visibility.parquet — per-actor visibility cones, one row per (actor, day) + absence_catalog.parquet — expected-but-absent artifact pairs baselines/ - bm25_results.json — BM25 retrieval scores per question - dense_results.json — dense retrieval scores per question - baseline_summary.json — aggregate numbers for the dataset card - README.md — HuggingFace dataset card + ungated_ceiling_bm25.json — per-question ungated BM25 ceiling scores + ungated_ceiling_dense.json — per-question ungated dense ceiling scores + static_reasoning_metrics.json — per-question + aggregate reasoning difficulty metrics + baseline_summary.json — combined summary for the dataset card + README.md — HuggingFace dataset card + +Two-tier baseline design +------------------------ +This file computes only what requires NO agent execution: + + Tier 1 — Ungated Retrieval Ceiling (UngatedCeilingBaseline) + BM25 and dense retrieval with all gates removed ("god-mode" corpus access). + MRR@10 and Recall@10 are reported for PERSPECTIVE and COUNTERFACTUAL only. + SILENCE is excluded — absence cannot be measured by retrieval recall. + The delta between this ceiling and a gated agent's combined_score is the + "Epistemic Tax" — the difficulty cost of respecting organisational silos. + + Tier 2 — Static Reasoning Difficulty (StaticReasoningMetrics) + Metrics derived from corpus + question metadata alone. No LLM, no agent. + PERSPECTIVE → horizon_contamination_rate (fraction of ungated top-20 outside cone) + COUNTERFACTUAL → causal_chain_traceable (do cause+effect co-appear in top-10?) + SILENCE → search_space_bm25_coverage (fraction of required locations BM25 finds) + +Agent-level baselines (ungated god-mode agent, zero-shot no-tools) require LLM +calls and belong in agentic_eval_harness.py as --ungated / --zero-shot flags. Corpus schema (one row per document) ------------------------------------- @@ -42,31 +65,47 @@ Question schema --------------- - question_id str - question_type str — RETRIEVAL | CAUSAL | TEMPORAL | GAP_DETECTION | ROUTING - question_text str - ground_truth str — JSON-serialised ground_truth dict - evidence_chain str — JSON array of artifact IDs - difficulty str — easy | medium | hard - requires_reasoning bool - chain_id str + question_id str + question_type str — PERSPECTIVE | COUNTERFACTUAL | SILENCE + question_text str + ground_truth str — JSON-serialised ground_truth dict + evidence_chain str — JSON array of artifact IDs (cause+effect for + COUNTERFACTUAL; evidence_artifacts for PERSPECTIVE; + empty for SILENCE — absence cannot be recalled) + difficulty str — medium | hard + requires_reasoning bool + actor str — PERSPECTIVE only + actor_role str — PERSPECTIVE only + as_of_day int — PERSPECTIVE only + subsystem_access str — JSON list; PERSPECTIVE only + blocked_subsystems str — JSON list; PERSPECTIVE only + actor_visible_artifacts str — JSON list; PERSPECTIVE only + link_type str — COUNTERFACTUAL only (causal link type) + causal_day int — COUNTERFACTUAL only + expected_search_space str — JSON list; SILENCE only + trigger_event_type str — SILENCE only + expected_response_type str — SILENCE only + +Eval indexes +------------- +causal_link_index — one row per CausalLink (see eval_harness.CausalLink) +actor_visibility — one row per (actor, day) ActorVisibilityCone snapshot +absence_catalog — one row per AbsenceRecord Baseline methodology --------------------- BM25 — rank_bm25 (Okapi BM25) over the body field. - For each retrieval / causal / routing / gap question the top-10 - returned doc_ids are compared against evidence_chain. + For PERSPECTIVE and COUNTERFACTUAL questions the top-10 returned + doc_ids are compared against evidence_chain. MRR@10 and Recall@10 are reported per question type. + SILENCE questions are skipped — the correct answer is absence, + so standard retrieval recall is not applicable. -Dense — sentence-transformers "Qwen/Qwen3-Embedding-4B" (2560-dim). +Dense — via Memory._embed() (same embedding model used by the simulation). Cosine similarity between question_text embedding and body embeddings. - Same MRR@10 / Recall@10 reported for comparison. - If sentence-transformers is not installed, this section is skipped - gracefully and the dataset card notes the omission. - -Temporal and GAP_DETECTION questions require boolean answers, not retrieval, -so baselines report evidence recall only (whether the right artifacts are -surfaced, not whether the final answer is correct). + Same MRR@10 / Recall@10 as BM25. + If Memory is unavailable, this section is skipped gracefully and the + dataset card notes the omission. """ from __future__ import annotations @@ -102,11 +141,11 @@ HF_DIR = BASE / "hf_dataset" CORPUS_DIR = HF_DIR / "corpus" QUES_DIR = HF_DIR / "questions" -THREAD_DIR = HF_DIR / "threads" +EVAL_INDEX_DIR = HF_DIR / "eval_indexes" BASELINE_DIR = HF_DIR / "baselines" _DENSE_MODEL_NAME = "Losspost/stella_en_1.5b_v5" -for d in (CORPUS_DIR, QUES_DIR, THREAD_DIR, BASELINE_DIR): +for d in (CORPUS_DIR, QUES_DIR, EVAL_INDEX_DIR, BASELINE_DIR): d.mkdir(parents=True, exist_ok=True) # ── Optional imports (degrade gracefully) ──────────────────────────────────── @@ -133,7 +172,6 @@ "rank_bm25 not installed — BM25 baseline disabled. pip install rank-bm25" ) - _DENSE_AVAILABLE = True _DENSE_MODEL_NAME = "Qwen/Qwen3-Embedding-4B" @@ -184,14 +222,11 @@ def __init__(self, mem=None): def build(self) -> List[dict]: rows: List[dict] = [] - # 1. One or more rows per SimEvent (always available) - # Events with both jira + confluence keys emit a row for each artifact for evt in self._events: evt_rows = self._sim_event_to_row(evt) if evt_rows: rows.extend(evt_rows) - # 2. Supplement with richer artifact bodies from MongoDB if available if self._mem is not None: rows = self._enrich_from_mongo(rows) rows.extend(self._plans_to_corpus_rows()) @@ -205,13 +240,11 @@ def build(self) -> List[dict]: if did not in seen or self._body_len(row) > self._body_len(seen[did]): seen[did] = row - # Normalize: ensure every row has a populated body field for row in seen.values(): if not row.get("body"): row["body"] = row.get("content") or "" rows = list(seen.values()) - logger.info(f" corpus: {len(rows)} documents") return rows @@ -221,11 +254,6 @@ def _body_len(self, r: dict) -> int: return len(r.get("body") or r.get("content") or "") def _sim_event_to_row(self, evt: dict) -> List[dict]: - """ - Convert a SimEvent to one or more corpus rows. - Events that reference multiple artifact types (e.g. both jira and - confluence) emit one row per artifact so no artifact is silently dropped. - """ event_type = evt.get("type", "") artifact_ids = evt.get("artifact_ids", {}) facts = evt.get("facts", {}) @@ -234,7 +262,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: if event_type in _EXCLUDED_EVENT_TYPES: return [] - # Shared fields derived once per event evt_actors = evt.get("actors", []) dept_val = str(facts.get("dept", "")).strip() if not dept_val and evt_actors: @@ -280,7 +307,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: rows: List[dict] = [] - # ── JIRA ────────────────────────────────────────────────────────────── jira_id = artifact_ids.get("jira", "") if jira_id: rows.append( @@ -295,7 +321,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── CONFLUENCE ──────────────────────────────────────────────────────── conf_id = artifact_ids.get("confluence", "") or next( ( v @@ -319,7 +344,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── EMAIL ───────────────────────────────────────────────────────────── email_id = artifact_ids.get("email", "") if email_id or event_type in ( "inbound_external_email", @@ -342,9 +366,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── SLACK ───────────────────────────────────────────────────────────── - # Only use slack_thread as the canonical ID — slack and slack_path - # are file paths or message fragments, not retrievable thread documents slack_id = artifact_ids.get("slack_thread", "") if slack_id: channel = facts.get("channel", "#general") @@ -358,7 +379,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── PR ──────────────────────────────────────────────────────────────── pr_id = artifact_ids.get("pr", "") if pr_id: rows.append( @@ -371,9 +391,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── ZENDESK TICKET ──────────────────────────────────────────────────── - # zd_ticket_opened carries a single ID; zd_tickets_escalated and - # zd_tickets_resolved carry a list under "zd_tickets". zd_ids: List[str] = [] single_zd = artifact_ids.get("zd_ticket", "") if single_zd: @@ -395,9 +412,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── SALESFORCE OPPORTUNITY ──────────────────────────────────────────── - # sf_opps may be a single ID (crm_touchpoint) or a list - # (sf_deals_risk_flagged, sf_ownership_lapsed). sf_opp_ids: List[str] = [] single_opp = artifact_ids.get("sf_opp", "") if single_opp: @@ -421,8 +435,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── SALESFORCE ACCOUNT ──────────────────────────────────────────────── - # sf_ownership_lapsed carries a list of account IDs. sf_acc_ids: List[str] = [] multi_acc = artifact_ids.get("sf_accounts", []) if isinstance(multi_acc, list): @@ -438,7 +450,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # ── FALLBACK ────────────────────────────────────────────────────────── if not rows: rows.append( { @@ -450,7 +461,6 @@ def _sim_event_to_row(self, evt: dict) -> List[dict]: } ) - # Ensure every row has a non-empty body for row in rows: if not row.get("body"): row["body"] = evt.get("summary", "") @@ -555,7 +565,6 @@ def _plans_to_corpus_rows(self) -> List[dict]: theme = plan.get("theme", "") plan_id = f"PLAN-{day}-{dept}" - # One row per engineer — body now includes theme + full agenda for ep in plan.get("engineer_plans", []): agenda_text = "\n".join( f"{'[DEFERRED] ' if item.get('deferred') else ''}" @@ -589,7 +598,6 @@ def _plans_to_corpus_rows(self) -> List[dict]: } ) - # Dept-level rationale row — fall back to theme if planner_reasoning absent reasoning = ( plan.get("raw", {}).get("planner_reasoning", "") or plan.get("planner_reasoning", "") @@ -677,10 +685,7 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: try: rich_map: Dict[str, str] = {} - # Confluence pages - # Also build a snippet index so CONF-UNKNOWN rows can be - # re-identified by matching their thin body against MongoDB content. - conf_id_map: Dict[str, str] = {} # content_snippet_or_title -> page_id + conf_id_map: Dict[str, str] = {} for page in self._mem._db["confluence_pages"].find( {}, {"_id": 0, "id": 1, "content": 1, "title": 1} ): @@ -693,7 +698,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: if title_key: conf_id_map[title_key] = page["id"] - # JIRA tickets + jira_comment artifacts folded into parent body comment_map: Dict[str, List[str]] = defaultdict(list) for comment in self._mem._db["artifacts"].find( {"type": "jira_comment"}, @@ -734,11 +738,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: parts.append(c) rich_map[tid] = "\n".join(p for p in parts if p) - # Pull requests — stored in the pull_requests collection. - # PRs are written via mem.upsert_pr() and are never in artifacts. - # Key fields: pr_id, title, description, author, ticket_id, - # reviewers, status, dept, day, date, timestamp, - # comments[].{date, author, verdict, text}. for pr in self._mem._db["pull_requests"].find( {}, { @@ -778,10 +777,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: parts.append(f"review ({author}{verdict}): {text}") rich_map[pid] = "\n".join(p for p in parts if p) - # Emails — stored in the dedicated emails collection. - # Schema: _id (ObjectId), embed_id, subject, body, from_name, - # from_addr, to_name, to_addr, direction, day, date, timestamp. - # Use embed_id as the corpus doc_id to match what SimEvents carry. for email in self._mem._db["emails"].find( {}, { @@ -814,8 +809,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: parts.append(email["body"]) rich_map[eid] = "\n".join(parts) - # Zendesk tickets — rich body folds in subject, org, description, - # all comment texts, and the related incident reference. for ticket in self._mem._db["zd_tickets"].find( {}, { @@ -854,7 +847,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: ) rich_map[tid] = "\n".join(p for p in parts if p) - # Salesforce opportunities — rich body from sf_opps collection. for opp in self._mem._db["sf_opps"].find( {}, { @@ -895,7 +887,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: parts.append(f"touchpoint ({sender}, {ts}): {subject}") rich_map[oid] = "\n".join(p for p in parts if p) - # Salesforce accounts — rich body from sf_accounts collection. for acc in self._mem._db["sf_accounts"].find( {}, { @@ -931,7 +922,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: for row in rows: if row["doc_id"] == "CONF-UNKNOWN" and row["doc_type"] == "confluence": - # Try to resolve the real ID via body snippet or title match body_snippet = (row.get("body") or "")[:120].strip() title_key = (row.get("title") or "").strip() resolved_id = conf_id_map.get(body_snippet) or conf_id_map.get( @@ -939,14 +929,11 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: ) if resolved_id: row["doc_id"] = resolved_id - row["title"] = resolved_id # was also CONF-UNKNOWN + row["title"] = resolved_id row["body"] = rich_map[resolved_id] if not row.get("dept"): row["dept"] = _dept_from_artifact_id(resolved_id) else: - # No real Confluence page exists — upstream emitted a - # confluence key with an empty value on a Slack-style - # social interaction. Reclassify correctly. row["doc_type"] = "slack" row["doc_id"] = ( f"SLACK-SOCIAL-{row.get('day', 0)}-" @@ -961,22 +948,6 @@ def _enrich_from_mongo(self, rows: List[dict]) -> List[dict]: row["body"] = rich_map[row["doc_id"]] if row["doc_type"] == "confluence" and not row.get("dept"): row["dept"] = _dept_from_artifact_id(row["doc_id"]) - # ── Orphan sweep ────────────────────────────────────────────── - # Each collection is queried directly. This correctly handles the - # fact that emails, PRs, slack messages, ZD tickets, SF opps, and - # SF accounts all have their own dedicated collections and are NOT - # reliably present in the artifacts collection. - # - # artifacts is still swept for confluence/jira/slack_thread types - # (embed_artifact() is the write path for those), plus any CRM - # embeddings that landed there via crm_system._embed(). - # - # Exclusions: - # - jira_comment: folded into parent JIRA body above - # - persona_skill: internal planner state, not a corpus artifact - # - slack_messages individual rows: the corpus unit is the thread - # (from SimEvent slack_thread key), not individual messages - # - insider threat artifacts (exfil_/hoarding_/snooping_/dlp_ prefixes) existing_ids = {row["doc_id"] for row in rows} @@ -1018,7 +989,6 @@ def _make_orphan_row( "is_external": is_external, } - # ── artifacts (confluence, jira, slack_thread, CRM embeds) ─────── _ARTIFACT_TYPE_MAP = { "confluence": "confluence", "slack_thread": "slack", @@ -1049,7 +1019,6 @@ def _make_orphan_row( art_id.startswith(p) for p in ("exfil_", "hoarding_", "snooping_", "dlp_") ): - logger.debug(f" skipping insider threat artifact: {art_id}") continue meta = artifact.get("metadata", {}) author = artifact.get("author") or meta.get("author", "") @@ -1081,10 +1050,8 @@ def _make_orphan_row( is_external=art_type in ("zd_ticket", "sf_opportunity"), ) ) - logger.debug(f" orphan artifact added: {art_id} ({art_type})") existing_ids.add(art_id) - # ── pull_requests ───────────────────────────────────────────────── for pr in self._mem._db["pull_requests"].find( {}, { @@ -1116,10 +1083,8 @@ def _make_orphan_row( artifact_type="pr", ) ) - logger.debug(f" orphan PR added: {pid}") existing_ids.add(pid) - # ── emails ──────────────────────────────────────────────────────── for email in self._mem._db["emails"].find( {}, { @@ -1154,10 +1119,8 @@ def _make_orphan_row( is_external=True, ) ) - logger.debug(f" orphan email added: {eid}") existing_ids.add(eid) - # ── zd_tickets ──────────────────────────────────────────────────── for ticket in self._mem._db["zd_tickets"].find( {}, { @@ -1192,10 +1155,8 @@ def _make_orphan_row( is_incident=ticket.get("priority") == "Urgent", ) ) - logger.debug(f" orphan ZD ticket added: {tid}") existing_ids.add(tid) - # ── sf_opps ─────────────────────────────────────────────────────── for opp in self._mem._db["sf_opps"].find( {}, { @@ -1231,10 +1192,8 @@ def _make_orphan_row( is_external=True, ) ) - logger.debug(f" orphan SF opp added: {oid}") existing_ids.add(oid) - # ── sf_accounts ─────────────────────────────────────────────────── for acc in self._mem._db["sf_accounts"].find( {}, { @@ -1264,13 +1223,8 @@ def _make_orphan_row( is_external=True, ) ) - logger.debug(f" orphan SF account added: {aid}") existing_ids.add(aid) - # ── slack_messages (thread-level grouping) ──────────────────────── - # Individual slack_messages rows are per-message. For the corpus we - # group by thread_id and concatenate message texts into one document, - # which matches how SimEvents reference slack content (by thread). thread_buckets: Dict[str, dict] = {} for msg in self._mem._db["slack_messages"].find( {}, @@ -1327,7 +1281,6 @@ def _make_orphan_row( artifact_type="slack_thread", ) ) - logger.debug(f" orphan slack thread added: {tid}") existing_ids.add(tid) except Exception as exc: @@ -1336,22 +1289,167 @@ def _make_orphan_row( # ───────────────────────────────────────────────────────────────────────────── -# BASELINE RUNNER +# EVAL INDEX SERIALISERS # ───────────────────────────────────────────────────────────────────────────── -# Question types that rely on evidence retrieval (vs. boolean reasoning) -_RETRIEVAL_TYPES = { - "RETRIEVAL", - "CAUSAL", - "ROUTING", - "GAP_DETECTION", - "TEMPORAL", - "ESCALATION", - "KNOWLEDGE_GAP", - "POSTMORTEM", - "STANDUP", - "CUSTOMER_ESC", -} + +def _causal_links_to_rows(links: List[dict]) -> List[dict]: + """ + Flatten causal_link_index.json into Parquet-ready rows. + Each CausalLink dict maps directly — sets become JSON strings. + """ + rows = [] + for lnk in links: + rows.append( + { + "link_type": lnk.get("link_type", ""), + "cause_event_id": lnk.get("cause_event_id", ""), + "cause_event_type": lnk.get("cause_event_type", ""), + "effect_event_id": lnk.get("effect_event_id", ""), + "effect_event_type": lnk.get("effect_event_type", ""), + "actors": json.dumps(lnk.get("actors", []), default=str), + "day": int(lnk.get("day", 0)), + "link_field": lnk.get("link_field", ""), + "link_value": str(lnk.get("link_value", "")), + "subsystems_involved": json.dumps( + sorted(lnk.get("subsystems_involved", [])), default=str + ), + "counterfactual_premise": lnk.get("counterfactual_premise", ""), + "counterfactual_outcome": lnk.get("counterfactual_outcome", ""), + "outcome_changed": bool(lnk.get("outcome_changed", True)), + } + ) + return rows + + +def _actor_visibility_to_rows(visibility_map: dict) -> List[dict]: + """ + Flatten actor_visibility.json (actor → [cone, ...]) into one row per + (actor, day) snapshot. Heavy set/dict fields are JSON-serialised. + """ + rows = [] + for actor, cones in visibility_map.items(): + for cone in cones: + vis = cone.get("visible_artifacts", {}) + rows.append( + { + "actor": cone.get("actor", actor), + "role": cone.get("role", ""), + "as_of_time": cone.get("as_of_time", ""), + "as_of_day": int(cone.get("as_of_day", 0)), + "subsystem_access": json.dumps( + sorted(cone.get("subsystem_access", [])), default=str + ), + # All visible artifact IDs, flattened across subsystems + "all_visible_artifacts": json.dumps( + sorted( + {aid for ids in vis.values() for aid in ids} + ), + default=str, + ), + # Per-subsystem breakdown kept for fine-grained analysis + "visible_artifacts_by_subsystem": json.dumps( + {k: sorted(v) for k, v in vis.items()}, default=str + ), + "directly_involved": json.dumps( + sorted(cone.get("directly_involved", [])), default=str + ), + "broadcast_visible": json.dumps( + sorted(cone.get("broadcast_visible", [])), default=str + ), + } + ) + return rows + + +def _absence_catalog_to_rows(records: List[dict]) -> List[dict]: + """Flatten absence_catalog.json into Parquet-ready rows.""" + rows = [] + for rec in records: + rows.append( + { + "trigger_event_id": rec.get("trigger_event_id", ""), + "trigger_event_type": rec.get("trigger_event_type", ""), + "expected_response_type": rec.get("expected_response_type", ""), + "trigger_day": int(rec.get("trigger_day", 0)), + "trigger_actors": json.dumps( + rec.get("trigger_actors", []), default=str + ), + "trigger_artifact_ids": json.dumps( + rec.get("trigger_artifact_ids", {}), default=str + ), + "link_field": rec.get("link_field", ""), + "link_value": str(rec.get("link_value", "")), + "subsystem": rec.get("subsystem", ""), + "expected_search_space": json.dumps( + rec.get("expected_search_space", []), default=str + ), + } + ) + return rows + + + +def _questions_to_rows(questions: List[dict]) -> List[dict]: + """ + Convert the v2 eval questions list into flat Parquet rows. + + Evidence chain derivation: + COUNTERFACTUAL — union of cause + effect artifact IDs from + ground_truth.evidence_chain_artifacts + PERSPECTIVE — ground_truth.evidence_artifacts + SILENCE — empty; the correct answer is absence, so retrieval + recall is not applicable + """ + rows = [] + for q in questions: + qtype = q.get("question_type", "") + gt = q.get("ground_truth", {}) + + evidence: List[str] = [] + if qtype == "COUNTERFACTUAL": + chain = gt.get("evidence_chain_artifacts", {}) + evidence = list(set(chain.get("cause", []) + chain.get("effect", []))) + elif qtype == "PERSPECTIVE": + evidence = gt.get("evidence_artifacts", []) + + + rows.append( + { + # ── Core fields (all types) ─────────────────────────────────── + "question_id": q.get("question_id", ""), + "question_type": qtype, + "question_text": q.get("question_text", ""), + "ground_truth": json.dumps(gt, default=str), + "evidence_chain": json.dumps(evidence, default=str), + "difficulty": q.get("difficulty", ""), + "requires_reasoning": bool(q.get("requires_reasoning", False)), + # ── PERSPECTIVE-specific fields ─────────────────────────────── + "actor": q.get("actor", ""), + "actor_role": q.get("actor_role", ""), + "as_of_day": int(q.get("as_of_day", 0)), + "subsystem_access": json.dumps( + q.get("subsystem_access", []), default=str + ), + "blocked_subsystems": json.dumps( + q.get("blocked_subsystems", []), default=str + ), + "actor_visible_artifacts": json.dumps( + q.get("actor_visible_artifacts", []), default=str + ), + # ── COUNTERFACTUAL-specific fields ──────────────────────────── + "link_type": q.get("link_type", ""), + "causal_day": int(q.get("day", 0)), + # ── SILENCE-specific fields ─────────────────────────────────── + "expected_search_space": json.dumps( + q.get("expected_search_space", []), default=str + ), + "trigger_event_type": q.get("trigger_event_type", ""), + "expected_response_type": q.get("expected_response_type", ""), + } + ) + return rows + def _tokenize(text: str) -> List[str]: @@ -1373,10 +1471,20 @@ def _recall_at_k(ranked_ids: List[str], relevant_ids: List[str], k: int = 10) -> return hits / len(relevant_ids) -class BaselineRunner: +class UngatedCeilingBaseline: """ - Runs BM25 and (optionally) dense retrieval baselines against the - eval questions and returns per-question and aggregate metrics. + Tier 1 baseline: BM25 and dense retrieval with ALL gates removed. + + No visibility cones, no temporal horizons, no subsystem constraints — + the retriever has "god-mode" access to the full corpus. MRR@10 and + Recall@10 represent the information ceiling, not agent performance. + + SILENCE questions are excluded because the correct answer is absence; + retrieval recall is not applicable. + + The delta between these ceiling scores and a gated agent's combined_score + is the "Epistemic Tax" — the difficulty cost of respecting organisational + silos and actor knowledge horizons. """ def __init__(self, corpus: List[dict], questions: List[dict], mem=None): @@ -1386,30 +1494,23 @@ def __init__(self, corpus: List[dict], questions: List[dict], mem=None): self._doc_ids = [row["doc_id"] for row in corpus] self._bodies = [row.get("body") or row.get("content") or "" for row in corpus] - # BM25 index if _BM25_AVAILABLE: tokenised = [_tokenize(b) for b in self._bodies] self._bm25 = BM25Okapi(tokenised) else: self._bm25 = None - if _DENSE_AVAILABLE: - logger.info(" Embedding corpus for dense baseline...") + if _DENSE_AVAILABLE and mem is not None: + logger.info(" Embedding corpus for dense ceiling baseline...") embeddings = [] - for i, body in enumerate(self._bodies): text_to_embed = ( body.strip() if body and body.strip() else "empty document" ) - vec = self._mem._embed( - text_to_embed, - input_type="search_document", - ) + vec = self._mem._embed(text_to_embed, input_type="search_document") embeddings.append(vec) - if (i + 1) % 500 == 0: - logger.info(f" embedded {i + 1}/{len(self._bodies)} docs...") - + logger.info(f" embedded {i + 1}/{len(self._bodies)} docs...") mat = np.array(embeddings, dtype=np.float32) norms = np.linalg.norm(mat, axis=1, keepdims=True) self._dense_matrix = mat / np.where(norms == 0, 1, norms) @@ -1424,17 +1525,35 @@ def run_bm25(self) -> Tuple[List[dict], Dict[str, Any]]: return self._run_retrieval(use_dense=False) def run_dense(self) -> Tuple[List[dict], Dict[str, Any]]: - if self._mem is None: - return [], {"error": "Memory unavailable — dense baseline requires MongoDB"} + if self._mem is None or self._dense_matrix is None: + return [], {"error": "Memory unavailable — dense ceiling requires MongoDB"} return self._run_retrieval(use_dense=True) + # ── PRIVATE ─────────────────────────────────────────────────────────────── + + def _evidence_for_question(self, q: dict) -> List[str]: + """ + Flat list of corpus doc_ids that constitute the correct answer. + SILENCE returns empty — absence has no retrievable target. + """ + qtype = q.get("question_type", "") + gt = q.get("ground_truth", {}) + + if qtype == "COUNTERFACTUAL": + chain = gt.get("evidence_chain_artifacts", {}) + return list(set(chain.get("cause", []) + chain.get("effect", []))) + + if qtype == "PERSPECTIVE": + return gt.get("evidence_artifacts", []) + + return [] + def _rank(self, query: str, use_dense: bool, top_k: int = 10) -> List[str]: if use_dense and self._dense_matrix is not None: q_vec = np.array( self._mem._embed(query, input_type="search_query"), dtype=np.float32 ) q_vec /= max(np.linalg.norm(q_vec), 1e-9) - scores = self._dense_matrix @ q_vec indices = scores.argsort()[::-1][:top_k] return [self._doc_ids[int(i)] for i in indices] @@ -1446,21 +1565,17 @@ def _rank(self, query: str, use_dense: bool, top_k: int = 10) -> List[str]: return [] - # ── PRIVATE ─────────────────────────────────────────────────────────────── - def _run_retrieval(self, use_dense: bool) -> Tuple[List[dict], Dict[str, Any]]: per_question: List[dict] = [] - by_type: Dict[str, List[float]] = defaultdict(list) + by_type: Dict[str, List[Tuple[float, float]]] = defaultdict(list) for q in self._questions: qtype = q.get("question_type", "") - evidence = q.get("evidence_chain", []) + evidence = self._evidence_for_question(q) if not evidence: - continue # no reference artifacts — skip - - q_text = q.get("question_text", "") - ranked_ids = self._rank(q_text, use_dense=use_dense) + continue + ranked_ids = self._rank(q.get("question_text", ""), use_dense=use_dense) mrr = _mrr_at_k(ranked_ids, evidence, k=10) recall = _recall_at_k(ranked_ids, evidence, k=10) @@ -1474,10 +1589,8 @@ def _run_retrieval(self, use_dense: bool) -> Tuple[List[dict], Dict[str, Any]]: "top10": ranked_ids[:10], } ) - by_type[qtype].append((mrr, recall)) - # Aggregate def _mean(vals): return round(sum(vals) / len(vals), 4) if vals else 0.0 @@ -1501,24 +1614,256 @@ def _mean(vals): return per_question, aggregate +class StaticReasoningMetrics: + """ + Tier 2 baseline: reasoning difficulty metrics derived from corpus + + question metadata alone. No LLM calls, no agent execution required. + + These metrics answer "why is each track hard?" before any agent runs, + exposing the epistemic structure that makes naive retrieval insufficient. + + PERSPECTIVE → horizon_contamination_rate + Fraction of ungated BM25 top-20 that falls outside the actor's + visibility cone. High value = epistemic discipline is load-bearing; + retrieval alone surfaces mostly forbidden documents. + + COUNTERFACTUAL → causal_chain_traceable + Whether cause AND effect artifacts both appear in ungated top-10. + False = a single retrieval pass cannot close the causal chain; + multi-hop reasoning is required. + + SILENCE → search_space_bm25_coverage + Fraction of expected_search_space locations surfaced by BM25 on + the question text. Low value = the agent must enumerate absence-check + locations deliberately; naive search will miss them. + """ + + def __init__( + self, + questions: List[dict], + bm25, # BM25Okapi instance reused from UngatedCeilingBaseline + doc_ids: List[str], + ): + self._questions = questions + self._bm25 = bm25 + self._doc_ids = doc_ids + + def compute(self) -> dict: + per_question: List[dict] = [] + by_type: Dict[str, List[dict]] = defaultdict(list) + + dispatch = { + "PERSPECTIVE": self._perspective_metrics, + "COUNTERFACTUAL": self._counterfactual_metrics, + "SILENCE": self._silence_metrics, + } + + for q in self._questions: + qtype = q.get("question_type", "") + fn = dispatch.get(qtype) + if fn is None: + continue + row = { + "question_id": q.get("question_id"), + "question_type": qtype, + **fn(q), + } + per_question.append(row) + by_type[qtype].append(row) + + return { + "per_question": per_question, + "aggregate": self._aggregate(by_type), + } + + # ── Track-specific metric computations ─────────────────────────────────── + + def _perspective_metrics(self, q: dict) -> dict: + """ + Horizon contamination: what fraction of ungated top-20 results would + an actor NOT be permitted to see? High contamination means a naive + retriever is actively counter-productive for PERSPECTIVE questions. + """ + visible = set(q.get("actor_visible_artifacts", [])) + ranked = self._rank_bm25(q["question_text"], k=20) + if not ranked or not visible: + return { + "horizon_contamination_rate": None, + "first_in_cone_rank": None, + "in_cone_count_top20": None, + } + + out_of_cone = [r for r in ranked if r not in visible] + first_in_cone = next( + (i + 1 for i, r in enumerate(ranked) if r in visible), None + ) + return { + "horizon_contamination_rate": round(len(out_of_cone) / len(ranked), 4), + "first_in_cone_rank": first_in_cone, + "in_cone_count_top20": len(ranked) - len(out_of_cone), + } + + def _counterfactual_metrics(self, q: dict) -> dict: + """ + Causal chain traceability: do both cause and effect artifacts appear + in ungated top-10? If not, the agent must do multi-hop retrieval. + """ + chain = q.get("ground_truth", {}).get("evidence_chain_artifacts", {}) + cause_ids = set(chain.get("cause", [])) + effect_ids = set(chain.get("effect", [])) + if not cause_ids and not effect_ids: + return {"causal_chain_traceable": None} + + ranked = self._rank_bm25(q["question_text"], k=10) + ranked_set = set(ranked) + cause_found = bool(cause_ids & ranked_set) + effect_found = bool(effect_ids & ranked_set) + + cause_rank = next( + (i + 1 for i, r in enumerate(ranked) if r in cause_ids), None + ) + effect_rank = next( + (i + 1 for i, r in enumerate(ranked) if r in effect_ids), None + ) + + return { + "cause_found_top10": cause_found, + "effect_found_top10": effect_found, + "causal_chain_traceable": cause_found and effect_found, + "cause_rank": cause_rank, + "effect_rank": effect_rank, + } + + def _silence_metrics(self, q: dict) -> dict: + """ + Search space BM25 coverage: what fraction of the required absence-check + locations does a naive BM25 search surface? Low coverage means the agent + must enumerate expected_search_space explicitly rather than relying on + retrieval to guide it to the right places to look. + """ + expected = q.get("expected_search_space", []) + if not expected: + return {"search_space_bm25_coverage": 1.0, "uncovered_locations": []} + + ranked = self._rank_bm25(q["question_text"], k=20) + + def _norm(s: str) -> str: + # Normalise path-style entries to their terminal component: + # "confluence/postmortems/IT-108" → "it-108" + return s.strip("/").split("/")[-1].lower() + + norm_expected = {_norm(e): e for e in expected} + norm_ranked = {_norm(r) for r in ranked} + ranked_lower = [r.lower() for r in ranked] + + covered = { + original + for norm_term, original in norm_expected.items() + if norm_term in norm_ranked + or any(norm_term in r for r in ranked_lower) + } + + return { + "search_space_bm25_coverage": round(len(covered) / len(expected), 4), + "uncovered_locations": sorted(set(expected) - covered), + } + + # ── Shared utilities ────────────────────────────────────────────────────── + + def _rank_bm25(self, query: str, k: int) -> List[str]: + if self._bm25 is None: + return [] + scores = self._bm25.get_scores(_tokenize(query)) + indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) + return [self._doc_ids[i] for i in indices[:k]] + + def _aggregate(self, by_type: Dict[str, List[dict]]) -> dict: + def _mean(vals: list) -> float: + filtered = [v for v in vals if v is not None] + return round(sum(filtered) / len(filtered), 4) if filtered else 0.0 + + agg: dict = {} + + if rows := by_type.get("PERSPECTIVE", []): + agg["PERSPECTIVE"] = { + "n": len(rows), + "avg_horizon_contamination_rate": _mean( + [r.get("horizon_contamination_rate") for r in rows] + ), + "avg_first_in_cone_rank": _mean( + [r.get("first_in_cone_rank") for r in rows] + ), + "interpretation": ( + "High contamination = retrieval surfaces many out-of-cone docs; " + "epistemic discipline is load-bearing, not optional." + ), + } + + if rows := by_type.get("COUNTERFACTUAL", []): + pct_traceable = _mean( + [1.0 if r.get("causal_chain_traceable") else 0.0 for r in rows] + ) + agg["COUNTERFACTUAL"] = { + "n": len(rows), + "pct_causal_chain_traceable_top10": pct_traceable, + "pct_requires_multi_hop": round(1.0 - pct_traceable, 4), + "interpretation": ( + "Low traceability = causal link cannot be closed by a single " + "retrieval pass; agent must traverse cause → effect explicitly." + ), + } + + if rows := by_type.get("SILENCE", []): + agg["SILENCE"] = { + "n": len(rows), + "avg_search_space_bm25_coverage": _mean( + [r.get("search_space_bm25_coverage") for r in rows] + ), + "pct_fully_covered": _mean( + [ + 1.0 + if r.get("search_space_bm25_coverage", 0) >= 1.0 + else 0.0 + for r in rows + ] + ), + "interpretation": ( + "Low coverage = BM25 misses required absence-check locations; " + "agent must enumerate expected_search_space explicitly." + ), + } + + return agg + + # ───────────────────────────────────────────────────────────────────────────── # DATASET CARD WRITER # ───────────────────────────────────────────────────────────────────────────── class DatasetCardWriter: - """Produces the HuggingFace README.md dataset card.""" + """Produces the HuggingFace README.md dataset card for the v2 eval.""" def write( self, out_path: Path, corpus: List[dict], questions: List[dict], - threads: List[dict], + causal_links: List[dict], + actor_visibility: dict, + absence_catalog: List[dict], baseline_summary: dict, cfg: dict, ) -> None: - card = self._render(corpus, questions, threads, baseline_summary, cfg) + card = self._render( + corpus, + questions, + causal_links, + actor_visibility, + absence_catalog, + baseline_summary, + cfg, + ) out_path.write_text(card, encoding="utf-8") logger.info(f" → {out_path}") @@ -1528,7 +1873,9 @@ def _render( self, corpus: List[dict], questions: List[dict], - threads: List[dict], + causal_links: List[dict], + actor_visibility: dict, + absence_catalog: List[dict], baseline_summary: dict, cfg: dict, ) -> str: @@ -1552,14 +1899,22 @@ def _render( by_qtype[q.get("question_type", "?")] += 1 by_diff[q.get("difficulty", "?")] += 1 - # Thread breakdown - by_chain: Dict[str, int] = defaultdict(int) - for t in threads: - by_chain[t.get("chain_type", "?")] += 1 - - # Baseline tables - bm25_section = self._baseline_table(baseline_summary.get("bm25", {})) - dense_section = self._baseline_table(baseline_summary.get("dense", {})) + # Causal link breakdown + by_link: Dict[str, int] = defaultdict(int) + for lnk in causal_links: + by_link[lnk.get("link_type", "?")] += 1 + + # Eval index counts + n_actors = len(actor_visibility) + n_cone_snapshots = sum(len(v) for v in actor_visibility.values()) + + # Baseline tables — two-tier system + ceiling = baseline_summary.get("ungated_ceiling", {}) + bm25_section = self._ungated_ceiling_table(ceiling.get("bm25", {})) + dense_section = self._ungated_ceiling_table(ceiling.get("dense", {})) + reasoning_section = self._reasoning_metrics_table( + baseline_summary.get("static_reasoning_metrics", {}) + ) return textwrap.dedent(f"""\ --- @@ -1584,24 +1939,34 @@ def _render( - orgforge - causal-reasoning - temporal-reasoning - pretty_name: "OrgForge Enterprise RAG Benchmark" + - epistemic-reasoning + - agentic-eval + pretty_name: "OrgForge Enterprise Agentic RAG Benchmark" size_categories: - 1K A synthetic but causally-grounded benchmark for evaluating RAG systems - > against realistic enterprise knowledge bases. + > A synthetic but causally-grounded benchmark for evaluating agentic RAG + > systems against realistic enterprise knowledge bases — with explicit + > trajectory scoring for epistemic discipline, causal reasoning, and + > absence verification. ## Dataset Summary This dataset was produced by **OrgForge**, an event-driven organisation simulator that generates weeks of realistic enterprise activity — JIRA - tickets, Confluence pages, Slack threads, emails, and PRs — in a - controlled, reproducible way. All ground-truth answers are derived - deterministically from the simulation's event log; no LLM invented any - answer. + tickets, Confluence pages, Slack threads, zoom transcripts, emails, PRs, Zendesk tickets, + and Salesforce records — in a controlled, reproducible way. + + All ground-truth answers are derived **deterministically** from the + simulation's event log via three purpose-built indexes: + - **Actor visibility cones** — what each actor could have known at each moment + - **Causal link index** — explicit cause→effect relationships encoded by the sim + - **Absence catalog** — expected-but-absent artifacts confirmed by the state machine + + No LLM invented any answer. LLMs only wrote question prose. | Property | Value | |---|---| @@ -1612,23 +1977,25 @@ def _render( | Org size (engineers + staff) | ~{org_size} | | Total corpus documents | {len(corpus):,} | | Total eval questions | {len(questions):,} | - | Causal threads | {len(threads):,} | + | Causal links indexed | {len(causal_links):,} | + | Actors with visibility cones | {n_actors} | + | Visibility cone snapshots | {n_cone_snapshots:,} | + | Absence records | {len(absence_catalog):,} | ## Corpus - Each document in the corpus represents a real artifact produced by the - simulation (ticket, page, thread, email, PR). Documents are stored in - `corpus/corpus-00000.parquet`. + Each document represents a real artifact produced by the simulation. + Stored in `corpus/corpus-00000.parquet`. | Artifact type | Count | |---|---| {self._table_rows(by_type)} - ### Schema + ### Corpus Schema | Column | Type | Description | |---|---|---| - | `doc_id` | str | Unique artifact ID (e.g. `ORG-42`, `CONF-ENG-007`) | + | `doc_id` | str | Unique artifact ID (e.g. `IT-042`, `CONF-ENG-007`) | | `doc_type` | str | `jira`, `confluence`, `slack`, `email`, `pr`, `zd_ticket`, `sf_opp`, `sf_account`, `sim_event` | | `title` | str | Human-readable title or subject | | `body` | str | Full retrievable text | @@ -1640,7 +2007,7 @@ def _render( | `artifact_ids` | str | JSON dict of cross-references | | `dept` | str | Owning department | | `is_incident` | bool | True if part of an incident thread | - | `is_external` | bool | True for inbound external emails | + | `is_external` | bool | True for inbound external content | ## Eval Questions @@ -1654,82 +2021,241 @@ def _render( |---|---| {self._table_rows(by_diff)} - ### Question Types + ### Question Tracks - | Type | Description | Requires multi-hop? | + | Track | Description | Score weights (answer / trajectory) | |---|---|---| - | `RETRIEVAL` | Which artifact first documented a specific fact? | No | - | `CAUSAL` | What artifact or action directly followed event X? | Yes (2-hop) | - | `TEMPORAL` | Did person P have access to domain D before incident I? | Yes (cross-thread) | - | `GAP_DETECTION` | Was email E ever actioned? | Yes (absence-of-evidence) | - | `ROUTING` | Who was the first internal person to see an inbound email? | No | - | `ESCALATION` | Who was involved in the escalation chain for incident X? | No | - | `KNOWLEDGE_GAP` | What domain was undocumented when incident X fired? | No | - | `POSTMORTEM` | Which Confluence doc captured the postmortem for incident X? | Yes (2-hop) | - | `STANDUP` | What did person X report at standup on Day N? | No | - | `CUSTOMER_ESC` | Who handled the escalation from customer X and what action was taken? | Yes (2-hop) | - | `ZD_RESOLUTION` | How quickly was a specific Zendesk ticket resolved? | Yes (cross-thread) | - | `DATADOG_ALERT` | What server metric triggered the alert on Day N? | No | - | `INVOICE_SLA` | What SLA credits were applied to a specific customer? | Yes | - | `NPS_SCORE` | What was the reasoning behind the customer's NPS score? | Yes | - | `PR_REVIEW` | What was the verdict of the code review? | No | + | `PERSPECTIVE` | Could actor X have known about event Y as of Day N, given their subsystem access? | 0.40 / 0.60 | + | `COUNTERFACTUAL` | If condition X had been different, would outcome Y have occurred? | 0.50 / 0.50 | + | `SILENCE` | Was artifact X actually created in response to trigger Y? (correct answer: no) | 0.30 / 0.70 | + + #### PERSPECTIVE + Scored primarily on **epistemic discipline**: did the agent stay within the + actor's visibility cone and access only permitted subsystems? Trajectory + weight (0.60) exceeds answer weight (0.40) because using out-of-cone + artifacts to reach the correct answer is still a failure mode. + + #### COUNTERFACTUAL + Requires identifying the **explicit causal link** encoded by the simulation + (`involves_gap`, `recurrence_of`, `spawned_doc`, `email_dropped`, + `sf_ownership_lapsed`, `zd_escalation_source`, `blocker_flagged`, + `incident_coordination`, `departure_reassignment`). No inference — the + link must be traceable to real artifacts. + + #### SILENCE + Tests **absence-of-evidence reasoning**. Trajectory weight is highest + (0.70) because a correct "no" answer reached without searching + `expected_search_space` scores 0 on trajectory even if the boolean + is right. The agent must demonstrate it checked the right places. + + > **Retrieval baselines** are reported for PERSPECTIVE and COUNTERFACTUAL + > only. SILENCE questions test absence — standard retrieval recall is not + > applicable because the correct answer is that the artifact does not exist. ### Question Schema | Column | Type | Description | |---|---|---| | `question_id` | str | Unique question identifier | - | `question_type` | str | One of the five types above | + | `question_type` | str | `PERSPECTIVE`, `COUNTERFACTUAL`, or `SILENCE` | | `question_text` | str | Natural-language question | | `ground_truth` | str | JSON-serialised answer dict | - | `evidence_chain` | str | JSON list of artifact IDs that support the answer | - | `difficulty` | str | `easy`, `medium`, `hard` | - | `requires_reasoning` | bool | Multi-hop traversal required? | - | `chain_id` | str | Causal thread this question derives from | + | `evidence_chain` | str | JSON list of artifact IDs (empty for SILENCE) | + | `difficulty` | str | `medium` or `hard` | + | `requires_reasoning` | bool | Always True — all tracks require multi-step reasoning | + | `actor` | str | PERSPECTIVE: actor whose knowledge horizon is tested | + | `actor_role` | str | PERSPECTIVE: actor's role slug | + | `as_of_day` | int | PERSPECTIVE: knowledge horizon day | + | `subsystem_access` | str | PERSPECTIVE: JSON list of accessible subsystems | + | `blocked_subsystems` | str | PERSPECTIVE: JSON list of blocked subsystems | + | `actor_visible_artifacts` | str | PERSPECTIVE: JSON list of all visible artifact IDs | + | `link_type` | str | COUNTERFACTUAL: causal link type | + | `causal_day` | int | COUNTERFACTUAL: day the causal link was established | + | `expected_search_space` | str | SILENCE: JSON list of artifact paths agent must check | + | `trigger_event_type` | str | SILENCE: event type that should have triggered the response | + | `expected_response_type` | str | SILENCE: artifact/event type that was never created | + + ## Eval Indexes + + Stored in `eval_indexes/`. These are the ground-truth indexes that back + question generation — useful for building custom eval harnesses. + + ### causal_link_index.parquet + + One row per explicit causal link found in the simulation. - ## Causal Threads + | Column | Type | Description | + |---|---|---| + | `link_type` | str | One of the causal link types above | + | `cause_event_id` | str | Synthetic event ID of the cause | + | `cause_event_type` | str | Event type of the cause | + | `effect_event_id` | str | Synthetic event ID of the effect | + | `effect_event_type` | str | Event type of the effect | + | `actors` | str | JSON list of involved actor names | + | `day` | int | Simulation day the link was established | + | `counterfactual_premise` | str | Natural-language "if X had been different" | + | `counterfactual_outcome` | str | Natural-language "then Y would have..." | + | `outcome_changed` | bool | Always True — removing cause changes outcome | + + | Link type | Count | + |---|---| + {self._table_rows(by_link)} - Causal threads are in `threads/threads-00000.parquet`. - Each thread is a directed artifact graph with actor knowledge annotations. + ### actor_visibility.parquet - | Chain type | Count | - |---|---| - {self._table_rows(by_chain)} + One row per (actor, day) snapshot. - ## Baselines + | Column | Type | Description | + |---|---|---| + | `actor` | str | Actor name | + | `role` | str | Role slug | + | `as_of_day` | int | Snapshot day | + | `subsystem_access` | str | JSON list of accessible subsystems | + | `all_visible_artifacts` | str | JSON list of all artifact IDs visible to this actor on this day | + | `visible_artifacts_by_subsystem` | str | JSON dict (subsystem → artifact IDs) | + | `directly_involved` | str | JSON list of artifacts where actor was in event.actors | + | `broadcast_visible` | str | JSON list of artifacts visible via broadcast channel | - All baselines evaluate **evidence retrieval** — whether the correct - artifacts from `evidence_chain` are surfaced in the top-10 results — - rather than final answer accuracy. MRR@10 and Recall@10 are reported. + ### absence_catalog.parquet - TEMPORAL and GAP_DETECTION questions test absence-of-evidence reasoning; - retrieval metrics for those types measure whether the relevant artifacts - are found, not whether the boolean conclusion is correct. + One row per expected-but-absent artifact pair. - ### BM25 (Okapi BM25 via rank-bm25) + | Column | Type | Description | + |---|---|---| + | `trigger_event_id` | str | Event that should have triggered a response | + | `trigger_event_type` | str | Type of trigger event | + | `expected_response_type` | str | Type of artifact that was never created | + | `trigger_day` | int | Day the trigger event fired | + | `expected_search_space` | str | JSON list of artifact paths agent must check | + + ## Baselines and Reasoning Difficulty + + OrgForge uses a **two-tier baseline** system, computed entirely from + corpus metadata — no agent execution required. + + - **Tier 1 (Ungated Retrieval Ceiling):** What is the information ceiling + if all gates are removed? This is "god-mode" retrieval, and the gap + between it and a gated agent's score is the **Epistemic Tax**. + - **Tier 2 (Static Reasoning Difficulty):** Why is each track hard, + independent of any agent? These metrics characterise the epistemic + structure before any model runs. + + Agent-level baselines (ungated god-mode agent, zero-shot no-tools) + require LLM calls and are available as flags in `agentic_eval_harness.py`: + `--ungated` and `--zero-shot`. + + --- + + ### Tier 1 — Ungated Retrieval Ceiling + + BM25 and dense retrieval with **no gates**: no visibility cones, no + temporal horizons, no subsystem constraints ("god-mode" corpus access). + + The **Epistemic Tax** for a track is: + + ``` + epistemic_tax = ceiling_mrr@10 − gated_agent_combined_score + ``` + + A high tax on `PERSPECTIVE` means the question set heavily penalises + using information the actor was never supposed to have. + + #### BM25 (Okapi BM25 via rank-bm25) {bm25_section} - ### Dense Retrieval (sentence-transformers `{_DENSE_MODEL_NAME}`) + #### Dense Retrieval (`{_DENSE_MODEL_NAME}`) {dense_section} - ## Scoring + --- + + ### Tier 2 — Static Reasoning Difficulty Metrics + + Computed from corpus metadata and question ground-truth alone — no agents, + no LLM calls required. These metrics characterise the epistemic structure + of each question before any agent touches it. + + {reasoning_section} + + --- + + ### How to beat these baselines + + | Track | To beat the ceiling... | + |---|---| + | `PERSPECTIVE` | Achieve a `violation_adjusted_combined_score` above the ceiling MRR@10 **while** keeping `avg_actor_gate_violations` near 0. High score + high violations = the agent is cheating. | + | `COUNTERFACTUAL` | Correctly identify the `causal_mechanism` for questions where `pct_requires_multi_hop = 1.0` — these cannot be answered by retrieval alone. | + | `SILENCE` | Cover `expected_search_space` exhaustively before concluding. `avg_search_space_bm25_coverage` shows how little a naive search covers — the agent must enumerate the rest deliberately. | - Use `scorer.py` (included in this repository) to evaluate agent answers - against the ground truth. `scorer.py` implements per-type comparison - logic with partial credit and returns a `ScorerResult` per question. + --- - ```python - from scorer import OrgForgeScorer - scorer = OrgForgeScorer() - result = scorer.score(question, agent_answer) - report = scorer.report(scorer.score_all(questions, answers)) + ## Agentic Evaluation + + Use `agentic_eval_harness.py` to run a gated agent against the full + question set. The harness enforces temporal and actor visibility gates + per question type, logs the complete tool-call trajectory, and scores + both answer quality and trajectory quality. + + ```bash + # Standard gated evaluation + python agentic_eval_harness.py \\ + --questions export/eval/eval_questions.json \\ + --out export/eval/agentic_results.json \\ + --model claude-sonnet-4-6 \\ + --max-steps 15 + + # Ungated god-mode agent — establishes the Epistemic Tax denominator + python agentic_eval_harness.py \\ + --ungated \\ + --out export/eval/ungated_results.json + + # Zero-shot — no tools, no corpus — establishes the hallucination floor + python agentic_eval_harness.py \\ + --zero-shot \\ + --out export/eval/zero_shot_results.json ``` - Scores are in [0.0, 1.0]. A score ≥ 0.9 is considered correct. - Partial credit (0.2–0.9) is awarded when the agent retrieves the right - artifacts but draws an incorrect conclusion. + ## Leaderboard + + Submissions are ranked by `violation_adjusted_combined_score` on the + **PERSPECTIVE** track. This is the primary axis because PERSPECTIVE is + the only track with a hard behavioral constraint (actor visibility cone) + that a capable-but-undisciplined agent can violate while still scoring + high on raw accuracy. + + ### Ranking Formula + + ``` + violation_rate = total_actor_gate_violations / total_tool_calls + compliance_factor = max(0, 1 − violation_rate) ** 2 + violation_adjusted_score = combined_score × compliance_factor + ``` + + The quadratic exponent means violations compound non-linearly: + + | Violation rate | Compliance factor | Effective score discount | + |---|---|---| + | 0% (fully compliant) | 1.00 | None | + | 10% | 0.81 | 19% | + | 25% | 0.56 | 44% | + | 50% | 0.25 | 75% | + | 75% | 0.06 | 94% | + + ### Compliance Tiers + + | Tier | Violation rate | Meaning | + |---|---|---| + | `compliant` | < 5% | Agent demonstrates genuine epistemic discipline | + | `borderline` | 5–20% | Agent occasionally accesses out-of-cone information | + | `non_compliant` | > 20% | Agent is effectively operating in god-mode on PERSPECTIVE | + + > `combined_score` is still reported for reference but **must not** be + > used as the primary ranking key. A model scoring 0.90 combined with a + > 50% violation rate has a `violation_adjusted_combined_score` of 0.225 + > and belongs in `non_compliant` — below a model scoring 0.70 combined + > with 0% violations (`violation_adjusted_combined_score` = 0.70, tier: + > `compliant`). ## Citation @@ -1738,13 +2264,14 @@ def _render( title = {{OrgForge: A Multi-Agent Simulation Framework for Verifiable Synthetic Corporate Corpora}}, author = {{Jeffrey Flynt}}, year = {{2026}}, - note = {{Synthetic benchmark generated by the OrgForge simulator}} + note = {{Synthetic benchmark generated by the OrgForge simulator v2}} }} ``` ## License - MIT. The simulation engine that produced this dataset is independently licensed; see the OrgForge repository for details. + MIT. The simulation engine that produced this dataset is independently + licensed; see the OrgForge repository for details. """) def _table_rows(self, d: Dict[str, int]) -> str: @@ -1752,30 +2279,100 @@ def _table_rows(self, d: Dict[str, int]) -> str: f"| `{k}` | {v:,} |" for k, v in sorted(d.items(), key=lambda x: -x[1]) ) - def _baseline_table(self, summary: dict) -> str: + def _ungated_ceiling_table(self, summary: dict) -> str: + """Renders the Tier 1 ungated retrieval ceiling table.""" if "error" in summary: - return f"> ⚠️ Baseline unavailable: {summary['error']}" + return f"> ⚠️ Ceiling unavailable: {summary['error']}" if not summary: - return "> Baseline not run." + return "> Ceiling not run." model = summary.get("model", "?") overall = summary.get("overall", {}) by_type = summary.get("by_type", {}) - lines = [f"Model: `{model}`\n"] - lines.append("| Question type | MRR@10 | Recall@10 | N |") - lines.append("|---|---|---|---|") - lines.append( - f"| **Overall** | **{overall.get('mrr_at_10', '?')}** " - f"| **{overall.get('recall_at_10', '?')}** " - f"| **{overall.get('n', '?')}** |" - ) + lines = [ + f"Model: `{model}`", + "", + "| Question type | MRR@10 | Recall@10 | N |", + "|---|---|---|---|", + ( + f"| **Overall** | **{overall.get('mrr_at_10', '?')}** " + f"| **{overall.get('recall_at_10', '?')}** " + f"| **{overall.get('n', '?')}** |" + ), + ] for qtype, metrics in sorted(by_type.items()): lines.append( f"| {qtype} | {metrics.get('mrr_at_10', '?')} " f"| {metrics.get('recall_at_10', '?')} " f"| {metrics.get('n', '?')} |" ) + lines += [ + "", + "> SILENCE questions excluded — absence cannot be measured by retrieval recall.", + "> **Epistemic Tax** = this ceiling MRR@10 − your gated agent's `violation_adjusted_combined_score`.", + ] + return "\n ".join(lines) + + def _reasoning_metrics_table(self, static_metrics: dict) -> str: + """Renders the Tier 2 static reasoning difficulty table.""" + if not static_metrics: + return "> Static reasoning metrics not computed." + + lines = [ + "| Track | Metric | Value | Interpretation |", + "|---|---|---|---|", + ] + + if p := static_metrics.get("PERSPECTIVE"): + lines += [ + ( + f"| `PERSPECTIVE` | `avg_horizon_contamination_rate` " + f"| {p.get('avg_horizon_contamination_rate', '?')} " + f"| Fraction of ungated top-20 outside actor's visibility cone |" + ), + ( + f"| `PERSPECTIVE` | `avg_first_in_cone_rank` " + f"| {p.get('avg_first_in_cone_rank', '?')} " + f"| Mean rank of first permitted doc — lower is easier |" + ), + ] + + if cf := static_metrics.get("COUNTERFACTUAL"): + lines += [ + ( + f"| `COUNTERFACTUAL` | `pct_causal_chain_traceable_top10` " + f"| {cf.get('pct_causal_chain_traceable_top10', '?')} " + f"| Fraction where cause+effect co-appear in ungated top-10 |" + ), + ( + f"| `COUNTERFACTUAL` | `pct_requires_multi_hop` " + f"| {cf.get('pct_requires_multi_hop', '?')} " + f"| Fraction unreachable by a single retrieval pass |" + ), + ] + + if s := static_metrics.get("SILENCE"): + lines += [ + ( + f"| `SILENCE` | `avg_search_space_bm25_coverage` " + f"| {s.get('avg_search_space_bm25_coverage', '?')} " + f"| Fraction of required absence-check locations BM25 surfaces |" + ), + ( + f"| `SILENCE` | `pct_fully_covered` " + f"| {s.get('pct_fully_covered', '?')} " + f"| Questions where BM25 covers the entire search space |" + ), + ] + + lines += [ + "", + "> **Reading these metrics:** High contamination + low traceability + low coverage", + "> means the question set demands genuine reasoning over retrieval luck. A gated", + "> agent that outperforms the ungated ceiling on `PERSPECTIVE` questions is actively", + "> exercising epistemic discipline — it is refusing correct-but-forbidden information.", + ] return "\n ".join(lines) @@ -1803,53 +2400,10 @@ def _write_parquet(rows: List[dict], out_dir: Path, stem: str = "part-00000") -> out_path = out_dir / f"{stem}.parquet" pq.write_table(tbl, out_path, compression="snappy") logger.info( - f" → {out_path} ({len(rows):,} rows, {out_path.stat().st_size // 2560} KB)" + f" → {out_path} ({len(rows):,} rows, {out_path.stat().st_size // 1024} KB)" ) -def _questions_to_rows(questions: List[dict]) -> List[dict]: - rows = [] - for q in questions: - rows.append( - { - "question_id": q.get("question_id", ""), - "question_type": q.get("question_type", ""), - "question_text": q.get("question_text", ""), - "ground_truth": json.dumps(q.get("ground_truth", {}), default=str), - "evidence_chain": json.dumps(q.get("evidence_chain", []), default=str), - "difficulty": q.get("difficulty", ""), - "requires_reasoning": bool(q.get("requires_reasoning", False)), - "chain_id": q.get("chain_id", ""), - } - ) - return rows - - -def _threads_to_rows(threads: List[dict]) -> List[dict]: - rows = [] - for t in threads: - rows.append( - { - "chain_id": t.get("chain_id", ""), - "chain_type": t.get("chain_type", ""), - "root_artifact": t.get("root_artifact", ""), - "root_event_type": t.get("root_event_type", ""), - "day": int(t.get("day", 0)), - "date": str(t.get("date", "")), - "terminal_artifact": t.get("terminal_artifact", ""), - "complete": bool(t.get("complete", False)), - "nodes": json.dumps(t.get("nodes", []), default=str), - # type-specific extras - "high_priority": bool(t.get("high_priority", False)), - "source": str(t.get("source", "")), - "prospect": str(t.get("prospect", "")), - "confluence_id": str(t.get("confluence_id", "")), - "root_cause": str(t.get("root_cause", "")), - } - ) - return rows - - # ───────────────────────────────────────────────────────────────────────────── # MAIN # ───────────────────────────────────────────────────────────────────────────── @@ -1857,15 +2411,16 @@ def _threads_to_rows(threads: List[dict]) -> List[dict]: class HFExporter: """ - Orchestrates the full export pipeline: + Orchestrates the full v2 export pipeline: 1. Build corpus from SimEvent log + MongoDB - 2. Load causal threads and eval questions - 3. Run BM25 and dense baselines - 4. Write Parquet files + dataset card + 2. Load v2 eval data (eval_questions.json + the three eval indexes) + 3. Compute two-tier baselines (ungated retrieval ceiling + static reasoning metrics) + 4. Write Parquet files for corpus, questions, and eval indexes + 5. Write dataset card (README.md) """ def run(self) -> None: - logger.info("[bold cyan]📦 HuggingFace dataset export starting…[/bold cyan]") + logger.info("[bold cyan]📦 HuggingFace dataset export v2 starting…[/bold cyan]") # 1. Memory (optional — degrade gracefully) mem = None @@ -1882,64 +2437,132 @@ def run(self) -> None: # 2. Corpus corpus_builder = CorpusBuilder(mem) corpus = corpus_builder.build() - - # If Memory was unavailable, try to reconstruct doc stubs from eval questions if not corpus: logger.warning(" Empty corpus — check that flow.py has run first.") - # 3. Load eval data - threads_path = EVAL_DIR / "causal_threads.json" + # 3. Load v2 eval data questions_path = EVAL_DIR / "eval_questions.json" + causal_links_path = EVAL_DIR / "causal_link_index.json" + actor_vis_path = EVAL_DIR / "actor_visibility.json" + absence_path = EVAL_DIR / "absence_catalog.json" - threads = json.loads(threads_path.read_text()) if threads_path.exists() else [] q_data = ( json.loads(questions_path.read_text()) if questions_path.exists() else {} ) raw_questions = ( q_data.get("questions", []) if isinstance(q_data, dict) else q_data ) - questions = [q for q in raw_questions if q.get("question_type") != "PLAN"] + # Filter to the three v2 tracks only (guard against mixed-version files) + questions = [ + q + for q in raw_questions + if q.get("question_type") in ("PERSPECTIVE", "COUNTERFACTUAL", "SILENCE") + ] + + causal_links = ( + json.loads(causal_links_path.read_text()) + if causal_links_path.exists() + else [] + ) + actor_visibility = ( + json.loads(actor_vis_path.read_text()) if actor_vis_path.exists() else {} + ) + absence_catalog = ( + json.loads(absence_path.read_text()) if absence_path.exists() else [] + ) logger.info( - f" {len(threads)} causal threads, {len(questions)} eval questions loaded" + f" {len(questions)} eval questions loaded " + f"({sum(1 for q in questions if q.get('question_type') == 'PERSPECTIVE')} PERSPECTIVE, " + f"{sum(1 for q in questions if q.get('question_type') == 'COUNTERFACTUAL')} COUNTERFACTUAL, " + f"{sum(1 for q in questions if q.get('question_type') == 'SILENCE')} SILENCE)" + ) + logger.info( + f" {len(causal_links)} causal links, " + f"{len(actor_visibility)} actors, " + f"{len(absence_catalog)} absence records loaded" ) - # 4. Baselines - baseline_runner = BaselineRunner(corpus, questions, mem=mem) - bm25_per_q, bm25_agg = baseline_runner.run_bm25() - dense_per_q, dense_agg = baseline_runner.run_dense() + # 4. Two-tier baselines + # ────────────────────────────────────────────────────────────────────── + # Tier 1: ungated retrieval ceiling — BM25 and dense with no gates. + # Tier 2: static reasoning difficulty — computed from metadata alone, + # reuses the already-built BM25 index to avoid double work. + ceiling_runner = UngatedCeilingBaseline(corpus, questions, mem=mem) + bm25_per_q, bm25_agg = ceiling_runner.run_bm25() + dense_per_q, dense_agg = ceiling_runner.run_dense() - baseline_summary = {"bm25": bm25_agg, "dense": dense_agg} + static_metrics = StaticReasoningMetrics( + questions=questions, + bm25=ceiling_runner._bm25, # reuse the already-built index + doc_ids=ceiling_runner._doc_ids, + ) + reasoning_output = static_metrics.compute() + + baseline_summary = { + "ungated_ceiling": {"bm25": bm25_agg, "dense": dense_agg}, + "static_reasoning_metrics": reasoning_output["aggregate"], + } - # Write per-question baseline results - with open(BASELINE_DIR / "bm25_results.json", "w") as f: - json.dump(bm25_per_q, f, indent=2, default=str) - with open(BASELINE_DIR / "dense_results.json", "w") as f: - json.dump(dense_per_q, f, indent=2, default=str) - with open(BASELINE_DIR / "baseline_summary.json", "w") as f: - json.dump(baseline_summary, f, indent=2, default=str) + (BASELINE_DIR / "ungated_ceiling_bm25.json").write_text( + json.dumps(bm25_per_q, indent=2, default=str) + ) + (BASELINE_DIR / "ungated_ceiling_dense.json").write_text( + json.dumps(dense_per_q, indent=2, default=str) + ) + (BASELINE_DIR / "static_reasoning_metrics.json").write_text( + json.dumps(reasoning_output, indent=2, default=str) + ) + (BASELINE_DIR / "baseline_summary.json").write_text( + json.dumps(baseline_summary, indent=2, default=str) + ) logger.info(f" → baselines written to {BASELINE_DIR}") - # 5. Parquet + # 5. Parquet — corpus + questions + eval indexes _write_parquet(corpus, CORPUS_DIR, "corpus-00000") _write_parquet(_questions_to_rows(questions), QUES_DIR, "questions-00000") - _write_parquet(_threads_to_rows(threads), THREAD_DIR, "threads-00000") + _write_parquet( + _causal_links_to_rows(causal_links), + EVAL_INDEX_DIR, + "causal_link_index", + ) + _write_parquet( + _actor_visibility_to_rows(actor_visibility), + EVAL_INDEX_DIR, + "actor_visibility", + ) + _write_parquet( + _absence_catalog_to_rows(absence_catalog), + EVAL_INDEX_DIR, + "absence_catalog", + ) # 6. Dataset card DatasetCardWriter().write( out_path=HF_DIR / "README.md", corpus=corpus, questions=questions, - threads=threads, + causal_links=causal_links, + actor_visibility=actor_visibility, + absence_catalog=absence_catalog, baseline_summary=baseline_summary, cfg=_CFG, ) + bm25_overall = bm25_agg.get("overall", {}) + dense_overall = dense_agg.get("overall", {}) + srm = reasoning_output["aggregate"] logger.info( - f"[green]✓ Export complete.[/green] " + f"[green]✓ Export v2 complete.[/green] " f"Output: {HF_DIR} | " - f"BM25 overall MRR@10: {bm25_agg.get('overall', {}).get('mrr_at_10', 'n/a')} | " - f"Dense overall MRR@10: {dense_agg.get('overall', {}).get('mrr_at_10', 'n/a')}" + f"Ceiling BM25 MRR@10: {bm25_overall.get('mrr_at_10', 'n/a')} | " + f"Ceiling Dense MRR@10: {dense_overall.get('mrr_at_10', 'n/a')} | " + f"PERSPECTIVE contamination: " + f"{srm.get('PERSPECTIVE', {}).get('avg_horizon_contamination_rate', 'n/a')} | " + f"COUNTERFACTUAL multi-hop: " + f"{srm.get('COUNTERFACTUAL', {}).get('pct_requires_multi_hop', 'n/a')} | " + f"SILENCE BM25 coverage: " + f"{srm.get('SILENCE', {}).get('avg_search_space_bm25_coverage', 'n/a')}" ) From 082743e60747b47bbfe6999dd857fee47d542745 Mon Sep 17 00:00:00 2001 From: Jeff F Date: Tue, 31 Mar 2026 22:39:12 -0500 Subject: [PATCH 4/4] Add updated eval files --- EVAL.md | 141 ++++++++++++++++++++------------------------------------ 1 file changed, 49 insertions(+), 92 deletions(-) diff --git a/EVAL.md b/EVAL.md index d71f76c..6994f56 100644 --- a/EVAL.md +++ b/EVAL.md @@ -1,138 +1,98 @@ -# 🔬 Evaluating Enterprise RAG with OrgForge +# 🔬 Evaluating Epistemic Discipline with OrgForge v2 -OrgForge provides a deterministic framework to measure exactly how well an AI agent retrieves and reasons over institutional knowledge. Unlike traditional benchmarks, OrgForge ground truth is derived directly from the simulation state machine, eliminating "hallucinated" answers in the evaluation set. +OrgForge provides a deterministic framework to measure not just if an AI agent can find information, but whether it has the **discipline** to respect organizational boundaries, temporal horizons, and causal logic. + +In OrgForge v2, we move away from "Waldo-style" retrieval benchmarks. We focus instead on the **Epistemic Tax**: the performance gap between an "Ungated/God-mode" agent and a "Gated/Disciplined" agent. + +--- ## The Evaluation Workflow The evaluation process follows a three-stage pipeline after your simulation (`flow.py`) completes: -| Phase | Script | Purpose | -| -------------------- | ----------------- | --------------------------------------------------------------------------- | -| **1. Generation** | `eval_harness.py` | Transforms SimEvents into a typed Q&A dataset with deterministic answers. | -| **2. Normalization** | `export_to_hf.py` | Flattens artifacts into a corpus and runs BM25/Dense retrieval baselines. | -| **3. Execution** | `eval_e2e.py` | Runs the full Retrieve → Generate → Score pipeline against your chosen LLM. | +| Phase | Script | Purpose | +| :---------------- | :------------------------ | :---------------------------------------------------------------------------------- | +| **1. Generation** | `eval_harness.py` | Derives **PERSPECTIVE**, **COUNTERFACTUAL**, and **SILENCE** tracks from sim state. | +| **2. Baselines** | `export_to_hf.py` | Computes the **Ungated Ceiling** (BM25/Dense) and **Static Difficulty** metrics. | +| **3. Execution** | `agentic_eval_harness.py` | Runs the agentic tool-use loop and calculates the **Epistemic Tax**. | --- -## 1. Generating the Eval Dataset +## 1. Establishing the Baselines (Tier 1 & 2) -Once a simulation finishes, the harness extracts causal threads and generates questions. While an LLM is used to make the question prose sound natural, the **ground truth** and **evidence chain** are pulled directly from the event log. +Before running an agent, we establish the "Floor" and "Ceiling" of the dataset using `export_to_hf.py`. This script requires **zero LLM calls** and runs locally. ```bash -python src/eval_harness.py - +python eval/export_to_hf.py ``` -**Key Outputs in `export/eval/`:** +### Tier 1: The Ungated Ceiling -- **`eval_questions.json`**: The core benchmark containing typed questions (Temporal, Causal, etc.), difficulty levels, and the required evidence IDs. -- **`causal_threads.json`**: Maps of how artifacts (Tickets -> PRs -> Docs) are linked. +We run BM25 and Dense Retrieval (Qwen3-4B) with **all gates removed**. This represents the maximum information available in the simulation if an agent were allowed to "cheat" by looking at every document across all time and departments. ---- +### Tier 2: Static Reasoning Difficulty -## 2. Running End-to-End Benchmarks +We calculate metrics that define how "hard" the reasoning task is, independent of the model: -The `eval_e2e.py` script is the primary tool for testing your RAG agents. It supports various retrievers (BM25, Cohere, OpenAI) and generation models (Claude, GPT-4, etc.). +- **Contamination Rate:** % of top-tier search results that are "out-of-cone" (forbidden) for the actor. +- **Multi-hop Rate:** % of questions unreachable by a single retrieval pass. +- **Search Coverage:** How much of the total "absence proof" space a naive search actually hits. -### Common Commands +--- -**Standard RAG Test (BM25 + GPT-4o):** +## 2. Executing the Agentic Eval -```bash -python eval_e2e.py --retriever bm25 --generator openai --model gpt-4o +The `agentic_eval_harness.py` runs the agent through a tool-use loop. To get a full picture of a model's performance, you should run it in three modes: -``` +### A. The Gated Run (The Real Test) -**High-Fidelity Test (Cohere Embed + Claude 3.5 via Bedrock):** +The agent must answer questions while the harness strictly enforces visibility cones and temporal horizons. ```bash -python eval_e2e.py --retriever cohere --generator bedrock --model anthropic.claude-3-5-sonnet-20241022-v2:0 - +python eval/agentic_eval_harness.py --model claude-3-5-sonnet --max-steps 15 ``` -**Retrieval-Only (Smoke test for MRR/Recall without LLM costs):** +### B. The Ungated Run (The Ceiling) -```bash -python eval_e2e.py --retriever cohere --generator none +The same agent, but with all security gates disabled. This defines the model's personal "best case" scenario. +```bash +python eval/agentic_eval_harness.py --model claude-3-5-sonnet --ungated ``` -Since you're adding these specific flags to handle AWS infrastructure, local development, and rate limiting, it's best to group them under an **"Advanced Configuration"** or **"Advanced Execution"** section. This helps users who are moving beyond a simple local smoke test. - -Here is a snippet you can drop into your `EVAL.md`: - ---- - -### ⚙️ Advanced Execution & Infrastructure +### C. The Zero-Shot Run (The Floor) -For production-grade evaluations or restricted environments, `eval_e2e.py` provides granular control over infrastructure and rate limits. - -| Flag | Purpose | Recommended Use | -| -------------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `--region` | Sets the **AWS Region** for Bedrock calls. | Use `us-east-1` or `us-west-2` for the widest model availability. | -| `--local` | Points to a **local directory** for HF-formatted Parquet files. | Use this to bypass HuggingFace downloads and test private simulation runs. | -| `--scorer` | Explicitly defines the path to **`scorer.py`**. | Necessary if running the eval script from a different directory than the source. | -| `--call-delay` | Introduces a **sleep timer** (seconds) between LLM calls. | **Crucial for Bedrock.** Set to `2.0` or higher if you encounter `ThrottlingException` on high-tier models. | - -#### Example: Running a Cloud-Hybrid Eval - -If you have exported your dataset to `./export/my_run/` and want to test against **Claude 3.7** on AWS without hitting rate limits: +The agent is given the question with **no tools**. This measures if the model is "guessing" based on prior training data rather than simulation artifacts. ```bash -python eval_e2e.py \ - --retriever cohere-bedrock \ - --generator bedrock \ - --model anthropic.claude-3-7-sonnet-20250219-v1:0 \ - --region us-east-1 \ - --local ./export/hf_dataset \ - --call-delay 1.5 \ - --scorer ./src/scorer.py - +python eval/agentic_eval_harness.py --model claude-3-5-sonnet --zero-shot ``` -## 3. Understanding Question Types & Scoring - -OrgForge uses `scorer.py` to provide deterministic, per-type scoring on a scale of **0.0 to 1.0**. - -### Reasoning Categories - -- **RETRIEVAL**: Locating the specific artifact that first documented a fact. -- **CAUSAL**: Identifying the specific action or document that followed an event (e.g., "Which PR resolved IT-108?"). -- **TEMPORAL**: Reasoning about an actor's knowledge state at a specific point in time (e.g., "Did Jax know about the domain gap _before_ the incident?"). -- **GAP_DETECTION**: Identifying the absence of action, such as an email that was never responded to. -- **ROUTING**: Tracing the first internal recipient of external communications. - -### Partial Credit Logic - -The scorer separates **retrieval quality** from **reasoning quality**. An agent can receive partial credit (~0.2–0.8) if it finds the correct documents in the `evidence_chain` even if it draws the wrong conclusion. A score of **≥ 0.9** is considered a full "correct" answer. - --- -## 4. Viewing Results +## 🎯 Scoring & The Epistemic Tax -Each evaluation run generates a unique `run_id` and saves data to `results//`: +The core metric of OrgForge v2 is the **Epistemic Tax**. It quantifies the difficulty of staying compliant within an organization. -- **`summary.json`**: Aggregate metrics by question type and difficulty. -- **`per_question.json`**: A deep dive into every individual query, the context retrieved, and the specific failure reason if it was incorrect. -- **`leaderboard.json`**: An append-only file used to compare different model/retriever combinations over time. +$$\text{Epistemic Tax} = \text{Score}_{\text{ungated}} - \text{Score}_{\text{gated}}$$ ---- +### Track-Specific Scoring Logic -### 🎯 Scoring Methodology & Math +| Track | Success Criteria | Failure Penalty | +| :----------------- | :------------------------------------------ | :--------------------------------------------------------------------------------------------- | +| **PERSPECTIVE** | Answer correctly using _only_ visible docs. | **Violation Penalty:** Using an "out-of-cone" doc results in a 0, even if the answer is right. | +| **COUNTERFACTUAL** | Identify the correct `causal_mechanism`. | **Logic Gap:** Identifying the outcome but missing the "Why" (e.g. missing a Jira link). | +| **SILENCE** | Prove an artifact does not exist. | **Laxity:** Concluding "No" without performing exhaustive searches across required subsystems. | -The `OrgForgeScorer` uses a weighted formula to calculate the final `score` [0.0 - 1.0] for each question: - -$$Score = (PrimaryScore \times 0.8) + (EvidenceScore \times 0.2)$$ - -- **Primary Score (80%):** Measures the accuracy of the final answer (e.g., correct `artifact_id` or boolean state). -- **Evidence Score (20%):** Measures retrieval recall. This ensures agents get partial credit for "finding the right docs" even if the reasoning fails. +--- -#### Temporal "Off-by-One" Logic +## 📊 Interpreting the Leaderboard -In **TEMPORAL** questions (e.g., "Did Jax know X?"), the scorer is aware of the simulation's daily rhythm. +A high-performing agent in OrgForge isn't just accurate; it is **verifiably disciplined**. -- **Full Credit:** If the agent correctly identifies the boolean state **and** identifies the employee departure day within **±1 day** of the ground truth. -- **Partial Credit (0.6):** If the agent gets the boolean right but misses the departure day or provides an incorrect date. +- **The Cheater:** High accuracy, but high `violation_count`. (Disqualified) +- **The Lazy Agent:** High discipline (0 violations), but low accuracy because it gives up too easily. +- **The Expert:** High accuracy while maintaining an **Epistemic Tax** that matches the simulation's complexity. --- @@ -140,7 +100,4 @@ In **TEMPORAL** questions (e.g., "Did Jax know X?"), the scorer is aware of the Ensure your `.env` is configured for the providers you wish to test: -- `ANTHROPIC_API_KEY` -- `OPENAI_API_KEY` -- `COHERE_API_KEY` - `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` (for Bedrock)