diff --git a/.gitignore b/.gitignore index f1e205d..8556805 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ backend/tests/ # Temp docs, backups, work logs docs/ +.worktrees/ *.bak stop_dev.sh diff --git a/backend/app/models/idea.py b/backend/app/models/idea.py index 71225fc..08ca8c6 100644 --- a/backend/app/models/idea.py +++ b/backend/app/models/idea.py @@ -23,6 +23,8 @@ class IdeaSessionStatus(str, Enum): """Idea session lifecycle states.""" PENDING = "pending" RUNNING = "running" + AWAITING_EVIDENCE = "awaiting_evidence" + AWAITING_IDEAS = "awaiting_ideas" COMPLETED = "completed" FAILED = "failed" CANCELLED = "cancelled" @@ -81,7 +83,7 @@ class IdeaSessionConfig(BaseModel): description="Optional search budget for BFTS; defaults to maxPapers if unset" ) maxReviewIterations: int = Field( - default=2, + default=3, ge=1, le=5, description="Maximum internal idea reviewer repair iterations before final handoff" @@ -419,6 +421,16 @@ class RawPaper(BaseModel): citationCount: int = 0 abstract: str = "" source: List[str] = Field(default_factory=list, description="List of sources: semantic_scholar, arxiv, local, openalex, crossref") + retrievalRoles: List[str] = Field(default_factory=list) + matchedQueries: List[str] = Field(default_factory=list) + evidenceTier: str = Field( + default="unclassified", + description="direct, transferable, rejected, unclassified", + ) + decisiveAnchors: List[str] = Field(default_factory=list) + relevanceComponents: Dict[str, float] = Field(default_factory=dict) + rejectionReason: str = "" + mustCiteOverride: bool = False normalizedTitleHash: str = Field(default="", description="SHA256 of normalized title for dedup") references: List[str] = Field(default_factory=list, description="Paper IDs cited by this paper") citedBy: List[str] = Field(default_factory=list, description="Paper IDs citing this paper") diff --git a/backend/app/modules/idea/bfts_search.py b/backend/app/modules/idea/bfts_search.py index f0c4d8d..3672a72 100644 --- a/backend/app/modules/idea/bfts_search.py +++ b/backend/app/modules/idea/bfts_search.py @@ -14,6 +14,7 @@ import heapq import logging import re +import math from collections import defaultdict from typing import List, Optional, Set, Tuple, Dict, Any from datetime import UTC, datetime @@ -53,6 +54,61 @@ "graphGrounding": 0.10, } +# --- P0: Semantic token-overlap helpers (replaces keyword heuristics) --- + +_STOPWORDS: Set[str] = { + "the", "a", "an", "and", "or", "of", "in", "to", "for", "with", + "on", "at", "by", "from", "that", "this", "it", "is", "are", "be", + "we", "our", "can", "has", "have", "not", "but", "as", "its", + "using", "based", "approach", "method", "model", "proposed", + "novel", "new", "improve", "via", "towards", "beyond", +} + +def _token_ngrams(text: str, n: int = 2) -> Set[str]: + """Extract overlapping n-gram token sets from text for semantic comparison.""" + tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", text.lower()) + tokens = [t for t in tokens if t not in _STOPWORDS] + if not tokens: + return set() + ngrams: Set[str] = set() + for i in range(len(tokens) - n + 1): + ngrams.add(" ".join(tokens[i:i + n])) + # Also include unigrams for fallback coverage + if n == 2: + ngrams.update(tokens) + return ngrams + + +def _semantic_overlap(text_a: str, text_b: str) -> float: + """Compute semantic overlap score (0-1) using 2-gram token Jaccard. + + This is a fast, deterministic proxy for semantic similarity that works + across languages (CJK + English) and requires no embedding model. + + Returns a float in [0, 1]. + """ + if not text_a or not text_b: + return 0.0 + ng_a = _token_ngrams(text_a) + ng_b = _token_ngrams(text_b) + if not ng_a or not ng_b: + return 0.0 + intersection = len(ng_a & ng_b) + union = len(ng_a | ng_b) + return intersection / union if union > 0 else 0.0 + + +def _normalized_hypothesis(node: Any) -> str: + """Extract a normalized text representation of the idea for scoring.""" + parts = [] + if node.title: + parts.append(node.title) + if node.hypothesis: + parts.append(node.hypothesis) + if node.abstract: + parts.append(node.abstract[:300]) + return " ".join(parts) + def _build_literature_context(structured_papers: List[StructuredPaper], limit: int = 8) -> str: """Format selected papers as context string for LLM prompts.""" @@ -466,6 +522,13 @@ def run(self) -> List[IdeaCandidate]: child = self._expand_node(parent, max_reflection) if child: + # P0: dedup pruning — skip near-duplicate ideas + if self._is_duplicate(child): + child.status = "pruned" + self.nodes.append(child) # keep for tracking + parent.isExpanded = True + continue + self._add_child(parent, child) expansion_count += 1 @@ -560,6 +623,31 @@ def _add_child(self, parent: IdeaNode, child: IdeaNode) -> None: self._beam, (-child.combinedScore, child.nodeId, child.depth) ) + def _is_duplicate(self, new_node: IdeaNode) -> bool: + """P0: Check if new_node is a near-duplicate of any existing node. + + Uses n-gram Jaccard similarity on title+hypothesis. Skips the node + if similarity exceeds pruneDuplicateThreshold (default 0.82). + """ + threshold = getattr(self.config, 'pruneDuplicateThreshold', 0.82) or 0.82 + new_text = _normalized_hypothesis(new_node) + if not new_text: + return False + + existing_nodes = [n for n in self.nodes if n.nodeId != new_node.nodeId] + for existing in existing_nodes: + existing_text = _normalized_hypothesis(existing) + if not existing_text: + continue + sim = _semantic_overlap(new_text, existing_text) + if sim > threshold: + logger.info( + f"BFTS: pruning duplicate node (sim={sim:.3f} > " + f"threshold={threshold}): '{new_node.title[:50] if new_node.title else '?'}...'" + ) + return True + return False + def _probe_topic_terms(self, node: IdeaNode) -> List[str]: """Extract topic-bearing terms for targeted literature probes.""" text = " ".join([ @@ -884,38 +972,69 @@ def _score_node(self, node: IdeaNode) -> None: ) * 10.0 # Scale back to 0-10 def _estimate_novelty(self, node: IdeaNode, prior: float) -> float: - """Estimate novelty: heuristic based on hypothesis text.""" - import re - # Check for novel-sounding keywords - novel_kw = ["novel", "new", "unexplored", "first", "towards", "beyond"] - hyp = (node.hypothesis or "").lower() - title = (node.title or "").lower() - kw_score = sum(1 for kw in novel_kw if kw in hyp or kw in title) - text_score = min(1.0, kw_score / 3.0) - return min(10.0, max(0.0, (prior + text_score) * 5.0)) + """Estimate novelty: semantic overlap with seed_query (lower overlap = more novel). + + P0 improvement: replaces keyword matching ("novel", "new", ...) with + n-gram semantic overlap against the seed. Lower overlap means the idea + spans vocabulary not already in the query — a proxy for genuine novelty. + """ + seed_text = self.seed_query or "" + idea_text = _normalized_hypothesis(node) + overlap = _semantic_overlap(seed_text, idea_text) + + # Score inversion: high overlap → low novelty, low overlap → high novelty + # Map [0, 1] overlap → [2, 10] novelty (anchored at 6 for moderate overlap ~0.4) + novelty_from_overlap = max(2.0, min(10.0, 10.0 - overlap * 8.0)) + + # Blend with path-seed prior (30% prior, 70% overlap-based) + return 0.3 * (prior * 10.0) + 0.7 * novelty_from_overlap def _estimate_feasibility(self, node: IdeaNode, prior: float) -> float: - """Estimate feasibility: based on experiment count and concreteness.""" + """Estimate feasibility: experiment concreteness + literature grounding. + + P0 improvement: adds literature probe evidence (actual paper support) + alongside experiment count. + """ exp_count = len(node.experiments or []) - has_concrete_experiments = exp_count >= 1 score = prior * 10.0 - if has_concrete_experiments: - score += min(2.0, exp_count * 0.5) + if exp_count >= 2: + score += 2.5 + elif exp_count >= 1: + score += 1.5 + # Bonus for literature-probe backing (evidence from actual papers) + if node.literatureProbeIds: + score += min(1.5, len(node.literatureProbeIds) * 0.3) return min(10.0, max(0.0, score)) def _estimate_impact(self, node: IdeaNode) -> float: - """Estimate impact: based on hypothesis ambition.""" - hyp = (node.hypothesis or "").lower() - impact_kw = ["improving", "outperforms", "state-of-the-art", "SOTA", "significant"] - kw_score = sum(1 for kw in impact_kw if kw in hyp) - return min(10.0, 5.0 + kw_score * 1.5) + """Estimate impact: hypothesis specificity and reference depth. + + P0 improvement: replaces keyword matching ("improving", "SOTA", ...) + with a composite of semantic clarity and citation anchoring. + """ + hyp = (node.hypothesis or "") + hyp_len = len(hyp.split()) + # Longer, more detailed hypotheses tend to be more impactful + length_score = min(1.0, hyp_len / 60.0) * 3.0 + # References indicate domain grounding + ref_count = len(node.references or []) if hasattr(node, 'references') else 0 + ref_score = min(2.0, ref_count * 0.4) + # Base + return min(10.0, 5.0 + length_score + ref_score) def _estimate_specificity(self, node: IdeaNode) -> float: - """Estimate specificity: 0-1, based on title concreteness.""" + """Estimate specificity: concept density in title and hypothesis. + + P0 improvement: replaces capital-letter heuristic with n-gram + concept density — counts unique semantic tokens normalized by length. + """ title = node.title or "" - # Count specific nouns / named entities (heuristic) - words = re.findall(r'\b[A-Z][a-z]{2,}\b', title) - return min(1.0, len(words) / 5.0) + hyp = node.hypothesis or "" + combined = f"{title} {hyp}" + tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", combined.lower()) + unique_meaningful = {t for t in tokens if t not in _STOPWORDS} + density = len(unique_meaningful) / max(1, len(tokens)) if tokens else 0 + return min(1.0, density * 1.5) def _estimate_evidence_support(self, node: IdeaNode) -> float: """Estimate evidence support from path seed prior plus reflection/probe evidence.""" diff --git a/backend/app/modules/idea/evidence_relevance.py b/backend/app/modules/idea/evidence_relevance.py new file mode 100644 index 0000000..00b4309 --- /dev/null +++ b/backend/app/modules/idea/evidence_relevance.py @@ -0,0 +1,421 @@ +"""Deterministic topic relevance utilities for Idea evidence retrieval.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import hashlib +import re +from typing import Any, Iterable, Mapping, Sequence + +from app.services.search_service import SearchResult, tokenize_topic_text + + +class EvidenceTier(str, Enum): + DIRECT = "direct" + TRANSFERABLE = "transferable" + REJECTED = "rejected" + + +GENERIC_TERMS = frozenset({ + "and", + "analysis", + "application", + "approach", + "chamber", + "dream", + "evaluation", + "exploration", + "framework", + "for", + "from", + "generation", + "method", + "model", + "outcome", + "outcomes", + "potential", + "predict", + "predicting", + "prediction", + "red", + "research", + "study", + "system", + "the", + "using", + "with", +}) + +CONNECTOR_FACET_WEAK_TERMS = frozenset({"answering", "qa", "question"}) + + +@dataclass(frozen=True) +class TopicIntentProfile: + seed_anchors: tuple[str, ...] + required_seed_facets: tuple[tuple[str, ...], ...] + core_anchors: tuple[str, ...] + task_anchors: tuple[str, ...] + method_anchors: tuple[str, ...] + evaluation_anchors: tuple[str, ...] + generic_terms: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "seedAnchors": list(self.seed_anchors), + "requiredSeedFacets": [list(facet) for facet in self.required_seed_facets], + "coreAnchors": list(self.core_anchors), + "taskAnchors": list(self.task_anchors), + "methodAnchors": list(self.method_anchors), + "evaluationAnchors": list(self.evaluation_anchors), + "genericTerms": list(self.generic_terms), + } + + +@dataclass(frozen=True) +class EvidenceAssessment: + tier: EvidenceTier + score: float + decisive_anchors: tuple[str, ...] + score_components: Mapping[str, float] + rejection_reason: str = "" + + +@dataclass(frozen=True) +class DedupeOutcome: + results: tuple[SearchResult, ...] + merge_count: int + + +def _unique(values: Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(value for value in values if value)) + + +def _quoted_phrases(values: Iterable[str]) -> tuple[str, ...]: + phrases: list[str] = [] + for value in values: + phrases.extend( + match.strip().lower() + for match in re.findall(r"[\"']([^\"']{4,100})[\"']", value or "") + ) + return _unique(phrases) + + +def _hyphen_phrases(values: Iterable[str]) -> tuple[str, ...]: + phrases: list[str] = [] + for value in values: + phrases.extend( + match.lower().replace("-", " ") + for match in re.findall( + r"\b[a-zA-Z0-9]+(?:-[a-zA-Z0-9]+)+\b", + value or "", + ) + ) + return _unique(phrases) + + +def _discriminative_tokens(values: Iterable[str]) -> tuple[str, ...]: + return _unique( + token + for value in values + for token in tokenize_topic_text(value) + if token not in GENERIC_TERMS and len(token) >= 3 + ) + + +def _required_seed_facets(seed: str) -> tuple[tuple[str, ...], ...]: + parts = re.split(r"\s+for\s+", seed or "", maxsplit=1, flags=re.IGNORECASE) + if len(parts) != 2 or not all(part.strip() for part in parts): + return () + facets = [] + for part in parts: + anchors = _unique([ + *_hyphen_phrases([part]), + *( + token + for token in _discriminative_tokens([part]) + if token not in CONNECTOR_FACET_WEAK_TERMS + ), + ]) + if anchors: + facets.append(anchors) + return tuple(facets) if len(facets) == 2 else () + + +def build_topic_intent_profile( + *, + seed: str, + domain: str, + role_queries: Mapping[str, Sequence[str]], +) -> TopicIntentProfile: + domain_queries = list(role_queries.get("domain", ())) + task_queries = list(role_queries.get("task", ())) + method_queries = list(role_queries.get("method", ())) + evaluation_queries = list(role_queries.get("evaluation", ())) + core_phrases = _quoted_phrases([seed, domain, *domain_queries, *task_queries]) + hyphen_phrases = _hyphen_phrases([seed, domain, *domain_queries, *task_queries]) + seed_anchors = _unique([ + *_hyphen_phrases([seed]), + *_discriminative_tokens([seed]), + ]) + core_tokens = _discriminative_tokens([seed, domain, *domain_queries]) + return TopicIntentProfile( + seed_anchors=seed_anchors, + required_seed_facets=_required_seed_facets(seed), + core_anchors=_unique([*core_phrases, *hyphen_phrases, *core_tokens]), + task_anchors=_discriminative_tokens(task_queries), + method_anchors=_discriminative_tokens(method_queries), + evaluation_anchors=_discriminative_tokens(evaluation_queries), + generic_terms=tuple(sorted(GENERIC_TERMS)), + ) + + +def _anchor_in_text(anchor: str, text: str) -> bool: + if not anchor: + return False + if re.search(r"[\u4e00-\u9fff]", anchor) or " " in anchor: + return anchor in text + return re.search(rf"(? tuple[str, ...]: + return tuple(anchor for anchor in anchors if _anchor_in_text(anchor, text)) + + +def _covers_required_seed_facets( + text: str, + facets: Sequence[Sequence[str]], +) -> bool: + for facet in facets: + phrase_hits = _hits(text, tuple(anchor for anchor in facet if " " in anchor)) + token_anchors = tuple(anchor for anchor in facet if " " not in anchor) + token_hits = _hits(text, token_anchors) + required_tokens = min(2, len(token_anchors)) + if not phrase_hits and len(token_hits) < required_tokens: + return False + return True + + +def assess_search_result( + result: SearchResult, + profile: TopicIntentProfile, +) -> EvidenceAssessment: + text = f"{result.title} {result.abstract}".lower().replace("-", " ") + phrase_hits = tuple( + anchor + for anchor in profile.core_anchors + if " " in anchor and _anchor_in_text(anchor, text) + ) + core_hits = _hits( + text, + tuple(anchor for anchor in profile.core_anchors if " " not in anchor), + ) + task_hits = _hits(text, profile.task_anchors) + method_hits = _hits(text, profile.method_anchors) + evaluation_hits = _hits(text, profile.evaluation_anchors) + seed_hits = _hits(text, profile.seed_anchors) + + components = { + "corePhrase": min(0.55, 0.55 * len(phrase_hits)), + "coreTerms": min(0.35, 0.12 * len(core_hits)), + "task": min(0.25, 0.10 * len(task_hits)), + "methodEvaluation": min( + 0.20, + 0.06 * (len(method_hits) + len(evaluation_hits)), + ), + "provider": min( + 0.10, + max(0.0, float(result.relevance_score or 0.0)) * 0.10, + ), + } + score = min(1.0, sum(components.values())) + decisive = _unique([ + *phrase_hits, + *core_hits, + *task_hits, + *method_hits, + *evaluation_hits, + ]) + has_supporting_signal = bool(task_hits or method_hits or evaluation_hits) + has_strong_seed_signal = any( + " " in anchor or len(anchor) >= 7 + for anchor in seed_hits + ) + seed_phrase_anchors = tuple( + anchor for anchor in profile.seed_anchors if " " in anchor + ) + if profile.required_seed_facets: + seed_facet_coverage = _covers_required_seed_facets( + text, + profile.required_seed_facets, + ) + elif len(seed_phrase_anchors) >= 2: + matched_seed_phrases = _hits(text, seed_phrase_anchors) + phrase_component_tokens = { + token + for phrase in matched_seed_phrases + for token in phrase.split() + } + independent_seed_hits = { + anchor + for anchor in seed_hits + if " " not in anchor and anchor not in phrase_component_tokens + } + seed_facet_coverage = ( + len(matched_seed_phrases) >= 2 + or ( + bool(matched_seed_phrases) + and len(independent_seed_hits) >= 2 + ) + ) + else: + seed_facet_coverage = True + + if ( + phrase_hits or (len(core_hits) >= 2 and has_strong_seed_signal) + ) and has_supporting_signal and seed_facet_coverage: + tier = EvidenceTier.DIRECT + rejection_reason = "" + elif len(task_hits) >= 2 and bool(method_hits or evaluation_hits): + tier = EvidenceTier.TRANSFERABLE + rejection_reason = "" + else: + tier = EvidenceTier.REJECTED + rejection_reason = "generic_overlap_only" if text.strip() else "missing_text" + + return EvidenceAssessment( + tier=tier, + score=round(score, 4), + decisive_anchors=decisive, + score_components=components, + rejection_reason=rejection_reason, + ) + + +def _normalized_title_key(title: str) -> str: + normalized = re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", (title or "").lower()) + return "title:" + hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def search_result_identity_keys(result: SearchResult) -> tuple[str, ...]: + keys: list[str] = [] + if result.doi: + keys.append(f"doi:{result.doi.lower().strip()}") + if result.arxiv_id: + keys.append(f"arxiv:{result.arxiv_id.lower().strip()}") + semantic_id = re.search(r"SemanticScholarID:(\w+)", result.url or "") + if semantic_id: + keys.append(f"s2:{semantic_id.group(1).lower()}") + keys.append(_normalized_title_key(result.title)) + return tuple(keys) + + +def raw_paper_identity_keys(paper: Any) -> tuple[str, ...]: + keys: list[str] = [] + doi = getattr(paper, "doi", None) + arxiv_id = getattr(paper, "arxivId", None) + semantic_id = getattr(paper, "semanticScholarId", None) + if doi: + keys.append(f"doi:{str(doi).lower().strip()}") + if arxiv_id: + keys.append(f"arxiv:{str(arxiv_id).lower().strip()}") + if semantic_id: + keys.append(f"s2:{str(semantic_id).lower().strip()}") + keys.append(_normalized_title_key(getattr(paper, "title", ""))) + return tuple(keys) + + +def _append_unique(current: list[str], incoming: Iterable[str]) -> None: + for value in incoming: + if value and value not in current: + current.append(value) + + +def _merge_result(target: SearchResult, incoming: SearchResult) -> None: + _append_unique(target.retrieval_roles, incoming.retrieval_roles) + _append_unique(target.matched_queries, incoming.matched_queries) + _append_unique( + target.retrieval_sources, + incoming.retrieval_sources or [incoming.source], + ) + if len(incoming.abstract or "") > len(target.abstract or ""): + target.abstract = incoming.abstract + if not target.doi and incoming.doi: + target.doi = incoming.doi + if not target.arxiv_id and incoming.arxiv_id: + target.arxiv_id = incoming.arxiv_id + if not target.url and incoming.url: + target.url = incoming.url + target.relevance_score = max(target.relevance_score, incoming.relevance_score) + target.citation_count = max(target.citation_count or 0, incoming.citation_count or 0) + + +def deduplicate_search_results(results: Sequence[SearchResult]) -> DedupeOutcome: + unique: list[SearchResult] = [] + index: dict[str, SearchResult] = {} + merge_count = 0 + for result in results: + keys = search_result_identity_keys(result) + target = next((index[key] for key in keys if key in index), None) + if target is not None: + _merge_result(target, result) + merge_count += 1 + else: + target = result + unique.append(target) + for key in (*search_result_identity_keys(target), *keys): + index[key] = target + return DedupeOutcome(tuple(unique), merge_count) + + +_TIER_PRIORITY = { + EvidenceTier.REJECTED.value: 0, + "unclassified": 1, + EvidenceTier.TRANSFERABLE.value: 2, + EvidenceTier.DIRECT.value: 3, +} + + +def better_evidence_tier(current: str, incoming: str) -> str: + return max( + (current, incoming), + key=lambda tier: _TIER_PRIORITY.get(tier, 0), + ) + + +def role_requirements_for_paper_type(paper_type: str) -> dict[str, int]: + normalized = (paper_type or "algorithm").lower() + if normalized in {"survey", "position", "theory"}: + return {"domainOrTask": 2, "method": 0, "evaluation": 0} + if normalized in {"benchmark", "evaluation", "reproducibility"}: + return {"domainOrTask": 2, "method": 0, "evaluation": 2} + return {"domainOrTask": 2, "method": 1, "evaluation": 1} + + +def semantically_eligible_roles( + tier: str, + roles: Sequence[str], +) -> tuple[str, ...]: + if tier == EvidenceTier.DIRECT.value: + return _unique(roles) + if tier == EvidenceTier.TRANSFERABLE.value: + return _unique(role for role in roles if role in {"method", "evaluation"}) + if tier == "unclassified": + return _unique(roles) + return () + + +def evidence_tier_allows_dimension(tier: str, dimension: str) -> bool: + if tier in {EvidenceTier.DIRECT.value, "unclassified"}: + return True + if tier != EvidenceTier.TRANSFERABLE.value: + return False + direct_only = { + "background", + "claim", + "domain", + "gap", + "high_risk_qa", + "novelty", + } + return dimension not in direct_only diff --git a/backend/app/modules/idea/ideas_api.py b/backend/app/modules/idea/ideas_api.py index cd6da34..d908abf 100644 --- a/backend/app/modules/idea/ideas_api.py +++ b/backend/app/modules/idea/ideas_api.py @@ -56,7 +56,7 @@ class CreateSessionRequest(BaseModel): constraints: Optional[List[str]] = None mustCiteList: Optional[List[str]] = None searchBudget: Optional[int] = Field(default=None, ge=10, le=500) - maxReviewIterations: int = Field(default=2, ge=1, le=5) + maxReviewIterations: int = Field(default=3, ge=1, le=5) class SessionResponse(BaseModel): @@ -478,6 +478,28 @@ async def start_session( ) +@router.post( + "/sessions/{session_id}/resume", + response_model=SessionResponse, + summary="Resume Idea Session", + description="Resume a session waiting for evidence or approved ideas.", +) +async def resume_session( + session_id: str, + background_tasks: BackgroundTasks, +) -> SessionResponse: + service = get_idea_service() + try: + session = service.resume_session(session_id) + background_tasks.add_task(service.run_pipeline, session_id) + return _session_to_response(session) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + + @router.post( "/sessions/{session_id}/cancel", response_model=SessionResponse, diff --git a/backend/app/modules/idea/service.py b/backend/app/modules/idea/service.py index 3a3c0c4..8974427 100644 --- a/backend/app/modules/idea/service.py +++ b/backend/app/modules/idea/service.py @@ -7,9 +7,10 @@ import logging import os import threading +from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime -from typing import Optional, List, Dict, Any +from typing import Optional, List, Dict, Any, Tuple from app.modules.idea.contracts import ( IdeaSession, @@ -78,9 +79,21 @@ from app.modules.idea.reasoning_kg import ReasoningKGBuilder from app.modules.idea.graph_linker import GraphLinker from app.modules.idea.path_seed import PathSeedGenerator +from app.modules.idea.evidence_relevance import ( + EvidenceTier, + assess_search_result, + better_evidence_tier, + build_topic_intent_profile, + deduplicate_search_results, + evidence_tier_allows_dimension, + raw_paper_identity_keys, + role_requirements_for_paper_type, + search_result_identity_keys, + semantically_eligible_roles, +) from app.llm.provider_client import get_provider_client, ChatMessage, ProviderError from app.llm.task_scheduler import get_llm_task_scheduler -from app.services.search_service import get_search_service, SearchResult +from app.services.search_service import get_search_service, SearchResult, tokenize_topic_text from app.services.ranking_service import get_ranking_service from app.services import prompts import json @@ -110,6 +123,33 @@ def __init__( self.step_artifacts = artifacts or [] +class RecoverableIdeaError(StepContextError): + """Pipeline stop that preserves work and can be resumed later.""" + + waiting_status: IdeaSessionStatus + resume_from: str + + +class AwaitingEvidenceError(RecoverableIdeaError): + waiting_status = IdeaSessionStatus.AWAITING_EVIDENCE + resume_from = "evidenceGate" + + +class AwaitingLiteratureEvidenceError(RecoverableIdeaError): + waiting_status = IdeaSessionStatus.AWAITING_EVIDENCE + resume_from = "literatureSearch" + + +class AwaitingTranslationError(RecoverableIdeaError): + waiting_status = IdeaSessionStatus.AWAITING_EVIDENCE + resume_from = "expandQuery" + + +class AwaitingIdeasError(RecoverableIdeaError): + waiting_status = IdeaSessionStatus.AWAITING_IDEAS + resume_from = "ideaBrainstorm" + + def _utcnow() -> datetime: return datetime.now(UTC) @@ -235,7 +275,7 @@ def _rank_results_for_topic( search_queries: List[str], ) -> List[SearchResult]: topic_text = " ".join([seed, domain, *search_queries]).lower() - tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9-]{2,}|[\u4e00-\u9fff]+", topic_text) + tokens = tokenize_topic_text(topic_text) stopwords = { "and", "the", "for", "with", "from", "that", "this", "into", "using", "based", "how", "can", "are", "what", "when", "where", @@ -330,7 +370,7 @@ def _topic_terms_from_seed(seed: Any, domain: Any = "", extra_terms: Optional[Li "是否", "如何", "研究", "方法", "模型", "系统", } terms: List[str] = [] - for token in re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]{2,}", topic_text): + for token in tokenize_topic_text(topic_text): if token in stopwords: continue if token not in terms: @@ -401,7 +441,11 @@ def _paper_alignment_score(paper: Any, topic_terms: List[str]) -> float: ]: if phrase in text: phrase_bonus += 0.12 - relevance_score = float(getattr(paper, "relevanceScore", 0.0) or 0.0) + relevance_score = float( + getattr(paper, "relevanceScore", None) + or getattr(paper, "relevance_score", 0.0) + or 0.0 + ) return min(1.0, hits / max(4, min(10, len(topic_terms))) + phrase_bonus + relevance_score * 0.25) @@ -421,20 +465,40 @@ def _evaluate_paper_quality_gate( papers: List[Any], stage: str, extra_terms: Optional[List[str]] = None, + paper_roles: Optional[Dict[str, List[str]]] = None, + paper_type: str = "algorithm", ) -> Dict[str, Any]: """Lightweight paper relevance gate used before idea generation.""" topic_terms = _topic_terms_from_seed(seed, domain, extra_terms) total = len(papers) - scored = [ - { - "paperId": getattr(paper, "id", ""), + scored: List[Dict[str, Any]] = [] + for paper in papers: + paper_id = getattr(paper, "rawPaperId", getattr(paper, "id", "")) + raw_roles = list(dict.fromkeys( + (paper_roles or {}).get(paper_id, []) + or getattr(paper, "retrievalRoles", None) + or getattr(paper, "retrieval_roles", []) + or [] + )) + tier = str( + getattr(paper, "evidenceTier", None) + or getattr(paper, "evidence_tier", "unclassified") + ) + eligible_roles = list(semantically_eligible_roles(tier, raw_roles)) + score = 0.0 if tier == EvidenceTier.REJECTED.value else _paper_alignment_score( + paper, + topic_terms, + ) + scored.append({ + "paperId": paper_id, "title": getattr(paper, "title", ""), - "score": round(_paper_alignment_score(paper, topic_terms), 3), + "score": round(score, 3), "sources": _paper_sources(paper), - } - for paper in papers - ] + "roles": eligible_roles, + "retrievalRoles": raw_roles, + "evidenceTier": tier, + }) scored.sort(key=lambda item: item["score"], reverse=True) aligned = [item for item in scored if item["score"] >= 0.32] external = [ @@ -470,6 +534,26 @@ def _evaluate_paper_quality_gate( min_papers = int(os.getenv("FAROS_PAPER_GATE_MIN_PAPERS", "4")) min_aligned = int(os.getenv("FAROS_PAPER_GATE_MIN_ALIGNED", "3")) + role_names = ("domain", "task", "method", "evaluation") + role_counts = { + role: sum( + 1 + for item in scored + if role in item["roles"] and item["score"] >= 0.12 + ) + for role in role_names + } + role_aware = any(item["retrievalRoles"] for item in scored) + role_requirements = role_requirements_for_paper_type(paper_type) + role_issues: List[str] = [] + if role_aware: + if role_counts["domain"] + role_counts["task"] < role_requirements["domainOrTask"]: + role_issues.append("insufficient domain/task evidence") + if role_requirements["method"] and role_counts["method"] < role_requirements["method"]: + role_issues.append("insufficient method evidence") + if role_requirements["evaluation"] and role_counts["evaluation"] < role_requirements["evaluation"]: + role_issues.append("insufficient evaluation evidence") + role_coverage_passed = role_aware and not role_issues errors: List[str] = [] warnings: List[str] = [] if total < min_papers: @@ -478,6 +562,8 @@ def _evaluate_paper_quality_gate( errors.append(f"{stage}: too few papers are semantically aligned with the seed query ({len(aligned)} < {min_aligned})") if scored and avg_top_score < 0.30: errors.append(f"{stage}: top papers have weak topic alignment (avg={avg_top_score:.2f})") + if role_aware and role_issues: + errors.append(f"{stage}: role-aware evidence coverage failed: {', '.join(role_issues)}") if total and not external: warnings.append(f"{stage}: all retrieved papers are from local fallback sources") if total and len(aligned) < max(2, total // 4): @@ -497,6 +583,13 @@ def _evaluate_paper_quality_gate( "sourceQuality": source_quality, "providerFallbackRisk": provider_fallback_risk, "avgTopAlignment": round(avg_top_score, 3), + "roleCoverage": { + "enabled": role_aware, + "passed": role_coverage_passed, + "counts": role_counts, + "requirements": role_requirements, + "issues": role_issues, + }, "topicTerms": topic_terms[:12], "topPapers": scored[:8], } @@ -671,6 +764,7 @@ def _evaluate_evidence_gate_v2( gap_outputs: Optional[Dict[str, Any]], stage: str, extra_terms: Optional[List[str]] = None, + paper_roles: Optional[Dict[str, List[str]]] = None, ) -> Dict[str, Any]: """Hard pre-brainstorm evidence gate. @@ -684,6 +778,8 @@ def _evaluate_evidence_gate_v2( papers=structured_papers, stage=stage, extra_terms=extra_terms, + paper_roles=paper_roles, + paper_type=paper_type, ) coverage = _paper_type_coverage(structured_papers, literature_map) required_coverage = _paper_type_coverage_requirements(paper_type) @@ -703,7 +799,10 @@ def _evaluate_evidence_gate_v2( errors.append( f"{stage}: insufficient external evidence papers ({external_count} < {min_external})" ) - if aligned_count < min_aligned: + role_coverage_passed = bool( + (base_gate.get("roleCoverage") or {}).get("passed", False) + ) + if aligned_count < min_aligned and not role_coverage_passed: errors.append( f"{stage}: insufficient topic-aligned structured papers ({aligned_count} < {min_aligned})" ) @@ -744,6 +843,63 @@ def _env_bool(name: str, default: bool) -> bool: return value.strip().lower() in {"1", "true", "yes", "on"} +def _deep_read_max_papers() -> int: + try: + configured = int(os.getenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "24")) + except ValueError: + configured = 24 + return max(4, min(40, configured)) + + +def _limit_deep_read_selection( + selected_paper_ids: List[str], + raw_by_id: Dict[str, RawPaper], + *, + limit: int, +) -> List[str]: + direct: List[str] = [] + transferable: List[str] = [] + must_cite: List[str] = [] + for paper_id in dict.fromkeys(selected_paper_ids): + paper = raw_by_id.get(paper_id) + if not paper: + continue + if paper.mustCiteOverride: + must_cite.append(paper_id) + elif paper.evidenceTier == EvidenceTier.TRANSFERABLE.value: + transferable.append(paper_id) + elif paper.evidenceTier != EvidenceTier.REJECTED.value: + direct.append(paper_id) + + transfer_cap = max(0, limit // 3) + regular = [ + *direct[: max(0, limit - transfer_cap)], + *transferable[:transfer_cap], + ] + return list(dict.fromkeys([*regular, *must_cite])) + + +def _tag_repair_results(results: List[SearchResult], query: str) -> None: + lowered_query = query.lower() + roles = ["repair"] + for role in ("domain", "task", "method", "evaluation"): + if role in lowered_query: + roles.append(role) + for result in results: + for role in roles: + if role not in result.retrieval_roles: + result.retrieval_roles.append(role) + if query not in result.matched_queries: + result.matched_queries.append(query) + + +def _record_step_result(trace: WorkflowTrace, result: StepResult) -> None: + trace.steps.append(result) + trace.totalSteps = len(trace.steps) + trace.successfulSteps = sum(step.status == "ok" for step in trace.steps) + trace.failedSteps = sum(step.status == "failed" for step in trace.steps) + + def _as_string_list(value: Any, *, limit: int = 8) -> List[str]: if isinstance(value, str): if not value.strip(): @@ -778,10 +934,25 @@ def _coverage_dimension_key(value: Any, fallback: str) -> str: return key or fallback +def _verify_coverage_dimension_support( + *, + dimension: str, + supporting_paper_ids: List[str], + paper_tiers: Dict[str, str], +) -> List[str]: + return [ + paper_id + for paper_id in dict.fromkeys(supporting_paper_ids) + if paper_id in paper_tiers + and evidence_tier_allows_dimension(paper_tiers[paper_id], dimension) + ] + + def _normalize_coverage_report( data: Optional[Dict[str, Any]], *, available_paper_ids: set[str], + paper_tiers: Optional[Dict[str, str]] = None, source: str = "llm", ) -> Dict[str, Any]: """Normalize LLM-first coverage output and enforce citation integrity.""" @@ -796,6 +967,10 @@ def _normalize_coverage_report( dimensions: List[Dict[str, Any]] = [] warnings = _as_string_list(data.get("warnings", []), limit=8) repair_queries = _as_string_list(data.get("repairQueries", []), limit=8) + effective_tiers = { + paper_id: (paper_tiers or {}).get(paper_id, "unclassified") + for paper_id in available_paper_ids + } for index, raw in enumerate(raw_dimensions if isinstance(raw_dimensions, list) else []): if not isinstance(raw, dict): @@ -807,10 +982,15 @@ def _normalize_coverage_report( raw.get("supportingPaperIds", raw.get("paperIds", [])), limit=12, ) - verified_paper_ids = [ + known_paper_ids = [ paper_id for paper_id in raw_paper_ids if paper_id in available_paper_ids ] + verified_paper_ids = _verify_coverage_dimension_support( + dimension=key, + supporting_paper_ids=known_paper_ids, + paper_tiers=effective_tiers, + ) dropped_ids = [ paper_id for paper_id in raw_paper_ids if paper_id not in available_paper_ids @@ -819,6 +999,15 @@ def _normalize_coverage_report( warnings.append( f"{key}: ignored unknown paper IDs: {', '.join(dropped_ids[:4])}" ) + disallowed_ids = [ + paper_id for paper_id in known_paper_ids + if paper_id not in verified_paper_ids + ] + if disallowed_ids: + warnings.append( + f"{key}: ignored paper IDs whose evidence tier cannot support " + f"this dimension: {', '.join(disallowed_ids[:4])}" + ) status = str(raw.get("status", "") or "").strip().lower() if required and not verified_paper_ids: status = "missing" @@ -955,6 +1144,7 @@ def _rule_seed_coverage_report( structured_papers: List[StructuredPaper], literature_map: Optional[LiteratureMap], gap_outputs: Dict[str, Any], + paper_tiers: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: """Rule fallback for LLM-first evidence coverage.""" @@ -997,6 +1187,7 @@ def _rule_seed_coverage_report( ), }, available_paper_ids={paper.id for paper in structured_papers}, + paper_tiers=paper_tiers, source="rule_fallback", ) if gap_outputs: @@ -1116,6 +1307,211 @@ def _ensure_gap_outputs( return clean_gap_analysis[:5], clean_prioritized[:5], clean_opportunities[:5] +# Typed-fallback research opportunities for gap analysis diversity enforcement +_TYPED_RESEARCH_OPPORTUNITIES = [ + ("method", "Develop a novel algorithmic mechanism or architecture for {seed}"), + ("benchmark", "Build comprehensive evaluation benchmarks and protocols for {seed}"), + ("system", "Design an end-to-end system integrating {seed} into a practical pipeline"), + ("safety", "Improve robustness, safety, or failure analysis of {seed} methods"), + ("application", "Apply {seed} to a concrete high-value domain-specific scenario"), +] + + +def _enforce_min_opportunities( + *, + opportunities: List[str], + seed: str, + paper_type: str, + gap_analysis: List[Any], + min_count: int = 3, +) -> List[str]: + """Ensure at least min_count independent research opportunities exist. + + Uses lightweight n-gram Jaccard deduplication to detect and remove + semantically near-duplicate opportunities, then supplements with + typed fallbacks if the result still falls below min_count. + """ + clean = [str(o).strip() for o in (opportunities or []) if str(o).strip()] + if len(clean) <= 1: + return _typed_opportunity_fallbacks(seed, paper_type, clean, min_count, gap_analysis) + + # Deduplicate semantically similar opportunities + deduped = _deduplicate_opportunities(clean) + if len(deduped) >= min_count: + return deduped[:min_count] + + # Supplement from gap_analysis content if available + from_gaps = [] + for gap in (gap_analysis or [])[:5]: + if isinstance(gap, dict): + approaches = gap.get("approaches", []) + if isinstance(approaches, list): + for a in approaches[:2]: + from_gaps.append(f"Explore {seed} via: {a}") + + for candidate in from_gaps: + if len(deduped) >= min_count: + break + if not _is_similar_to_any(candidate, deduped): + deduped.append(candidate) + + if len(deduped) >= min_count: + return deduped[:min_count] + + # Fall back to typed opportunities + typed = _typed_opportunity_fallbacks(seed, paper_type, deduped, min_count, gap_analysis) + return typed[:min_count] + + +def _typed_opportunity_fallbacks( + seed: str, + paper_type: str, + existing: List[str], + min_count: int, + gap_analysis: List[Any], +) -> List[str]: + """Generate typed fallback research opportunities.""" + result = list(existing) + + # Derive gap-specific opportunities from the gap_analysis + gap_specific = [] + for gap in (gap_analysis or []): + if isinstance(gap, dict): + gap_desc = str(gap.get("gap", "") or "").strip() + if gap_desc and len(gap_desc) > 10: + gap_specific.append(f"Address: {gap_desc[:120]}") + for gs in gap_specific: + if len(result) >= min_count: + break + if not _is_similar_to_any(gs, result): + result.append(gs) + + # Reorder typed fallbacks based on paper_type priority + typed_items = list(_TYPED_RESEARCH_OPPORTUNITIES) + if paper_type in {"safety", "reliability"}: + typed_items = sorted(typed_items, key=lambda x: 0 if x[0] == "safety" else 1) + elif paper_type in {"benchmark", "evaluation"}: + typed_items = sorted(typed_items, key=lambda x: 0 if x[0] == "benchmark" else 1) + elif paper_type in {"system", "application"}: + typed_items = sorted(typed_items, key=lambda x: 0 if x[0] in ("system", "application") else 1) + + for _, template in typed_items: + if len(result) >= min_count: + break + candidate = template.format(seed=seed) + if not _is_similar_to_any(candidate, result): + result.append(candidate) + + return result[:max(min_count, len(result))] + + +def _deduplicate_opportunities(items: List[str], threshold: float = 0.78) -> List[str]: + """Remove near-duplicate opportunities using n-gram Jaccard similarity.""" + if len(items) <= 1: + return list(items) + + def _ngrams(text: str, n: int = 3) -> set: + words = text.lower().split() + if len(words) <= n: + return set(words) + return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} + + kept: List[str] = [items[0]] + kept_ngrams = [_ngrams(items[0])] + for item in items[1:]: + item_ng = _ngrams(item) + is_dup = False + for kng in kept_ngrams: + if not item_ng or not kng: + continue + inter = len(item_ng & kng) + union = len(item_ng | kng) + if union > 0 and inter / union >= threshold: + is_dup = True + break + if not is_dup: + kept.append(item) + kept_ngrams.append(item_ng) + return kept + + +def _is_similar_to_any(candidate: str, existing: List[str], threshold: float = 0.75) -> bool: + """Check if candidate is semantically similar to any existing item.""" + if not existing: + return False + + def _ngrams(text: str, n: int = 3) -> set: + words = text.lower().split() + if len(words) <= n: + return set(words) + return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} + + cand_ng = _ngrams(candidate) + if not cand_ng: + return False + for item in existing: + item_ng = _ngrams(item) + if not item_ng: + continue + inter = len(cand_ng & item_ng) + union = len(cand_ng | item_ng) + if union > 0 and inter / union >= threshold: + return True + return False + + +def _deduplicate_research_directions_by_focus( + directions: List[Dict[str, Any]], + threshold: float = 0.78, +) -> List[Dict[str, Any]]: + """Remove near-duplicate research directions using n-gram Jaccard on focus+rationale text. + + Directions of the same type with highly overlapping focus/rationale are merged, + keeping the first (LLM-generated) one and discarding later near-duplicates. + """ + if len(directions) <= 1: + return list(directions) + + def _ngrams(text: str, n: int = 3) -> set: + words = text.lower().split() + if len(words) <= n: + return set(words) + return {" ".join(words[i:i + n]) for i in range(len(words) - n + 1)} + + kept: List[Dict[str, Any]] = [directions[0]] + kept_texts: List[set] = [_ngrams( + f"{directions[0].get('focus', '')} {directions[0].get('rationale', '')}" + )] + + for direction in directions[1:]: + text = f"{direction.get('focus', '')} {direction.get('rationale', '')}" + item_ng = _ngrams(text) + if not item_ng: + kept.append(direction) + kept_texts.append(item_ng) + continue + + is_dup = False + for kng in kept_texts: + if not kng: + continue + inter = len(item_ng & kng) + union = len(item_ng | kng) + if union > 0 and inter / union >= threshold: + logger.debug( + "Dropping near-duplicate research direction: %s (focus: %s)", + direction.get("title", ""), + direction.get("focus", "")[:80], + ) + is_dup = True + break + if not is_dup: + kept.append(direction) + kept_texts.append(item_ng) + + return kept + + def _gap_relevance_score(gap: Any, focus_terms: List[str]) -> float: text = " ".join([ str(getattr(gap, "direction", "") or ""), @@ -1234,7 +1630,7 @@ def _candidate_similarity_key(candidate: IdeaCandidate) -> set[str]: } tokens = { token - for token in re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]{2,}", text) + for token in tokenize_topic_text(text) if token not in stopwords } alias_groups = [ @@ -1274,6 +1670,18 @@ def _normalized_topic_text(*parts: Any) -> str: _APPLICATION_DOMAIN_DRIFT_GROUPS = [ + { + "label": "climate science or weather modeling", + "phrases": [ + "climate science", + "climate model", + "climate modeling", + "climate modelling", + "weather model", + "weather forecasting", + ], + "terms": {"climate", "weather", "meteorological"}, + }, { "label": "carbon market / climate finance", "phrases": [ @@ -1360,15 +1768,22 @@ def _topic_phrase_tokens(text: str) -> List[str]: ] -def _candidate_unrequested_application_phrase_issues(seed_query: str, candidate: IdeaCandidate) -> List[str]: - seed_tokens = set(_topic_phrase_tokens(seed_query)) +def _candidate_unrequested_application_phrase_issues( + seed_query: str, + candidate: IdeaCandidate, + english_search_queries: Optional[List[str]] = None, +) -> List[str]: + seed_text = seed_query + if english_search_queries and re.search(r'[\u4e00-\u9fff]', seed_query or ""): + seed_text = " ".join([seed_query, *english_search_queries]) + seed_tokens = set(_topic_phrase_tokens(seed_text)) if not seed_tokens: return [] - phrase_candidates: List[str] = [] - for source_text in [ - _normalized_topic_text(candidate.title), - _normalized_topic_text(candidate.problem), - _normalized_topic_text(candidate.keyInsight), + phrase_candidates: List[tuple[str, bool]] = [] + for source_text, is_title in [ + (_normalized_topic_text(candidate.title), True), + (_normalized_topic_text(candidate.problem), False), + (_normalized_topic_text(candidate.keyInsight), False), ]: for match in re.finditer( r"\b(?:for|in|on|within|across)\s+([a-z][a-z0-9\s]{4,80}?)(?:[.;,:]|\bwith\b|\busing\b|\buse\b|\buses\b|\bneeds\b|\bshould\b|\bcan\b|$)", @@ -1376,10 +1791,10 @@ def _candidate_unrequested_application_phrase_issues(seed_query: str, candidate: ): phrase = " ".join(match.group(1).split()) if phrase: - phrase_candidates.append(phrase) + phrase_candidates.append((phrase, is_title)) issues: List[str] = [] - for phrase in phrase_candidates[:8]: + for phrase, is_title in phrase_candidates[:8]: tokens = _topic_phrase_tokens(phrase) if len(tokens) < 2: continue @@ -1390,10 +1805,9 @@ def _candidate_unrequested_application_phrase_issues(seed_query: str, candidate: any(token in _APPLICATION_PHRASE_SUFFIXES for token in tokens) or tokens[-1] in _APPLICATION_PHRASE_SUFFIXES ) - if not has_application_shape: - continue novelty_ratio = len(novel_tokens) / max(1, len(tokens)) - if novelty_ratio >= 0.6: + title_domain_shape = is_title and novelty_ratio >= 0.75 + if (has_application_shape and novelty_ratio >= 0.6) or title_domain_shape: issues.append( "Candidate shows topic drift: unrequested application phrase " f"'{phrase}' is central but absent from the seed query." @@ -1457,7 +1871,11 @@ def _candidate_topic_drift_issues( issues.append( f"Candidate shows topic drift: unrequested {label} anchor '{marker}' is central but absent from the seed query." ) - issues.extend(_candidate_unrequested_application_phrase_issues(seed_query, candidate)) + issues.extend(_candidate_unrequested_application_phrase_issues( + seed_query, + candidate, + english_search_queries=english_search_queries, + )) return list(dict.fromkeys(issues)) @@ -1892,7 +2310,187 @@ def _build_literature_repair_queries( queries.append(query) if len(queries) >= limit: break - return queries or [seed] + return self._anchor_repair_queries(session, queries or [seed]) + + def _anchor_repair_queries( + self, + session: IdeaSession, + queries: List[str], + ) -> List[str]: + """Keep CJK coverage repair queries anchored to the translated topic.""" + seed = session.config.seedQuery.strip() + if not re.search(r"[\u4e00-\u9fff]", seed): + return list(dict.fromkeys(query for query in queries if query.strip())) + + role_queries = self._get_step_output( + session, + "expandQuery", + "searchQueriesByRole", + {}, + ) or {} + english_queries = self._get_step_output( + session, + "expandQuery", + "englishSearchQueries", + [], + ) or [] + anchors = [ + str(query).strip() + for role in ("domain", "task") + for query in (role_queries.get(role, []) if isinstance(role_queries, dict) else []) + if str(query).strip() + ] + anchors.extend(str(query).strip() for query in english_queries if str(query).strip()) + anchor = next(iter(dict.fromkeys(anchors)), "") + if not anchor: + return list(dict.fromkeys(query for query in queries if query.strip())) + + anchored: List[str] = [] + for raw_query in queries: + query = re.sub(r"\s+", " ", str(raw_query or "")).strip() + if not query: + continue + suffix = re.sub(re.escape(seed), " ", query, flags=re.IGNORECASE) + suffix = re.sub(r"\s+", " ", suffix).strip() + if anchor.lower() in query.lower(): + combined = query + else: + combined = " ".join(part for part in [anchor, suffix] if part).strip() + if combined and combined not in anchored: + anchored.append(combined) + return anchored + + def _core_search_queries(self, session: IdeaSession) -> List[str]: + """Return domain/task queries used as immutable topic anchors.""" + role_queries = self._get_step_output( + session, + "expandQuery", + "searchQueriesByRole", + {}, + ) or {} + core: List[str] = [] + if isinstance(role_queries, dict): + for role in ("domain", "task"): + for query in _as_string_list(role_queries.get(role, []), limit=3): + if query not in core: + core.append(query) + if core: + return core + english = _as_string_list( + self._get_step_output(session, "expandQuery", "englishSearchQueries", []), + limit=4, + ) + if english: + return english + expanded = _as_string_list( + self._get_step_output(session, "expandQuery", "expandedTerms", []), + limit=4, + ) + return expanded or [session.config.seedQuery] + + @staticmethod + def _cjk_query_roles(data: Dict[str, Any]) -> Dict[str, List[str]]: + roles: Dict[str, List[str]] = { + "domain": [], + "task": [], + "method": [], + "evaluation": [], + } + nested = data.get("englishQueryRoles", {}) + if not isinstance(nested, dict): + nested = {} + field_names = { + "domain": "domainQueries", + "task": "taskQueries", + "method": "methodQueries", + "evaluation": "evaluationQueries", + } + for role, field_name in field_names.items(): + values = nested.get(role, data.get(field_name, [])) + roles[role] = _as_string_list(values, limit=2) + return roles + + @staticmethod + def _fallback_english_query_roles( + *, + seed: str, + domain: str, + expanded_terms: List[str], + data: Optional[Dict[str, Any]] = None, + ) -> Dict[str, List[str]]: + provided = IdeaGenerationService._cjk_query_roles(data or {}) + if any(provided.values()): + return provided + + queries = list(dict.fromkeys( + str(query).strip() for query in expanded_terms if str(query).strip() + )) + method_markers = ( + "algorithm", + "approach", + "framework", + "mechanism", + "method", + "model", + "technique", + "using", + ) + evaluation_markers = ( + "benchmark", + "challenge", + "evaluation", + "experiment", + "limitation", + "metric", + "reliability", + ) + method_queries = [ + query for query in queries + if any(marker in query.lower() for marker in method_markers) + ][:2] + evaluation_queries = [ + query for query in queries + if any(marker in query.lower() for marker in evaluation_markers) + ][:2] + domain_query = " ".join( + part for part in [seed, domain if domain != "general" else ""] if part + ).strip() + return { + "domain": list(dict.fromkeys([domain_query or seed, *queries[:1]]))[:2], + "task": queries[:2] or [seed], + "method": method_queries or [f"{seed} methods techniques algorithms"], + "evaluation": evaluation_queries or [ + f"{seed} evaluation benchmark metrics limitations" + ], + } + + def _translate_cjk_query_roles( + self, + session: IdeaSession, + client: Any, + ) -> tuple[Dict[str, List[str]], int]: + messages = [ + ChatMessage( + role="system", + content=( + "Translate a Chinese research topic into role-specific English academic " + "queries. Preserve named works, entities, domain, and research intent." + ), + ), + ChatMessage( + role="user", + content=( + f"Chinese topic: {session.config.seedQuery}\n" + f"Paper type: {session.config.paperType}\n\n" + "Return JSON with domainQueries, taskQueries, methodQueries, and " + "evaluationQueries. Each field must contain one or two English queries." + ), + ), + ] + response = client.chat(messages, model=session.config.model, max_tokens=350) + return self._cjk_query_roles(_extract_json_object(response.text) or {}), int( + getattr(response, "latency_ms", 0) or 0 + ) def _load_evidence_gate_inputs(self, session: IdeaSession) -> tuple[List[StructuredPaper], Optional[LiteratureMap], Dict[str, Any]]: structured_papers = self.structured_storage.list_by_session(session.id) @@ -1905,6 +2503,14 @@ def _load_evidence_gate_inputs(self, session: IdeaSession) -> tuple[List[Structu break return structured_papers, literature_map, gap_outputs + def _paper_roles_for_session(self, session_id: str) -> Dict[str, List[str]]: + roles: Dict[str, List[str]] = {} + for paper in self.raw_paper_storage.list_by_session(session_id): + paper_roles = list(getattr(paper, "retrievalRoles", []) or []) + if paper_roles: + roles[paper.id] = paper_roles + return roles + def _evaluate_evidence_gate_for_session( self, session: IdeaSession, @@ -1914,8 +2520,9 @@ def _evaluate_evidence_gate_for_session( seed = session.config.seedQuery domain = session.config.domain or "" paper_type = session.config.paperType - extra_terms = self._get_step_output(session, "expandQuery", "expandedTerms", []) + extra_terms = self._core_search_queries(session) structured_papers, literature_map, gap_outputs = self._load_evidence_gate_inputs(session) + paper_roles = self._paper_roles_for_session(session.id) rule_gate = _evaluate_evidence_gate_v2( seed=seed, domain=domain, @@ -1925,6 +2532,7 @@ def _evaluate_evidence_gate_for_session( gap_outputs=gap_outputs, stage=stage, extra_terms=extra_terms, + paper_roles=paper_roles, ) coverage_report = self._run_evidence_coverage_llm_reviewer( session=session, @@ -1994,7 +2602,7 @@ def _step_evidence_gate(self, session: IdeaSession) -> tuple: if not final_gate.get("passed", False): errors = "; ".join(final_gate.get("errors", [])[:4]) - raise StepContextError( + raise AwaitingEvidenceError( f"Evidence Gate 2.0 failed before idea generation: {errors}", inputs=inputs, outputs=outputs, @@ -2039,6 +2647,7 @@ def _repair_evidence_pool_before_idea_brainstorm( query, limit=max(8, min(24, session.config.maxPapers // max(1, len(queries)))), ) + _tag_repair_results(batch, query) results.extend(batch) except Exception as exc: logger.warning("Pre-idea evidence repair search failed for '%s': %s", query, exc) @@ -2093,12 +2702,17 @@ def _run_evidence_coverage_llm_reviewer( ) -> Dict[str, Any]: """LLM-first coverage planner/mapper with rule verification fallback.""" + paper_tiers = { + paper.id: paper.evidenceTier + for paper in self.raw_paper_storage.list_by_session(session.id) + } rule_fallback = _rule_seed_coverage_report( seed=session.config.seedQuery, paper_type=session.config.paperType, structured_papers=structured_papers, literature_map=literature_map, gap_outputs=gap_outputs, + paper_tiers=paper_tiers, ) if not _env_bool("FAROS_EVIDENCE_COVERAGE_LLM_ENABLED", True): rule_fallback["warnings"] = [ @@ -2120,6 +2734,7 @@ def _run_evidence_coverage_llm_reviewer( "paperId": paper.id, "title": paper.title, "source": paper.source, + "evidenceTier": paper_tiers.get(paper.id, "unclassified"), "summary": paper.summary[:650], "abstract": paper.abstract[:450], "claims": [claim.text[:240] for claim in paper.claims[:4]], @@ -2261,6 +2876,7 @@ def _run_evidence_coverage_llm_reviewer( report = _normalize_coverage_report( data, available_paper_ids={paper.id for paper in structured_papers}, + paper_tiers=paper_tiers, source="llm", ) report["latencyMs"] = response.latency_ms @@ -2482,6 +3098,23 @@ def start_session(self, session_id: str) -> IdeaSession: ) return self.session_storage.update(session) + + def resume_session(self, session_id: str) -> IdeaSession: + """Resume a session paused for evidence or candidate regeneration.""" + session = self.session_storage.get(session_id) + if not session: + raise ValueError(f"Session {session_id} not found") + if session.status not in { + IdeaSessionStatus.AWAITING_EVIDENCE, + IdeaSessionStatus.AWAITING_IDEAS, + }: + raise ValueError(f"Cannot resume session in {session.status} state") + session.status = IdeaSessionStatus.RUNNING + session.endedAt = None + session.errorMessage = None + if session.trace: + session.trace.endedAt = None + return self.session_storage.update(session) def cancel_session(self, session_id: str) -> IdeaSession: """Cancel a running session.""" @@ -2517,7 +3150,7 @@ def get_candidates(self, session_id: str, view: str = "final") -> List[IdeaCandi session = self.session_storage.get(session_id) final_ids = list(getattr(session, "finalCandidateIds", []) or []) if session else [] if not final_ids: - return candidates + return [] by_id = {candidate.id: candidate for candidate in candidates} return [by_id[candidate_id] for candidate_id in final_ids if candidate_id in by_id] @@ -2866,7 +3499,7 @@ def _select_final_candidates( Internal generation may keep many candidates for exploration and repair. This selector exposes only diverse, review-passed ideas by default. """ - target_count = max(1, min(2, max_count, len(ranked))) + target_count = 2 scored = sorted( ranked, key=lambda candidate: self._direction_aware_quality_score( @@ -3010,8 +3643,7 @@ def _target_final_candidate_count( ) -> int: """Target at least two user-facing ideas when the pool can support it.""" - configured_max = max(1, getattr(session.config, "maxCandidates", 1) or 1) - return max(1, min(2, configured_max, len(ranked))) + return 2 def _current_final_ready_candidates( self, @@ -3428,38 +4060,62 @@ def run_pipeline(self, session_id: str) -> IdeaSession: raise ValueError(f"Session must be in RUNNING state, got {session.status}") try: - # Step 1: Expand Query - session = self._run_step(session, "expandQuery", self._step_expand_query) - - # Step 2: Literature Search - session = self._run_step(session, "literatureSearch", self._step_literature_search) - - # Step 3: Novelty Check - session = self._run_step(session, "noveltyCheck", self._step_novelty_check) - - # Step 4: Gap Analysis - session = self._run_step(session, "gapAnalysis", self._step_gap_analysis) - - # Step 5: Evidence Gate 2.0 (repair literature before idea generation) - session = self._run_step(session, "evidenceGate", self._step_evidence_gate) - - # Step 6: Idea Brainstorm (uses LLM) - session = self._run_step(session, "ideaBrainstorm", self._step_idea_brainstorm) - - # Step 7: Rank Candidates - session = self._run_step(session, "rankCandidates", self._step_rank_candidates) + pipeline_steps = [ + ("expandQuery", self._step_expand_query), + ("literatureSearch", self._step_literature_search), + ("noveltyCheck", self._step_novelty_check), + ("gapAnalysis", self._step_gap_analysis), + ("evidenceGate", self._step_evidence_gate), + ("ideaBrainstorm", self._step_idea_brainstorm), + ("rankCandidates", self._step_rank_candidates), + ("finalizeSession", self._step_finalize), + ] + resume_from = str((session.qualityLoopSummary or {}).get("resumeFrom", "")) + start_index = next( + ( + index + for index, (step_name, _) in enumerate(pipeline_steps) + if step_name == resume_from + ), + 0, + ) + for step_name, step_func in pipeline_steps[start_index:]: + session = self._run_step(session, step_name, step_func) - # Step 8: Finalize - session = self._run_step(session, "finalizeSession", self._step_finalize) + if len(session.finalCandidateIds) < 2: + raise AwaitingIdeasError( + "Idea session cannot complete with fewer than two approved candidates" + ) # Mark completed session.status = IdeaSessionStatus.COMPLETED + session.qualityLoopSummary = { + **(session.qualityLoopSummary or {}), + "qualityStatus": "ready", + "requiresRegeneration": False, + } session.endedAt = _utcnow() if session.trace: session.trace.endedAt = _utcnow() return self.session_storage.update(session) + except RecoverableIdeaError as e: + logger.warning("Pipeline paused for session %s: %s", session_id, e) + session = self.session_storage.get(session_id) or session + session.status = e.waiting_status + session.errorMessage = str(e) + session.endedAt = None + session.qualityLoopSummary = { + **(session.qualityLoopSummary or {}), + "qualityStatus": e.waiting_status.value, + "requiresRegeneration": True, + "resumeFrom": e.resume_from, + "blockingReason": str(e), + } + if session.trace: + session.trace.endedAt = None + return self.session_storage.update(session) except Exception as e: logger.error(f"Pipeline failed for session {session_id}: {e}") session.status = IdeaSessionStatus.FAILED @@ -3498,9 +4154,7 @@ def _run_step( ) if session.trace: - session.trace.steps.append(step_result) - session.trace.totalSteps += 1 - session.trace.successfulSteps += 1 + _record_step_result(session.trace, step_result) return self.session_storage.update(session) @@ -3524,9 +4178,7 @@ def _run_step( ) if session.trace: - session.trace.steps.append(step_result) - session.trace.totalSteps += 1 - session.trace.failedSteps += 1 + _record_step_result(session.trace, step_result) self.session_storage.update(session) raise @@ -3547,7 +4199,8 @@ def _step_expand_query(self, session: IdeaSession) -> tuple: user_prompt = prompts.EXPAND_QUERY_USER.format( seed_query=seed, paper_type=paper_type, - domain=domain + domain=domain, + literature_context="", # P1: RAG-ready — populated on retry ) is_cjk = bool(re.search(r'[\u4e00-\u9fff]', seed)) if is_cjk: @@ -3576,16 +4229,51 @@ def _step_expand_query(self, session: IdeaSession) -> tuple: ) expanded_terms = _clean_query_terms(raw_queries, seed) english_search_queries: list[str] = [] + search_queries_by_role: Dict[str, List[str]] = { + "domain": [], "task": [], "method": [], "evaluation": [], + } + translation_status = "not_required" + translation_latency_ms = 0 if is_cjk: english_search_queries = [ str(q).strip() for q in data.get("englishSearchQueries", []) if isinstance(q, str) and q.strip() ][:5] + search_queries_by_role = self._cjk_query_roles(data) + if not english_search_queries and any(search_queries_by_role.values()): + english_search_queries = list(dict.fromkeys( + query + for role in ("domain", "task", "method", "evaluation") + for query in search_queries_by_role[role] + ))[:6] + if not any(search_queries_by_role.values()) and english_search_queries: + search_queries_by_role["domain"] = english_search_queries[:2] + search_queries_by_role["task"] = english_search_queries[2:4] + if not english_search_queries: + search_queries_by_role, translation_latency_ms = self._translate_cjk_query_roles( + session, + client, + ) + english_search_queries = list(dict.fromkeys( + query + for role in ("domain", "task", "method", "evaluation") + for query in search_queries_by_role[role] + ))[:6] + translation_status = "fallback" if english_search_queries else "missing" + else: + translation_status = "primary" # Prepend English queries to expanded terms so they get # searched first against international databases if english_search_queries: expanded_terms = english_search_queries + expanded_terms + else: + search_queries_by_role = self._fallback_english_query_roles( + seed=seed, + domain=domain, + expanded_terms=expanded_terms, + data=data, + ) key_concepts = [ str(item).strip() for item in data.get("keyConcepts", []) @@ -3650,6 +4338,9 @@ def _step_expand_query(self, session: IdeaSession) -> tuple: "queryPlan": query_plan.model_dump(), "llmLatencyMs": response.latency_ms, "englishSearchQueries": english_search_queries if is_cjk else [], + "searchQueriesByRole": search_queries_by_role, + "translationStatus": translation_status, + "translationLatencyMs": translation_latency_ms, } except Exception as e: @@ -3663,25 +4354,36 @@ def _step_expand_query(self, session: IdeaSession) -> tuple: # fails, the seed is used as-is (OpenAlex may still find results). is_cjk = bool(re.search(r'[\u4e00-\u9fff]', seed)) english_search_queries: list[str] = [] + search_queries_by_role: Dict[str, List[str]] = { + "domain": [], "task": [], "method": [], "evaluation": [], + } if is_cjk: try: client = get_provider_client(session.config.providerName) - trans_messages = [ - ChatMessage(role="system", content="You are a translation assistant. Translate the given Chinese research topic into 3 English academic search queries."), - ChatMessage(role="user", content=f"Chinese topic: {seed}\n\nReturn JSON: {{\"queries\": [\"English query 1\", \"English query 2\", \"English query 3\"]}}"), - ] - trans_resp = client.chat(trans_messages, model=session.config.model, max_tokens=200) - trans_data = _extract_json_object(trans_resp.text) or {} - english_search_queries = [ - str(q).strip() - for q in trans_data.get("queries", []) - if isinstance(q, str) and q.strip() - ] + search_queries_by_role, translation_latency_ms = self._translate_cjk_query_roles( + session, + client, + ) + english_search_queries = list(dict.fromkeys( + query + for role in ("domain", "task", "method", "evaluation") + for query in search_queries_by_role[role] + ))[:6] if english_search_queries: expanded_terms = english_search_queries + expanded_terms logger.info(f"CJK fallback translation succeeded: {english_search_queries}") except Exception as trans_e: logger.warning(f"CJK fallback translation also failed: {trans_e}") + search_queries_by_role = { + "domain": [], "task": [], "method": [], "evaluation": [], + } + translation_latency_ms = 0 + else: + search_queries_by_role = self._fallback_english_query_roles( + seed=seed, + domain=domain, + expanded_terms=expanded_terms, + ) query_plan = QueryPlan( refinedQuestion=seed, @@ -3704,8 +4406,20 @@ def _step_expand_query(self, session: IdeaSession) -> tuple: "error": str(e), "cjkTranslationApplied": bool(english_search_queries), "englishSearchQueries": english_search_queries, + "searchQueriesByRole": search_queries_by_role, + "translationStatus": ( + "fallback" if english_search_queries else "missing" + ) if is_cjk else "not_required", + "translationLatencyMs": translation_latency_ms if is_cjk else 0, } + if re.search(r"[\u4e00-\u9fff]", seed) and not outputs.get("englishSearchQueries"): + raise AwaitingTranslationError( + "CJK query expansion did not produce usable English academic queries", + inputs=inputs, + outputs=outputs, + ) + return inputs, outputs, [] def _step_literature_search(self, session: IdeaSession) -> tuple: @@ -3719,76 +4433,113 @@ def _step_literature_search(self, session: IdeaSession) -> tuple: seed = session.config.seedQuery max_papers = session.config.maxPapers - # Get expanded terms from Step 1 - search_queries = [seed] - if session.trace: - for step in session.trace.steps: - if step.name == "expandQuery" and step.outputs.get("expandedTerms"): - search_queries = step.outputs["expandedTerms"] - break + role_queries = self._get_step_output( + session, + "expandQuery", + "searchQueriesByRole", + {}, + ) or {} + query_specs: List[tuple[str, str]] = [] + if isinstance(role_queries, dict): + for role in ("domain", "task", "method", "evaluation"): + for query in _as_string_list(role_queries.get(role, []), limit=2): + if all(existing_query != query for _, existing_query in query_specs): + query_specs.append((role, query)) + if not query_specs: + expanded = _as_string_list( + self._get_step_output(session, "expandQuery", "expandedTerms", []), + limit=3, + ) + query_specs = [("core", query) for query in (expanded or [seed])] + search_queries = [query for _, query in query_specs] + core_queries = self._core_search_queries(session) + profile = build_topic_intent_profile( + seed=seed, + domain=session.config.domain or "", + role_queries=role_queries if isinstance(role_queries, dict) else {}, + ) + must_cite_refs = [ + str(value).lower().strip() + for value in (session.config.mustCiteList or []) + if str(value).strip() + ] # Search across sources search_service = get_search_service() all_results: List[SearchResult] = [] - for query in search_queries[:3]: + per_query_limit = max(5, min(20, max_papers // max(1, len(query_specs)))) + query_result_counts: Dict[str, int] = {} + for role, query in query_specs: try: - results = search_service.search(query, limit=max_papers) + results = search_service.search(query, limit=per_query_limit) + for result in results: + if role not in result.retrieval_roles: + result.retrieval_roles.append(role) + if query not in result.matched_queries: + result.matched_queries.append(query) all_results.extend(results) + query_result_counts[f"{role}:{query}"] = len(results) logger.info(f"Search for '{query}' returned {len(results)} results") except Exception as e: + query_result_counts[f"{role}:{query}"] = 0 logger.warning(f"Search failed for '{query}': {e}") - def _dedupe_rank_filter(results: List[SearchResult], queries: List[str]) -> tuple[List[SearchResult], int, int]: - # Dedup chain: doi > arxivId > semanticScholarId > normalized title hash - seen_dois: set = set() - seen_arxiv_ids: set = set() - seen_s2_ids: set = set() - seen_title_hashes: set = set() - unique: List[SearchResult] = [] - - for result in results: - if result.doi and result.doi in seen_dois: - continue - if result.arxiv_id and result.arxiv_id in seen_arxiv_ids: - continue - s2_id = None - if result.source == "semantic_scholar" and result.url: - s2_match = re.search(r'SemanticScholarID:(\w+)', result.url) - if s2_match: - s2_id = s2_match.group(1) - if s2_id and s2_id in seen_s2_ids: - continue - title_hash = _compute_title_hash(result.title) - if title_hash in seen_title_hashes: - continue - - if result.doi: - seen_dois.add(result.doi) - if result.arxiv_id: - seen_arxiv_ids.add(result.arxiv_id) - if s2_id: - seen_s2_ids.add(s2_id) - seen_title_hashes.add(title_hash) - unique.append(result) - - unique = _rank_results_for_topic( - unique, - seed=seed, - domain=session.config.domain or "", - search_queries=queries, + def _matches_must_cite(result: SearchResult) -> bool: + haystack = " ".join( + str(value) + for value in [result.doi, result.arxiv_id, result.url, result.title] + if value + ).lower() + return any(reference in haystack for reference in must_cite_refs) + + def _dedupe_assess_rank(results: List[SearchResult]) -> tuple: + dedupe = deduplicate_search_results(results) + persistable: List[SearchResult] = [] + gate_eligible: List[SearchResult] = [] + rejected: List[SearchResult] = [] + for result in dedupe.results: + assessment = assess_search_result(result, profile) + result.evidence_tier = assessment.tier.value + result.decisive_anchors = list(assessment.decisive_anchors) + result.relevance_components = dict(assessment.score_components) + result.rejection_reason = assessment.rejection_reason + result.relevance_score = assessment.score + if assessment.tier is not EvidenceTier.REJECTED: + persistable.append(result) + gate_eligible.append(result) + else: + result.must_cite_override = _matches_must_cite(result) + rejected.append(result) + if result.must_cite_override: + persistable.append(result) + persistable.sort(key=lambda item: item.relevance_score, reverse=True) + gate_eligible.sort(key=lambda item: item.relevance_score, reverse=True) + return ( + persistable, + gate_eligible, + rejected, + dedupe.merge_count, + len(dedupe.results), ) - ranked = len(unique) - filtered, dropped = _filter_results_for_topic(unique) - return filtered, dropped, ranked - unique_results, filtered_out_count, ranked_count = _dedupe_rank_filter(all_results, search_queries) + ( + unique_results, + gate_eligible_results, + rejected_results, + duplicate_merge_count, + ranked_count, + ) = _dedupe_assess_rank(all_results) + filtered_out_count = len([ + result for result in rejected_results if not result.must_cite_override + ]) raw_quality_gate = _evaluate_paper_quality_gate( seed=seed, domain=session.config.domain or "", - papers=unique_results, + papers=gate_eligible_results, stage="literatureSearch.initial", - extra_terms=search_queries, + extra_terms=core_queries, + paper_type=session.config.paperType, ) repair_queries: List[str] = [] repair_attempted = False @@ -3802,20 +4553,68 @@ def _dedupe_rank_filter(results: List[SearchResult], queries: List[str]) -> tupl for query in repair_queries: try: results = search_service.search(query, limit=max(8, max_papers // max(1, len(repair_queries)))) + _tag_repair_results(results, query) all_results.extend(results) + query_result_counts[f"repair:{query}"] = len(results) logger.info(f"Repair search for '{query}' returned {len(results)} results") except Exception as e: + query_result_counts[f"repair:{query}"] = 0 logger.warning(f"Repair search failed for '{query}': {e}") - unique_results, filtered_out_count, ranked_count = _dedupe_rank_filter( - all_results, - [*search_queries, *repair_queries], - ) + ( + unique_results, + gate_eligible_results, + rejected_results, + duplicate_merge_count, + ranked_count, + ) = _dedupe_assess_rank(all_results) + filtered_out_count = len([ + result for result in rejected_results if not result.must_cite_override + ]) raw_quality_gate = _evaluate_paper_quality_gate( seed=seed, domain=session.config.domain or "", - papers=unique_results, + papers=gate_eligible_results, stage="literatureSearch.repaired", - extra_terms=[*search_queries, *repair_queries], + extra_terms=core_queries, + paper_type=session.config.paperType, + ) + + if not raw_quality_gate.get("passed", False): + diagnostic_outputs = { + "searchQueries": search_queries, + "coreSearchQueries": core_queries, + "searchQueriesByRole": role_queries, + "queryResultCounts": query_result_counts, + "topicIntentProfile": profile.to_dict(), + "resultCountBeforeDedup": len(all_results), + "uniqueResultCount": ranked_count, + "duplicateMergeCount": duplicate_merge_count, + "evidenceTierCounts": { + "direct": sum( + result.evidence_tier == "direct" for result in unique_results + ), + "transferable": sum( + result.evidence_tier == "transferable" for result in unique_results + ), + "rejected": len(rejected_results), + }, + "rejectionReasonCounts": dict(Counter( + result.rejection_reason for result in rejected_results + )), + "filteredOutCount": filtered_out_count, + "paperQualityGate": raw_quality_gate, + "repairAttempted": repair_attempted, + "repairQueries": repair_queries, + } + errors = "; ".join(raw_quality_gate.get("errors", [])[:4]) + raise AwaitingLiteratureEvidenceError( + f"Literature evidence is insufficient before deep reading: {errors}", + inputs={ + "seedQuery": seed, + "maxPapers": max_papers, + "searchQueries": search_queries, + }, + outputs=diagnostic_outputs, ) if ranked_count and not unique_results: @@ -3828,8 +4627,9 @@ def _dedupe_rank_filter(results: List[SearchResult], queries: List[str]) -> tupl unique_results = unique_results[:max_papers] sources_used: List[str] = [] for result in unique_results: - if result.source not in sources_used: - sources_used.append(result.source) + for source in result.retrieval_sources or [result.source]: + if source and source not in sources_used: + sources_used.append(source) # Create RawPaper objects raw_papers: List[RawPaper] = [] @@ -3860,7 +4660,14 @@ def _dedupe_rank_filter(results: List[SearchResult], queries: List[str]) -> tupl semanticScholarId=s2_id, citationCount=result.citation_count or 0, abstract=result.abstract or "", - source=[result.source] if result.source else [], + source=list(result.retrieval_sources or ([result.source] if result.source else [])), + retrievalRoles=list(result.retrieval_roles), + matchedQueries=list(result.matched_queries), + evidenceTier=result.evidence_tier, + decisiveAnchors=list(result.decisive_anchors), + relevanceComponents=dict(result.relevance_components), + rejectionReason=result.rejection_reason, + mustCiteOverride=result.must_cite_override, normalizedTitleHash=title_hash, relevanceScore=min(1.0, max(0.0, base_score)), ) @@ -3893,13 +4700,33 @@ def _dedupe_rank_filter(results: List[SearchResult], queries: List[str]) -> tupl ) self.graph_storage.create(graph) - inputs = {"seedQuery": seed, "maxPapers": max_papers, "searchQueries": search_queries[:3]} + retrieval_role_counts = { + role: sum(1 for result in unique_results if role in result.retrieval_roles) + for role in ("domain", "task", "method", "evaluation", "repair") + } + inputs = {"seedQuery": seed, "maxPapers": max_papers, "searchQueries": search_queries} outputs = { "paperCount": len(raw_papers), "rawPaperIds": [p.id for p in raw_papers], "graphId": graph.id, "sourcesUsed": sources_used, - "searchQueries": search_queries[:3], + "searchQueries": search_queries, + "coreSearchQueries": core_queries, + "searchQueriesByRole": role_queries, + "queryResultCounts": query_result_counts, + "retrievalRoleCounts": retrieval_role_counts, + "topicIntentProfile": profile.to_dict(), + "resultCountBeforeDedup": len(all_results), + "uniqueResultCount": ranked_count, + "duplicateMergeCount": duplicate_merge_count, + "evidenceTierCounts": { + "direct": sum(result.evidence_tier == "direct" for result in unique_results), + "transferable": sum(result.evidence_tier == "transferable" for result in unique_results), + "rejected": len(rejected_results), + }, + "rejectionReasonCounts": dict(Counter( + result.rejection_reason for result in rejected_results + )), "filteredOutCount": filtered_out_count, "minExternalRelevance": float(os.getenv("FAROS_MIN_EXTERNAL_RELEVANCE", "0.12")), "minLocalRelevance": float(os.getenv("FAROS_MIN_LOCAL_RELEVANCE", "0.28")), @@ -3946,17 +4773,29 @@ def _step_novelty_check( graph = self.graph_builder.cluster_papers(graph) # Step 3b: Select papers by role - num_select = min(40, max(5, len(raw_papers) // 2)) + num_select = min(_deep_read_max_papers(), len(raw_papers)) graph, selected_paper_ids = self.graph_builder.select_papers( graph, num_select=num_select, must_cite_list=must_cite_list ) + raw_by_id = {paper.id: paper for paper in raw_papers} + selected_paper_ids = _limit_deep_read_selection( + selected_paper_ids, + raw_by_id, + limit=num_select, + ) + selected_set = set(selected_paper_ids) + graph = graph.model_copy(update={ + "nodes": [ + node.model_copy(update={"isSelected": node.paperId in selected_set}) + for node in graph.nodes + ] + }) forced_selected_ids: List[str] = [] if forced_raw_paper_ids: - raw_by_id = {paper.id: paper for paper in raw_papers} topic_terms = _topic_terms_from_seed( seed, session.config.domain or "", - self._get_step_output(session, "expandQuery", "expandedTerms", []), + self._core_search_queries(session), ) forced_candidates = [ paper for paper_id in forced_raw_paper_ids @@ -4016,13 +4855,14 @@ def _step_novelty_check( domain=session.config.domain or "", papers=selected_raw, stage="noveltyCheck.selectedRaw", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=paper_type, ) if not selected_quality_gate["passed"] and raw_papers: topic_terms = _topic_terms_from_seed( seed, session.config.domain or "", - self._get_step_output(session, "expandQuery", "expandedTerms", []), + self._core_search_queries(session), ) aligned_raw = sorted( raw_papers, @@ -4040,7 +4880,8 @@ def _step_novelty_check( domain=session.config.domain or "", papers=selected_raw, stage="noveltyCheck.selectedRaw.repaired", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=paper_type, ) # Step 3c: Deep-read only papers that are not already structured. @@ -4115,7 +4956,8 @@ def _step_novelty_check( domain=session.config.domain or "", papers=structured_papers, stage="noveltyCheck.structured", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=paper_type, ) # Step 3d: Build LiteratureMap @@ -4362,6 +5204,12 @@ def _step_gap_analysis(self, session: IdeaSession) -> tuple: opportunities = [] try: + # P1: Build RAG-enhanced literature context with gap signals + rag_context = self._build_rag_literature_context(session) + if rag_context: + # Merge RAG context into literature summary for richer prompts + lit_summary = f"{rag_context}\n\n---\n{lit_summary}" + client = get_provider_client(session.config.providerName) user_prompt = prompts.GAP_ANALYSIS_USER.format( seed_query=seed, @@ -4401,6 +5249,14 @@ def _step_gap_analysis(self, session: IdeaSession) -> tuple: seed_query=seed, ) + # Enforce minimum 3 independent research opportunities for multi-direction exploration + opportunities = _enforce_min_opportunities( + opportunities=opportunities, + seed=seed, + paper_type=paper_type, + gap_analysis=gap_analysis, + ) + inputs = {"topic": seed, "literatureCount": len(literature)} outputs = { # Phase 2 outputs @@ -4432,8 +5288,11 @@ def _fallback_gap_analysis(self, session: IdeaSession, seed: str, literature) -> f"Theoretical foundations of {seed}", ], "researchOpportunities": [ - f"Novel architectures for {seed}", - f"Efficient training methods for {seed}", + f"Novel architectures or algorithmic mechanisms for {seed}", + f"Comprehensive benchmarks and evaluation protocols for {seed}", + f"Robustness, safety, and failure analysis of {seed} methods", + f"End-to-end systems integrating {seed} into practical pipelines", + f"Domain-specific applications of {seed} with measurable outcomes", ], } return inputs, outputs, [] @@ -4564,35 +5423,48 @@ def _step_idea_brainstorm_bfts(self, session: IdeaSession) -> tuple: pruneDuplicateThreshold=bfts_config.pruneDuplicateThreshold, scoreWeights=bfts_config.scoreWeights, ) - for index, direction in enumerate(research_directions): - direction_seed_query = self._direction_seed_query(seed, direction) - direction_literature_context = ( + + # P1: Parallel directional BFTS — run each direction's tree + # search concurrently using ThreadPoolExecutor. + import os as _os + max_dir_workers = min( + len(research_directions), + int(_os.getenv("FAROS_BFTS_DIRECTION_CONCURRENCY", "3")) + ) + + def _run_direction_tree( + index: int, + direction: Dict[str, Any], + ) -> Tuple[int, List[IdeaCandidate], Dict[str, Any]]: + """Run one directional BFTS tree. Returns (index, candidates, summary).""" + dir_seed_query = self._direction_seed_query(seed, direction) + dir_lit_context = ( f"{literature_context}\n\n" f"Research Direction: {direction.get('title', '')}\n" f"Type: {direction.get('type', '')}\n" f"Focus: {direction.get('focus', '')}\n" f"Rationale: {direction.get('rationale', '')}" ) - direction_path_seed = path_seeds[index % len(path_seeds)] + dir_path_seed = path_seeds[index % len(path_seeds)] + dir_candidates: List[IdeaCandidate] = [] + dir_summary: Dict[str, Any] = {**direction} try: tree = BFTSSearchTree( session_id=session.id, bfts_config=direction_config, provider_name=session.config.providerName, model=session.config.model, - path_seeds=[direction_path_seed], + path_seeds=[dir_path_seed], structured_papers=structured_papers, - literature_context=direction_literature_context, - seed_query=direction_seed_query, + literature_context=dir_lit_context, + seed_query=dir_seed_query, paper_type=f"{paper_type} / {direction.get('type', '')}", ) - direction_candidates = tree.run() - self._tag_candidates_with_research_direction(direction_candidates, direction) - candidates.extend(direction_candidates) - direction_summaries.append({ - **direction, - "pathSeedId": direction_path_seed.seedId, - "generatedCandidateCount": len(direction_candidates), + dir_candidates = tree.run() + self._tag_candidates_with_research_direction(dir_candidates, direction) + dir_summary.update({ + "pathSeedId": dir_path_seed.seedId, + "generatedCandidateCount": len(dir_candidates), }) except Exception as direction_error: logger.warning( @@ -4600,12 +5472,45 @@ def _step_idea_brainstorm_bfts(self, session: IdeaSession) -> tuple: direction.get("id"), direction_error, ) - direction_summaries.append({ - **direction, - "pathSeedId": direction_path_seed.seedId, + dir_summary.update({ + "pathSeedId": dir_path_seed.seedId, "generatedCandidateCount": 0, "error": str(direction_error), }) + return index, dir_candidates, dir_summary + + from concurrent.futures import ThreadPoolExecutor, as_completed + dir_results_map: Dict[int, tuple] = {} + + if max_dir_workers <= 1 or len(research_directions) <= 1: + # Sequential fallback + for idx, direction in enumerate(research_directions): + idx, dir_cands, dir_summ = _run_direction_tree(idx, direction) + dir_results_map[idx] = (dir_cands, dir_summ) + else: + with ThreadPoolExecutor(max_workers=max_dir_workers) as executor: + future_to_idx = {} + for idx, direction in enumerate(research_directions): + future = executor.submit(_run_direction_tree, idx, direction) + future_to_idx[future] = idx + for future in as_completed(future_to_idx): + try: + idx, dir_cands, dir_summ = future.result() + dir_results_map[idx] = (dir_cands, dir_summ) + except Exception as e: + idx = future_to_idx[future] + logger.warning(f"Direction BFTS worker failed for index {idx}: {e}") + dir_results_map[idx] = ([], { + **research_directions[idx], + "error": str(e), + "generatedCandidateCount": 0, + }) + + # Collect results in original order + for idx in range(len(research_directions)): + dir_cands, dir_summ = dir_results_map.get(idx, ([], {})) + candidates.extend(dir_cands) + direction_summaries.append(dir_summ) used_directional_bfts = bool(candidates and research_directions) if not candidates: @@ -4703,6 +5608,78 @@ def _build_bfts_literature_context( ) return "\n\n".join(lines) + def _build_rag_context_for_prompt( + self, session: IdeaSession, limit: int = 5 + ) -> str: + """P1: Build RAG context snippet for LLM prompts from top selected papers. + + Extracts the most relevant structured papers for the session and formats + their titles, key claims, and open questions as context. Injected into + expandQuery/gapAnalysis prompts to improve domain precision. + """ + try: + papers = self.structured_storage.list_by_session(session.id) + if not papers: + return "" + except Exception: + return "" + + papers = sorted( + papers, + key=lambda p: getattr(p, 'relevanceScore', 0) or 0, + reverse=True, + )[:limit] + if not papers: + return "" + + lines = ["Relevant Background Literature:"] + for i, sp in enumerate(papers): + title = (sp.title or "(untitled)")[:120] + year = sp.year or "N/A" + claims = "" + if sp.claims: + claims = "; ".join(c.text[:80] for c in sp.claims[:2]) + lines.append(f"[{i+1}] {title} ({year}) — {claims}") + return "\n".join(lines) + + def _build_rag_literature_context(self, session: IdeaSession) -> str: + """Build literature context with gap signals for gap analysis prompts. + + P1: Enhanced version — includes limitations, open questions, and + contradictions from structured papers to provide richer context. + """ + try: + papers = self.structured_storage.list_by_session(session.id) + if not papers: + return "" + except Exception: + return "" + + papers = sorted( + papers, + key=lambda p: getattr(p, 'relevanceScore', 0) or 0, + reverse=True, + )[:6] + if not papers: + return "" + + lines = ["Key Literature Context (with identified gaps):"] + for i, sp in enumerate(papers): + title = (sp.title or "(untitled)")[:120] + year = sp.year or "N/A" + parts = [] + if sp.claims: + parts.append("Claims: " + "; ".join(c.text[:60] for c in sp.claims[:1])) + gaps = [ + *getattr(sp, 'openQuestions', [])[:1], + *getattr(sp, 'limitations', [])[:1], + *getattr(sp, 'methodWeaknesses', [])[:1], + ] + if gaps: + parts.append("Gaps: " + "; ".join(str(g)[:80] for g in gaps)) + lines.append(f"[{i+1}] {title} ({year})\n " + " | ".join(parts)) + return "\n".join(lines) + def _direction_decomposition_enabled(self) -> bool: return _env_bool("FAROS_IDEA_DIRECTION_DECOMPOSITION", True) @@ -4832,6 +5809,12 @@ def _normalize_seed_research_directions( if len(directions) >= target: break + # Semantic deduplication: remove directions whose focus/rationale + # is too similar (n-gram Jaccard >= 0.78) to an already-kept direction + directions = _deduplicate_research_directions_by_focus(directions) + + # Ensure ≥3 directions by filling with typed fallbacks + filled_types: set[str] = {d["type"] for d in directions} if len(directions) < min(3, target): for fallback in self._fallback_seed_research_directions( seed, @@ -4840,8 +5823,9 @@ def _normalize_seed_research_directions( ): if len(directions) >= target: break - if fallback["type"] in {direction["type"] for direction in directions}: + if fallback["type"] in filled_types: continue + filled_types.add(fallback["type"]) directions.append(fallback) return directions[:target] @@ -5473,9 +6457,10 @@ def _step_rank_candidates(self, session: IdeaSession) -> tuple: domain=domain, papers=structured_papers, stage="ideaReview.structuredPapers", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=session.config.paperType, ) - max_review_iterations = max(1, min(5, getattr(session.config, "maxReviewIterations", 2))) + max_review_iterations = max(1, min(5, getattr(session.config, "maxReviewIterations", 3))) target_final_count = self._target_final_candidate_count(session, ranked) review_iteration_summaries: List[Dict[str, Any]] = [] repaired_candidate_ids: set[str] = set() @@ -5532,7 +6517,7 @@ def _replace_candidate_analysis( gate_reports, final_ready_ids=final_ready_ids, paper_quality_gate=paper_quality_gate, - max_targets=1, + max_targets=max(1, target_final_count - len(final_ready)), skipped_candidate_ids=repaired_candidate_ids, ) if not repair_targets: @@ -5608,7 +6593,8 @@ def _replace_candidate_analysis( domain=domain, papers=structured_papers, stage=f"ideaReview.iteration{review_iteration + 1}.afterLiteratureRepair", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=session.config.paperType, ) if structured_papers: for candidate in ranked[:min(5, len(ranked))]: @@ -5921,9 +6907,9 @@ def _replace_candidate_analysis( "qualityLoopSummary": session.qualityLoopSummary, } - if gate_reports and idea_review_passed_count == 0: - raise StepContextError( - "Idea review gate failed: no candidate passed after internal repair iterations", + if len(session.finalCandidateIds) < 2: + raise AwaitingIdeasError( + "Idea review gate requires at least two approved final candidates", inputs=inputs, outputs=outputs, ) @@ -5956,42 +6942,113 @@ def _persist_repair_search_results( search_queries: Optional[List[str]] = None, ) -> Dict[str, Any]: existing_raw = self.raw_paper_storage.list_by_session(session.id) - seen_dois = {paper.doi for paper in existing_raw if paper.doi} - seen_arxiv_ids = {paper.arxivId for paper in existing_raw if paper.arxivId} - seen_title_hashes = {paper.normalizedTitleHash for paper in existing_raw if paper.normalizedTitleHash} + existing_by_identity: Dict[str, RawPaper] = {} + for paper in existing_raw: + for identity in raw_paper_identity_keys(paper): + existing_by_identity[identity] = paper created_raw_ids: List[str] = [] + updated_raw_ids: List[str] = [] created_literature_ids: List[str] = [] - - ranked = _rank_results_for_topic( - results, + queries = search_queries or [session.config.seedQuery] + role_queries = self._get_step_output( + session, + "expandQuery", + "searchQueriesByRole", + {}, + ) or {} + if not isinstance(role_queries, dict) or not any(role_queries.values()): + role_queries = { + "domain": [session.config.seedQuery, session.config.domain or ""], + "task": [session.config.seedQuery], + "method": queries, + "evaluation": queries, + } + profile = build_topic_intent_profile( seed=session.config.seedQuery, domain=session.config.domain or "", - search_queries=search_queries or [session.config.seedQuery], + role_queries=role_queries, ) - filtered, filtered_out_count = _filter_results_for_topic(ranked) - filtered.sort( + dedupe = deduplicate_search_results(results) + eligible: List[SearchResult] = [] + rejected: List[SearchResult] = [] + for result in dedupe.results: + assessment = assess_search_result(result, profile) + result.evidence_tier = assessment.tier.value + result.decisive_anchors = list(assessment.decisive_anchors) + result.relevance_components = dict(assessment.score_components) + result.rejection_reason = assessment.rejection_reason + result.relevance_score = assessment.score + if assessment.tier is EvidenceTier.REJECTED: + rejected.append(result) + else: + eligible.append(result) + eligible.sort( key=lambda result: _repair_result_priority(result, session.config.paperType), reverse=True, ) - for result in filtered: + matched_existing_count = 0 + for result in eligible: title_hash = _compute_title_hash(result.title) - if result.doi and result.doi in seen_dois: - continue - if result.arxiv_id and result.arxiv_id in seen_arxiv_ids: - continue - if title_hash in seen_title_hashes: - continue - seen_title_hashes.add(title_hash) - if result.doi: - seen_dois.add(result.doi) - if result.arxiv_id: - seen_arxiv_ids.add(result.arxiv_id) - s2_id = None if result.source == "semantic_scholar" and result.url: s2_match = re.search(r'SemanticScholarID:(\w+)', result.url) if s2_match: s2_id = s2_match.group(1) + existing = next( + ( + existing_by_identity[identity] + for identity in search_result_identity_keys(result) + if identity in existing_by_identity + ), + None, + ) + if existing: + updated = existing.model_copy(update={ + "authors": existing.authors or result.authors, + "year": existing.year or result.year, + "venue": existing.venue or result.venue, + "url": existing.url or result.url or "", + "doi": existing.doi or result.doi, + "arxivId": existing.arxivId or result.arxiv_id, + "semanticScholarId": existing.semanticScholarId or s2_id, + "citationCount": max(existing.citationCount, result.citation_count or 0), + "abstract": ( + result.abstract + if len(result.abstract or "") > len(existing.abstract or "") + else existing.abstract + ), + "source": list(dict.fromkeys([ + *existing.source, + *(result.retrieval_sources or [result.source]), + ])), + "retrievalRoles": list(dict.fromkeys([ + *existing.retrievalRoles, + *result.retrieval_roles, + ])), + "matchedQueries": list(dict.fromkeys([ + *existing.matchedQueries, + *result.matched_queries, + ])), + "evidenceTier": better_evidence_tier( + existing.evidenceTier, + result.evidence_tier, + ), + "decisiveAnchors": list(dict.fromkeys([ + *existing.decisiveAnchors, + *result.decisive_anchors, + ])), + "relevanceComponents": dict(result.relevance_components), + "rejectionReason": result.rejection_reason, + "mustCiteOverride": existing.mustCiteOverride or result.must_cite_override, + "relevanceScore": max(existing.relevanceScore, result.relevance_score), + }) + self.raw_paper_storage.update(updated) + updated_raw_ids.append(updated.id) + matched_existing_count += 1 + for identity in raw_paper_identity_keys(updated): + existing_by_identity[identity] = updated + continue + raw_paper = RawPaper( id=generate_raw_paper_id(), sessionId=session.id, @@ -5999,18 +7056,27 @@ def _persist_repair_search_results( authors=result.authors, year=result.year, venue=result.venue, - url=result.url, + url=result.url or "", doi=result.doi, arxivId=result.arxiv_id, semanticScholarId=s2_id, citationCount=result.citation_count or 0, abstract=result.abstract or "", - source=[result.source] if result.source else [], + source=list(result.retrieval_sources or ([result.source] if result.source else [])), + retrievalRoles=list(result.retrieval_roles), + matchedQueries=list(result.matched_queries), + evidenceTier=result.evidence_tier, + decisiveAnchors=list(result.decisive_anchors), + relevanceComponents=dict(result.relevance_components), + rejectionReason=result.rejection_reason, + mustCiteOverride=result.must_cite_override, normalizedTitleHash=title_hash, relevanceScore=min(1.0, max(0.0, result.relevance_score)), ) self.raw_paper_storage.create(raw_paper) created_raw_ids.append(raw_paper.id) + for identity in raw_paper_identity_keys(raw_paper): + existing_by_identity[identity] = raw_paper lit_item = LiteratureItem( id=generate_literature_id(), @@ -6019,7 +7085,7 @@ def _persist_repair_search_results( authors=result.authors, venue=result.venue, year=result.year, - url=result.url, + url=result.url or "", doi=result.doi, arxivId=result.arxiv_id, snippet=(result.abstract or "")[:500], @@ -6032,11 +7098,21 @@ def _persist_repair_search_results( all_raw = self.raw_paper_storage.list_by_session(session.id) if all_raw: graph = self.graph_builder.build_graph_v0(session_id=session.id, raw_papers=all_raw) - self.graph_storage.create(graph) + existing_graph = self.graph_storage.get_by_session(session.id) + if existing_graph: + graph = graph.model_copy(update={"id": existing_graph.id}) + self.graph_storage.update(graph) + else: + self.graph_storage.create(graph) return { "createdRawPaperIds": created_raw_ids, + "updatedRawPaperIds": updated_raw_ids, "createdLiteratureIds": created_literature_ids, - "filteredOutCount": filtered_out_count, + "filteredOutCount": len(rejected), + "duplicateMergeCount": dedupe.merge_count + matched_existing_count, + "evidenceTierCounts": dict(Counter( + result.evidence_tier for result in [*eligible, *rejected] + )), "rawPaperCountAfterRepair": len(all_raw), } @@ -6059,6 +7135,7 @@ def _repair_literature_pool_for_idea_quality( for query in queries: try: batch = search_service.search(query, limit=max(8, min(24, session.config.maxPapers // max(1, len(queries))))) + _tag_repair_results(batch, query) results.extend(batch) except Exception as exc: logger.warning("Idea-stage literature repair search failed for '%s': %s", query, exc) @@ -6093,7 +7170,8 @@ def _repair_literature_pool_for_idea_quality( domain=session.config.domain or "", papers=repaired_structured, stage="ideaReview.literatureRepair.structured", - extra_terms=self._get_step_output(session, "expandQuery", "expandedTerms", []), + extra_terms=self._core_search_queries(session), + paper_type=session.config.paperType, ) return { "attempted": True, @@ -6933,8 +8011,25 @@ def _regenerate_candidate_from_review( """Generate one improved candidate from idea-stage review feedback.""" client = get_provider_client(session.config.providerName) + blocking_issues = [ + str(issue) + for issue in review_gate.get("blockingIssues", []) + if str(issue).strip() + ] + off_topic_repair = any( + marker in issue.lower() + for issue in blocking_issues + for marker in ("topic drift", "weak topic overlap") + ) + forbidden_application_anchors = list(dict.fromkeys( + anchor.strip() + for issue in blocking_issues + if "topic drift" in issue.lower() + for anchor in re.findall(r"'([^']+)'", issue) + if anchor.strip() + )) review_context = { - "sourceCandidate": { + "sourceCandidate": {} if off_topic_repair else { "title": base_candidate.title, "problem": base_candidate.problem, "hypothesisStatement": base_candidate.hypothesisStatement, @@ -6944,18 +8039,36 @@ def _regenerate_candidate_from_review( "scores": base_candidate.scoreBreakdown, }, "reviewGate": review_gate, - "critique": critique.model_dump() if critique else {}, - "priorWork": [item.model_dump() for item in prior_work[:3]], + "critique": critique.model_dump() if critique and not off_topic_repair else {}, + "priorWork": ( + [item.model_dump() for item in prior_work[:3]] + if not off_topic_repair else [] + ), "seedQuery": session.config.seedQuery, "domain": session.config.domain, "paperType": session.config.paperType, + "forbiddenApplicationAnchors": forbidden_application_anchors, + "hardConstraints": ( + [ + "Treat seedQuery as the complete research boundary.", + "Do not introduce any application domain absent from seedQuery, even as an example, dataset, or evaluation setting.", + "Remove every forbiddenApplicationAnchor from all generated fields.", + ] + if off_topic_repair else [] + ), + "regenerationRequirements": [ + "Make the approach operationally specific: name system components and their control flow.", + "Define seed-aligned task or dataset selection criteria without inventing unavailable resources.", + "Specify baselines, metrics, ablations, and validation steps.", + "State measurable expected outcomes and principal failure tests.", + ], "researchDirection": { "id": self._candidate_direction_id(base_candidate), "type": self._candidate_direction_type(base_candidate), "title": self._candidate_direction_title(base_candidate), "notes": base_candidate.draftPlan.notes if base_candidate.draftPlan else "", }, - "literatureContext": literature_context[:8000], + "literatureContext": "" if off_topic_repair else literature_context[:8000], } messages = [ ChatMessage( @@ -6965,6 +8078,10 @@ def _regenerate_candidate_from_review( "Return JSON only. Do not claim executed experiments. Do not invent paper IDs. " "The new idea must preserve useful parts of the source candidate while directly addressing " "reviewGate warnings, blocking issues, and suggested improvements. " + "For topic-drift repairs, seedQuery is a hard boundary: do not introduce an absent " + "application domain in any field, including examples, datasets, or evaluation settings. " + "The approach must specify components, task or dataset selection criteria, baselines, " + "metrics, ablations, validation steps, and measurable expected outcomes. " "If researchDirection is provided, keep the regenerated idea inside that direction; " "for example, a benchmark direction should remain a benchmark idea." ), diff --git a/backend/app/services/plan_package_builder.py b/backend/app/services/plan_package_builder.py index e3cb416..893095b 100644 --- a/backend/app/services/plan_package_builder.py +++ b/backend/app/services/plan_package_builder.py @@ -42,6 +42,7 @@ PlanStep, ) from app.services.plan_package_templates import get_plan_template +from app.services.plan_package_specificity import hypothesis_is_falsifiable from app.storage.plan_package_storage import generate_plan_package_id @@ -299,6 +300,125 @@ def add(value: Any, *, require_relevance: bool = False) -> None: return metrics[:limit] or ["primary_metric"] +def _planned_metric_target(metric: str) -> str: + lowered = metric.lower() + if any( + term in lowered + for term in ["latency", "cost", "error", "failure", "violation", "hallucination"] + ): + return "<= strongest baseline under the same evaluation protocol" + if any( + term in lowered + for term in [ + "agreement", + "accuracy", + "faithfulness", + "coverage", + "recall", + "precision", + "score", + "rate", + "throughput", + ] + ): + return ">= strongest baseline with a positive mean delta on the preregistered split" + return "mean_delta versus strongest baseline must be positive on the preregistered primary evaluation" + + +def _planned_baseline_methods( + candidate: IdeaCandidate, + literature_survey: PlanLiteratureSurvey, +) -> List[str]: + names: List[str] = [] + for prior in candidate.closestPriorWork or []: + name = _first_text_value( + _get(prior, "method", ""), + _get(prior, "name", ""), + _get(prior, "title", ""), + ) + if name: + names.append(name) + + for paper in sorted( + literature_survey.papers, + key=lambda item: item.relevanceScore, + reverse=True, + ): + paper_names: List[str] = [] + for method in paper.methods: + name = _first_text_value( + _get(method, "name", ""), + _get(method, "method", ""), + ) + role_text = " ".join( + [ + str(_get(method, "role", "")), + str(_get(method, "description", "")), + ] + ).lower() + if name and any( + term in role_text for term in ["baseline", "control", "comparison"] + ): + paper_names.append(name) + if paper.role == "baseline" and not paper_names: + paper_names.append(paper.title) + names.extend(paper_names) + if names: + break + return _unique(names)[:5] + + +def _strengthen_plan_hypothesis( + hypothesis: str, + *, + primary_metric: str, + baseline_label: str, + paper_type: str, +) -> str: + base = hypothesis.strip().rstrip(".") + if paper_type == "survey" or hypothesis_is_falsifiable(base): + return hypothesis.strip() + return ( + f"{base}. Relative to {baseline_label}, this predicts a positive improvement " + f"in the primary metric {primary_metric}; reject the hypothesis if the mean " + "delta is not positive on the preregistered primary evaluation." + ) + + +def _deterministic_plan_core( + *, + candidate: IdeaCandidate, + literature_survey: PlanLiteratureSurvey, + paper_type: str, + hypothesis: str = "", +) -> tuple[str, str]: + template = get_plan_template(paper_type) + metrics = _planned_validation_metrics( + candidate=candidate, + literature_survey=literature_survey, + literature_map=None, + template_metrics=template.recommendedMetrics, + ) + primary_metric = metrics[0] + baselines = _planned_baseline_methods(candidate, literature_survey) + baseline_label = ( + ", ".join(baselines) + if baselines + else "the strongest eligible baseline from investigated prior work" + ) + question = ( + f"Can {candidate.title} improve {primary_metric} over {baseline_label} " + f"for the bounded problem: {candidate.problem}?" + ) + hypothesis = _strengthen_plan_hypothesis( + hypothesis or _candidate_hypothesis(candidate), + primary_metric=primary_metric, + baseline_label=baseline_label, + paper_type=template.paperType, + ) + return question, hypothesis + + def _seed_mentions_rag_safety_text(text: str) -> bool: lowered = text.lower() return ( @@ -502,6 +622,38 @@ def build_literature_survey( methods = [_method_dict(method) for method in sp.methods] findings = [_finding_dict(finding) for finding in sp.findings] claims = [_claim_dict(claim) for claim in sp.claims] + existing_method_names = { + str(method.get("name", "")).strip().lower() + for method in methods + if isinstance(method, dict) + } + for baseline in _unique([*sp.baselines, *sp.baselineMethods]): + if baseline.lower() in existing_method_names: + continue + methods.append( + { + "name": baseline, + "description": "Deep-reader recommended comparison method.", + "role": "baseline", + } + ) + existing_method_names.add(baseline.lower()) + existing_metric_text = { + str(claim.get("text", "")).strip().lower() + for claim in claims + if isinstance(claim, dict) + } + for metric in _unique([*sp.metrics, *sp.recommendedMetrics]): + if metric.lower() in existing_metric_text: + continue + claims.append( + { + "text": metric, + "claimType": "metric", + "source": "deep_reader_recommendation", + } + ) + existing_metric_text.add(metric.lower()) relevance_score, relevance_signals, relevance_reason = _paper_relevance( title=sp.title, summary=summary, @@ -985,6 +1137,12 @@ def build_default_stages( literature_map=None, template_metrics=template.recommendedMetrics, ) + baseline_methods = _planned_baseline_methods(candidate, literature_survey) + baseline_label = ( + ", ".join(baseline_methods) + if baseline_methods + else "a frozen baseline selected from the closest investigated prior work" + ) evidence_refs = [ PlanEvidenceRef(type="paper", id=paper.paperId, source=paper.source) @@ -1269,8 +1427,14 @@ def fit_constrained_stages(source_stages: List[PlanStage]) -> List[PlanStage]: id="step-1-2", order=2, title="Select implementation gap and baseline scope", - desc="Confirm the selected GAP, record supporting papers or graph signals, and define the baseline/control methods for downstream comparison.", - method="Use gap.items[], evidenceTrace, closestPriorWork, and literatureSurvey roles to bind the gap to evidence IDs and fair baselines.", + desc=( + "Confirm the selected GAP, record supporting papers or graph signals, " + f"and define {baseline_label} for downstream comparison." + ), + method=( + "Use gap.items[], evidenceTrace, closestPriorWork, and literatureSurvey " + f"roles to bind the gap to evidence IDs and compare against {baseline_label}." + ), inputFrom=["step-1-1"], outputs=[ PlanOutput(type="checkpoint", name="selected_gap.json", desc="Selected gap and supporting evidence", requiredFor=["review"]), @@ -1286,8 +1450,14 @@ def fit_constrained_stages(source_stages: List[PlanStage]) -> List[PlanStage]: id="step-1-3", order=3, title="Define baseline comparison scope", - desc="List the baseline or control methods that downstream validation should compare against the proposed idea.", - method="Use closestPriorWork, literatureSurvey roles, and selected GAP evidence to define fair comparison groups.", + desc=( + "Freeze the comparison scope around " + f"{baseline_label} before downstream validation." + ), + method=( + "Use closestPriorWork, literatureSurvey roles, and selected GAP evidence " + f"to define matched settings for {baseline_label}." + ), inputFrom=["step-1-2"], outputs=[PlanOutput(type="table", name="baseline_comparison_scope.csv", desc="Baseline/control methods and comparison rationale", requiredFor=["validation", "paper", "review"])], expected=[PlanExpectedMetric(metric="baseline_count", target=">= 1", desc="At least one baseline or control comparison is declared.")], @@ -1348,11 +1518,18 @@ def fit_constrained_stages(source_stages: List[PlanStage]) -> List[PlanStage]: order=1, title="Specify validation metrics", desc="Define planned metrics and target criteria for downstream validation.", - method="Use candidate expected metrics and literature-derived baselines when available.", + method=( + "Use candidate expected metrics and compare every primary outcome against " + f"{baseline_label} under the same evaluation protocol." + ), inputFrom=["step-2-2"], outputs=[PlanOutput(type="metrics", name="validation_metrics.json", desc="Planned metrics and target values", requiredFor=["validation", "paper"])], expected=[ - PlanExpectedMetric(metric=str(metric), target="specified before implementation", desc="Pre-registered expected metric.") + PlanExpectedMetric( + metric=str(metric), + target=_planned_metric_target(str(metric)), + desc="Pre-registered expected metric.", + ) for metric in metrics[:5] ], ), @@ -1726,10 +1903,14 @@ def build_plan_package( if candidate.draftPlan: research_question = candidate.draftPlan.researchQuestion hypothesis = candidate.draftPlan.hypothesis - research_question = research_question or ( - f"How can {candidate.title} address: {candidate.problem}?" + deterministic_question, deterministic_hypothesis = _deterministic_plan_core( + candidate=candidate, + literature_survey=literature_survey, + paper_type=paper_type, + hypothesis=hypothesis, ) - hypothesis = hypothesis or _candidate_hypothesis(candidate) + research_question = research_question or deterministic_question + hypothesis = deterministic_hypothesis proposed_method = _candidate_method(candidate, critique) expected_outcome = _candidate_expected_outcome(candidate, critique) diff --git a/backend/app/services/plan_package_generation.py b/backend/app/services/plan_package_generation.py new file mode 100644 index 0000000..b1986b4 --- /dev/null +++ b/backend/app/services/plan_package_generation.py @@ -0,0 +1,317 @@ +"""Internal segmented generation for the unchanged PlanPackage contract.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Callable + +from app.models.plan_package import PlanPackage, PlanStage +from app.services.plan_package_llm_schema import ( + validate_llm_plan_core_output, + validate_llm_plan_stage_output, +) +from app.services.plan_package_plan_quality import build_single_plan_design_brief + + +JsonCall = Callable[[str, str, int], dict[str, Any]] +SegmentValidator = Callable[ + [Any], + tuple[dict[str, Any] | None, list[str]], +] + +_PROTECTED_CONSTANTS = { + "ideaSessionId", + "ideaCandidateId", + "planStage", + "seedQuery", + "domain", + "paperType", +} + + +@dataclass +class PlanSegmentGenerationResult: + research_question: str + hypothesis: str + constants: dict[str, Any] + stages: list[PlanStage] + core_used: bool = False + llm_stage_ids: list[str] = field(default_factory=list) + fallback_stage_ids: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +def _call_with_one_repair( + *, + segment_name: str, + prompt: str, + max_tokens: int, + call_json: JsonCall, + validator: SegmentValidator, +) -> dict[str, Any]: + issues: list[str] = [] + last_exception: Exception | None = None + for attempt in range(2): + repair = "" + if attempt: + repair = ( + "\nRepair these validation issues and return the complete segment JSON: " + + "; ".join(issues) + ) + try: + raw = call_json(segment_name, prompt + repair, max_tokens) + except Exception as exc: + last_exception = exc + issues = [f"{type(exc).__name__}: {exc}"] + continue + last_exception = None + parsed, issues = validator(raw) + if parsed is not None and not issues: + return parsed + if last_exception is not None: + raise last_exception + raise ValueError("; ".join(issues) or "segment validation failed") + + +def _selected_gap(package: PlanPackage) -> dict[str, Any]: + selected = next( + (item for item in package.gap.items if item.id == package.gap.selectedGapId), + None, + ) + if selected: + return selected.model_dump() + return {"id": package.gap.selectedGapId, "statement": package.gap.summary} + + +def _allowed_evidence(package: PlanPackage) -> dict[str, list[str]]: + return { + "candidate": [package.idea.id], + "gap": [item.id for item in package.gap.items], + "paper": [paper.paperId for paper in package.literatureSurvey.papers], + "principle": ["principle"], + } + + +def _compact_prompt_value(value: Any) -> Any: + if isinstance(value, str): + text = value.strip() + if len(text) <= 1800: + return text + return text[:1797].rstrip() + "..." + if isinstance(value, dict): + return { + str(key): _compact_prompt_value(item) + for key, item in value.items() + } + if isinstance(value, list): + if all( + isinstance(item, (str, int, float, bool)) or item is None + for item in value + ): + return [_compact_prompt_value(item) for item in value] + return [_compact_prompt_value(item) for item in value[:12]] + return value + + +def build_core_prompt(package: PlanPackage) -> str: + context = { + "seedQuery": package.constants.get("seedQuery", ""), + "paperType": package.constants.get("paperType", "generic"), + "idea": package.idea.model_dump(), + "selectedGap": _selected_gap(package), + "principle": package.principle.model_dump(), + "currentConstants": package.constants, + "allowedEvidence": _allowed_evidence(package), + "returnShape": { + "researchQuestion": "string", + "hypothesis": "falsifiable string", + "constants": {}, + }, + } + return ( + "Strengthen only the researchQuestion, hypothesis, and constants of one " + "PlanPackage. The question must name the object, method, comparison, " + "outcome, and boundary. The hypothesis must name a mechanism, expected " + "direction, metric, and rejection condition. Do not invent datasets, " + "benchmark values, evidence IDs, or executed results. Return JSON only.\n" + + json.dumps( + _compact_prompt_value(context), + ensure_ascii=False, + separators=(",", ":"), + ) + ) + + +def build_stage_prompt( + package: PlanPackage, + fallback_stage: PlanStage, + result: PlanSegmentGenerationResult, +) -> str: + blueprint = build_single_plan_design_brief( + package, + max_stages=len(package.stages), + max_steps_per_stage=max(len(fallback_stage.steps), 1), + ) + context = { + "researchQuestion": result.research_question, + "hypothesis": result.hypothesis, + "constants": result.constants, + "idea": package.idea.model_dump(), + "selectedGap": _selected_gap(package), + "principle": package.principle.model_dump(), + "stageRole": fallback_stage.model_dump(), + "planBlueprint": blueprint, + "allowedEvidence": _allowed_evidence(package), + "topPapers": [ + { + "paperId": paper.paperId, + "title": paper.title, + "summary": paper.summary, + "methods": paper.methods[:4], + "limitations": paper.limitations[:4], + } + for paper in sorted( + package.literatureSurvey.papers, + key=lambda item: item.relevanceScore, + reverse=True, + )[:6] + ], + "returnShape": {"stage": fallback_stage.model_dump()}, + } + return ( + "Generate exactly one stage for the supplied stageRole. Preserve its " + "scientific purpose. Every step needs a concrete method, planned " + "artifacts, measurable expected targets, and valid evidence IDs. Use " + "numeric, baseline-relative, or statistical targets; never use readiness " + "placeholders. Do not claim executed results and do not invent datasets " + "or evidence. Return JSON only.\n" + + json.dumps( + _compact_prompt_value(context), + ensure_ascii=False, + separators=(",", ":"), + ) + ) + + +def _materialize_stage( + parsed_stage: dict[str, Any], + fallback_stage: PlanStage, +) -> PlanStage: + stage_data = dict(parsed_stage) + stage_data["id"] = fallback_stage.id + stage_data["order"] = fallback_stage.order + stage_data["dependsOn"] = list(fallback_stage.dependsOn) + steps: list[dict[str, Any]] = [] + for index, raw_step in enumerate(stage_data.get("steps", []), start=1): + step_data = dict(raw_step) + step_data["id"] = f"step-{fallback_stage.order}-{index}" + step_data["order"] = index + steps.append(step_data) + stage_data["steps"] = steps + return PlanStage.model_validate(stage_data) + + +def generate_stage_segment( + *, + package: PlanPackage, + fallback_stage: PlanStage, + core_result: PlanSegmentGenerationResult, + call_json: JsonCall, + max_steps_per_stage: int, +) -> PlanStage: + parsed = _call_with_one_repair( + segment_name=fallback_stage.id, + prompt=build_stage_prompt(package, fallback_stage, core_result), + max_tokens=1600, + call_json=call_json, + validator=validate_llm_plan_stage_output, + ) + stage = _materialize_stage(parsed["stage"], fallback_stage) + stage.steps = stage.steps[:max_steps_per_stage] + return stage + + +def normalize_stage_graph(stages: list[PlanStage]) -> list[PlanStage]: + step_id_map: dict[str, str] = {} + normalized = [ + stage.model_copy(deep=True) + for stage in sorted(stages, key=lambda item: item.order) + ] + for stage_index, stage in enumerate(normalized, start=1): + stage.id = f"stage-{stage_index}" + stage.order = stage_index + stage.dependsOn = [] if stage_index == 1 else [f"stage-{stage_index - 1}"] + for step_index, step in enumerate(stage.steps, start=1): + old_id = step.id + new_id = f"step-{stage_index}-{step_index}" + if old_id: + step_id_map[old_id] = new_id + step.id = new_id + step.order = step_index + + known_ids = {step.id for stage in normalized for step in stage.steps} + previous_step_id = "" + for stage in normalized: + for step in stage.steps: + mapped = [step_id_map.get(item, item) for item in step.inputFrom] + step.inputFrom = [ + item for item in mapped if item in known_ids and item != step.id + ] + if not step.inputFrom and previous_step_id: + step.inputFrom = [previous_step_id] + previous_step_id = step.id + return normalized + + +def generate_plan_segments( + *, + package: PlanPackage, + call_json: JsonCall, + max_steps_per_stage: int, +) -> PlanSegmentGenerationResult: + result = PlanSegmentGenerationResult( + research_question=package.researchQuestion, + hypothesis=package.hypothesis, + constants=dict(package.constants), + stages=[stage.model_copy(deep=True) for stage in package.stages], + ) + try: + core = _call_with_one_repair( + segment_name="core", + prompt=build_core_prompt(package), + max_tokens=1400, + call_json=call_json, + validator=validate_llm_plan_core_output, + ) + result.research_question = core["researchQuestion"].strip() + result.hypothesis = core["hypothesis"].strip() + result.constants.update( + (key, value) + for key, value in core.get("constants", {}).items() + if key not in _PROTECTED_CONSTANTS + ) + result.core_used = True + except Exception as exc: + result.warnings.append(f"segment_fallback:core:{type(exc).__name__}") + + merged_stages: list[PlanStage] = [] + for fallback_stage in package.stages: + try: + stage = generate_stage_segment( + package=package, + fallback_stage=fallback_stage, + core_result=result, + call_json=call_json, + max_steps_per_stage=max_steps_per_stage, + ) + merged_stages.append(stage) + result.llm_stage_ids.append(stage.id) + except Exception as exc: + merged_stages.append(fallback_stage.model_copy(deep=True)) + result.fallback_stage_ids.append(fallback_stage.id) + result.warnings.append( + f"segment_fallback:{fallback_stage.id}:{type(exc).__name__}" + ) + result.stages = normalize_stage_graph(merged_stages) + return result diff --git a/backend/app/services/plan_package_llm_schema.py b/backend/app/services/plan_package_llm_schema.py index 643903b..d74295b 100644 --- a/backend/app/services/plan_package_llm_schema.py +++ b/backend/app/services/plan_package_llm_schema.py @@ -113,6 +113,20 @@ class PlanPackageLLMOutput(BaseModel): model_config = ConfigDict(extra="forbid") +class LLMPlanCoreOutput(BaseModel): + researchQuestion: str + hypothesis: str + constants: Dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class LLMPlanStageOutput(BaseModel): + stage: LLMPlanStage + + model_config = ConfigDict(extra="forbid") + + _ALLOWED_TOP_LEVEL = { "researchQuestion", "hypothesis", @@ -332,6 +346,54 @@ def _validation_error_messages(exc: ValidationError) -> List[str]: return messages +def validate_llm_plan_core_output( + raw: Any, +) -> Tuple[Optional[Dict[str, Any]], List[str]]: + if not isinstance(raw, dict): + return None, ["Core output must be one JSON object"] + forbidden = sorted( + set(raw) - {"researchQuestion", "hypothesis", "constants"} + ) + if forbidden: + return None, ["Core output contains forbidden keys: " + ", ".join(forbidden)] + nulls = _null_paths(raw) + if nulls: + return None, [ + "Core output contains null values; omit the field or use empty arrays/objects/strings: " + + ", ".join(nulls[:12]) + ] + try: + parsed = LLMPlanCoreOutput.model_validate(raw) + except ValidationError as exc: + return None, _validation_error_messages(exc) + return parsed.model_dump(), [] + + +def validate_llm_plan_stage_output( + raw: Any, +) -> Tuple[Optional[Dict[str, Any]], List[str]]: + if not isinstance(raw, dict): + return None, ["Stage output must be one JSON object"] + forbidden = sorted(set(raw) - {"stage"}) + if forbidden: + return None, ["Stage output contains forbidden keys: " + ", ".join(forbidden)] + repaired = dict(raw) + if isinstance(repaired.get("stage"), dict): + wrapped = _repair_plan_stage_shapes({"stages": [repaired["stage"]]}) + repaired["stage"] = wrapped["stages"][0] + nulls = _null_paths(repaired) + if nulls: + return None, [ + "Stage output contains null values; omit the field or use empty arrays/objects/strings: " + + ", ".join(nulls[:12]) + ] + try: + parsed = LLMPlanStageOutput.model_validate(repaired) + except ValidationError as exc: + return None, _validation_error_messages(exc) + return parsed.model_dump(), [] + + def validate_llm_plan_output( raw: Any, *, diff --git a/backend/app/services/plan_package_review_loop.py b/backend/app/services/plan_package_review_loop.py new file mode 100644 index 0000000..4ee3fce --- /dev/null +++ b/backend/app/services/plan_package_review_loop.py @@ -0,0 +1,229 @@ +"""Internal reviewer issue routing for PlanPackage repair and presentation.""" + +from __future__ import annotations + +import re +from collections import OrderedDict +from typing import Literal + +from app.models.plan_package import PlanPackage, PlanReviewerIssue + + +IssueRoute = Literal[ + "plan_repairable", + "upstream_blocking", + "user_decision_required", + "diagnostic_only", +] + +_DIAGNOSTIC_TERMS = { + "timeout", + "provider", + "non-json", + "invalid json", + "llm reviewer unavailable", + "repair round", +} +_USER_DECISION_TERMS = { + "user must choose", + "human decision", + "budget approval", + "scope decision", + "owner confirmation", +} + + +def route_review_issue(issue: PlanReviewerIssue) -> IssueRoute: + path = issue.sectionPath.lower() + text = issue.message.lower() + if path.startswith("reviewreports") or any( + term in text for term in _DIAGNOSTIC_TERMS + ): + return "diagnostic_only" + if any(term in text for term in _USER_DECISION_TERMS): + return "user_decision_required" + if path.startswith( + ( + "evidencetrace", + "upstream", + "source.search", + "source.path", + "literaturesurvey", + "gap", + "principle", + ) + ): + return "upstream_blocking" + return "plan_repairable" + + +def _issue_category(message: str) -> str: + text = message.lower() + categories = { + "hypothesis_falsifiability": [ + "falsif", + "rejection condition", + "reject the hypothesis", + ], + "metric_target": ["metric target", "expected target", "generic target"], + "baseline": ["baseline", "control comparison"], + "evidence": ["evidence", "citation", "paper id", "claim id"], + "topic": ["topic", "seed query", "drift", "relevance"], + } + for category, terms in categories.items(): + if any(term in text for term in terms): + return category + tokens = re.findall(r"[a-z0-9_]+", text) + return " ".join(tokens[:5]) + + +def _fingerprint(issue: PlanReviewerIssue) -> tuple[str, str]: + return issue.sectionPath.lower(), _issue_category(issue.message) + + +def dedupe_review_issues( + issues: list[PlanReviewerIssue], +) -> list[PlanReviewerIssue]: + grouped: OrderedDict[tuple[str, str], PlanReviewerIssue] = OrderedDict() + for issue in issues: + key = _fingerprint(issue) + existing = grouped.get(key) + if existing is None or ( + existing.severity != "blocking" and issue.severity == "blocking" + ): + grouped[key] = issue + return list(grouped.values()) + + +def repair_targets(issues: list[PlanReviewerIssue]) -> list[str]: + targets: list[str] = [] + for issue in issues: + if route_review_issue(issue) != "plan_repairable": + continue + path = issue.sectionPath.lower() + if path.startswith("researchquestion"): + target = "researchQuestion" + elif path.startswith("hypothesis"): + target = "hypothesis" + elif path.startswith("constants"): + target = "constants" + else: + target = "stages" + if target not in targets: + targets.append(target) + return targets + + +def repair_stage_ids( + issues: list[PlanReviewerIssue], + available_stage_ids: list[str], +) -> list[str]: + selected: list[str] = [] + for issue in issues: + if route_review_issue(issue) != "plan_repairable": + continue + match = re.match(r"stages\[([^\]]+)\]", issue.sectionPath.lower()) + if not match: + continue + selector = match.group(1) + if selector.isdigit(): + index = int(selector) + stage_id = ( + available_stage_ids[index] + if 0 <= index < len(available_stage_ids) + else "" + ) + else: + stage_id = next( + ( + item + for item in available_stage_ids + if item.lower() == selector + ), + "", + ) + if stage_id and stage_id not in selected: + selected.append(stage_id) + return selected + + +def _gate_issues(package: PlanPackage) -> list[PlanReviewerIssue]: + issues: list[PlanReviewerIssue] = [] + error_messages = set(package.qualityGate.errors) + for index, message in enumerate( + [*package.qualityGate.errors, *package.qualityGate.warnings] + ): + section, separator, detail = message.partition(":") + issues.append( + PlanReviewerIssue( + id=f"visible-{index}", + severity="blocking" if message in error_messages else "warning", + sectionPath=section.strip() if separator else "package", + message=detail.strip() if separator else message, + ) + ) + return issues + + +def final_user_issues(package: PlanPackage) -> list[PlanReviewerIssue]: + issues: list[PlanReviewerIssue] = [] + if package.metaReview: + issues.extend(package.metaReview.blockingIssues) + issues.extend( + issue + for issue in package.metaReview.warnings + if route_review_issue(issue) + in {"upstream_blocking", "user_decision_required"} + ) + issues.extend(_gate_issues(package)) + return dedupe_review_issues( + [ + issue + for issue in issues + if route_review_issue(issue) != "diagnostic_only" + and ( + issue.severity == "blocking" + or route_review_issue(issue) + in {"upstream_blocking", "user_decision_required"} + ) + ] + ) + + +def user_visible_concerns(package: PlanPackage) -> list[str]: + routes = {route_review_issue(issue) for issue in final_user_issues(package)} + concerns: list[str] = [] + if "upstream_blocking" in routes: + concerns.append( + "The upstream evidence trace or literature support is incomplete, " + "so this plan remains a draft." + ) + if "user_decision_required" in routes: + concerns.append( + "A research-scope, dataset, or resource decision still requires " + "owner confirmation." + ) + if "plan_repairable" in routes: + concerns.append( + "The plan could not fully resolve an experimental-specificity " + "requirement within the repair budget." + ) + return concerns[:3] + + +def required_user_actions(package: PlanPackage) -> list[str]: + routes = {route_review_issue(issue) for issue in final_user_issues(package)} + actions: list[str] = [] + if "upstream_blocking" in routes: + actions.append( + "Complete the missing evidence selection or trace before approval." + ) + if "user_decision_required" in routes: + actions.append( + "Confirm the unresolved scope, dataset, or resource constraint." + ) + if "plan_repairable" in routes: + actions.append( + "Regenerate the affected plan section with additional constraints." + ) + return actions[:3] diff --git a/backend/app/services/plan_package_reviewers.py b/backend/app/services/plan_package_reviewers.py index 3ece015..fc82848 100644 --- a/backend/app/services/plan_package_reviewers.py +++ b/backend/app/services/plan_package_reviewers.py @@ -16,6 +16,11 @@ PlanReviewerReport, ) from app.services.plan_package_plan_quality import missing_plan_roles +from app.services.plan_package_review_loop import route_review_issue +from app.services.plan_package_specificity import ( + metric_target_is_concrete, + plan_specificity_issues, +) def _issue( @@ -314,7 +319,12 @@ def feasibility_reviewer(package: PlanPackage) -> PlanReviewerReport: blocking.append(_issue("No implementation stages are defined.", section_path="stages", severity="blocking")) return _make_report("FeasibilityReviewer", 0.0, blocking, warnings, suggestions) - generic_tokens = {"readiness", "primary_metric", "specified before implementation", "default plan step"} + generic_tokens = { + "readiness", + "primary_metric", + "specified before implementation", + "default plan step", + } total_steps = 0 detailed_steps = 0 artifact_steps = 0 @@ -329,12 +339,26 @@ def feasibility_reviewer(package: PlanPackage) -> PlanReviewerReport: detailed_steps += 1 if step.outputs: artifact_steps += 1 - if step.expected and not all( - expected.metric.lower() in generic_tokens or expected.target.lower() in generic_tokens + if step.expected and all( + expected.metric.strip().lower() not in generic_tokens + and metric_target_is_concrete(expected.target) for expected in step.expected ): metric_steps += 1 + feasibility_specificity = [ + issue + for issue in plan_specificity_issues(package) + if issue.sectionPath == "hypothesis" + or issue.sectionPath.endswith(".outputs") + or issue.sectionPath.endswith(".expected") + ] + blocking.extend(feasibility_specificity) + if feasibility_specificity: + suggestions.append( + "Repair the hypothesis and plan steps with falsifiable outcomes, artifacts, and evaluation criteria." + ) + if total_steps and detailed_steps / total_steps < 0.67: blocking.append(_issue( "Too many steps are generic or underspecified.", @@ -383,21 +407,25 @@ def metric_reviewer(package: PlanPackage) -> PlanReviewerReport: blocking.append(_issue("No expected metrics are defined.", section_path="stages[].steps[].expected", severity="blocking")) return _make_report("MetricReviewer", 0.0, blocking, warnings, suggestions) - generic = {"readiness", "primary_metric", "specified before implementation", "planned_metric"} - def is_concrete_target(target: str) -> bool: - normalized = target.strip().lower() - if normalized in generic: - return False - if re.search(r"(>=|<=|>|<|=|±|\d)", normalized): - return True - return len(normalized) >= 6 + generic = {"readiness", "primary_metric", "planned_metric"} concrete = [ item for item in expected_items if item.metric.strip().lower() not in generic - and is_concrete_target(item.target) + and metric_target_is_concrete(item.target) + ] + metric_specificity = [ + issue + for issue in plan_specificity_issues(package) + if ".expected[" in issue.sectionPath + and issue.sectionPath.endswith((".metric", ".target")) ] + blocking.extend(metric_specificity) + if metric_specificity: + suggestions.append( + "Replace generic metric names and targets with numeric, baseline-relative, or statistical criteria." + ) topic_terms = _topic_terms(package) metric_hits = _hit_count( " ".join(f"{item.metric} {item.target} {item.desc}" for item in expected_items), @@ -581,10 +609,12 @@ def apply_review_to_quality_gate( review_error_messages = [ f"{issue.sectionPath}: {issue.message}" if issue.sectionPath else issue.message for issue in meta.blockingIssues + if route_review_issue(issue) != "diagnostic_only" ] review_warning_messages = [ f"{issue.sectionPath}: {issue.message}" if issue.sectionPath else issue.message for issue in meta.warnings + if route_review_issue(issue) != "diagnostic_only" ] existing_errors = set(gate.errors) existing_warnings = set(gate.warnings) diff --git a/backend/app/services/plan_package_service.py b/backend/app/services/plan_package_service.py index d644b77..8d740ec 100644 --- a/backend/app/services/plan_package_service.py +++ b/backend/app/services/plan_package_service.py @@ -28,6 +28,12 @@ PlanStep, ) from app.services.plan_package_builder import build_contribution_statements, build_plan_package +from app.services.plan_package_generation import ( + PlanSegmentGenerationResult, + generate_plan_segments, + generate_stage_segment, + normalize_stage_graph, +) from app.services.plan_package_llm_schema import llm_plan_output_schema_hint, validate_llm_plan_output from app.services.plan_package_plan_quality import ( build_plan_blueprint, @@ -37,6 +43,12 @@ ) from app.services.plan_package_readiness import evaluate_downstream_readiness from app.services.plan_package_revisor import build_plan_revision_patch +from app.services.plan_package_review_loop import ( + dedupe_review_issues, + repair_stage_ids, + repair_targets, + route_review_issue, +) from app.services.plan_package_reviewers import apply_review_to_quality_gate from app.services.plan_package_validator import validate_plan_package from app.services.plan_package_views import build_plan_package_handoff, build_plan_package_presentation @@ -526,38 +538,60 @@ def _auto_repair_plan_from_review( ) -> None: """Automatically repair plan-owned fields from reviewer findings.""" - if max_repair_rounds <= 0: - return - for repair_index in range(max_repair_rounds): + repair_budget = min(max(0, max_repair_rounds), 2) + for repair_index in range(repair_budget): if package.qualityGate.agentApproved and package.qualityGate.implementationReady: return - revision_patch = build_plan_revision_patch( - package, - include_feedback=False, - allow_narrative=False, + current_issues = dedupe_review_issues( + [ + *( + package.metaReview.blockingIssues + if package.metaReview + else [] + ), + *(package.metaReview.warnings if package.metaReview else []), + ] ) - targets = self._normalize_revision_targets(revision_patch.changedSections) if revision_patch.changedSections else [] - if not targets and not revision_patch.upstreamBlocked: - targets = self._infer_plan_repair_targets_from_review(package) - if revision_patch.upstreamBlocked: - message = "PlanPackage auto repair blocked by upstream issue: " + "; ".join(revision_patch.unresolvedIssues[:3]) + if any( + route_review_issue(issue) == "upstream_blocking" + for issue in current_issues + ): + message = "review_repair_blocked:upstream" if message not in package.generation.warnings: package.generation.warnings.append(message) - if message not in package.qualityGate.warnings: - package.qualityGate.warnings.append(message) return + targets = repair_targets(current_issues) if not targets: return + stage_ids = repair_stage_ids( + current_issues, + [stage.id for stage in package.stages], + ) + if "stages" in targets and not stage_ids: + stage_ids = [stage.id for stage in package.stages] + reports_before_repair = [ + report.model_dump(mode="json") + for report in package.reviewReports + ] try: previous_repair_rounds = package.generation.repairRounds - self._apply_llm_plan_fields( - package, - session, - max_stages=max_stages, - max_steps_per_stage=max_steps_per_stage, - max_repair_rounds=1, - target_sections=targets, - ) + core_targets = [target for target in targets if target != "stages"] + if core_targets: + self._apply_llm_plan_fields( + package, + session, + max_stages=max_stages, + max_steps_per_stage=max_steps_per_stage, + max_repair_rounds=1, + target_sections=core_targets, + ) + if stage_ids: + self._repair_llm_stage_fields( + package, + session, + stage_ids=stage_ids, + max_steps_per_stage=max_steps_per_stage, + ) package.contributionStatement = build_contribution_statements( candidate=self._candidate_from_package(package), gap=package.gap, @@ -570,10 +604,15 @@ def _auto_repair_plan_from_review( parentPackageId=package.packageId, changedSections=[*targets, "contributionStatement", "qualityGate", "reviewReports"], feedbackIds=[], - summary="Auto-repaired PlanPackage from reviewer findings.", + summary="Auto-repaired internal reviewer findings.", generationMode="hybrid", repairRounds=max(0, package.generation.repairRounds - previous_repair_rounds), - patchSummary=revision_patch.model_dump(), + patchSummary={ + "repairIndex": repair_index + 1, + "targets": targets, + "stageIds": stage_ids, + "reviewReportsBeforeRepair": reports_before_repair, + }, ) ) package.qualityGate = validate_plan_package(package) @@ -587,8 +626,6 @@ def _auto_repair_plan_from_review( message = f"PlanPackage auto repair failed on round {repair_index + 1}: {exc}" if message not in package.generation.warnings: package.generation.warnings.append(message) - if message not in package.qualityGate.warnings: - package.qualityGate.warnings.append(message) return def _infer_plan_repair_targets_from_review(self, package: PlanPackage) -> List[str]: @@ -663,9 +700,21 @@ def _apply_review_mode( reviewer_mode: str, ): mode = self._normalize_reviewer_mode(reviewer_mode) + upstream_blockers = self._upstream_plan_blockers(package) + for blocker in upstream_blockers: + message = f"upstream: {blocker}" + if message not in gate.errors: + gate.errors.append(message) + diagnostic = f"upstream_blocked:{blocker}" + if diagnostic not in package.generation.warnings: + package.generation.warnings.append(diagnostic) gate = apply_review_to_quality_gate(package, gate) package.qualityGate = gate - llm_reports = self._run_llm_reviewers(package, reviewer_mode=mode) + llm_reports = ( + [] + if upstream_blockers + else self._run_llm_reviewers(package, reviewer_mode=mode) + ) if llm_reports: gate = apply_review_to_quality_gate(package, gate, extra_reports=llm_reports) gate = self._apply_downstream_readiness(package, gate) @@ -673,6 +722,47 @@ def _apply_review_mode( package.generation.llmReviewerUsed = self._llm_review_used(llm_reports) return gate + def _upstream_plan_blockers(self, package: PlanPackage) -> List[str]: + blockers: List[str] = [] + if not all( + [ + package.source.searchTreeId, + package.source.searchNodeId, + package.source.pathSeedId, + package.evidenceTrace.searchNodeId, + package.evidenceTrace.pathSeedId, + ] + ): + blockers.append("upstream evidence trace identity is incomplete") + if ( + package.source.searchNodeId + and package.evidenceTrace.searchNodeId + and package.source.searchNodeId != package.evidenceTrace.searchNodeId + ): + blockers.append("source and evidence trace search nodes do not match") + if ( + package.source.pathSeedId + and package.evidenceTrace.pathSeedId + and package.source.pathSeedId != package.evidenceTrace.pathSeedId + ): + blockers.append("source and evidence trace path seeds do not match") + if not package.literatureSurvey.papers: + blockers.append("investigated literature is empty") + selected_gap = next( + ( + item + for item in package.gap.items + if item.id == package.gap.selectedGapId + ), + None, + ) + if selected_gap is None or not ( + selected_gap.supportedByPaperIds + or selected_gap.supportedByClaimIds + ): + blockers.append("selected GAP lacks upstream paper or claim support") + return blockers + def _apply_downstream_readiness(self, package: PlanPackage, gate: Any): readiness = evaluate_downstream_readiness(package) package.downstreamReadiness = readiness @@ -1192,7 +1282,7 @@ def create_from_idea_session( package.generation.repairRounds = 0 package.generation.fallbackUsed = mode == "deterministic" package.generation.promptVersion = ( - "plan-package-single-implementation-planner-v2" + "plan-package-segmented-implementation-planner-v1" if mode == "hybrid" else "plan-package-adapter-v1" ) @@ -1211,19 +1301,24 @@ def create_from_idea_session( "PlanPackage.literatureSurvey", ] - if mode == "hybrid": + upstream_blockers = self._upstream_plan_blockers(package) + if mode == "hybrid" and not upstream_blockers: try: - self._apply_llm_plan_fields( + self._apply_segmented_llm_plan_fields( package, session, - max_stages=max_stages, max_steps_per_stage=max_steps_per_stage, - max_repair_rounds=max_repair_rounds, ) except Exception as exc: logger.warning("LLM plan field generation failed: %s", exc, exc_info=True) generation_warnings.append(f"LLM plan field generation failed: {exc}") package.generation.fallbackUsed = True + elif mode == "hybrid": + package.generation.fallbackUsed = True + generation_warnings.extend( + f"upstream_blocked:{blocker}" + for blocker in upstream_blockers + ) package.contributionStatement = build_contribution_statements( candidate=candidate, @@ -1234,8 +1329,12 @@ def create_from_idea_session( package.schemaVersion = "plan-package/v4" package.qualityGate = validate_plan_package(package) package.qualityGate = self._apply_review_mode(package, package.qualityGate, reviewer_mode=reviewer_mode) - package.qualityGate.warnings.extend(generation_warnings) - package.generation.warnings.extend(generation_warnings) + existing_generation_warnings = set(package.generation.warnings) + package.generation.warnings.extend( + warning + for warning in generation_warnings + if warning not in existing_generation_warnings + ) if mode == "hybrid": self._auto_repair_plan_from_review( package, @@ -1463,6 +1562,153 @@ def _apply_llm_plan_fields( package.generation.schemaRepairRounds += schema_repair_rounds raise ValueError("LLM plan field generation failed validation: " + "; ".join(last_issues)) + def _apply_segmented_llm_plan_fields( + self, + package: PlanPackage, + session: IdeaSession, + *, + max_steps_per_stage: int, + ) -> None: + client = get_provider_client(session.config.providerName) + scheduler = get_llm_task_scheduler() + package.generation.providerName = session.config.providerName + package.generation.model = session.config.model + package.generation.promptVersion = "plan-package-segmented-implementation-planner-v1" + + def call_json( + segment_name: str, + prompt: str, + max_tokens: int, + ) -> Dict[str, Any]: + task_name = ( + "plan_package_core" + if segment_name == "core" + else "plan_package_stage" + ) + system_prompt = ( + "Generate PlanPackage core JSON only." + if segment_name == "core" + else "Generate exactly one PlanPackage stage JSON only." + ) + response = scheduler.run( + task_name, + lambda: client.chat( + messages=[ + ChatMessage( + role="system", + content=( + system_prompt + + " Do not invent paper IDs, claim IDs, datasets, " + "benchmark values, or executed results." + ), + ), + ChatMessage(role="user", content=prompt), + ], + model=session.config.model, + temperature=0.2, + max_tokens=max_tokens, + response_format={"type": "json_object"}, + ), + timeout_seconds=_plan_llm_timeout_seconds(), + ) + parsed = _extract_json(response.text or "") + if parsed is None: + raise ValueError(f"{segment_name} returned invalid JSON") + return parsed + + result = generate_plan_segments( + package=package, + call_json=call_json, + max_steps_per_stage=max_steps_per_stage, + ) + package.researchQuestion = result.research_question + package.hypothesis = result.hypothesis + package.constants.update(sanitize_constants(result.constants)) + package.stages = result.stages + package.generation.llmUsedSections = ( + ["implementationPlan"] + if result.core_used or result.llm_stage_ids + else [] + ) + package.generation.fallbackUsed = bool( + result.fallback_stage_ids or not result.core_used + ) + existing_warnings = set(package.generation.warnings) + package.generation.warnings.extend( + warning + for warning in result.warnings + if warning not in existing_warnings + ) + + def _repair_llm_stage_fields( + self, + package: PlanPackage, + session: IdeaSession, + *, + stage_ids: List[str], + max_steps_per_stage: int, + ) -> None: + client = get_provider_client(session.config.providerName) + scheduler = get_llm_task_scheduler() + + def call_json( + segment_name: str, + prompt: str, + max_tokens: int, + ) -> Dict[str, Any]: + response = scheduler.run( + "plan_package_stage_repair", + lambda: client.chat( + messages=[ + ChatMessage( + role="system", + content=( + "Repair one PlanPackage stage. Return JSON only and " + "do not invent evidence, datasets, benchmark values, " + "or executed results." + ), + ), + ChatMessage(role="user", content=prompt), + ], + model=session.config.model, + temperature=0.0, + max_tokens=max_tokens, + response_format={"type": "json_object"}, + ), + timeout_seconds=_plan_llm_timeout_seconds(), + ) + parsed = _extract_json(response.text or "") + if parsed is None: + raise ValueError(f"{segment_name} repair returned invalid JSON") + return parsed + + core_result = PlanSegmentGenerationResult( + research_question=package.researchQuestion, + hypothesis=package.hypothesis, + constants=dict(package.constants), + stages=[stage.model_copy(deep=True) for stage in package.stages], + core_used=True, + ) + replacements: Dict[str, PlanStage] = {} + for fallback_stage in package.stages: + if fallback_stage.id not in stage_ids: + continue + replacements[fallback_stage.id] = generate_stage_segment( + package=package, + fallback_stage=fallback_stage, + core_result=core_result, + call_json=call_json, + max_steps_per_stage=max_steps_per_stage, + ) + if replacements: + package.stages = normalize_stage_graph( + [ + replacements.get(stage.id, stage).model_copy(deep=True) + for stage in package.stages + ] + ) + package.generation.repairRounds += 1 + def _apply_parsed_plan_fields( self, package: PlanPackage, diff --git a/backend/app/services/plan_package_specificity.py b/backend/app/services/plan_package_specificity.py new file mode 100644 index 0000000..290a1cd --- /dev/null +++ b/backend/app/services/plan_package_specificity.py @@ -0,0 +1,125 @@ +"""Deterministic content-quality rules for existing PlanPackage fields.""" + +from __future__ import annotations + +import hashlib +import re + +from app.models.plan_package import PlanPackage, PlanReviewerIssue + + +_GENERIC_TARGETS = { + "specified before implementation", + "primary_metric", + "readiness", + "planned_metric", + "higher", + "lower", + "better", + "improved", +} +_GENERIC_METRICS = { + "primary_metric", + "planned_metric", + "readiness", + "default plan step", +} +_MEASURABLE_TARGET = re.compile( + r"(?:\d|>=|<=|>|<|baseline|control|confidence interval|\bci\b|" + r"non[- ]?inferior|statistical|all preregistered|zero failures|" + r"\btrue\b|\bfalse\b|\bpass\b|complete|基线|对照|置信区间|不劣)", + re.IGNORECASE, +) +_DIRECTION = re.compile( + r"(?:increase|decrease|improve|reduce|higher|lower|positive|negative|" + r"non[- ]?inferior|提升|提高|改善|增强|降低|增加|减少|优于|不劣)", + re.IGNORECASE, +) +_FALSIFIER = re.compile( + r"(?:reject|falsif|not positive|fails? if|unless|no improvement|" + r"拒绝|证伪|不成立|未提升|没有改善)", + re.IGNORECASE, +) +_MEASURE = re.compile( + r"(?:metric|score|accuracy|faithfulness|rate|latency|cost|throughput|" + r"effect size|coverage|recall|precision|指标|准确率|忠实度|错误率|" + r"延迟|成本|吞吐|覆盖率|召回率|精确率)", + re.IGNORECASE, +) + + +def metric_target_is_concrete(target: str) -> bool: + normalized = " ".join(str(target or "").strip().lower().split()) + return bool( + normalized + and normalized not in _GENERIC_TARGETS + and _MEASURABLE_TARGET.search(normalized) + ) + + +def hypothesis_is_falsifiable(hypothesis: str) -> bool: + text = str(hypothesis or "").strip() + return bool( + _DIRECTION.search(text) + and _MEASURE.search(text) + and _FALSIFIER.search(text) + ) + + +def _issue(section_path: str, message: str) -> PlanReviewerIssue: + digest = hashlib.sha1( + f"{section_path}|{message}".encode("utf-8") + ).hexdigest()[:12] + return PlanReviewerIssue( + id=f"specificity:{digest}", + severity="blocking", + sectionPath=section_path, + message=message, + ) + + +def plan_specificity_issues(package: PlanPackage) -> list[PlanReviewerIssue]: + issues: list[PlanReviewerIssue] = [] + paper_type = str(package.constants.get("paperType", "")).strip().lower() + if paper_type != "survey" and not hypothesis_is_falsifiable(package.hypothesis): + issues.append( + _issue( + "hypothesis", + "Hypothesis must state a measurable direction and a falsification condition.", + ) + ) + + for stage_index, stage in enumerate(package.stages): + for step_index, step in enumerate(stage.steps): + base = f"stages[{stage_index}].steps[{step_index}]" + if not step.outputs: + issues.append( + _issue( + f"{base}.outputs", + "Step must declare at least one concrete artifact.", + ) + ) + if not step.expected: + issues.append( + _issue( + f"{base}.expected", + "Step must declare at least one evaluation criterion.", + ) + ) + for metric_index, expected in enumerate(step.expected): + metric_path = f"{base}.expected[{metric_index}]" + if expected.metric.strip().lower() in _GENERIC_METRICS: + issues.append( + _issue( + f"{metric_path}.metric", + "Metric name must describe a scientific or operational outcome.", + ) + ) + if not metric_target_is_concrete(expected.target): + issues.append( + _issue( + f"{metric_path}.target", + "Metric target must be numeric, baseline-relative, or statistically testable.", + ) + ) + return issues diff --git a/backend/app/services/plan_package_validator.py b/backend/app/services/plan_package_validator.py index 8e03110..badf522 100644 --- a/backend/app/services/plan_package_validator.py +++ b/backend/app/services/plan_package_validator.py @@ -4,6 +4,7 @@ from app.models.plan_package import PlanOutputType, PlanPackage, PlanQualityGate from app.services.plan_package_plan_quality import missing_plan_roles +from app.services.plan_package_specificity import plan_specificity_issues def _has_text(value: str | None) -> bool: @@ -487,6 +488,11 @@ def validate_plan_package(package: PlanPackage) -> PlanQualityGate: if raw_method and package.principle.mechanism and raw_method not in package.principle.mechanism: warnings.append("principle.mechanism differs from IdeaCandidate.proposedMethod adapter source") + warnings.extend( + f"specificity.{issue.sectionPath}: {issue.message}" + for issue in plan_specificity_issues(package) + ) + schema_valid = not schema_errors evidence_valid = not evidence_errors return PlanQualityGate( diff --git a/backend/app/services/plan_package_views.py b/backend/app/services/plan_package_views.py index 8d6767f..b19cc3b 100644 --- a/backend/app/services/plan_package_views.py +++ b/backend/app/services/plan_package_views.py @@ -9,6 +9,7 @@ PlanPackage, PlanPackageHandoff, PlanPackagePresentation, + PlanPackageStatus, PlanPresentationBackground, PlanPresentationDebugRef, PlanPresentationEvidenceSummary, @@ -20,6 +21,10 @@ PlanReadableStage, PlanReadableStep, ) +from app.services.plan_package_review_loop import ( + required_user_actions, + user_visible_concerns, +) def _compact_text(value: Any) -> str: @@ -145,32 +150,21 @@ def _confidence(package: PlanPackage) -> str: def _review_concerns(package: PlanPackage) -> List[str]: - concerns = [ - issue.message - for issue in (package.metaReview.blockingIssues if package.metaReview else []) - ] - if not concerns: - concerns.extend(package.qualityGate.errors[:4]) - if not concerns and package.metaReview: - concerns.extend(issue.message for issue in package.metaReview.warnings[:4]) - return concerns[:6] + return user_visible_concerns(package) def _next_actions(package: PlanPackage) -> List[str]: - if package.status == "approved" or str(package.status).endswith("APPROVED"): + if package.status == PlanPackageStatus.APPROVED: return [ - "Hand off the compact package to code, paper, and review modules.", - "Keep the full package available only for audit/debug traceability.", + "Hand off the compact package to downstream modules.", + "Keep the full package available for audit and debugging.", ] + actions = required_user_actions(package) + if actions: + return actions if package.qualityGate.agentApproved: - return [ - "Ask a human owner to approve or add targeted feedback.", - "Use the Revise action for any required wording or metric fixes.", - ] - return [ - "Resolve reviewer blocking issues before downstream handoff.", - "Add targeted human feedback, then run Revise and Review again.", - ] + return ["Review and approve the finalized research plan."] + return ["Retry internal plan optimization before approval."] def build_plan_package_presentation(package: PlanPackage) -> PlanPackagePresentation: @@ -186,12 +180,7 @@ def build_plan_package_presentation(package: PlanPackage) -> PlanPackagePresenta f"The proposed mechanism is: {(package.principle.mechanism or package.idea.proposedMethod).strip()}" ) - evidence_weak_points = list(package.qualityGate.errors[:4]) - evidence_weak_points.extend(package.qualityGate.warnings[:4]) - evidence_weak_points.extend( - f"{issue.module}: {issue.message}" - for issue in package.downstreamReadiness.blockingIssues[:4] - ) + evidence_weak_points: List[str] = [] if selected and selected.whyUnsolved: evidence_weak_points.append(selected.whyUnsolved) @@ -233,8 +222,8 @@ def build_plan_package_presentation(package: PlanPackage) -> PlanPackagePresenta evidenceSummary=PlanPresentationEvidenceSummary( confidence=_confidence(package), summary=( - f"{len(key_papers)} key papers are surfaced for user-facing evidence. " - f"Reviewer score is {package.qualityGate.overallScore:.2f}." + f"{len(key_papers)} key papers are surfaced with the selected " + "GAP and method evidence trace." ), supportingPapers=key_papers[:4], weakPoints=evidence_weak_points[:8], @@ -243,7 +232,7 @@ def build_plan_package_presentation(package: PlanPackage) -> PlanPackagePresenta decision=package.qualityGate.reviewDecision, score=package.qualityGate.overallScore, mainConcerns=_review_concerns(package), - requiredFixes=(package.metaReview.requiredRepairs[:6] if package.metaReview else []), + requiredFixes=required_user_actions(package), reviewerMode=package.generation.reviewerMode, llmReviewerUsed=package.generation.llmReviewerUsed, ), diff --git a/backend/app/services/prompts.py b/backend/app/services/prompts.py index c43a8b5..2b5117a 100644 --- a/backend/app/services/prompts.py +++ b/backend/app/services/prompts.py @@ -20,6 +20,7 @@ Research Topic: {seed_query} Paper Type: {paper_type} Domain: {domain} +{literature_context} Respond in JSON format: {{ @@ -41,9 +42,15 @@ EXPAND_QUERY_CJK_SUFFIX = """ IMPORTANT: The research topic above is in Chinese (CJK characters). -In addition to the fields above, please ALSO provide English translations for academic search: +In addition to the fields above, provide role-specific English academic queries: "englishSearchQueries": ["English search query 1", "English search query 2", ...], + "englishQueryRoles": { + "domain": ["queries about the named object/domain"], + "task": ["queries about the research task or gap"], + "method": ["queries about applicable methods"], + "evaluation": ["queries about evaluation and limitations"] + }, "englishKeyConcepts": ["English concept 1", "English concept 2", ...] These English queries will be used to search international academic databases. @@ -119,6 +126,18 @@ 4. Expected impact if addressed 5. Feasibility assessment +CRITICAL — Diversity Requirement for researchOpportunities: +You MUST generate at least 3 research opportunities that are INDEPENDENT of each other. +Each opportunity must differ from the others in at least TWO of these dimensions: +- problem formulation (what question is being asked) +- core methodology (what technical approach is used — e.g. contrastive learning, graph algorithms, probabilistic models, RL, etc.) +- evaluation setting (how success is measured) +- application domain (what real-world problem it addresses) + +Do NOT generate multiple opportunities that all revolve around the same technical +paradigm (e.g. all using "contrastive learning" or all using "attention mechanisms"). +Instead, cover genuinely distinct paradigms from the literature review. + Respond in JSON format: {{ "gapAnalysis": [ @@ -152,12 +171,23 @@ Key Literature: {key_papers} -Create 3-5 complementary research directions. Cover different contribution styles when relevant: -- method -- benchmark -- system -- safety_reliability -- application +IMPORTANT: You MUST create at least 3 research directions that are INDEPENDENT +from each other — each direction must explore a genuinely different technical +paradigm or problem angle. They must NOT all revolve around the same core +methodology (e.g. all "contrastive learning" or all "graph attention"). + +Each direction must differ from the others in at least TWO of these dimensions: +- problem formulation +- core methodology / technical paradigm +- evaluation setting +- application domain + +Cover DIFFERENT contribution styles when relevant: +- method (novel algorithm, architecture, or training approach) +- benchmark (new evaluation tasks, metrics, datasets) +- system (end-to-end pipeline, integration, deployment) +- safety_reliability (robustness, faithfulness, failure analysis) +- application (domain-specific adaptation with real-world use case) Each direction must stay faithful to the seed topic and cite why it is worth exploring. diff --git a/backend/app/services/ranking_service.py b/backend/app/services/ranking_service.py index 536b1cc..b11499f 100644 --- a/backend/app/services/ranking_service.py +++ b/backend/app/services/ranking_service.py @@ -10,8 +10,10 @@ import json import logging +import os import random import hashlib +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List, Dict, Any, Optional, Tuple from dataclasses import dataclass @@ -234,13 +236,22 @@ def _llm_rank( provider_name: str, model: str, ) -> List[RankingResult]: - """Rank using LLM-based scoring.""" - client = get_provider_client(provider_name) + """Rank using LLM-based scoring — P1: parallelized with ThreadPoolExecutor. + + Previously each candidate was scored sequentially. Now all candidates + are scored in parallel, controlled by FAROS_RANKING_CONCURRENCY (default 4). + """ weights = PAPER_TYPE_WEIGHTS.get(paper_type, PAPER_TYPE_WEIGHTS["default"]) - results = [] - - for candidate in candidates: + max_workers = min(len(candidates), int(os.getenv("FAROS_RANKING_CONCURRENCY", "4"))) + + # Pre-compute indices to preserve ordering + results_map: Dict[int, RankingResult] = {} + + def _score_one(idx: int, candidate: IdeaCandidate) -> Tuple[int, RankingResult]: try: + from app.llm.provider_client import get_provider_client, ChatMessage + client = get_provider_client(provider_name) + user_prompt = RANKING_USER_PROMPT.format( paper_type=paper_type, domain=domain or "general", @@ -252,25 +263,49 @@ def _llm_rank( ref_count=len(candidate.references), has_experiments="Yes" if candidate.requiredExperiments else "No", ) - + messages = [ ChatMessage(role="system", content=RANKING_SYSTEM_PROMPT), ChatMessage(role="user", content=user_prompt), ] - + response = client.chat(messages, model=model, max_tokens=800) - - # Parse response result = self._parse_llm_response(candidate.id, response.text, weights) - results.append(result) - + return idx, result except Exception as e: logger.warning(f"Failed to score candidate {candidate.id}: {e}") - # Use heuristic for this candidate heuristic_result = self._heuristic_score_single(candidate, seed_query, paper_type) - results.append(heuristic_result) - - return results + return idx, heuristic_result + + if max_workers <= 1 or len(candidates) <= 1: + # Fall back to sequential for small batches + results = [] + for i, candidate in enumerate(candidates): + _, result = _score_one(i, candidate) + results.append(result) + return results + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_idx: Dict[Any, int] = {} + for i, candidate in enumerate(candidates): + future = executor.submit(_score_one, i, candidate) + future_to_idx[future] = i + + for future in as_completed(future_to_idx): + try: + idx, result = future.result() + results_map[idx] = result + except Exception as e: + idx = future_to_idx[future] + logger.warning(f"Ranking worker failed for candidate[{idx}]: {e}") + # Use heuristic fallback for failed worker + heuristic_result = self._heuristic_score_single( + candidates[idx], seed_query, paper_type, + ) + results_map[idx] = heuristic_result + + # Preserve original order + return [results_map[i] for i in range(len(candidates))] def _parse_llm_response( self, candidate_id: str, response_text: str, weights: Dict[str, float] diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index fca41f6..436c00d 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -17,8 +17,9 @@ import re import ssl import time +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List, Dict, Any, Optional -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime import urllib.request import urllib.parse @@ -64,6 +65,26 @@ def _env_bool(name: str, default: bool = False) -> bool: return raw.strip().lower() in {"1", "true", "yes", "on"} +def tokenize_topic_text(text: str) -> List[str]: + """Tokenize English words and Han text for lightweight topic matching.""" + normalized = (text or "").lower().replace("-", " ") + tokens: List[str] = [] + + def _add(token: str) -> None: + if token and token not in tokens: + tokens.append(token) + + for token in re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}", normalized): + _add(token) + for span in re.findall(r"[\u4e00-\u9fff]+", normalized): + if len(span) < 2: + continue + _add(span) + for index in range(len(span) - 1): + _add(span[index:index + 2]) + return tokens + + @dataclass class SearchResult: """A single search result.""" @@ -78,6 +99,18 @@ class SearchResult: citation_count: Optional[int] source: str # "semantic_scholar", "arxiv", "local" relevance_score: float = 0.0 + retrieval_roles: List[str] = field(default_factory=list) + matched_queries: List[str] = field(default_factory=list) + evidence_tier: str = "unclassified" + decisive_anchors: List[str] = field(default_factory=list) + relevance_components: Dict[str, float] = field(default_factory=dict) + rejection_reason: str = "" + must_cite_override: bool = False + retrieval_sources: List[str] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.source and self.source not in self.retrieval_sources: + self.retrieval_sources.append(self.source) class SemanticScholarSearch: @@ -96,6 +129,12 @@ def __init__(self, api_key: Optional[str] = None): # Unauthenticated free tier is 100 requests / 5 minutes. Keep a small # margin so multi-query idea sessions do not immediately hit 429. self.min_request_interval = 1.0 if api_key else 3.2 + try: + self.rate_limit_cooldown_seconds = float( + os.getenv("FAROS_SEMANTIC_SCHOLAR_429_COOLDOWN_SECONDS", "180") + ) + except ValueError: + self.rate_limit_cooldown_seconds = 180.0 def _rate_limit(self): """Ensure we don't exceed rate limits.""" @@ -135,6 +174,11 @@ def _make_request(self, endpoint: str, params: Dict[str, Any]) -> Optional[Dict] logger.warning("Semantic Scholar rate limited; retrying in %.1fs", delay) time.sleep(delay) continue + if e.code == 429: + self.disabled_until = time.time() + max( + 30.0, + self.rate_limit_cooldown_seconds, + ) logger.warning(f"Semantic Scholar API error: {e.code} - {e.reason}") return None except urllib.error.URLError as e: @@ -289,7 +333,7 @@ def _build_query_variants(self, query: str) -> List[str]: phrase_parts.append(f'all:"{phrase}"') consumed_terms.update(phrase.split()) - tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9-]{2,}|[\u4e00-\u9fff]+", normalized) + tokens = tokenize_topic_text(normalized) stopwords = { "and", "the", "for", "with", "from", "that", "this", "into", "using", "based", "how", "can", "are", "what", "when", "where", @@ -580,7 +624,7 @@ def _query_terms(self, query: str) -> List[str]: "model", "models", "large", "learning", "neural", } terms: List[str] = [] - for token in re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", normalized): + for token in tokenize_topic_text(normalized): if token in stopwords: continue if token not in terms: @@ -596,9 +640,9 @@ def _compute_relevance(self, paper: Dict, query: str) -> float: title = paper.get("title", "").lower().replace("-", " ") abstract = paper.get("abstract", "").lower().replace("-", " ") keywords_text = " ".join(str(k).lower().replace("-", " ") for k in paper.get("keywords", [])) - title_terms = set(re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", title)) - keyword_terms = set(re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", keywords_text)) - abstract_terms = set(re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", abstract)) + title_terms = set(tokenize_topic_text(title)) + keyword_terms = set(tokenize_topic_text(keywords_text)) + abstract_terms = set(tokenize_topic_text(abstract)) title_hits = len(query_terms & title_terms) keyword_hits = len(query_terms & keyword_terms) @@ -795,7 +839,8 @@ def search( arxiv_id=None, citation_count=citation_count, source="openalex", - relevance_score=float(work.get("relevance_score") or 0.0), + # OpenAlex scores are ranking magnitudes, not normalized probabilities. + relevance_score=0.0, )) return results[:limit] @@ -918,64 +963,57 @@ def search( sources: Optional[List[str]] = None ) -> List[SearchResult]: """ - Search for papers across all available sources. - + Search for papers across all available sources **in parallel**. + Args: query: Search query limit: Maximum results per source - sources: Optional list of sources to use ["semantic_scholar", "arxiv", "local"] - + sources: Optional list of sources to use + Returns: Combined list of SearchResult objects, deduplicated by title """ - all_results = [] + all_results: List[SearchResult] = [] sources = sources or ["semantic_scholar", "arxiv", "openalex", "cnki", "wanfang", "local"] - - # Try Semantic Scholar - if "semantic_scholar" in sources and self.semantic_scholar: - try: - results = self.semantic_scholar.search(query, limit) - all_results.extend(results) - logger.info(f"Semantic Scholar returned {len(results)} results") - except Exception as e: - logger.warning(f"Semantic Scholar search failed: {e}") - - # Try ArXiv - if "arxiv" in sources and self.arxiv: - try: - results = self.arxiv.search(query, limit) - all_results.extend(results) - logger.info(f"ArXiv returned {len(results)} results") - except Exception as e: - logger.warning(f"ArXiv search failed: {e}") - - # Try OpenAlex (free, no key required, indexes global papers) - if "openalex" in sources and self.openalex: - try: - results = self.openalex.search(query, limit) - all_results.extend(results) - logger.info(f"OpenAlex returned {len(results)} results") - except Exception as e: - logger.warning(f"OpenAlex search failed: {e}") - - # Try CNKI (requires API key, disabled if not configured) - if "cnki" in sources and self.cnki: - try: - results = self.cnki.search(query, limit) - all_results.extend(results) - logger.info(f"CNKI returned {len(results)} results") - except Exception as e: - logger.warning(f"CNKI search failed: {e}") - - # Try WanFang (requires API key, disabled if not configured) - if "wanfang" in sources and self.wanfang: - try: - results = self.wanfang.search(query, limit) - all_results.extend(results) - logger.info(f"WanFang returned {len(results)} results") - except Exception as e: - logger.warning(f"WanFang search failed: {e}") - + + # Build parallel search tasks (exclude local corpus — it's a sync offline fallback) + parallel_sources = [s for s in sources if s != "local"] + source_map: Dict[str, Any] = { + "semantic_scholar": self.semantic_scholar, + "arxiv": self.arxiv, + "openalex": self.openalex, + "cnki": self.cnki, + "wanfang": self.wanfang, + } + + search_tasks: List[tuple] = [] + for source_name in parallel_sources: + searcher = source_map.get(source_name) + if searcher: + search_tasks.append((source_name, searcher)) + + if search_tasks: + # P0: Parallelize external search sources. + # Each source has its own rate-limit logic, so running them concurrently + # cuts wall-clock time from sum(intervals) to max(interval). + max_workers = min(len(search_tasks), int(os.getenv("FAROS_SEARCH_PARALLELISM", "5"))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_source: Dict[Any, str] = {} + for source_name, searcher in search_tasks: + future = executor.submit( + self._safe_search_source, source_name, searcher, query, limit, + ) + future_to_source[future] = source_name + + for future in as_completed(future_to_source): + source_name = future_to_source[future] + try: + results: List[SearchResult] = future.result() + all_results.extend(results) + logger.info(f"{source_name} returned {len(results)} results for '{query[:60]}'") + except Exception as e: + logger.warning(f"{source_name} search failed for '{query[:60]}': {e}") + external_results_count = len(all_results) # Try local corpus only as a strict offline fallback by default. Mixing @@ -989,7 +1027,7 @@ def search( logger.info(f"Local corpus returned {len(results)} results") except Exception as e: logger.warning(f"Local search failed: {e}") - + # Deduplicate by normalized title seen_titles = set() unique_results = [] @@ -998,9 +1036,19 @@ def search( if normalized_title not in seen_titles: seen_titles.add(normalized_title) unique_results.append(result) - + return unique_results[:limit * 2] # Return up to 2x limit after dedup + @staticmethod + def _safe_search_source( + source_name: str, + searcher: Any, + query: str, + limit: int, + ) -> List[SearchResult]: + """Search a single source. Static so it is picklable for ThreadPoolExecutor.""" + return searcher.search(query, limit) + # Global service instance _search_service: Optional[SearchService] = None diff --git a/backend/app/storage/idea_storage.py b/backend/app/storage/idea_storage.py index c16a685..d2e5e7f 100644 --- a/backend/app/storage/idea_storage.py +++ b/backend/app/storage/idea_storage.py @@ -359,6 +359,18 @@ def create(self, paper: RawPaper) -> RawPaper: return paper + def update(self, paper: RawPaper) -> RawPaper: + """Atomically update an existing raw paper.""" + path = self._get_path(paper.id) + if not path.exists(): + raise ValueError(f"RawPaper {paper.id} not found") + + data = paper.model_dump() + data['createdAt'] = data['createdAt'].isoformat() if isinstance(data['createdAt'], datetime) else data['createdAt'] + _write_json_atomic(path, data, default=str) + + return paper + def get(self, paper_id: str) -> Optional[RawPaper]: """Get raw paper by ID.""" path = self._get_path(paper_id) diff --git a/backend/scripts/run_idea_plan_eval.py b/backend/scripts/run_idea_plan_eval.py index 9cf6943..1f2b6d6 100644 --- a/backend/scripts/run_idea_plan_eval.py +++ b/backend/scripts/run_idea_plan_eval.py @@ -10,6 +10,7 @@ from app.models.idea import IdeaSessionConfig from app.modules.idea.service import IdeaGenerationService from app.services.plan_package_service import get_plan_package_service +from app.services.plan_package_views import build_plan_package_presentation SEEDS = [ @@ -40,6 +41,16 @@ "baselineSeconds": 1439, "baselineNote": "previous run: final=2, evidence=0.95, candidates similar", }, + { + "label": "D_cjk_named_work_negative_stress", + "seed": "预测红楼梦可能结局", + "domain": "", + "paperType": "system", + "maxCandidates": 3, + "baselineSeconds": 975, + "baselineNote": "negative pollution stress; awaiting_evidence is valid", + "negativeStress": True, + }, ] @@ -128,6 +139,61 @@ def null_paths(value, prefix=""): return paths +def plan_quality_summary(package, elapsed_seconds): + plan_owned = { + "researchQuestion": package.researchQuestion, + "hypothesis": package.hypothesis, + "constants": package.constants, + "stages": [stage.model_dump(mode="json") for stage in package.stages], + } + expected = [ + item + for stage in plan_owned["stages"] + for step in stage.get("steps", []) + for item in step.get("expected", []) + ] + text = json.dumps(plan_owned, ensure_ascii=False).lower() + placeholders = [ + value + for value in [ + "specified before implementation", + "primary_metric", + "readiness", + "planned_metric", + "default plan step", + ] + if value in text + ] + segment_fallbacks = [ + warning + for warning in package.generation.warnings + if warning.startswith("segment_fallback:") + ] + presentation = build_plan_package_presentation(package) + return { + "elapsedSeconds": round(float(elapsed_seconds), 3), + "status": str( + package.status.value + if hasattr(package.status, "value") + else package.status + ), + "fallbackUsed": package.generation.fallbackUsed, + "segmentFallbacks": segment_fallbacks, + "llmUsedSections": package.generation.llmUsedSections, + "repairRounds": package.generation.repairRounds, + "schemaRepairRounds": package.generation.schemaRepairRounds, + "llmReviewerUsed": package.generation.llmReviewerUsed, + "implementationReady": package.qualityGate.implementationReady, + "downstreamReady": package.qualityGate.downstreamReady, + "stageCount": len(package.stages), + "stepCount": sum(len(stage.steps) for stage in package.stages), + "expectedMetricCount": len(expected), + "placeholderValues": placeholders, + "criticalNullPaths": null_paths(plan_owned), + "userConcernCount": len(presentation.reviewSummary.mainConcerns), + } + + def candidate_quality(candidate): critique = candidate.critique if isinstance(critique, dict): @@ -153,6 +219,89 @@ def candidate_quality(candidate): } +def retrieval_quality(literature_outputs): + return { + "resultCountBeforeDedup": literature_outputs.get("resultCountBeforeDedup", 0), + "uniqueResultCount": literature_outputs.get("uniqueResultCount", 0), + "duplicateMergeCount": literature_outputs.get("duplicateMergeCount", 0), + "evidenceTierCounts": literature_outputs.get("evidenceTierCounts", {}), + "rejectionReasonCounts": literature_outputs.get("rejectionReasonCounts", {}), + "retrievalRoleCounts": literature_outputs.get("retrievalRoleCounts", {}), + "topicIntentProfile": literature_outputs.get("topicIntentProfile", {}), + } + + +def freeze_checks( + *, + session, + spec, + literature_outputs, + novelty_outputs, + evidence_gate, + deep_read_limit, +): + status_value = getattr(session.status, "value", str(session.status)) + is_negative_stress = bool(spec.get("negativeStress", False)) + raw_gate = literature_outputs.get("paperQualityGate", {}) + return { + "positiveSeedCompleted": is_negative_stress or status_value == "completed", + "completionHasTwoIdeas": ( + status_value != "completed" or len(session.finalCandidateIds or []) >= 2 + ), + "waitingStateIsRecoverable": status_value in { + "completed", + "awaiting_evidence", + "awaiting_ideas", + }, + "rawRoleCoverageEnabled": bool( + (raw_gate.get("roleCoverage") or {}).get("enabled", False) + ), + "structuredRoleCoverageEnabled": ( + is_negative_stress and status_value == "awaiting_evidence" + ) or bool((evidence_gate.get("roleCoverage") or {}).get("enabled", False)), + "deepReadBounded": int( + novelty_outputs.get("deepReadRequestedCount", 0) or 0 + ) <= int(deep_read_limit), + } + + +def build_closure_report(summary): + hard_checks = [ + check + for seed in summary.get("seeds", []) + for check in seed.get("freezeChecks", {}).values() + ] + return { + "decision": ( + "freeze" if hard_checks and all(hard_checks) else "continue_closure" + ), + "technicalSeeds": [ + { + "label": seed["label"], + "sessionId": seed.get("sessionId"), + "status": seed.get("status"), + "finalCandidateIds": seed.get("finalCandidateIds", []), + "performance": seed.get("performance", {}), + "retrievalQuality": seed.get("retrievalQuality", {}), + "freezeChecks": seed.get("freezeChecks", {}), + } + for seed in summary.get("seeds", []) + ], + "externalProviderIncidents": [ + seed.get("sessionError") + for seed in summary.get("seeds", []) + if seed.get("sessionError") + and "provider" in seed.get("sessionError", "").lower() + ], + "remainingRisks": [ + f"{seed['label']}:{name}" + for seed in summary.get("seeds", []) + for name, passed in seed.get("freezeChecks", {}).items() + if not passed + ], + } + + def main() -> None: summary_path = os.environ["FAROS_EVAL_SUMMARY"] summary_dir = os.path.dirname(os.path.abspath(summary_path)) @@ -210,7 +359,7 @@ def main() -> None: service.start_session(session.id) session = service.run_pipeline(session.id) item["ideaSeconds"] = round(time.perf_counter() - seed_start, 2) - item["status"] = str(session.status) + item["status"] = getattr(session.status, "value", str(session.status)) item["sessionError"] = session.errorMessage item["finalCandidateIds"] = list(session.finalCandidateIds or []) item["hiddenCandidateIds"] = list(session.hiddenCandidateIds or []) @@ -229,6 +378,7 @@ def main() -> None: rank_out = steps.get("rankCandidates", {}).get("outputs", {}) evidence_out = steps.get("evidenceGate", {}).get("outputs", {}) evidence_gate = evidence_out.get("evidenceGate", evidence_out) + literature_out = steps.get("literatureSearch", {}).get("outputs", {}) repair_novelty_out = pick(evidence_out, ["repairReport", "noveltyOutputs"], {}) or {} quality_loop = quality_loop_summary(rank_out, session) review_usage = reviewer_usage(rank_out) @@ -256,6 +406,17 @@ def main() -> None: "regeneratedCandidateIds": rank_out.get("regeneratedCandidateIds"), "reviewerUsage": review_usage, } + item["retrievalQuality"] = retrieval_quality(literature_out) + item["freezeChecks"] = freeze_checks( + session=session, + spec=spec, + literature_outputs=literature_out, + novelty_outputs=novelty_out, + evidence_gate=evidence_gate, + deep_read_limit=int( + os.getenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "24") + ), + ) item["qualitySignals"] = { "finalCandidateCount": len(session.finalCandidateIds or []), "targetFinalCandidateCount": quality_loop.get("targetFinalCandidateCount"), @@ -313,9 +474,10 @@ def main() -> None: reviewer_mode="hybrid", max_repair_rounds=2, ) + plan_elapsed = time.perf_counter() - plan_start item["planPackage"] = { "packageId": package.packageId, - "seconds": round(time.perf_counter() - plan_start, 2), + "seconds": round(plan_elapsed, 2), "schemaVersion": package.schemaVersion, "status": str(package.status), "qualityGate": package.qualityGate.model_dump() @@ -356,6 +518,9 @@ def main() -> None: if hasattr(package.generation, "model_dump") else {}, } + item["planPackage"].update( + plan_quality_summary(package, plan_elapsed) + ) item["finishedAt"] = utcnow() print( json.dumps( @@ -385,9 +550,27 @@ def main() -> None: json.dump(summary, f, ensure_ascii=False, indent=2, default=str) summary["finishedAt"] = utcnow() + closure_report = build_closure_report(summary) + report_path = os.path.join(summary_dir, "idea-closure-report.json") + summary["closureReport"] = report_path with open(summary_path, "w", encoding="utf-8") as f: json.dump(summary, f, ensure_ascii=False, indent=2, default=str) - print(json.dumps({"event": "eval_done", "summary": summary_path}, ensure_ascii=False), flush=True) + with open(report_path, "w", encoding="utf-8") as f: + json.dump(closure_report, f, ensure_ascii=False, indent=2, default=str) + print( + json.dumps( + { + "event": "eval_done", + "summary": summary_path, + "closureReport": report_path, + "decision": closure_report["decision"], + }, + ensure_ascii=False, + ), + flush=True, + ) + if closure_report["decision"] != "freeze": + raise SystemExit(1) if __name__ == "__main__": diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 75c6a31..7dd58e9 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -9,6 +9,8 @@ import pytest +from tests.plan_package_test_data import make_plan_package + # Ensure the backend package is importable sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -83,3 +85,8 @@ def temp_import_error_repo(): ) yield str(repo) + + +@pytest.fixture +def plan_package(): + return make_plan_package() diff --git a/backend/tests/plan_package_test_data.py b/backend/tests/plan_package_test_data.py new file mode 100644 index 0000000..3b97615 --- /dev/null +++ b/backend/tests/plan_package_test_data.py @@ -0,0 +1,235 @@ +from app.models.plan_package import ( + PlanBackground, + PlanContributionStatement, + PlanEvidenceRef, + PlanEvidenceTrace, + PlanExpectedMetric, + PlanGap, + PlanGapItem, + PlanIdeaSummary, + PlanLiteratureCoverage, + PlanLiteraturePaperSummary, + PlanLiteratureSurvey, + PlanOutput, + PlanPackage, + PlanPrinciple, + PlanSource, + PlanStage, + PlanStep, +) + + +def make_plan_package() -> PlanPackage: + paper_id = "paper-rag-1" + return PlanPackage( + packageId="ppkg_test", + source=PlanSource( + ideaSessionId="idea-test", + ideaCandidateId="cand-test", + searchTreeId="tree-test", + searchNodeId="node-test", + pathSeedId="path-test", + literatureMapId="lit-test", + ), + idea=PlanIdeaSummary( + id="cand-test", + title="Citation-faithful refusal-aware RAG", + problem="High-risk QA needs faithful citations and reliable refusal.", + hypothesisStatement="Claim verification improves citation faithfulness.", + proposedMethod="Verify answer claims against cited passages and refuse unsupported answers.", + expectedOutcome="Higher citation faithfulness without weaker refusal accuracy.", + closestPriorWork=[{"paperId": paper_id, "method": "vanilla RAG"}], + ), + background=PlanBackground( + summary="High-risk RAG requires claim-level evidence verification.", + motivation="Unsupported clinical answers can cause harmful decisions.", + currentLimitations=["Vanilla RAG can cite passages that do not support answer claims."], + evidenceRefs=[PlanEvidenceRef(type="paper", id=paper_id, source="structured")], + ), + literatureSurvey=PlanLiteratureSurvey( + summary="Prior work studies citation attribution and abstention separately.", + coverage=PlanLiteratureCoverage( + rawPaperCount=1, + selectedPaperCount=1, + structuredPaperCount=1, + ), + papers=[ + PlanLiteraturePaperSummary( + paperId=paper_id, + structuredPaperId=paper_id, + source="structured", + title="Citation Attribution for Retrieval-Augmented Generation", + summary="Evaluates claim-level citation faithfulness against vanilla RAG.", + relevanceScore=0.95, + relevanceSignals=["citation", "faithfulness", "RAG", "baseline"], + relevanceReason="Directly supports the selected method and metric.", + methods=[{"name": "vanilla RAG", "role": "baseline"}], + claims=[ + { + "claimId": "claim-rag-1", + "text": "Attribution precision measures citation support.", + } + ], + ) + ], + ), + gap=PlanGap( + summary="Joint citation verification and refusal evaluation remains under-specified.", + selectedGapId="gap-1", + items=[ + PlanGapItem( + id="gap-1", + kind="selected", + statement="Existing RAG evaluations do not jointly test claim support and refusal.", + severity="high", + existingCoverage="Citation and refusal are usually evaluated separately.", + unresolvedIssue="Their interaction on insufficient-evidence questions remains unclear.", + proposedEntry="Add a claim verifier with an evidence-aware refusal rule.", + boundary="High-risk question answering with retrieved textual evidence.", + validationNeeds=["citation faithfulness", "refusal accuracy"], + whyUnsolved="Existing evaluations lack a joint protocol.", + supportedByPaperIds=[paper_id], + supportedByClaimIds=["claim-rag-1"], + ) + ], + ), + principle=PlanPrinciple( + summary="Verify every answer claim before allowing a supported response.", + mechanism="Align claims to cited spans and refuse when support is insufficient.", + noveltyClaim="Jointly bind citation attribution and refusal to the same evidence test.", + assumptions=[ + "The retrieved corpus contains answer-supporting evidence when the question is answerable." + ], + risks=["An over-conservative verifier can reduce answer coverage."], + ), + contributionStatement=[ + PlanContributionStatement( + id="contribution-1", + type="method", + statement="A joint claim-verification and refusal mechanism.", + noveltyBasis="Prior work evaluates citation and refusal separately.", + validationStageIds=["stage-3"], + validationStepIds=["step-3-1"], + evidenceRefs=[PlanEvidenceRef(type="paper", id=paper_id, source="structured")], + ) + ], + researchQuestion="Can claim verification improve citation faithfulness over vanilla RAG on high-risk QA?", + hypothesis=( + "Compared with vanilla RAG, claim verification increases citation faithfulness; " + "reject the hypothesis if the mean delta is not positive on the preregistered split." + ), + constants={ + "seedQuery": "citation-faithful medical RAG for high-risk clinical question answering", + "paperType": "algorithmic_method", + "baseline": "vanilla RAG", + "datasetProtocol": "fixed preregistered train, validation, and test split", + "randomSeeds": [13, 29, 47], + }, + stages=[ + PlanStage( + id="stage-1", + order=1, + title="Evidence and baseline grounding", + goal="Bind the selected GAP to the strongest available comparison.", + method="Use the selected paper and GAP to define vanilla RAG as the control.", + steps=[ + PlanStep( + id="step-1-1", + order=1, + title="Freeze baseline scope", + desc="Record the control method and shared evaluation inputs.", + method="Use vanilla RAG with the same corpus, prompts, split, and evaluator.", + outputs=[ + PlanOutput( + type="table", + name="baseline_scope.csv", + requiredFor=["validation", "paper"], + ) + ], + expected=[PlanExpectedMetric(metric="baseline_count", target=">= 1")], + evidenceRefs=[ + PlanEvidenceRef(type="paper", id=paper_id, source="structured") + ], + ) + ], + ), + PlanStage( + id="stage-2", + order=2, + title="Verifier implementation", + goal="Specify the claim verifier and refusal rule.", + method="Map answer claims to cited spans and emit supported, unsupported, or refuse.", + dependsOn=["stage-1"], + steps=[ + PlanStep( + id="step-2-1", + order=1, + title="Implement claim verification", + desc="Define inputs, outputs, and refusal thresholds.", + method="Apply the same verifier configuration to every preregistered run.", + inputFrom=["step-1-1"], + outputs=[ + PlanOutput( + type="code", + name="claim_verifier.py", + requiredFor=["code", "validation"], + ) + ], + expected=[PlanExpectedMetric(metric="artifact_count", target=">= 1")], + evidenceRefs=[ + PlanEvidenceRef(type="candidate", id="cand-test", source="idea") + ], + ) + ], + ), + PlanStage( + id="stage-3", + order=3, + title="Controlled validation", + goal="Test citation faithfulness and refusal accuracy against vanilla RAG.", + method="Run paired evaluation on the same preregistered split and random seeds.", + dependsOn=["stage-2"], + steps=[ + PlanStep( + id="step-3-1", + order=1, + title="Measure primary outcomes", + desc="Compare paired citation and refusal metrics.", + method="Report mean deltas and 95% confidence intervals across fixed seeds.", + inputFrom=["step-2-1"], + outputs=[ + PlanOutput( + type="metrics", + name="validation_metrics.json", + requiredFor=["validation", "paper"], + ) + ], + expected=[ + PlanExpectedMetric( + metric="citation faithfulness", + target="mean_delta > 0 with 95% confidence interval excluding 0", + ), + PlanExpectedMetric( + metric="refusal accuracy", + target=">= vanilla RAG on the preregistered split", + ), + ], + evidenceRefs=[ + PlanEvidenceRef(type="gap", id="gap-1", source="literature_map") + ], + ) + ], + ), + ], + evidenceTrace=PlanEvidenceTrace( + ideaCandidateId="cand-test", + searchNodeId="node-test", + pathSeedId="path-test", + literatureMapId="lit-test", + selectedPaperIds=[paper_id], + structuredPaperIds=[paper_id], + reasoningTrace=[ + {"from": "gap-1", "to": "cand-test", "relation": "addressed_by"} + ], + ), + ) diff --git a/backend/tests/test_idea_eval_telemetry.py b/backend/tests/test_idea_eval_telemetry.py new file mode 100644 index 0000000..ef8ceb6 --- /dev/null +++ b/backend/tests/test_idea_eval_telemetry.py @@ -0,0 +1,86 @@ +import importlib.util +from pathlib import Path +from types import SimpleNamespace + + +def _load_eval_module(): + path = Path(__file__).parents[1] / "scripts" / "run_idea_plan_eval.py" + spec = importlib.util.spec_from_file_location("run_idea_plan_eval", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_positive_seed_freeze_checks_require_completion_and_two_ideas(): + module = _load_eval_module() + session = SimpleNamespace( + status=SimpleNamespace(value="completed"), + finalCandidateIds=["cand_1", "cand_2"], + ) + + checks = module.freeze_checks( + session=session, + spec={"negativeStress": False}, + literature_outputs={ + "paperQualityGate": {"roleCoverage": {"enabled": True}}, + }, + novelty_outputs={"deepReadRequestedCount": 12}, + evidence_gate={"roleCoverage": {"enabled": True}}, + deep_read_limit=24, + ) + + assert checks == { + "positiveSeedCompleted": True, + "completionHasTwoIdeas": True, + "waitingStateIsRecoverable": True, + "rawRoleCoverageEnabled": True, + "structuredRoleCoverageEnabled": True, + "deepReadBounded": True, + } + + +def test_negative_stress_seed_may_wait_for_evidence_before_structured_gate(): + module = _load_eval_module() + session = SimpleNamespace( + status=SimpleNamespace(value="awaiting_evidence"), + finalCandidateIds=[], + ) + + checks = module.freeze_checks( + session=session, + spec={"negativeStress": True}, + literature_outputs={ + "paperQualityGate": {"roleCoverage": {"enabled": True}}, + }, + novelty_outputs={}, + evidence_gate={}, + deep_read_limit=24, + ) + + assert all(checks.values()) + + +def test_closure_report_continues_when_any_hard_check_fails(): + module = _load_eval_module() + summary = { + "seeds": [ + { + "label": "A", + "sessionId": "idea_a", + "status": "completed", + "finalCandidateIds": ["cand_1"], + "performance": {}, + "retrievalQuality": {}, + "freezeChecks": { + "positiveSeedCompleted": True, + "completionHasTwoIdeas": False, + }, + } + ] + } + + report = module.build_closure_report(summary) + + assert report["decision"] == "continue_closure" + assert report["remainingRisks"] == ["A:completionHasTwoIdeas"] diff --git a/backend/tests/test_idea_evidence_gate.py b/backend/tests/test_idea_evidence_gate.py new file mode 100644 index 0000000..a093543 --- /dev/null +++ b/backend/tests/test_idea_evidence_gate.py @@ -0,0 +1,308 @@ +from app.models.idea import RawPaper +from app.modules.idea.service import ( + _evaluate_evidence_gate_v2, + _evaluate_paper_quality_gate, + _normalize_coverage_report, + _verify_coverage_dimension_support, +) +from app.services.search_service import SearchResult + + +def _raw( + title: str, + *, + tier: str, + roles: list[str], + index: int, + relevance: float = 0.9, +) -> RawPaper: + return RawPaper( + id=f"raw_gate_{index}", + sessionId="idea_gate", + title=title, + abstract=title, + source=["openalex"], + retrievalRoles=roles, + evidenceTier=tier, + relevanceScore=relevance, + ) + + +def test_paper_quality_gate_exposes_local_only_fallback_risk(): + papers = [ + RawPaper( + id=f"raw_local_{index}", + sessionId="idea_local", + title=f"Citation faithful RAG local fallback paper {index}", + abstract="Citation faithful retrieval augmented generation with refusal and traceability.", + source=["local"], + relevanceScore=0.9, + ) + for index in range(4) + ] + + gate = _evaluate_paper_quality_gate( + seed="citation faithful RAG for high-risk QA", + domain="", + papers=papers, + stage="literatureSearch.initial", + ) + + assert gate["sourceQuality"] == "local_only" + assert gate["providerFallbackRisk"] == "high" + assert gate["localOnly"] is True + assert "local" in gate["sourcesUsed"] + + +def test_paper_quality_gate_accepts_role_composed_interdisciplinary_evidence(): + papers = [ + RawPaper( + id="raw_domain_1", + sessionId="idea_roles", + title="The known ending and narrative closure in Dream of the Red Chamber", + abstract="A study of Hongloumeng ending scholarship and narrative closure.", + source=["openalex"], + retrievalRoles=["domain"], + relevanceScore=0.9, + ), + RawPaper( + id="raw_domain_2", + sessionId="idea_roles", + title="Hongloumeng authorship and the unfinished ending", + abstract="Textual evidence about the ending of Dream of the Red Chamber.", + source=["openalex"], + retrievalRoles=["task"], + relevanceScore=0.85, + ), + RawPaper( + id="raw_method", + sessionId="idea_roles", + title="Computational narrative completion for unfinished literary works", + abstract="A method for narrative closure reconstruction using character constraints.", + source=["arxiv"], + retrievalRoles=["method"], + relevanceScore=0.8, + ), + RawPaper( + id="raw_evaluation", + sessionId="idea_roles", + title="Evaluating narrative coherence and character consistency", + abstract="Metrics and human evaluation for reconstructed story endings.", + source=["arxiv"], + retrievalRoles=["evaluation"], + relevanceScore=0.8, + ), + ] + + gate = _evaluate_paper_quality_gate( + seed="\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40", + domain="computational literary studies", + papers=papers, + stage="literatureSearch.initial", + extra_terms=[ + "Dream of the Red Chamber ending studies", + "Hongloumeng narrative closure reconstruction", + ], + ) + + assert gate["passed"] is True + assert gate["roleCoverage"]["passed"] is True + assert gate["roleCoverage"]["counts"] == { + "domain": 1, + "task": 1, + "method": 1, + "evaluation": 1, + } + + +def test_role_coverage_does_not_pass_with_rejected_query_hits(): + papers = [ + _raw("Clinical forecasting", tier="rejected", roles=["task"], index=1), + _raw("Chemical evaluation", tier="rejected", roles=["evaluation"], index=2), + _raw("Generic framework", tier="rejected", roles=["method"], index=3), + _raw("Generic survey", tier="rejected", roles=["domain"], index=4), + ] + + gate = _evaluate_paper_quality_gate( + seed="citation-faithful medical RAG", + domain="medical QA", + papers=papers, + stage="test", + paper_type="system", + ) + + assert gate["roleCoverage"]["enabled"] is True + assert gate["roleCoverage"]["passed"] is False + assert gate["alignedPaperCount"] == 0 + + +def test_survey_does_not_require_method_or_evaluation_query_role(): + papers = [ + _raw("RAG safety survey and open gaps", tier="direct", roles=["domain"], index=5), + _raw("Retrieval augmented generation safety limitations", tier="direct", roles=["task"], index=6), + _raw("RAG safety claims and synthesis", tier="direct", roles=["domain"], index=7), + _raw("Open questions in safe RAG", tier="direct", roles=["task"], index=8), + ] + + gate = _evaluate_paper_quality_gate( + seed="RAG safety survey", + domain="retrieval augmented generation", + papers=papers, + stage="test", + paper_type="survey", + ) + + assert gate["roleCoverage"]["requirements"]["method"] == 0 + assert gate["roleCoverage"]["requirements"]["evaluation"] == 0 + assert gate["roleCoverage"]["passed"] is True + + +def test_transferable_paper_cannot_fill_domain_role(): + paper = _raw( + "Narrative completion method for unfinished novel endings", + tier="transferable", + roles=["domain", "method"], + index=9, + ) + + gate = _evaluate_paper_quality_gate( + seed="Dream of the Red Chamber ending", + domain="computational literary studies", + papers=[paper], + stage="test", + paper_type="system", + ) + + assert gate["roleCoverage"]["counts"]["domain"] == 0 + assert gate["roleCoverage"]["counts"]["method"] == 1 + + +def test_role_coverage_cannot_bypass_weak_semantic_alignment(): + papers = [ + _raw("Medical clinical forecasting", tier="direct", roles=["domain"], index=10, relevance=0.0), + _raw("RAG chemical synthesis", tier="direct", roles=["task"], index=11, relevance=0.0), + _raw("Citation air filtration materials", tier="direct", roles=["method"], index=12, relevance=0.0), + _raw("Faithful hydraulic actuator benchmark", tier="direct", roles=["evaluation"], index=13, relevance=0.0), + ] + + gate = _evaluate_paper_quality_gate( + seed="citation-faithful medical RAG", + domain="medical QA", + papers=papers, + stage="test", + paper_type="system", + ) + + assert gate["roleCoverage"]["passed"] is True + assert gate["passed"] is False + + +def test_raw_search_results_expose_snake_case_role_coverage(): + roles = ["domain", "task", "method", "evaluation"] + papers = [] + for index, role in enumerate(roles): + paper = SearchResult( + title=f"Citation faithful medical RAG {role}", + authors=[], + abstract="Medical retrieval augmented generation citation refusal evaluation.", + year=2025, + venue="test", + url="", + doi=f"10.1000/{role}", + arxiv_id=None, + citation_count=0, + source="openalex", + relevance_score=0.9, + evidence_tier="direct", + retrieval_roles=[role], + ) + papers.append(paper) + + gate = _evaluate_paper_quality_gate( + seed="citation-faithful medical RAG", + domain="medical QA", + papers=papers, + stage="test", + paper_type="system", + ) + + assert gate["roleCoverage"]["enabled"] is True + assert gate["roleCoverage"]["counts"] == { + "domain": 1, + "task": 1, + "method": 1, + "evaluation": 1, + } + + +def test_evidence_gate_propagates_paper_type_to_role_requirements(): + papers = [ + _raw("RAG safety survey and open gaps", tier="direct", roles=["domain"], index=20), + _raw("Retrieval augmented generation safety limitations", tier="direct", roles=["task"], index=21), + _raw("RAG safety claims and synthesis", tier="direct", roles=["domain"], index=22), + _raw("Open questions in safe RAG", tier="direct", roles=["task"], index=23), + ] + + gate = _evaluate_evidence_gate_v2( + seed="RAG safety survey", + domain="retrieval augmented generation", + paper_type="survey", + structured_papers=papers, + literature_map=None, + gap_outputs={"gaps": ["RAG safety synthesis remains incomplete"]}, + stage="test", + ) + + assert gate["roleCoverage"]["requirements"]["method"] == 0 + assert gate["roleCoverage"]["requirements"]["evaluation"] == 0 + + +def test_transferable_paper_cannot_be_the_only_gap_support(): + verified = _verify_coverage_dimension_support( + dimension="gap", + supporting_paper_ids=["raw_transfer"], + paper_tiers={"raw_transfer": "transferable"}, + ) + + assert verified == [] + + +def test_transferable_paper_can_support_method_dimension(): + verified = _verify_coverage_dimension_support( + dimension="method", + supporting_paper_ids=["raw_transfer"], + paper_tiers={"raw_transfer": "transferable"}, + ) + + assert verified == ["raw_transfer"] + + +def test_coverage_report_drops_transferable_gap_support_but_keeps_method_support(): + report = _normalize_coverage_report( + { + "passed": True, + "overallEvidenceScore": 0.9, + "dimensions": [ + { + "key": "gap", + "required": True, + "score": 0.9, + "supportingPaperIds": ["raw_transfer"], + }, + { + "key": "method", + "required": True, + "score": 0.9, + "supportingPaperIds": ["raw_transfer"], + }, + ], + }, + available_paper_ids={"raw_transfer"}, + paper_tiers={"raw_transfer": "transferable"}, + ) + + dimensions = {item["key"]: item for item in report["dimensions"]} + assert dimensions["gap"]["supportingPaperIds"] == [] + assert dimensions["gap"]["status"] == "missing" + assert dimensions["method"]["supportingPaperIds"] == ["raw_transfer"] + assert report["passed"] is False diff --git a/backend/tests/test_idea_evidence_relevance.py b/backend/tests/test_idea_evidence_relevance.py new file mode 100644 index 0000000..8845423 --- /dev/null +++ b/backend/tests/test_idea_evidence_relevance.py @@ -0,0 +1,345 @@ +from app.modules.idea.evidence_relevance import ( + EvidenceTier, + assess_search_result, + build_topic_intent_profile, + deduplicate_search_results, +) +from app.services.search_service import SearchResult +from app.models.idea import RawPaper +from app.storage.idea_storage import RawPaperStorage + + +def _result(title: str, abstract: str = "") -> SearchResult: + return SearchResult( + title=title, + authors=[], + abstract=abstract, + year=2025, + venue="test", + url=None, + doi=None, + arxiv_id=None, + citation_count=0, + source="openalex", + ) + + +def _red_chamber_profile(): + return build_topic_intent_profile( + seed="预测红楼梦可能结局", + domain="", + role_queries={ + "domain": ["Literary analysis of 'Dream of the Red Chamber'"], + "task": ["Computational prediction of endings for 'Dream of the Red Chamber'"], + "method": ["Narrative completion using character constraints"], + "evaluation": ["Narrative coherence and character consistency evaluation"], + }, + ) + + +def test_named_work_is_preserved_as_one_core_anchor(): + profile = _red_chamber_profile() + + assert "dream of the red chamber" in profile.core_anchors + assert "dream" in profile.generic_terms + + +def test_hyphenated_seed_phrases_are_preserved_and_stopwords_are_not_anchors(): + profile = build_topic_intent_profile( + seed="citation-faithful medical RAG for high-risk clinical QA", + domain="medical NLP, retrieval-augmented generation", + role_queries={ + "domain": ["citation-faithful medical RAG"], + "task": ["high-risk clinical question answering"], + "method": ["retrieval-augmented generation verifier"], + "evaluation": ["citation faithfulness evaluation"], + }, + ) + + assert "citation faithful" in profile.seed_anchors + assert "high risk" in profile.seed_anchors + assert "for" not in profile.core_anchors + + +def test_direct_named_work_paper_is_eligible(): + assessment = assess_search_result( + _result( + "Multiple Authors Detection: A Quantitative Analysis of Dream of the Red Chamber", + "Computational evidence about authorship and the unfinished ending.", + ), + _red_chamber_profile(), + ) + + assert assessment.tier is EvidenceTier.DIRECT + assert "dream of the red chamber" in assessment.decisive_anchors + + +def test_transferable_narrative_method_is_eligible_but_not_direct(): + assessment = assess_search_result( + _result( + "Narrative completion for unfinished novels", + "Computational character constraints and coherence evaluation reconstruct plausible endings.", + ), + _red_chamber_profile(), + ) + + assert assessment.tier is EvidenceTier.TRANSFERABLE + + +def test_generic_clinical_forecasting_is_rejected(): + assessment = assess_search_result( + _result( + "Clinical time-series forecasting and analysis", + "A web platform predicts patient outcomes with configurable models.", + ), + _red_chamber_profile(), + ) + + assert assessment.tier is EvidenceTier.REJECTED + assert assessment.rejection_reason == "generic_overlap_only" + + +def test_generic_chemical_evaluation_is_rejected(): + assessment = assess_search_result( + _result("ChemEval: A multi-level chemical evaluation for language models"), + _red_chamber_profile(), + ) + + assert assessment.tier is EvidenceTier.REJECTED + + +def test_generic_autonomous_llm_agent_is_not_direct_scientific_discovery_evidence(): + profile = build_topic_intent_profile( + seed="LLM agents for scientific discovery", + domain="AI for science, autonomous research agents", + role_queries={ + "domain": ["LLM agents for scientific discovery"], + "task": ["autonomous research agents using LLMs"], + "method": ["LLM agent architectures and methods"], + "evaluation": ["scientific discovery agent evaluation"], + }, + ) + + assessment = assess_search_result( + _result( + "Agents: An Open-source Framework for Autonomous Language Agents", + "A framework for building and deploying LLM agents.", + ), + profile, + ) + + assert assessment.tier is not EvidenceTier.DIRECT + + +def test_scientific_discovery_agent_paper_remains_direct(): + profile = build_topic_intent_profile( + seed="LLM agents for scientific discovery", + domain="AI for science, autonomous research agents", + role_queries={ + "domain": ["LLM agents for scientific discovery"], + "task": ["autonomous research agents using LLMs"], + "method": ["LLM agent architectures and methods"], + "evaluation": ["scientific discovery agent evaluation"], + }, + ) + + assessment = assess_search_result( + _result( + "Autonomous LLM Agents for Scientific Discovery", + "Large language model agents formulate and evaluate scientific hypotheses.", + ), + profile, + ) + + assert assessment.tier is EvidenceTier.DIRECT + + +def test_generic_rag_paper_is_not_direct_medical_citation_evidence(): + profile = build_topic_intent_profile( + seed="citation-faithful medical RAG for high-risk clinical question answering", + domain="medical NLP, retrieval-augmented generation, clinical QA", + role_queries={ + "domain": ["citation-faithful medical RAG"], + "task": ["high-risk clinical question answering"], + "method": ["medical RAG attribution methods"], + "evaluation": ["clinical RAG citation faithfulness evaluation"], + }, + ) + + assessment = assess_search_result( + _result( + "A Survey of Graph Retrieval-Augmented Generation", + "Graph RAG methods, benchmarks, and limitations for customized language models.", + ), + profile, + ) + + assert assessment.tier is not EvidenceTier.DIRECT + + +def test_legal_citation_qa_is_not_direct_medical_clinical_evidence(): + profile = build_topic_intent_profile( + seed="citation-faithful medical RAG for high-risk clinical question answering", + domain="medical NLP, retrieval-augmented generation, clinical QA", + role_queries={ + "domain": ["citation-faithful medical RAG"], + "task": ["high-risk clinical question answering"], + "method": ["medical RAG attribution methods"], + "evaluation": ["clinical RAG citation faithfulness evaluation"], + }, + ) + + assessment = assess_search_result( + _result( + "Attribution-Aware Citation Quality for Legal QA", + "A citation-faithful retrieval-augmented method for legal question answering.", + ), + profile, + ) + + assert assessment.tier is not EvidenceTier.DIRECT + + +def test_high_risk_clinical_citation_rag_remains_direct(): + profile = build_topic_intent_profile( + seed="citation-faithful medical RAG for high-risk clinical question answering", + domain="medical NLP, retrieval-augmented generation, clinical QA", + role_queries={ + "domain": ["citation-faithful medical RAG"], + "task": ["high-risk clinical question answering"], + "method": ["medical RAG attribution methods"], + "evaluation": ["clinical RAG citation faithfulness evaluation"], + }, + ) + + assessment = assess_search_result( + _result( + "Citation-Faithful Medical RAG for High-Risk Clinical QA", + "The system retrieves medical evidence and verifies every clinical citation.", + ), + profile, + ) + + assert assessment.tier is EvidenceTier.DIRECT + + +def test_single_multi_agent_phrase_is_not_direct_research_automation_evidence(): + profile = build_topic_intent_profile( + seed="reliable multi-agent research automation with evidence-grounded planning and self-review", + domain="autonomous research agents", + role_queries={ + "domain": ["multi-agent research automation"], + "task": ["reliable autonomous research workflows"], + "method": ["evidence-grounded planning and self-review"], + "evaluation": ["multi-agent research reliability benchmark"], + }, + ) + + assessment = assess_search_result( + _result( + "Multi-Agent Workflow for Medical Intent Classification", + "A reliable agent framework evaluated on healthcare intent benchmarks.", + ), + profile, + ) + + assert assessment.tier is not EvidenceTier.DIRECT + + +def test_multi_phrase_research_automation_evidence_remains_direct(): + profile = build_topic_intent_profile( + seed="reliable multi-agent research automation with evidence-grounded planning and self-review", + domain="autonomous research agents", + role_queries={ + "domain": ["multi-agent research automation"], + "task": ["reliable autonomous research workflows"], + "method": ["evidence-grounded planning and self-review"], + "evaluation": ["multi-agent research reliability benchmark"], + }, + ) + + assessment = assess_search_result( + _result( + "Evidence-Grounded Planning and Self-Review for Multi-Agent Research Automation", + "The system improves reliable autonomous research workflows and evidence traceability.", + ), + profile, + ) + + assert assessment.tier is EvidenceTier.DIRECT + + +def test_deduplication_merges_roles_queries_sources_and_richer_metadata(): + first = _result("Citation-Enforced RAG") + first.doi = "10.1000/rag" + first.source = "semantic_scholar" + first.retrieval_sources = ["semantic_scholar"] + first.retrieval_roles = ["domain"] + first.matched_queries = ["citation faithful RAG"] + first.relevance_score = 0.4 + + second = _result( + "Citation-Enforced RAG", + "A richer abstract with refusal evaluation.", + ) + second.doi = "10.1000/rag" + second.source = "openalex" + second.retrieval_roles = ["method", "evaluation"] + second.matched_queries = ["RAG verifier", "RAG refusal benchmark"] + second.relevance_score = 0.8 + + outcome = deduplicate_search_results([first, second]) + + assert outcome.merge_count == 1 + assert len(outcome.results) == 1 + merged = outcome.results[0] + assert merged.retrieval_roles == ["domain", "method", "evaluation"] + assert merged.matched_queries == [ + "citation faithful RAG", + "RAG verifier", + "RAG refusal benchmark", + ] + assert merged.retrieval_sources == ["semantic_scholar", "openalex"] + assert merged.abstract == second.abstract + assert merged.relevance_score == 0.8 + + +def test_deduplication_uses_title_when_identifiers_are_split_across_sources(): + doi_result = _result("Evidence Grounded Research Agents") + doi_result.doi = "10.1000/agents" + arxiv_result = _result("Evidence-Grounded Research Agents") + arxiv_result.arxiv_id = "2601.00001" + arxiv_result.retrieval_roles = ["method"] + + outcome = deduplicate_search_results([doi_result, arxiv_result]) + + assert outcome.merge_count == 1 + assert len(outcome.results) == 1 + assert outcome.results[0].arxiv_id == "2601.00001" + + +def test_raw_paper_storage_updates_persisted_evidence_assessment(tmp_path): + storage = RawPaperStorage(data_dir=str(tmp_path)) + paper = RawPaper( + id="raw_assessment", + sessionId="idea_assessment", + title="Citation-Enforced RAG", + evidenceTier="direct", + decisiveAnchors=["citation", "rag"], + relevanceComponents={"coreTerms": 0.24}, + relevanceScore=0.8, + ) + storage.create(paper) + + updated = paper.model_copy(update={ + "retrievalRoles": ["domain", "method"], + "matchedQueries": ["citation faithful RAG", "RAG verifier"], + "evidenceTier": "direct", + }) + storage.update(updated) + loaded = storage.get(paper.id) + + assert loaded is not None + assert loaded.evidenceTier == "direct" + assert loaded.decisiveAnchors == ["citation", "rag"] + assert loaded.retrievalRoles == ["domain", "method"] diff --git a/backend/tests/test_idea_final_candidates.py b/backend/tests/test_idea_final_candidates.py index ed0648d..ed8958c 100644 --- a/backend/tests/test_idea_final_candidates.py +++ b/backend/tests/test_idea_final_candidates.py @@ -3,6 +3,8 @@ import time from types import SimpleNamespace +import pytest + from app.models.idea import ( BFTSConfig, BFTSHandoff, @@ -12,12 +14,23 @@ IdeaSession, IdeaSessionConfig, PriorWorkComparison, + RawPaper, ReasoningPathSeed, + StepResult, StructuredPaper, + WorkflowTrace, ) import app.modules.idea.service as idea_service_module from app.modules.idea.service import IdeaGenerationService -from app.storage.idea_storage import CandidateStorage, IdeaSessionStorage, generate_candidate_id +from app.services.search_service import SearchResult +from app.storage.idea_storage import ( + CandidateStorage, + IdeaSessionStorage, + LiteratureGraphStorage, + LiteratureStorage, + RawPaperStorage, + generate_candidate_id, +) def _candidate(candidate_id: str, session_id: str, *, title: str, novelty: float = 8.0) -> IdeaCandidate: @@ -63,6 +76,641 @@ def _service(tmp_path) -> IdeaGenerationService: return service +def test_idea_review_default_allows_post_literature_candidate_repair(): + config = IdeaSessionConfig(seedQuery="reliable research automation") + + assert config.maxReviewIterations == 3 + + +def test_cjk_expand_query_backfills_role_queries_when_primary_output_omits_english(monkeypatch): + service = object.__new__(IdeaGenerationService) + session = IdeaSession( + id="idea_cjk_query_roles", + config=IdeaSessionConfig( + seedQuery="\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40", + providerName="fake", + model="fake-model", + paperType="system", + ), + ) + + class FakeClient: + def __init__(self): + self.calls = 0 + + def chat(self, messages, **kwargs): + self.calls += 1 + if self.calls == 1: + return SimpleNamespace( + text=json.dumps({ + "refinedQuestion": "\u5982\u4f55\u8ba1\u7b97\u91cd\u5efa\u7ea2\u697c\u68a6\u7ed3\u5c40\uff1f", + "searchQueries": ["\u7ea2\u697c\u68a6\u7ed3\u5c40\u7814\u7a76"], + "keyConcepts": ["\u53d9\u4e8b\u95ed\u5408"], + }), + latency_ms=10, + ) + return SimpleNamespace( + text=json.dumps({ + "domainQueries": ["Dream of the Red Chamber ending studies"], + "taskQueries": ["Hongloumeng narrative closure reconstruction"], + "methodQueries": ["computational literary narrative completion"], + "evaluationQueries": ["narrative coherence character consistency evaluation"], + }), + latency_ms=8, + ) + + client = FakeClient() + monkeypatch.setattr(idea_service_module, "get_provider_client", lambda _provider: client) + + _, outputs, _ = service._step_expand_query(session) + + assert client.calls == 2 + assert outputs["translationStatus"] == "fallback" + assert outputs["searchQueriesByRole"]["domain"] == [ + "Dream of the Red Chamber ending studies" + ] + assert outputs["searchQueriesByRole"]["method"] == [ + "computational literary narrative completion" + ] + assert outputs["englishSearchQueries"][0] == "Dream of the Red Chamber ending studies" + + +def test_english_expand_query_backfills_roles_from_flat_queries(monkeypatch): + service = object.__new__(IdeaGenerationService) + session = IdeaSession( + id="idea_english_query_roles", + config=IdeaSessionConfig( + seedQuery="citation-faithful medical RAG for high-risk clinical QA", + domain="medical NLP, retrieval-augmented generation, clinical QA", + providerName="fake", + model="fake-model", + paperType="evaluation", + ), + ) + + class FakeClient: + def chat(self, messages, **kwargs): + return SimpleNamespace( + text=json.dumps({ + "refinedQuestion": "How can citation-faithful medical RAG be evaluated?", + "searchQueries": [ + "citation-faithful medical RAG for clinical QA", + "medical RAG refusal and evidence attribution methods", + "clinical RAG benchmark citation faithfulness evaluation", + ], + "keyConcepts": ["citation faithfulness", "refusal", "clinical QA"], + }), + latency_ms=10, + ) + + monkeypatch.setattr( + idea_service_module, + "get_provider_client", + lambda _provider: FakeClient(), + ) + + _, outputs, _ = service._step_expand_query(session) + + roles = outputs["searchQueriesByRole"] + assert roles["domain"] + assert roles["task"] + assert roles["method"] + assert roles["evaluation"] + assert "medical RAG refusal and evidence attribution methods" in roles["method"] + assert "clinical RAG benchmark citation faithfulness evaluation" in roles["evaluation"] + + +def test_cjk_expand_query_pauses_when_translation_fallback_is_empty(monkeypatch): + service = object.__new__(IdeaGenerationService) + session = IdeaSession( + id="idea_cjk_translation_missing", + config=IdeaSessionConfig( + seedQuery="\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40", + providerName="fake", + model="fake-model", + ), + ) + + class FakeClient: + def chat(self, messages, **kwargs): + return SimpleNamespace( + text=json.dumps({"searchQueries": ["\u7ea2\u697c\u68a6\u7ed3\u5c40\u7814\u7a76"]}), + latency_ms=5, + ) + + monkeypatch.setattr( + idea_service_module, + "get_provider_client", + lambda _provider: FakeClient(), + ) + + try: + service._step_expand_query(session) + except idea_service_module.RecoverableIdeaError as exc: + assert exc.waiting_status == idea_service_module.IdeaSessionStatus.AWAITING_EVIDENCE + assert exc.resume_from == "expandQuery" + else: + raise AssertionError("Missing CJK translation must pause the session") + + +def test_cjk_repair_queries_keep_english_core_topic_anchor(tmp_path): + service = _service(tmp_path) + session = IdeaSession( + id="idea_cjk_repair_anchor", + config=IdeaSessionConfig(seedQuery="\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40"), + trace=idea_service_module.WorkflowTrace( + sessionId="idea_cjk_repair_anchor", + startedAt=idea_service_module._utcnow(), + steps=[ + idea_service_module.StepResult( + name="expandQuery", + status="ok", + outputs={ + "englishSearchQueries": ["Dream of the Red Chamber ending studies"], + "searchQueriesByRole": { + "domain": ["Dream of the Red Chamber ending studies"], + "task": ["Hongloumeng narrative closure reconstruction"], + "method": [], + "evaluation": [], + }, + }, + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + ) + ], + ), + ) + + anchored = service._anchor_repair_queries( + session, + [ + "\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40 evaluation evidence", + "\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40 method evidence", + ], + ) + + assert anchored == [ + "Dream of the Red Chamber ending studies evaluation evidence", + "Dream of the Red Chamber ending studies method evidence", + ] + + +def test_core_search_queries_exclude_method_and_evaluation_only_roles(tmp_path): + service = _service(tmp_path) + session = IdeaSession( + id="idea_role_core_queries", + config=IdeaSessionConfig(seedQuery="\u9884\u6d4b\u7ea2\u697c\u68a6\u53ef\u80fd\u7ed3\u5c40"), + trace=idea_service_module.WorkflowTrace( + sessionId="idea_role_core_queries", + startedAt=idea_service_module._utcnow(), + steps=[ + idea_service_module.StepResult( + name="expandQuery", + status="ok", + outputs={ + "englishSearchQueries": [ + "Dream of the Red Chamber ending studies", + "Hongloumeng narrative closure reconstruction", + "computational literary narrative completion", + "narrative coherence evaluation", + ], + "searchQueriesByRole": { + "domain": ["Dream of the Red Chamber ending studies"], + "task": ["Hongloumeng narrative closure reconstruction"], + "method": ["computational literary narrative completion"], + "evaluation": ["narrative coherence evaluation"], + }, + }, + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + ) + ], + ), + ) + + assert service._core_search_queries(session) == [ + "Dream of the Red Chamber ending studies", + "Hongloumeng narrative closure reconstruction", + ] + + +def test_literature_search_rejects_generic_cross_domain_results(monkeypatch, tmp_path): + service = object.__new__(IdeaGenerationService) + service.raw_paper_storage = RawPaperStorage(data_dir=str(tmp_path)) + service.literature_storage = LiteratureStorage(data_dir=str(tmp_path)) + service.graph_storage = LiteratureGraphStorage(data_dir=str(tmp_path)) + service.graph_builder = idea_service_module.LiteratureGraphBuilder() + session = IdeaSession( + id="idea_literature_tiers", + config=IdeaSessionConfig( + seedQuery="预测红楼梦可能结局", + paperType="system", + maxPapers=24, + ), + trace=WorkflowTrace( + sessionId="idea_literature_tiers", + startedAt=idea_service_module._utcnow(), + steps=[ + StepResult( + name="expandQuery", + status="ok", + outputs={ + "englishSearchQueries": [ + "Literary analysis of Dream of the Red Chamber", + "Computational prediction of Dream of the Red Chamber endings", + ], + "searchQueriesByRole": { + "domain": ["Literary analysis of 'Dream of the Red Chamber'"], + "task": ["Computational prediction of endings for 'Dream of the Red Chamber'"], + "method": ["Narrative completion using character constraints"], + "evaluation": ["Narrative coherence and character consistency evaluation"], + }, + }, + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + ) + ], + ), + ) + + papers = [ + SearchResult( + title="Dream of the Red Chamber authorship and unfinished ending", + authors=[], + abstract="Computational evidence about Hongloumeng narrative closure.", + year=2024, + venue="test", + url="", + doi="10.1000/direct-1", + arxiv_id=None, + citation_count=1, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="Narrative completion for unfinished novels", + authors=[], + abstract="Computational character constraints reconstruct plausible endings.", + year=2023, + venue="test", + url="", + doi="10.1000/method-1", + arxiv_id=None, + citation_count=1, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="Evaluating narrative coherence and character consistency", + authors=[], + abstract="Computational evaluation metrics for generated novel endings.", + year=2023, + venue="test", + url="", + doi="10.1000/eval-1", + arxiv_id=None, + citation_count=1, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="Hongloumeng character relationships and narrative structure", + authors=[], + abstract="Dream of the Red Chamber characters constrain narrative outcomes.", + year=2022, + venue="test", + url="", + doi="10.1000/direct-2", + arxiv_id=None, + citation_count=1, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="Clinical time-series forecasting and analysis", + authors=[], + abstract="A web platform predicts patient outcomes with configurable models.", + year=2025, + venue="test", + url="", + doi="10.1000/clinical", + arxiv_id=None, + citation_count=0, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="ChemEval: A chemical evaluation for language models", + authors=[], + abstract="A multi-level benchmark for chemistry question answering.", + year=2025, + venue="test", + url="", + doi="10.1000/chem", + arxiv_id=None, + citation_count=0, + source="openalex", + relevance_score=0.9, + ), + ] + + class FakeSearchService: + def search(self, _query, limit=10): + return [paper for paper in papers[:limit]] + + monkeypatch.setattr( + idea_service_module, + "get_search_service", + lambda: FakeSearchService(), + ) + + _, outputs, _ = service._step_literature_search(session) + stored = service.raw_paper_storage.list_by_session(session.id) + + assert all(paper.evidenceTier in {"direct", "transferable"} for paper in stored) + assert not any("Clinical time-series" in paper.title for paper in stored) + assert not any("ChemEval" in paper.title for paper in stored) + assert outputs["evidenceTierCounts"]["rejected"] >= 2 + assert outputs["duplicateMergeCount"] > 0 + assert outputs["topicIntentProfile"]["coreAnchors"] + + +def test_literature_search_pauses_weak_pool_before_deep_read(monkeypatch, tmp_path): + service = object.__new__(IdeaGenerationService) + service.raw_paper_storage = RawPaperStorage(data_dir=str(tmp_path)) + service.literature_storage = LiteratureStorage(data_dir=str(tmp_path)) + service.graph_storage = LiteratureGraphStorage(data_dir=str(tmp_path)) + service.graph_builder = idea_service_module.LiteratureGraphBuilder() + session = IdeaSession( + id="idea_literature_waiting", + config=IdeaSessionConfig(seedQuery="预测红楼梦可能结局", maxPapers=20), + trace=WorkflowTrace( + sessionId="idea_literature_waiting", + startedAt=idea_service_module._utcnow(), + steps=[ + StepResult( + name="expandQuery", + status="ok", + outputs={ + "englishSearchQueries": ["Dream of the Red Chamber endings"], + "searchQueriesByRole": { + "domain": ["Literary analysis of 'Dream of the Red Chamber'"], + "task": ["Computational prediction of endings for 'Dream of the Red Chamber'"], + "method": ["Narrative completion using character constraints"], + "evaluation": ["Narrative coherence evaluation"], + }, + }, + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + ) + ], + ), + ) + + distractors = [ + SearchResult( + title="Clinical time-series forecasting and analysis", + authors=[], + abstract="Predicts patient outcomes.", + year=2025, + venue="test", + url="", + doi="10.1000/clinical-only", + arxiv_id=None, + citation_count=0, + source="openalex", + relevance_score=0.9, + ), + SearchResult( + title="ChemEval chemical evaluation", + authors=[], + abstract="Evaluates chemistry language models.", + year=2025, + venue="test", + url="", + doi="10.1000/chem-only", + arxiv_id=None, + citation_count=0, + source="openalex", + relevance_score=0.9, + ), + ] + + class FakeSearchService: + def search(self, _query, limit=10): + return distractors[:limit] + + monkeypatch.setattr( + idea_service_module, + "get_search_service", + lambda: FakeSearchService(), + ) + + with pytest.raises(idea_service_module.RecoverableIdeaError) as exc_info: + service._step_literature_search(session) + + assert exc_info.value.resume_from == "literatureSearch" + assert service.raw_paper_storage.list_by_session(session.id) == [] + + +def test_deep_read_limit_is_bounded(monkeypatch): + monkeypatch.setenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "6") + assert idea_service_module._deep_read_max_papers() == 6 + + monkeypatch.setenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "1") + assert idea_service_module._deep_read_max_papers() == 4 + + monkeypatch.setenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "100") + assert idea_service_module._deep_read_max_papers() == 40 + + monkeypatch.setenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "invalid") + assert idea_service_module._deep_read_max_papers() == 24 + + +def test_deep_read_selection_limits_transferable_quota_and_keeps_must_cite(): + papers = { + **{ + f"raw_direct_{index}": RawPaper( + id=f"raw_direct_{index}", + sessionId="idea_quota", + title=f"Direct paper {index}", + evidenceTier="direct", + ) + for index in range(5) + }, + **{ + f"raw_transfer_{index}": RawPaper( + id=f"raw_transfer_{index}", + sessionId="idea_quota", + title=f"Transferable paper {index}", + evidenceTier="transferable", + ) + for index in range(5) + }, + "raw_must": RawPaper( + id="raw_must", + sessionId="idea_quota", + title="Explicit must cite", + evidenceTier="rejected", + mustCiteOverride=True, + ), + } + selected = [ + "raw_transfer_0", + "raw_direct_0", + "raw_transfer_1", + "raw_direct_1", + "raw_transfer_2", + "raw_direct_2", + "raw_transfer_3", + "raw_direct_3", + "raw_transfer_4", + "raw_direct_4", + "raw_must", + ] + + limited = idea_service_module._limit_deep_read_selection( + selected, + papers, + limit=6, + ) + + assert len([paper_id for paper_id in limited if "transfer" in paper_id]) == 2 + assert len([paper_id for paper_id in limited if "direct" in paper_id]) == 4 + assert "raw_must" in limited + + +def test_repair_search_merges_existing_raw_paper_provenance(tmp_path): + service = object.__new__(IdeaGenerationService) + service.raw_paper_storage = RawPaperStorage(data_dir=str(tmp_path)) + service.literature_storage = LiteratureStorage(data_dir=str(tmp_path)) + service.graph_storage = LiteratureGraphStorage(data_dir=str(tmp_path)) + service.graph_builder = idea_service_module.LiteratureGraphBuilder() + session = IdeaSession( + id="idea_repair_upsert", + config=IdeaSessionConfig( + seedQuery="citation-faithful medical RAG", + domain="medical QA", + paperType="system", + ), + ) + existing = RawPaper( + id="raw_existing_repair", + sessionId=session.id, + title="Citation-Enforced Medical RAG", + abstract="Citation faithful retrieval augmented generation.", + doi="10.1000/repair-rag", + source=["semantic_scholar"], + retrievalRoles=["domain"], + matchedQueries=["citation faithful medical RAG"], + evidenceTier="direct", + decisiveAnchors=["citation", "medical", "rag"], + normalizedTitleHash=idea_service_module._compute_title_hash( + "Citation-Enforced Medical RAG" + ), + relevanceScore=0.7, + ) + service.raw_paper_storage.create(existing) + initial_graph = service.graph_builder.build_graph_v0( + session_id=session.id, + raw_papers=[existing], + ) + service.graph_storage.create(initial_graph) + duplicate = SearchResult( + title="Citation-Enforced Medical RAG", + authors=[], + abstract="A richer abstract with refusal evaluation and evidence traceability.", + year=2025, + venue="test", + url="", + doi="10.1000/repair-rag", + arxiv_id=None, + citation_count=4, + source="openalex", + relevance_score=0.9, + retrieval_roles=["method", "evaluation", "repair"], + matched_queries=["citation faithful medical RAG method evaluation"], + ) + + report = service._persist_repair_search_results( + session, + [duplicate], + search_queries=["citation faithful medical RAG method evaluation"], + ) + loaded = service.raw_paper_storage.get(existing.id) + + assert loaded is not None + assert service.raw_paper_storage.list_by_session(session.id) == [loaded] + assert loaded.retrievalRoles == ["domain", "method", "evaluation", "repair"] + assert loaded.matchedQueries == [ + "citation faithful medical RAG", + "citation faithful medical RAG method evaluation", + ] + assert loaded.source == ["semantic_scholar", "openalex"] + assert loaded.relevanceScore >= existing.relevanceScore + assert report["updatedRawPaperIds"] == [existing.id] + assert report["duplicateMergeCount"] == 1 + assert len(list(service.graph_storage.base_path.glob("lg_*.json"))) == 1 + assert service.graph_storage.get_by_session(session.id).id == initial_graph.id + + +def test_repair_results_keep_query_and_missing_dimension_role(): + result = SearchResult( + title="Citation faithful RAG verifier", + authors=[], + abstract="", + year=2025, + venue="test", + url="", + doi=None, + arxiv_id=None, + citation_count=0, + source="openalex", + ) + + idea_service_module._tag_repair_results( + [result], + "citation faithful RAG method evidence", + ) + + assert result.retrieval_roles == ["repair", "method"] + assert result.matched_queries == ["citation faithful RAG method evidence"] + + +def test_trace_counters_are_recomputed_from_recorded_attempts(): + trace = WorkflowTrace( + sessionId="idea_trace_attempts", + startedAt=idea_service_module._utcnow(), + ) + ok = StepResult( + name="literatureSearch", + status="ok", + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + ) + failed = StepResult( + name="evidenceGate", + status="failed", + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + durationSeconds=0.0, + error="insufficient evidence", + ) + + idea_service_module._record_step_result(trace, ok) + idea_service_module._record_step_result(trace, failed) + + assert trace.totalSteps == 2 + assert trace.successfulSteps == 1 + assert trace.failedSteps == 1 + assert trace.steps == [ok, failed] + + def test_get_candidates_defaults_to_final_candidate_ids(tmp_path): service = _service(tmp_path) session_id = "idea_final_view" @@ -265,6 +913,92 @@ def test_select_final_candidates_marks_insufficient_final_count(tmp_path): assert result["summary"]["requiresRegeneration"] is True +def test_target_final_candidate_count_remains_two_with_one_ranked_candidate(tmp_path): + service = _service(tmp_path) + session = IdeaSession( + id="idea_target_two", + config=IdeaSessionConfig( + seedQuery="citation faithful RAG", + maxCandidates=1, + ), + ) + only = _candidate("cand_only", session.id, title="Citation faithful RAG verifier") + + assert service._target_final_candidate_count(session, [only]) == 2 + + +def test_resume_session_transitions_waiting_session_to_running(tmp_path): + service = _service(tmp_path) + session = IdeaSession( + id="idea_resume_evidence", + config=IdeaSessionConfig(seedQuery="citation faithful RAG"), + status=idea_service_module.IdeaSessionStatus.AWAITING_EVIDENCE, + startedAt=idea_service_module._utcnow(), + endedAt=idea_service_module._utcnow(), + errorMessage="Evidence pool is incomplete.", + qualityLoopSummary={"resumeFrom": "evidenceGate"}, + ) + service.session_storage.create(session) + + resumed = service.resume_session(session.id) + + assert resumed.status == idea_service_module.IdeaSessionStatus.RUNNING + assert resumed.endedAt is None + assert resumed.errorMessage is None + + +def test_waiting_session_does_not_expose_internal_candidates(tmp_path): + service = _service(tmp_path) + session_id = "idea_waiting_hidden" + service.session_storage.create( + IdeaSession( + id=session_id, + config=IdeaSessionConfig(seedQuery="citation faithful RAG"), + status=idea_service_module.IdeaSessionStatus.AWAITING_IDEAS, + candidateIds=["cand_internal"], + finalCandidateIds=[], + ) + ) + service.candidate_storage.create( + _candidate("cand_internal", session_id, title="Internal rejected candidate") + ) + + assert service.get_candidates(session_id) == [] + + +def test_pipeline_persists_awaiting_evidence_instead_of_failed(monkeypatch, tmp_path): + service = _service(tmp_path) + service._pipeline_lock_guard = threading.Lock() + service._pipeline_locks = {} + session = IdeaSession( + id="idea_pipeline_waiting_evidence", + config=IdeaSessionConfig(seedQuery="citation faithful RAG"), + status=idea_service_module.IdeaSessionStatus.RUNNING, + startedAt=idea_service_module._utcnow(), + trace=idea_service_module.WorkflowTrace( + sessionId="idea_pipeline_waiting_evidence", + startedAt=idea_service_module._utcnow(), + ), + ) + service.session_storage.create(session) + + def ok_step(_session): + return {}, {}, [] + + def evidence_step(_session): + raise idea_service_module.AwaitingEvidenceError("Evidence is incomplete") + + for name in ["_step_expand_query", "_step_literature_search", "_step_novelty_check", "_step_gap_analysis"]: + monkeypatch.setattr(service, name, ok_step) + monkeypatch.setattr(service, "_step_evidence_gate", evidence_step) + + result = service.run_pipeline(session.id) + + assert result.status == idea_service_module.IdeaSessionStatus.AWAITING_EVIDENCE + assert result.qualityLoopSummary["resumeFrom"] == "evidenceGate" + assert result.endedAt is None + + def test_pool_repair_targets_second_candidate_when_top_already_passes(tmp_path): service = _service(tmp_path) session = IdeaSession( @@ -432,6 +1166,79 @@ def chat(self, *args, **kwargs): assert "directionType:benchmark" in regenerated.draftPlan.tags +def test_off_topic_regeneration_uses_seed_boundary_without_polluted_context( + tmp_path, + monkeypatch, +): + service = _service(tmp_path) + session = IdeaSession( + id="idea_regenerate_off_topic", + config=IdeaSessionConfig( + seedQuery="reliable multi-agent research automation with evidence-grounded planning and self-review", + providerName="fake", + model="fake-model", + ), + ) + base = _tag_direction( + _candidate( + "cand_medical_benchmark", + session.id, + title="Multi-agent benchmark for medical intent automation", + ), + "benchmark", + direction_id="dir-benchmark", + ) + base.proposedMethod = "Evaluate the benchmark on medical intents and legal reasoning." + captured = {} + + class _FakeResponse: + text = json.dumps({ + "ideas": [{ + "title": "Evidence-grounded multi-agent research automation benchmark", + "problem": "Research automation lacks reliability benchmarks.", + "keyInsight": "Evidence traces and self-review should be tested together.", + "approach": "Benchmark planning, evidence grounding, and self-review failures.", + "expectedOutcomes": ["A reproducible reliability benchmark"], + "risks": [], + "requiredExperiments": [], + }] + }) + + class _FakeClient: + def chat(self, messages, **kwargs): + captured["prompt"] = "\n".join(message.content for message in messages) + return _FakeResponse() + + monkeypatch.setattr( + idea_service_module, + "get_provider_client", + lambda provider_name: _FakeClient(), + ) + + regenerated = service._regenerate_candidate_from_review( + session=session, + base_candidate=base, + review_gate={ + "passed": False, + "blockingIssues": [ + "Candidate shows topic drift: unrequested clinical or medical deployment anchor 'medical' is central but absent from the seed query.", + "Candidate shows topic drift: unrequested legal domain anchor 'legal' is central but absent from the seed query.", + ], + "suggestedImprovements": ["Remove unrequested application domains."], + }, + critique=None, + prior_work=[], + literature_context="A medical intents paper and a legal reasoning paper.", + ) + + assert regenerated is not None + assert '"forbiddenApplicationAnchors": ["medical", "legal"]' in captured["prompt"] + assert '"sourceCandidate": {}' in captured["prompt"] + assert '"literatureContext": ""' in captured["prompt"] + assert "A medical intents paper" not in captured["prompt"] + assert "baselines, metrics, ablations, and validation steps" in captured["prompt"] + + def test_candidate_pool_failure_routes_drive_repair_actions(tmp_path): service = _service(tmp_path) candidate = _candidate( @@ -1124,6 +1931,37 @@ def test_rule_idea_reviewer_blocks_unrequested_application_domain_drift(tmp_path assert any("seed query" in instruction.lower() for instruction in report["repairInstructions"]) +def test_rule_idea_reviewer_blocks_unrequested_climate_science_drift(tmp_path): + service = _service(tmp_path) + seed_query = "reliable multi-agent research automation with evidence-grounded planning and self-review" + candidate = _candidate( + "cand_climate_drift", + "idea_topic_drift", + title="EARS-Climate: Reliable multi-agent planning for climate models", + ) + candidate.problem = "Climate science needs reliable automated model analysis." + candidate.keyInsight = "Use climate model datasets as the primary self-review setting." + candidate.proposedMethod = "Build evidence-grounded agents for weather and climate simulation." + + report = service._rule_idea_reviewer_report( + spec={"name": "IdeaFeasibilityReviewer"}, + candidate=candidate, + evidence=CandidateGraphEvidence( + candidateId=candidate.id, + supportingPaperIds=["raw_agent"], + supportingEntityIds=["ent_agent"], + supportingPathSeedIds=["rps_agent"], + ), + comparisons=[], + critique=None, + seed_query=seed_query, + allowed_evidence_refs=["raw_agent", "ent_agent", "rps_agent"], + ) + + assert report["passed"] is False + assert any("climate" in issue.lower() for issue in report["blockingIssues"]) + + def test_rule_idea_reviewer_blocks_generic_unrequested_application_phrase(tmp_path): service = _service(tmp_path) seed_query = "reliable multi-agent research automation with evidence-grounded planning and self-review" @@ -1165,6 +2003,81 @@ def test_rule_idea_reviewer_blocks_generic_unrequested_application_phrase(tmp_pa assert any("unrequested application" in issue.lower() for issue in report["blockingIssues"]) +def test_rule_idea_reviewer_blocks_unrequested_title_domain_phrase(tmp_path): + service = _service(tmp_path) + seed_query = "reliable multi-agent research automation with evidence-grounded planning and self-review" + candidate = _candidate( + "cand_social_science_drift", + "idea_topic_drift", + title="Evidential Multi-Agent Research Automation for Social Sciences", + ) + candidate.problem = "Research automation needs reliable evidence and self-review." + candidate.keyInsight = "Apply the framework to social sciences discovery workflows." + candidate.proposedMethod = "Build evidence-grounded planning and validation agents." + + report = service._rule_idea_reviewer_report( + spec={"name": "IdeaFeasibilityReviewer"}, + candidate=candidate, + evidence=CandidateGraphEvidence( + candidateId=candidate.id, + supportingPaperIds=["raw_agent"], + supportingEntityIds=["ent_agent"], + supportingPathSeedIds=["rps_agent"], + ), + comparisons=[], + critique=None, + seed_query=seed_query, + allowed_evidence_refs=["raw_agent", "ent_agent", "rps_agent"], + ) + + assert report["passed"] is False + assert any("social sciences" in issue.lower() for issue in report["blockingIssues"]) + + +def test_rule_idea_reviewer_blocks_generic_drift_for_cjk_seed_with_english_queries(tmp_path): + service = _service(tmp_path) + candidate = _candidate( + "cand_cjk_warehouse_drift", + "idea_cjk_topic_drift", + title="Reliable multi-agent self-review for warehouse logistics optimization", + ) + candidate.problem = ( + "Use evidence-grounded planning to coordinate warehouse routing, packing, " + "and inventory optimization." + ) + candidate.keyInsight = "Warehouse logistics becomes the main evaluation scenario." + + report = service._rule_idea_reviewer_report( + spec={"name": "IdeaFeasibilityReviewer"}, + candidate=candidate, + evidence=CandidateGraphEvidence( + candidateId=candidate.id, + supportingPaperIds=["raw_agent"], + supportingEntityIds=["ent_agent"], + supportingPathSeedIds=["rps_agent"], + ), + comparisons=[ + PriorWorkComparison( + candidateId=candidate.id, + comparedPaperIds=["raw_agent"], + differences=["Adds evidence-grounded planning and self-review."], + advantages=["Improves reliability."], + comparisonConfidence=0.8, + ) + ], + critique=None, + seed_query="\u57fa\u4e8e\u8bc1\u636e\u89c4\u5212\u548c\u81ea\u6211\u8bc4\u5ba1\u7684\u53ef\u9760\u591a\u667a\u80fd\u4f53\u7814\u7a76\u81ea\u52a8\u5316", + allowed_evidence_refs=["raw_agent", "ent_agent", "rps_agent"], + english_search_queries=[ + "reliable multi-agent research automation", + "evidence-grounded planning and self-review", + ], + ) + + assert report["passed"] is False + assert any("unrequested application" in issue.lower() for issue in report["blockingIssues"]) + + def test_revalidate_session_final_candidates_removes_stale_topic_drift(tmp_path): service = _service(tmp_path) session_id = "idea_revalidate_drift" diff --git a/backend/tests/test_idea_gap_analysis_regression.py b/backend/tests/test_idea_gap_analysis_regression.py new file mode 100644 index 0000000..e7ccb6f --- /dev/null +++ b/backend/tests/test_idea_gap_analysis_regression.py @@ -0,0 +1,65 @@ +from types import SimpleNamespace + +import app.modules.idea.service as idea_service_module +from app.models.idea import IdeaSession, IdeaSessionConfig, StructuredPaper +from app.modules.idea.service import IdeaGenerationService + + +def test_gap_analysis_enforces_multiple_opportunities_via_module_helper(monkeypatch): + class FakeClient: + def chat(self, messages, **kwargs): + return SimpleNamespace( + text=( + '{"gapAnalysis": [], "prioritizedGaps": [], ' + '"researchOpportunities": ["Evaluate citation faithfulness."]}' + ) + ) + + monkeypatch.setattr(idea_service_module, "get_provider_client", lambda provider: FakeClient()) + + service = object.__new__(IdeaGenerationService) + service.get_literature = lambda session_id: [ + SimpleNamespace(title="Citation-faithful RAG", year=2026, snippet="Evaluation study") + ] + literature_map = SimpleNamespace(id="lm_test", selectedPaperIds=["raw_citation"], gaps=[]) + service.map_storage = SimpleNamespace(get_by_session=lambda session_id: literature_map) + service.structured_storage = SimpleNamespace( + list_by_session=lambda session_id: [ + StructuredPaper( + id="raw_citation", + sessionId=session_id, + rawPaperId="raw_citation", + title="Citation-faithful RAG", + ) + ] + ) + service.reasoning_builder = SimpleNamespace( + build_reasoning_kg=lambda **kwargs: SimpleNamespace(id="rkg_test") + ) + service.reasoning_kg_storage = SimpleNamespace(create=lambda value: value) + service.graph_linker = SimpleNamespace(link_graphs=lambda **kwargs: []) + service.evidence_link_storage = SimpleNamespace(create=lambda value: value) + service.path_seed_gen = SimpleNamespace(generate_seeds=lambda **kwargs: []) + service.path_seed_storage = SimpleNamespace(create=lambda value: value) + service.handoff_storage = SimpleNamespace( + get_by_session=lambda session_id: None, + delete=lambda handoff_id: None, + create=lambda value: value, + ) + service._get_step_output = lambda session, step_name, key, default=None: default + service._build_rag_literature_context = lambda session: "" + + session = IdeaSession( + id="idea_gap_analysis", + config=IdeaSessionConfig( + providerName="fake", + model="fake-model", + seedQuery="Improve citation faithfulness in high-risk RAG QA.", + paperType="algorithm", + ), + ) + + _, outputs, errors = service._step_gap_analysis(session) + + assert errors == [] + assert len(outputs["researchOpportunities"]) >= 3 diff --git a/backend/tests/test_idea_plan_eval_script.py b/backend/tests/test_idea_plan_eval_script.py index 2c60c4b..1fe1276 100644 --- a/backend/tests/test_idea_plan_eval_script.py +++ b/backend/tests/test_idea_plan_eval_script.py @@ -77,3 +77,26 @@ def test_step_map_uses_recorded_duration_seconds_when_milliseconds_are_absent(): assert mapped["noveltyCheck"]["durationMs"] == 12500 assert mapped["noveltyCheck"]["durationSeconds"] == 12.5 + + +def test_plan_quality_summary_reports_segment_and_content_quality(plan_package): + plan_package.generation.fallbackUsed = True + plan_package.generation.llmUsedSections = ["implementationPlan"] + plan_package.generation.llmReviewerUsed = True + plan_package.generation.warnings = [ + "segment_fallback:stage-2:TimeoutError" + ] + plan_package.stages[2].steps[0].expected[0].target = ( + "specified before implementation" + ) + + summary = eval_script.plan_quality_summary(plan_package, 12.3456) + + assert summary["elapsedSeconds"] == 12.346 + assert summary["segmentFallbacks"] == [ + "segment_fallback:stage-2:TimeoutError" + ] + assert summary["placeholderValues"] == ["specified before implementation"] + assert summary["expectedMetricCount"] == 4 + assert summary["llmReviewerUsed"] is True + assert summary["criticalNullPaths"] == [] diff --git a/backend/tests/test_plan_package_contract_compatibility.py b/backend/tests/test_plan_package_contract_compatibility.py new file mode 100644 index 0000000..40fca0c --- /dev/null +++ b/backend/tests/test_plan_package_contract_compatibility.py @@ -0,0 +1,39 @@ +import hashlib +import json + +from app.models.plan_package import ( + PlanPackage, + PlanPackageHandoff, + PlanPackagePresentation, +) + + +def _schema_hash(model) -> str: + payload = json.dumps( + model.model_json_schema(), + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def test_plan_package_public_schemas_are_frozen(): + assert _schema_hash(PlanPackage) == ( + "1c8e042cef9b59caaac268a62e38914473e9bf61c3f990e6369e3182f6c4c975" + ) + assert _schema_hash(PlanPackageHandoff) == ( + "79bb44f881629465e3e1c1c8e62dfad524a7b912ad4a1406f98c2dec9c56db70" + ) + assert _schema_hash(PlanPackagePresentation) == ( + "6a0cf5271bb04207654a82759c60ea9ff304ddd0acb000d07a42df5dd9563de6" + ) + + +def test_plan_package_schema_versions_remain_compatible(): + package_schema = PlanPackage.model_json_schema() + handoff_schema = PlanPackageHandoff.model_json_schema() + + assert package_schema["properties"]["schemaVersion"]["default"] == "plan-package/v4" + assert handoff_schema["properties"]["schemaVersion"]["default"] == ( + "plan-package-handoff/v1" + ) diff --git a/backend/tests/test_plan_package_generation_segments.py b/backend/tests/test_plan_package_generation_segments.py new file mode 100644 index 0000000..301faea --- /dev/null +++ b/backend/tests/test_plan_package_generation_segments.py @@ -0,0 +1,126 @@ +from app.services.plan_package_generation import ( + PlanSegmentGenerationResult, + build_stage_prompt, + generate_plan_segments, +) + + +def test_one_stage_failure_preserves_other_llm_segments(plan_package): + calls = [] + + def fake_call(segment_name, _prompt, _max_tokens): + calls.append(segment_name) + if segment_name == "core": + return { + "researchQuestion": ( + "Can claim verification improve citation-faithful RAG over vanilla RAG?" + ), + "hypothesis": ( + "Claim verification increases citation faithfulness; " + "reject if the mean delta is not positive." + ), + "constants": {"baseline": "vanilla RAG"}, + } + if segment_name == "stage-2": + raise TimeoutError("stage timeout") + stage_index = int(segment_name.rsplit("-", 1)[-1]) - 1 + stage = plan_package.stages[stage_index].model_dump(mode="json") + stage["title"] = f"LLM {stage['title']}" + return {"stage": stage} + + result = generate_plan_segments( + package=plan_package, + call_json=fake_call, + max_steps_per_stage=3, + ) + + assert result.core_used is True + assert result.llm_stage_ids == ["stage-1", "stage-3"] + assert result.fallback_stage_ids == ["stage-2"] + assert [stage.id for stage in result.stages] == ["stage-1", "stage-2", "stage-3"] + assert result.stages[0].title.startswith("LLM ") + assert result.stages[1].title == plan_package.stages[1].title + assert result.stages[2].title.startswith("LLM ") + assert calls.count("stage-2") == 2 + + +def test_invalid_segment_is_repaired_without_regenerating_other_segments(plan_package): + stage_one_calls = 0 + + def fake_call(segment_name, _prompt, _max_tokens): + nonlocal stage_one_calls + if segment_name == "core": + return { + "researchQuestion": plan_package.researchQuestion, + "hypothesis": plan_package.hypothesis, + "constants": {}, + } + stage_index = int(segment_name.rsplit("-", 1)[-1]) - 1 + if segment_name == "stage-1": + stage_one_calls += 1 + if stage_one_calls == 1: + return {"stage": {"title": "Incomplete", "steps": ["broken"]}} + return { + "stage": plan_package.stages[stage_index].model_dump(mode="json") + } + + result = generate_plan_segments( + package=plan_package, + call_json=fake_call, + max_steps_per_stage=3, + ) + + assert stage_one_calls == 2 + assert result.fallback_stage_ids == [] + assert result.llm_stage_ids == ["stage-1", "stage-2", "stage-3"] + + +def test_segment_generation_does_not_overwrite_protected_constants(plan_package): + def fake_call(segment_name, _prompt, _max_tokens): + if segment_name == "core": + return { + "researchQuestion": plan_package.researchQuestion, + "hypothesis": plan_package.hypothesis, + "constants": { + "seedQuery": "drifted query", + "paperType": "survey", + "resourceBudget": "2 GPU hours", + }, + } + stage_index = int(segment_name.rsplit("-", 1)[-1]) - 1 + return { + "stage": plan_package.stages[stage_index].model_dump(mode="json") + } + + result = generate_plan_segments( + package=plan_package, + call_json=fake_call, + max_steps_per_stage=3, + ) + + assert result.constants["seedQuery"] == plan_package.constants["seedQuery"] + assert result.constants["paperType"] == plan_package.constants["paperType"] + assert result.constants["resourceBudget"] == "2 GPU hours" + + +def test_stage_prompt_compacts_long_context_without_dropping_evidence_ids(plan_package): + paper = plan_package.literatureSurvey.papers[0] + paper.summary = "long evidence summary " * 4000 + paper.methods = [ + {"name": "vanilla RAG", "description": "long method detail " * 4000} + ] + core_result = PlanSegmentGenerationResult( + research_question=plan_package.researchQuestion, + hypothesis=plan_package.hypothesis, + constants=dict(plan_package.constants), + stages=plan_package.stages, + ) + + prompt = build_stage_prompt( + plan_package, + plan_package.stages[0], + core_result, + ) + + assert len(prompt) < 30000 + assert paper.paperId in prompt diff --git a/backend/tests/test_plan_package_llm_schema.py b/backend/tests/test_plan_package_llm_schema.py index 3b7e57d..c657eda 100644 --- a/backend/tests/test_plan_package_llm_schema.py +++ b/backend/tests/test_plan_package_llm_schema.py @@ -1,4 +1,59 @@ -from app.services.plan_package_llm_schema import validate_llm_plan_output +from app.services.plan_package_llm_schema import ( + validate_llm_plan_core_output, + validate_llm_plan_output, + validate_llm_plan_stage_output, +) + + +def test_validate_llm_plan_core_output_rejects_stage_write(): + parsed, issues = validate_llm_plan_core_output( + { + "researchQuestion": "Can verification improve citation faithfulness?", + "hypothesis": ( + "Verification increases citation faithfulness; " + "reject if mean delta is not positive." + ), + "constants": {"baseline": "vanilla RAG"}, + "stages": [], + } + ) + + assert parsed is None + assert issues == ["Core output contains forbidden keys: stages"] + + +def test_validate_llm_plan_stage_output_accepts_one_existing_stage_shape(): + parsed, issues = validate_llm_plan_stage_output( + { + "stage": { + "title": "Controlled evaluation", + "goal": "Compare verification with vanilla RAG.", + "method": "Use the preregistered split and fixed retrieval corpus.", + "steps": [ + { + "title": "Measure citation faithfulness", + "desc": "Run both methods on the same questions.", + "method": "Score claim-level attribution with a frozen evaluator.", + "outputs": [ + {"type": "metrics", "name": "citation_metrics.json"} + ], + "expected": [ + { + "metric": "citation faithfulness", + "target": ( + "mean_delta > 0 with 95% confidence interval excluding 0" + ), + } + ], + } + ], + } + } + ) + + assert issues == [] + assert parsed is not None + assert parsed["stage"]["title"] == "Controlled evaluation" def test_validate_llm_plan_output_ignores_recoverable_expected_metrics_top_level(): diff --git a/backend/tests/test_plan_package_quality_regressions.py b/backend/tests/test_plan_package_quality_regressions.py index 7d46d5d..b6759df 100644 --- a/backend/tests/test_plan_package_quality_regressions.py +++ b/backend/tests/test_plan_package_quality_regressions.py @@ -1,4 +1,10 @@ -from app.models.idea import Claim, ExperimentSpec, IdeaCandidate +from app.models.idea import ( + Claim, + ExperimentSpec, + IdeaCandidate, + RawPaper, + StructuredPaper, +) from app.models.plan_package import ( PlanBackground, PlanContributionStatement, @@ -19,10 +25,16 @@ PlanStep, ) from app.services.plan_package_builder import ( + _deterministic_plan_core, build_contribution_statements, build_default_stages, + build_literature_survey, ) from app.services.plan_package_validator import validate_plan_package +from app.services.plan_package_specificity import ( + hypothesis_is_falsifiable, + metric_target_is_concrete, +) def _candidate(**overrides): @@ -131,6 +143,115 @@ def test_default_stages_filter_unrelated_claim_metrics(): assert not any("attention projection" in metric for metric in metrics) +def test_default_validation_metrics_never_use_generic_targets(): + stages = build_default_stages( + candidate=_candidate(), + literature_survey=_survey_with_claim_metrics(), + gap=_gap(), + principle=PlanPrinciple(summary="Verifier", mechanism="Verify cited spans."), + paper_type="algorithm", + max_stages=3, + max_steps_per_stage=3, + ) + targets = [ + expected.target + for stage in stages + for step in stage.steps + for expected in step.expected + ] + + assert "specified before implementation" not in targets + assert all(metric_target_is_concrete(target) for target in targets) + + +def test_default_stages_name_literature_baseline_when_available(): + survey = _survey_with_claim_metrics() + survey.papers[0].methods = [ + { + "name": "vanilla RAG", + "role": "baseline", + "description": "retrieval followed by generation without claim verification", + } + ] + + stages = build_default_stages( + candidate=_candidate(), + literature_survey=survey, + gap=_gap(), + principle=PlanPrinciple(summary="Verifier", mechanism="Verify cited spans."), + paper_type="algorithm", + max_stages=3, + max_steps_per_stage=3, + ) + baseline_text = " ".join( + text + for stage in stages + for text in [ + stage.title, + stage.goal, + stage.method, + *[ + f"{step.title} {step.desc} {step.method}" + for step in stage.steps + ], + ] + ).lower() + + assert "vanilla rag" in baseline_text + + +def test_literature_adapter_preserves_deep_reader_baselines_and_metrics(): + raw = RawPaper( + id="raw_deep", + sessionId="idea_test", + title="Citation-aware RAG evaluation", + abstract="Studies citation faithfulness and retrieval baselines.", + ) + structured = StructuredPaper( + id="raw_deep", + sessionId="idea_test", + rawPaperId="raw_deep", + title=raw.title, + summary=raw.abstract, + baselineMethods=["BM25 RAG"], + recommendedMetrics=["citation faithfulness"], + ) + + survey = build_literature_survey( + candidate=_candidate(), + raw_papers=[raw], + structured_papers=[structured], + literature_map=None, + probe_results=[], + ) + + paper = survey.papers[0] + assert any( + method.get("name") == "BM25 RAG" and method.get("role") == "baseline" + for method in paper.methods + ) + assert any( + claim.get("text") == "citation faithfulness" + and claim.get("claimType") == "metric" + for claim in paper.claims + ) + + +def test_deterministic_plan_core_is_comparative_and_falsifiable(): + survey = _survey_with_claim_metrics() + survey.papers[0].methods = [{"name": "vanilla RAG", "role": "baseline"}] + + question, hypothesis = _deterministic_plan_core( + candidate=_candidate(), + literature_survey=survey, + paper_type="algorithm", + ) + + assert "vanilla RAG" in question + assert "citation faithfulness" in question + assert hypothesis_is_falsifiable(hypothesis) + + def test_default_stages_prefer_candidate_experiment_metrics_over_claim_text(): candidate = _candidate( experimentSpecs=[ diff --git a/backend/tests/test_plan_package_review_loop.py b/backend/tests/test_plan_package_review_loop.py new file mode 100644 index 0000000..340455d --- /dev/null +++ b/backend/tests/test_plan_package_review_loop.py @@ -0,0 +1,359 @@ +import json +from types import SimpleNamespace + +from app.models.plan_package import ( + PlanMetaReview, + PlanQualityGate, + PlanReviewerIssue, + PlanReviewerReport, +) +from app.services.plan_package_review_loop import ( + dedupe_review_issues, + repair_stage_ids, + repair_targets, + route_review_issue, +) +from app.services.plan_package_reviewers import apply_review_to_quality_gate +from app.services.plan_package_service import PlanPackageService +from app.services.plan_package_validator import validate_plan_package +from app.services.plan_package_views import build_plan_package_presentation + + +def _issue(path, message, severity="blocking"): + return PlanReviewerIssue( + id=f"issue-{path}-{message}", + severity=severity, + sectionPath=path, + message=message, + ) + + +def test_review_issue_routing_keeps_plan_owned_repairs_internal(): + metric_issue = _issue( + "stages[1].steps[0].expected[0].target", + "Metric target is generic.", + ) + upstream_issue = _issue( + "evidenceTrace.pathSeedId", + "Required upstream path seed is missing.", + ) + + assert route_review_issue(metric_issue) == "plan_repairable" + assert route_review_issue(upstream_issue) == "upstream_blocking" + assert repair_targets([metric_issue]) == ["stages"] + assert repair_stage_ids( + [metric_issue], + ["stage-1", "stage-2", "stage-3"], + ) == ["stage-2"] + + +def test_review_issue_deduplication_merges_reviewer_wording(): + issues = dedupe_review_issues( + [ + _issue("hypothesis", "Hypothesis lacks a falsification condition."), + _issue( + "hypothesis", + "The hypothesis has no falsifiable rejection condition.", + ), + ] + ) + + assert len(issues) == 1 + assert issues[0].sectionPath == "hypothesis" + + +def test_internal_reviewer_diagnostic_is_not_copied_to_quality_gate(plan_package): + gate = validate_plan_package(plan_package) + diagnostic = PlanReviewerReport( + reviewer="MetricReviewer", + score=0.8, + passed=True, + warnings=[ + _issue( + "reviewReports", + "LLM reviewer unavailable for MetricReviewer: provider timeout", + severity="warning", + ) + ], + ) + + gate = apply_review_to_quality_gate( + plan_package, + gate, + extra_reports=[diagnostic], + ) + + assert not any("LLM reviewer unavailable" in item for item in gate.warnings) + assert any( + "LLM reviewer unavailable" in issue.message + for report in plan_package.reviewReports + for issue in report.warnings + ) + + +def test_stage_repair_does_not_rewrite_unaffected_stages(monkeypatch, plan_package): + before_stage_1 = plan_package.stages[0].model_dump(mode="json") + before_stage_3 = plan_package.stages[2].model_dump(mode="json") + repaired_stage = plan_package.stages[1].model_copy(deep=True) + repaired_stage.title = "Verifier implementation with frozen interfaces" + + class FakeClient: + def chat(self, **_kwargs): + return SimpleNamespace( + text=json.dumps({"stage": repaired_stage.model_dump(mode="json")}) + ) + + monkeypatch.setattr( + "app.services.plan_package_service.get_provider_client", + lambda _name: FakeClient(), + ) + monkeypatch.setattr( + "app.services.plan_package_service.get_llm_task_scheduler", + lambda: SimpleNamespace(run=lambda _name, fn, **_kwargs: fn()), + ) + service = PlanPackageService() + session = SimpleNamespace( + config=SimpleNamespace(providerName="fake", model="fake-model") + ) + + service._repair_llm_stage_fields( + plan_package, + session, + stage_ids=["stage-2"], + max_steps_per_stage=3, + ) + + assert plan_package.stages[0].model_dump(mode="json") == before_stage_1 + assert plan_package.stages[1].title == ( + "Verifier implementation with frozen interfaces" + ) + assert plan_package.stages[2].model_dump(mode="json") == before_stage_3 + + +def test_resolved_reviewer_issue_moves_to_revision_audit(monkeypatch, plan_package): + old_message = "Metric target is generic." + issue = _issue( + "stages[2].steps[0].expected[0].target", + old_message, + ) + plan_package.metaReview = PlanMetaReview( + overallScore=0.5, + decision="revise", + blockingIssues=[issue], + requiredRepairs=["Replace the generic target."], + ) + plan_package.reviewReports = [ + PlanReviewerReport( + reviewer="MetricReviewer", + score=0.5, + passed=False, + blockingIssues=[issue], + ) + ] + plan_package.qualityGate = PlanQualityGate( + schemaValid=True, + evidenceValid=True, + planSpecific=False, + errors=[f"{issue.sectionPath}: {old_message}"], + ) + service = PlanPackageService() + + def fake_stage_repair(package, _session, **_kwargs): + package.stages[2].steps[0].expected[0].target = ( + "mean_delta > 0 with 95% confidence interval excluding 0" + ) + package.generation.repairRounds += 1 + + def fake_review(package, gate, *, reviewer_mode): + assert reviewer_mode == "hybrid" + gate.topicRelevant = True + gate.citationFaithful = True + gate.planSpecific = True + gate.downstreamReady = True + gate.agentApproved = True + gate.implementationReady = True + gate.errors = [] + gate.warnings = [] + package.metaReview = PlanMetaReview( + overallScore=0.9, + decision="approve", + blockingIssues=[], + warnings=[], + ) + package.reviewReports = [ + PlanReviewerReport( + reviewer="MetricReviewer", + score=0.9, + passed=True, + ) + ] + return gate + + monkeypatch.setattr(service, "_repair_llm_stage_fields", fake_stage_repair) + monkeypatch.setattr(service, "_apply_review_mode", fake_review) + monkeypatch.setattr( + "app.services.plan_package_service.validate_plan_package", + lambda _package: PlanQualityGate(schemaValid=True, evidenceValid=True), + ) + monkeypatch.setattr( + "app.services.plan_package_service.build_contribution_statements", + lambda **_kwargs: plan_package.contributionStatement, + ) + session = SimpleNamespace( + config=SimpleNamespace(providerName="fake", model="fake-model") + ) + + service._auto_repair_plan_from_review( + plan_package, + session, + max_stages=3, + max_steps_per_stage=3, + max_repair_rounds=2, + reviewer_mode="hybrid", + ) + + assert old_message not in " ".join(plan_package.qualityGate.errors) + assert plan_package.metaReview.blockingIssues == [] + assert old_message in json.dumps( + plan_package.revisions[-1].patchSummary["reviewReportsBeforeRepair"] + ) + + +def test_presentation_hides_internal_reviewer_diagnostics(plan_package): + diagnostic = _issue( + "reviewReports", + "LLM reviewer unavailable for MetricReviewer: provider timeout", + severity="warning", + ) + plan_package.generation.warnings = ["segment_fallback:stage-2:TimeoutError"] + plan_package.qualityGate.warnings = [ + "reviewReports: LLM reviewer unavailable for MetricReviewer" + ] + plan_package.metaReview = PlanMetaReview( + overallScore=0.8, + decision="revise", + warnings=[diagnostic], + ) + + presentation = build_plan_package_presentation(plan_package) + rendered = " ".join( + [ + *presentation.reviewSummary.mainConcerns, + *presentation.reviewSummary.requiredFixes, + *presentation.evidenceSummary.weakPoints, + *presentation.nextActions, + ] + ).lower() + + assert "metricreviewer" not in rendered + assert "timeout" not in rendered + assert "provider" not in rendered + assert "schema" not in rendered + + +def test_presentation_consolidates_unresolved_user_actions(plan_package): + issues = [ + _issue( + "evidenceTrace.pathSeedId", + "Required upstream evidence path is missing.", + ), + _issue( + "literatureSurvey", + "Evidence coverage is insufficient.", + ), + _issue( + "constants.resourceBudget", + "User must choose the compute budget.", + ), + _issue( + "stages[2].steps[0].expected[0].target", + "Metric target is generic.", + ), + ] + plan_package.metaReview = PlanMetaReview( + overallScore=0.4, + decision="revise", + blockingIssues=issues, + ) + + presentation = build_plan_package_presentation(plan_package) + + assert len(presentation.reviewSummary.mainConcerns) == 3 + assert len(presentation.reviewSummary.requiredFixes) == 3 + assert len(set(presentation.reviewSummary.mainConcerns)) == 3 + + +def test_successful_presentation_has_no_review_problem_list(plan_package): + plan_package.qualityGate.agentApproved = True + plan_package.qualityGate.implementationReady = True + plan_package.qualityGate.errors = [] + plan_package.qualityGate.warnings = [] + plan_package.metaReview = PlanMetaReview( + overallScore=0.9, + decision="approve", + ) + + presentation = build_plan_package_presentation(plan_package) + + assert presentation.reviewSummary.mainConcerns == [] + assert presentation.reviewSummary.requiredFixes == [] + + +def test_missing_upstream_trace_is_detected_before_plan_llm(plan_package): + plan_package.source.searchNodeId = None + plan_package.source.pathSeedId = None + plan_package.evidenceTrace.searchNodeId = None + plan_package.evidenceTrace.pathSeedId = None + + blockers = PlanPackageService()._upstream_plan_blockers(plan_package) + + assert "upstream evidence trace identity is incomplete" in blockers + + +def test_upstream_blocker_skips_llm_reviewer(monkeypatch, plan_package): + plan_package.source.pathSeedId = None + plan_package.evidenceTrace.pathSeedId = None + service = PlanPackageService() + monkeypatch.setattr( + service, + "_run_llm_reviewers", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("LLM reviewer must not run") + ), + ) + + gate = service._apply_review_mode( + plan_package, + validate_plan_package(plan_package), + reviewer_mode="hybrid", + ) + + assert plan_package.generation.llmReviewerUsed is False + assert gate.implementationReady is False + assert any("upstream evidence trace" in item for item in gate.errors) + + +def test_readiness_flags_obey_final_gate_invariants(plan_package): + gate = plan_package.qualityGate + gate.schemaValid = True + gate.evidenceValid = True + gate.topicRelevant = True + gate.citationFaithful = True + gate.planSpecific = True + gate.agentApproved = True + service = PlanPackageService() + + reviewed_gate = service._apply_downstream_readiness(plan_package, gate) + + assert reviewed_gate.downstreamReady == ( + plan_package.downstreamReadiness.overallReady + ) + assert reviewed_gate.implementationReady == bool( + plan_package.downstreamReadiness.overallReady + and reviewed_gate.schemaValid + and reviewed_gate.evidenceValid + and reviewed_gate.topicRelevant + and reviewed_gate.citationFaithful + and reviewed_gate.planSpecific + and reviewed_gate.agentApproved + ) diff --git a/backend/tests/test_plan_package_service_timeout.py b/backend/tests/test_plan_package_service_timeout.py index 224294e..afa2475 100644 --- a/backend/tests/test_plan_package_service_timeout.py +++ b/backend/tests/test_plan_package_service_timeout.py @@ -167,3 +167,57 @@ def chat(self, *, messages, **kwargs): assert len(calls) == 2 assert [message.role for message in calls[1][0]] == ["system", "user", "user"] assert all(call_kwargs["max_tokens"] <= 4096 for _, call_kwargs in calls) + + +def test_segmented_service_keeps_successful_sections_when_one_stage_times_out( + monkeypatch, + plan_package, +): + stage_two_calls = 0 + + class FakeClient: + def chat(self, *, messages, **_kwargs): + nonlocal stage_two_calls + prompt = messages[-1].content + if prompt.startswith("Strengthen only"): + return SimpleNamespace( + text=json.dumps( + { + "researchQuestion": plan_package.researchQuestion, + "hypothesis": plan_package.hypothesis, + "constants": {"resourceBudget": "2 GPU hours"}, + } + ) + ) + context_json = prompt.split("\n", 1)[1].split("\nRepair", 1)[0] + context = json.loads(context_json) + stage = context["stageRole"] + if stage["id"] == "stage-2": + stage_two_calls += 1 + raise TimeoutError("stage timeout") + return SimpleNamespace(text=json.dumps({"stage": stage})) + + monkeypatch.setattr( + "app.services.plan_package_service.get_provider_client", + lambda _name: FakeClient(), + ) + monkeypatch.setattr( + "app.services.plan_package_service.get_llm_task_scheduler", + lambda: SimpleNamespace(run=lambda _task, fn, **_kwargs: fn()), + ) + session = SimpleNamespace( + config=SimpleNamespace(providerName="fake", model="fake-model") + ) + service = PlanPackageService() + + service._apply_segmented_llm_plan_fields( + plan_package, + session, + max_steps_per_stage=3, + ) + + assert plan_package.constants["resourceBudget"] == "2 GPU hours" + assert plan_package.generation.llmUsedSections == ["implementationPlan"] + assert plan_package.generation.fallbackUsed is True + assert "segment_fallback:stage-2:TimeoutError" in plan_package.generation.warnings + assert stage_two_calls == 2 diff --git a/backend/tests/test_plan_package_specificity.py b/backend/tests/test_plan_package_specificity.py new file mode 100644 index 0000000..d007d62 --- /dev/null +++ b/backend/tests/test_plan_package_specificity.py @@ -0,0 +1,71 @@ +from app.services.plan_package_specificity import ( + hypothesis_is_falsifiable, + metric_target_is_concrete, + plan_specificity_issues, +) +from app.services.plan_package_reviewers import metric_reviewer +from app.services.plan_package_validator import validate_plan_package + + +def test_metric_target_rejects_generic_placeholders(): + for target in [ + "specified before implementation", + "primary_metric", + "readiness", + "higher", + "better", + ]: + assert metric_target_is_concrete(target) is False + + +def test_metric_target_accepts_measurable_or_baseline_relative_criteria(): + for target in [ + ">= strongest baseline", + "mean_delta > 0 with 95% confidence interval excluding 0", + "non-inferior to control within 1 percentage point", + "failure rate <= 0.05", + "true", + ]: + assert metric_target_is_concrete(target) is True + + +def test_hypothesis_requires_direction_measure_and_falsifier(): + assert hypothesis_is_falsifiable("The verifier should improve the system.") is False + assert hypothesis_is_falsifiable( + "Compared with vanilla RAG, claim verification increases citation faithfulness; " + "the hypothesis is rejected if the mean delta is not positive on the preregistered split." + ) is True + + +def test_chinese_hypothesis_can_express_a_falsifiable_metric_direction(): + assert hypothesis_is_falsifiable( + "相比基础模型,证据校验将提高回答正确率;" + "若预注册测试集上的主要指标未显著改善,则该假设不成立。" + ) is True + + +def test_specificity_issues_point_to_exact_metric_path(plan_package): + plan_package.stages[2].steps[0].expected[0].target = "better" + + issues = plan_specificity_issues(plan_package) + + assert any( + issue.sectionPath == "stages[2].steps[0].expected[0].target" + for issue in issues + ) + + +def test_specificity_is_diagnostic_in_validator_and_blocking_in_metric_review(plan_package): + plan_package.stages[2].steps[0].expected[0].target = "better" + + gate = validate_plan_package(plan_package) + report = metric_reviewer(plan_package) + + assert any( + warning.startswith("specificity.stages[2].steps[0].expected[0].target") + for warning in gate.warnings + ) + assert any( + issue.sectionPath == "stages[2].steps[0].expected[0].target" + for issue in report.blockingIssues + ) diff --git a/backend/tests/test_search_service.py b/backend/tests/test_search_service.py new file mode 100644 index 0000000..8480cf4 --- /dev/null +++ b/backend/tests/test_search_service.py @@ -0,0 +1,71 @@ +import json +import time +import urllib.error + +from app.services import search_service as search_service_module +from app.services.search_service import LocalCorpusSearch, OpenAlexSearch, SemanticScholarSearch + + +def test_semantic_scholar_circuit_breaks_after_rate_limit(monkeypatch): + client = SemanticScholarSearch() + client.min_request_interval = 0.0 + calls = {"count": 0} + + def fake_urlopen(*args, **kwargs): + calls["count"] += 1 + raise urllib.error.HTTPError( + url="https://api.semanticscholar.org/graph/v1/paper/search", + code=429, + msg="Too Many Requests", + hdrs={}, + fp=None, + ) + + monkeypatch.setattr(search_service_module, "_urlopen", fake_urlopen) + monkeypatch.setattr(search_service_module.time, "sleep", lambda *_args, **_kwargs: None) + + assert client.search("citation faithful RAG", limit=1) == [] + assert client.disabled_until > time.time() + assert client.search("citation faithful RAG", limit=1) == [] + assert calls["count"] == 2 + + +def test_local_corpus_matches_equivalent_spaced_cjk_terms(): + client = LocalCorpusSearch() + paper = { + "title": "\u7ea2\u697c\u68a6 \u4eba\u7269\u5173\u7cfb \u7f51\u7edc\u5206\u6790", + "abstract": "\u7814\u7a76\u7ea2\u697c\u68a6\u4e2d\u7684\u4eba\u7269\u5173\u7cfb\u4e0e\u793e\u4f1a\u7f51\u7edc\u3002", + "keywords": ["\u7ea2\u697c\u68a6", "\u4eba\u7269\u5173\u7cfb", "\u7f51\u7edc\u5206\u6790"], + } + + assert client._compute_relevance(paper, "\u7ea2\u697c\u68a6\u4eba\u7269\u5173\u7cfb\u7f51\u7edc\u5206\u6790") >= 0.18 + + +def test_openalex_does_not_treat_provider_rank_as_normalized_relevance(monkeypatch): + payload = { + "results": [ + { + "id": "https://openalex.org/W1", + "title": "Unrelated high-ranked result", + "publication_year": 2024, + "relevance_score": 1223.5, + "authorships": [], + } + ] + } + + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return json.dumps(payload).encode("utf-8") + + monkeypatch.setattr(search_service_module, "_urlopen", lambda *_args, **_kwargs: FakeResponse()) + + results = OpenAlexSearch().search("Dream of the Red Chamber ending", limit=1) + + assert results[0].relevance_score == 0.0 diff --git a/docs/superpowers/plans/2026-07-12-idea-module-closure.md b/docs/superpowers/plans/2026-07-12-idea-module-closure.md new file mode 100644 index 0000000..e43e731 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-idea-module-closure.md @@ -0,0 +1,1306 @@ +# FAROS Idea Module Closure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Freeze the FAROS Idea pipeline after making evidence relevance generic and trustworthy, pausing weak evidence before bulk deep reading, and guaranteeing that completed sessions expose at least two approved, diverse candidates. + +**Architecture:** Add a focused `evidence_relevance` module that builds a deterministic topic profile, classifies papers as direct/transferable/rejected, and merges duplicate retrieval provenance. Keep orchestration in the existing Idea service, persist relevance diagnostics on `SearchResult` and `RawPaper`, and make both raw and structured evidence gates consume the same semantic eligibility and paper-type rules. + +**Tech Stack:** Python 3, Pydantic v2, dataclasses, pytest, FastAPI service/storage patterns already used by FAROS, Qwen-backed black-box evaluator, React/Vite only for the already-implemented waiting-state UI baseline. + +--- + +## File Map + +- Create `backend/app/modules/idea/evidence_relevance.py`: deterministic topic profile, evidence assessment, duplicate merge, and paper-type role requirements. +- Create `backend/tests/test_idea_evidence_relevance.py`: focused unit tests for anchors, tiers, deduplication, and role eligibility. This path is ignored by the repository-wide tests rule, so commits must use `git add -f`. +- Modify `backend/app/services/search_service.py`: carry evidence-tier diagnostics on provider-neutral `SearchResult`. +- Modify `backend/app/models/idea.py`: persist tier and score diagnostics on `RawPaper`; retain waiting states already present in the working tree. +- Modify `backend/app/modules/idea/service.py`: integrate topic profiles, early raw gate, bounded deep reading, type-aware role coverage, repair reuse, and telemetry. +- Modify `backend/app/storage/idea_storage.py`: add atomic RawPaper update needed to merge retrieval provenance during resume/repair. +- Modify `backend/tests/test_idea_evidence_gate.py`: raw/structured gate consistency and paper-type requirements. Force-add because this file is currently ignored and untracked. +- Modify `backend/tests/test_search_service.py`: preserve current provider regressions and evidence diagnostic defaults. Force-add because this file is currently ignored and untracked. +- Modify `backend/tests/test_idea_final_candidates.py`: early waiting/resume behavior and the existing two-candidate completion contract. +- Modify `backend/scripts/run_idea_plan_eval.py`: report relevance tiers, deduplication, deep-read cost, waiting outcomes, and freeze assertions without changing Plan generation behavior. +- Create `backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/idea-closure-report.json` at runtime only; do not commit generated data. + +## Task 1: Checkpoint the Existing Closure Foundation + +This task separates the already-implemented CJK query roles, waiting states, +resume endpoint/UI, search cooldown, and two-final-candidate contract from the +new relevance work. Do not edit unrelated team modules. + +**Files:** +- Modify: `backend/app/models/idea.py` +- Modify: `backend/app/modules/idea/ideas_api.py` +- Modify: `backend/app/modules/idea/service.py` +- Modify: `backend/app/services/prompts.py` +- Modify: `backend/app/services/search_service.py` +- Modify: `backend/tests/test_idea_final_candidates.py` +- Add existing ignored test: `backend/tests/test_idea_evidence_gate.py` +- Add existing ignored test: `backend/tests/test_search_service.py` +- Modify: `frontend/src/components/ideas/IdeaGenerationPanel.tsx` + +- [ ] **Step 1: Verify the current focused baseline** + +Run: + +```powershell +pytest -q backend/tests/test_idea_final_candidates.py backend/tests/test_idea_evidence_gate.py backend/tests/test_search_service.py backend/tests/test_plan_package_llm_schema.py backend/tests/test_plan_package_quality_regressions.py backend/tests/test_plan_package_service_timeout.py backend/tests/test_llm_task_scheduler.py +``` + +Expected: `60 passed` with no failures. + +- [ ] **Step 2: Verify backend syntax and frontend behavior** + +Run: + +```powershell +python -m py_compile backend/app/models/idea.py backend/app/modules/idea/ideas_api.py backend/app/modules/idea/service.py backend/app/services/prompts.py backend/app/services/search_service.py +Set-Location frontend +npm test -- --run +npm run typecheck +npm run build +Set-Location .. +``` + +Expected: Python compilation succeeds, frontend tests pass, TypeScript reports +no errors, and Vite produces a successful build. + +- [ ] **Step 3: Review the checkpoint diff for scope** + +Run: + +```powershell +git diff --check +git diff --stat +git diff -- backend/app/models/idea.py backend/app/modules/idea/ideas_api.py backend/app/modules/idea/service.py backend/app/services/prompts.py backend/app/services/search_service.py backend/tests/test_idea_final_candidates.py frontend/src/components/ideas/IdeaGenerationPanel.tsx +``` + +Expected: only Idea/search/waiting UI changes are present; no code, experiment, +paper, or review module changes appear. + +- [ ] **Step 4: Commit only the verified foundation** + +```powershell +git add backend/app/models/idea.py backend/app/modules/idea/ideas_api.py backend/app/modules/idea/service.py backend/app/services/prompts.py backend/app/services/search_service.py backend/tests/test_idea_final_candidates.py frontend/src/components/ideas/IdeaGenerationPanel.tsx +git add -f backend/tests/test_idea_evidence_gate.py backend/tests/test_search_service.py +git diff --cached --check +git commit -m "feat: stabilize idea evidence waiting and cjk retrieval" +``` + +Expected: one commit containing the verified baseline, while unrelated working +tree changes, if any, remain untouched. + +## Task 2: Build the Deterministic Topic Profile and Evidence Tiers + +**Files:** +- Create: `backend/app/modules/idea/evidence_relevance.py` +- Create: `backend/tests/test_idea_evidence_relevance.py` +- Modify: `backend/app/services/search_service.py` + +- [ ] **Step 1: Write failing tests for anchors and tiers** + +Create `backend/tests/test_idea_evidence_relevance.py` with these cases: + +```python +from app.modules.idea.evidence_relevance import ( + EvidenceTier, + assess_search_result, + build_topic_intent_profile, +) +from app.services.search_service import SearchResult + + +def result(title: str, abstract: str = "") -> SearchResult: + return SearchResult( + title=title, + authors=[], + abstract=abstract, + year=2025, + venue="test", + url=None, + doi=None, + arxiv_id=None, + citation_count=0, + source="openalex", + ) + + +def red_chamber_profile(): + return build_topic_intent_profile( + seed="预测红楼梦可能结局", + domain="", + role_queries={ + "domain": ["Literary analysis of 'Dream of the Red Chamber'"], + "task": ["Computational prediction of endings for 'Dream of the Red Chamber'"], + "method": ["Narrative completion using character constraints"], + "evaluation": ["Narrative coherence and character consistency evaluation"], + }, + ) + + +def test_named_work_is_preserved_as_one_core_anchor(): + profile = red_chamber_profile() + assert "dream of the red chamber" in profile.core_anchors + assert "dream" in profile.generic_terms + + +def test_direct_named_work_paper_is_eligible(): + assessment = assess_search_result( + result( + "Multiple Authors Detection: A Quantitative Analysis of Dream of the Red Chamber", + "Computational evidence about authorship and the unfinished ending.", + ), + red_chamber_profile(), + ) + assert assessment.tier is EvidenceTier.DIRECT + assert "dream of the red chamber" in assessment.decisive_anchors + + +def test_transferable_narrative_method_is_eligible_but_not_direct(): + assessment = assess_search_result( + result( + "Narrative completion for unfinished novels", + "Computational character constraints and coherence evaluation reconstruct plausible endings.", + ), + red_chamber_profile(), + ) + assert assessment.tier is EvidenceTier.TRANSFERABLE + + +def test_generic_clinical_forecasting_is_rejected(): + assessment = assess_search_result( + result( + "Clinical time-series forecasting and analysis", + "A web platform predicts patient outcomes with configurable models.", + ), + red_chamber_profile(), + ) + assert assessment.tier is EvidenceTier.REJECTED + assert assessment.rejection_reason == "generic_overlap_only" + + +def test_generic_chemical_evaluation_is_rejected(): + assessment = assess_search_result( + result("ChemEval: A multi-level chemical evaluation for language models"), + red_chamber_profile(), + ) + assert assessment.tier is EvidenceTier.REJECTED +``` + +- [ ] **Step 2: Run the new tests and verify RED** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py +``` + +Expected: collection fails because +`app.modules.idea.evidence_relevance` does not exist. + +- [ ] **Step 3: Implement the relevance types and deterministic profile** + +Create `backend/app/modules/idea/evidence_relevance.py` with these public types +and functions: + +```python +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import re +from typing import Iterable, Mapping, Sequence + +from app.services.search_service import SearchResult, tokenize_topic_text + + +class EvidenceTier(str, Enum): + DIRECT = "direct" + TRANSFERABLE = "transferable" + REJECTED = "rejected" + + +GENERIC_TERMS = frozenset({ + "analysis", "application", "approach", "evaluation", "exploration", + "framework", "generation", "method", "model", "outcome", "outcomes", + "potential", "predict", "predicting", "prediction", "research", "study", + "system", "using", "dream", "red", "chamber", +}) + + +@dataclass(frozen=True) +class TopicIntentProfile: + core_anchors: tuple[str, ...] + task_anchors: tuple[str, ...] + method_anchors: tuple[str, ...] + evaluation_anchors: tuple[str, ...] + generic_terms: tuple[str, ...] + + def to_dict(self) -> dict[str, list[str]]: + return { + "coreAnchors": list(self.core_anchors), + "taskAnchors": list(self.task_anchors), + "methodAnchors": list(self.method_anchors), + "evaluationAnchors": list(self.evaluation_anchors), + "genericTerms": list(self.generic_terms), + } + + +@dataclass(frozen=True) +class EvidenceAssessment: + tier: EvidenceTier + score: float + decisive_anchors: tuple[str, ...] + score_components: Mapping[str, float] + rejection_reason: str = "" + + +def _unique(values: Iterable[str]) -> tuple[str, ...]: + return tuple(dict.fromkeys(value for value in values if value)) + + +def _quoted_phrases(values: Iterable[str]) -> tuple[str, ...]: + phrases: list[str] = [] + for value in values: + phrases.extend( + match.strip().lower() + for match in re.findall(r"[\"']([^\"']{4,100})[\"']", value or "") + ) + return _unique(phrases) + + +def _discriminative_tokens(values: Iterable[str]) -> tuple[str, ...]: + return _unique( + token + for value in values + for token in tokenize_topic_text(value) + if token not in GENERIC_TERMS and len(token) >= 3 + ) + + +def build_topic_intent_profile( + *, + seed: str, + domain: str, + role_queries: Mapping[str, Sequence[str]], +) -> TopicIntentProfile: + domain_queries = list(role_queries.get("domain", ())) + task_queries = list(role_queries.get("task", ())) + method_queries = list(role_queries.get("method", ())) + evaluation_queries = list(role_queries.get("evaluation", ())) + core_phrases = _quoted_phrases([seed, domain, *domain_queries, *task_queries]) + core_tokens = _discriminative_tokens([seed, domain, *domain_queries]) + return TopicIntentProfile( + core_anchors=_unique([*core_phrases, *core_tokens]), + task_anchors=_discriminative_tokens(task_queries), + method_anchors=_discriminative_tokens(method_queries), + evaluation_anchors=_discriminative_tokens(evaluation_queries), + generic_terms=tuple(sorted(GENERIC_TERMS)), + ) + + +def _hits(text: str, anchors: Sequence[str]) -> tuple[str, ...]: + return tuple(anchor for anchor in anchors if anchor and anchor in text) + + +def assess_search_result( + result: SearchResult, + profile: TopicIntentProfile, +) -> EvidenceAssessment: + text = f"{result.title} {result.abstract}".lower().replace("-", " ") + phrase_hits = tuple(anchor for anchor in profile.core_anchors if " " in anchor and anchor in text) + core_hits = _hits(text, tuple(anchor for anchor in profile.core_anchors if " " not in anchor)) + task_hits = _hits(text, profile.task_anchors) + method_hits = _hits(text, profile.method_anchors) + evaluation_hits = _hits(text, profile.evaluation_anchors) + + components = { + "corePhrase": min(0.55, 0.55 * len(phrase_hits)), + "coreTerms": min(0.35, 0.12 * len(core_hits)), + "task": min(0.25, 0.10 * len(task_hits)), + "methodEvaluation": min(0.20, 0.06 * (len(method_hits) + len(evaluation_hits))), + "provider": min(0.10, max(0.0, float(result.relevance_score or 0.0)) * 0.10), + } + score = min(1.0, sum(components.values())) + decisive = _unique([*phrase_hits, *core_hits, *task_hits, *method_hits, *evaluation_hits]) + has_supporting_signal = bool(task_hits or method_hits or evaluation_hits) + + if (phrase_hits or len(core_hits) >= 2) and has_supporting_signal: + tier = EvidenceTier.DIRECT + reason = "" + elif len(task_hits) >= 2 and bool(method_hits or evaluation_hits): + tier = EvidenceTier.TRANSFERABLE + reason = "" + else: + tier = EvidenceTier.REJECTED + reason = "generic_overlap_only" if text.strip() else "missing_text" + + return EvidenceAssessment( + tier=tier, + score=round(score, 4), + decisive_anchors=decisive, + score_components=components, + rejection_reason=reason, + ) +``` + +Add provider-neutral diagnostic fields to `SearchResult`: + +```python +evidence_tier: str = "unclassified" +decisive_anchors: List[str] = field(default_factory=list) +relevance_components: Dict[str, float] = field(default_factory=dict) +rejection_reason: str = "" +must_cite_override: bool = False +``` + +- [ ] **Step 4: Verify deterministic normalization** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py backend/tests/test_search_service.py +``` + +Expected: all tests pass. Do not add a seed-specific branch; corrections must +change generic phrase/token behavior. + +- [ ] **Step 5: Commit the relevance core** + +```powershell +git add backend/app/modules/idea/evidence_relevance.py backend/app/services/search_service.py +git add -f backend/tests/test_idea_evidence_relevance.py +git diff --cached --check +git commit -m "feat: classify idea evidence by topic relevance" +``` + +## Task 3: Merge Duplicate Retrieval Provenance and Persist Assessments + +**Files:** +- Modify: `backend/app/modules/idea/evidence_relevance.py` +- Modify: `backend/app/services/search_service.py` +- Modify: `backend/app/models/idea.py` +- Modify: `backend/app/storage/idea_storage.py` +- Modify: `backend/tests/test_idea_evidence_relevance.py` + +- [ ] **Step 1: Write failing deduplication tests** + +Append: + +```python +from app.modules.idea.evidence_relevance import deduplicate_search_results + + +def test_deduplication_merges_roles_queries_sources_and_richer_metadata(): + first = result("Citation-Enforced RAG") + first.doi = "10.1000/rag" + first.source = "semantic_scholar" + first.retrieval_sources = ["semantic_scholar"] + first.retrieval_roles = ["domain"] + first.matched_queries = ["citation faithful RAG"] + first.relevance_score = 0.4 + + second = result("Citation-Enforced RAG", "A richer abstract with refusal evaluation.") + second.doi = "10.1000/rag" + second.source = "openalex" + second.retrieval_roles = ["method", "evaluation"] + second.matched_queries = ["RAG verifier", "RAG refusal benchmark"] + second.relevance_score = 0.8 + + outcome = deduplicate_search_results([first, second]) + + assert outcome.merge_count == 1 + assert len(outcome.results) == 1 + merged = outcome.results[0] + assert merged.retrieval_roles == ["domain", "method", "evaluation"] + assert merged.matched_queries == [ + "citation faithful RAG", "RAG verifier", "RAG refusal benchmark" + ] + assert merged.retrieval_sources == ["semantic_scholar", "openalex"] + assert merged.abstract == second.abstract + assert merged.relevance_score == 0.8 +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py::test_deduplication_merges_roles_queries_sources_and_richer_metadata +``` + +Expected: import or attribute failure for the missing deduplication API. + +- [ ] **Step 3: Implement canonical identity and merge behavior** + +Add `retrieval_sources` to `SearchResult` and initialize it in +`__post_init__`: + +```python +retrieval_sources: List[str] = field(default_factory=list) + +def __post_init__(self) -> None: + if self.source and self.source not in self.retrieval_sources: + self.retrieval_sources.append(self.source) +``` + +Add to `evidence_relevance.py`: + +```python +from dataclasses import dataclass +import hashlib + + +@dataclass(frozen=True) +class DedupeOutcome: + results: tuple[SearchResult, ...] + merge_count: int + + +def _identity_keys(result: SearchResult) -> tuple[str, ...]: + keys: list[str] = [] + if result.doi: + keys.append(f"doi:{result.doi.lower().strip()}") + if result.arxiv_id: + keys.append(f"arxiv:{result.arxiv_id.lower().strip()}") + semantic_id = re.search(r"SemanticScholarID:(\w+)", result.url or "") + if semantic_id: + keys.append(f"s2:{semantic_id.group(1).lower()}") + normalized = re.sub(r"[^a-z0-9\u4e00-\u9fff]+", "", result.title.lower()) + keys.append("title:" + hashlib.sha256(normalized.encode("utf-8")).hexdigest()) + return tuple(keys) + + +def _append_unique(current: list[str], incoming: Iterable[str]) -> None: + for value in incoming: + if value and value not in current: + current.append(value) + + +def _merge_result(target: SearchResult, incoming: SearchResult) -> SearchResult: + _append_unique(target.retrieval_roles, incoming.retrieval_roles) + _append_unique(target.matched_queries, incoming.matched_queries) + _append_unique(target.retrieval_sources, incoming.retrieval_sources or [incoming.source]) + if len(incoming.abstract or "") > len(target.abstract or ""): + target.abstract = incoming.abstract + if not target.doi and incoming.doi: + target.doi = incoming.doi + if not target.arxiv_id and incoming.arxiv_id: + target.arxiv_id = incoming.arxiv_id + target.relevance_score = max(target.relevance_score, incoming.relevance_score) + target.citation_count = max(target.citation_count or 0, incoming.citation_count or 0) + return target + + +def deduplicate_search_results(results: Sequence[SearchResult]) -> DedupeOutcome: + unique: list[SearchResult] = [] + index: dict[str, SearchResult] = {} + merge_count = 0 + for result in results: + keys = _identity_keys(result) + target = next((index[key] for key in keys if key in index), None) + if target is not None: + _merge_result(target, result) + merge_count += 1 + else: + target = result + unique.append(target) + for key in (*_identity_keys(target), *keys): + index[key] = target + return DedupeOutcome(tuple(unique), merge_count) + + +_TIER_PRIORITY = { + EvidenceTier.REJECTED.value: 0, + "unclassified": 1, + EvidenceTier.TRANSFERABLE.value: 2, + EvidenceTier.DIRECT.value: 3, +} + + +def better_evidence_tier(current: str, incoming: str) -> str: + return max((current, incoming), key=lambda tier: _TIER_PRIORITY.get(tier, 0)) +``` + +- [ ] **Step 4: Persist evidence diagnostics and support RawPaper updates** + +Add to `RawPaper`: + +```python +evidenceTier: str = Field( + default="unclassified", + description="direct, transferable, rejected, unclassified", +) +decisiveAnchors: List[str] = Field(default_factory=list) +relevanceComponents: Dict[str, float] = Field(default_factory=dict) +rejectionReason: str = "" +mustCiteOverride: bool = False +``` + +Add to `RawPaperStorage` using the same atomic writer as `create`: + +```python +def update(self, paper: RawPaper) -> RawPaper: + path = self._get_path(paper.id) + if not path.exists(): + raise ValueError(f"RawPaper {paper.id} not found") + data = paper.model_dump() + data["createdAt"] = ( + data["createdAt"].isoformat() + if isinstance(data["createdAt"], datetime) + else data["createdAt"] + ) + _write_json_atomic(path, data, default=str) + return paper +``` + +- [ ] **Step 5: Run model/storage/relevance tests** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py backend/tests/test_structured_paper_cache.py backend/tests/test_search_service.py +``` + +Expected: all tests pass. + +- [ ] **Step 6: Commit the provenance model** + +```powershell +git add backend/app/modules/idea/evidence_relevance.py backend/app/services/search_service.py backend/app/models/idea.py backend/app/storage/idea_storage.py +git add -f backend/tests/test_idea_evidence_relevance.py +git diff --cached --check +git commit -m "feat: merge idea retrieval provenance" +``` + +## Task 4: Integrate Tier-Aware Search and the Early Evidence Gate + +**Files:** +- Modify: `backend/app/modules/idea/service.py` +- Modify: `backend/tests/test_idea_final_candidates.py` +- Modify: `backend/tests/test_idea_evidence_gate.py` + +- [ ] **Step 1: Write a failing service-level pollution test** + +Add a test that constructs a session with stored `expandQuery` role outputs, +stubs `get_search_service().search`, and returns four direct/transferable papers +plus clinical and chemical distractors. Use a temporary service data directory. +The decisive assertions are: + +```python +inputs, outputs, _ = service._step_literature_search(session) +stored = service.raw_paper_storage.list_by_session(session.id) + +assert all(paper.evidenceTier in {"direct", "transferable"} for paper in stored) +assert not any("Clinical time-series" in paper.title for paper in stored) +assert not any("ChemEval" in paper.title for paper in stored) +assert outputs["evidenceTierCounts"]["rejected"] >= 2 +assert outputs["duplicateMergeCount"] >= 0 +assert outputs["topicIntentProfile"]["coreAnchors"] +``` + +- [ ] **Step 2: Run the service test and verify RED** + +Run: + +```powershell +pytest -q backend/tests/test_idea_final_candidates.py::test_literature_search_rejects_generic_cross_domain_results +``` + +Expected: failure because search outputs do not expose tier/profile telemetry and +rejected papers can still be persisted. + +- [ ] **Step 3: Replace the nested skip-only dedupe with the shared pipeline** + +In `_step_literature_search`: + +```python +from collections import Counter + +profile = build_topic_intent_profile( + seed=seed, + domain=session.config.domain or "", + role_queries=role_queries if isinstance(role_queries, dict) else {}, +) +must_cite_refs = [value.lower().strip() for value in (session.config.mustCiteList or [])] + +def matches_must_cite(result: SearchResult) -> bool: + haystack = " ".join( + value + for value in [result.doi, result.arxiv_id, result.url, result.title] + if value + ).lower() + return any(reference and reference in haystack for reference in must_cite_refs) + +def dedupe_assess_rank(results: List[SearchResult]): + dedupe = deduplicate_search_results(results) + persistable: list[SearchResult] = [] + gate_eligible: list[SearchResult] = [] + rejected: list[SearchResult] = [] + for result in dedupe.results: + assessment = assess_search_result(result, profile) + result.evidence_tier = assessment.tier.value + result.decisive_anchors = list(assessment.decisive_anchors) + result.relevance_components = dict(assessment.score_components) + result.rejection_reason = assessment.rejection_reason + result.relevance_score = assessment.score + if assessment.tier is not EvidenceTier.REJECTED: + persistable.append(result) + gate_eligible.append(result) + else: + result.must_cite_override = matches_must_cite(result) + rejected.append(result) + if result.must_cite_override: + persistable.append(result) + persistable.sort(key=lambda item: item.relevance_score, reverse=True) + gate_eligible.sort(key=lambda item: item.relevance_score, reverse=True) + return persistable, gate_eligible, rejected, dedupe.merge_count, len(dedupe.results) +``` + +Before placing a rejected result in the rejected-only list, compare its DOI, +arXiv ID, URL, and normalized title with `session.config.mustCiteList`. A match +sets `result.must_cite_override = True` and keeps the paper for storage and +DeepReader, while its `evidence_tier` remains `rejected`. It is excluded from +aligned counts and role coverage. This preserves an explicit user citation +without allowing it to satisfy the quality gate. + +Assign the returned collections explicitly: use `gate_eligible` for the raw +quality gate, use `persistable` for RawPaper/LiteratureItem creation, and use +`rejected` only for rejection telemetry. This prevents a must-cite override from +accidentally satisfying evidence coverage. + +Use this same function after initial search and after repair search. Remove the +old `seen_*` loop and do not call `_filter_results_for_topic` as a second, +provider-score-only eligibility decision. + +Tag repair results with both `repair` and the missing semantic dimension when +the repair query targets `domain`, `task`, `method`, or `evaluation`: + +```python +def repair_roles(query: str) -> list[str]: + lowered = query.lower() + roles = ["repair"] + for role in ("domain", "task", "method", "evaluation"): + if role in lowered and role not in roles: + roles.append(role) + return roles +``` + +Merge these roles into every result returned for that repair query before +deduplication. `limitation` remains a coverage dimension rather than a retrieval +role. + +- [ ] **Step 4: Persist semantic evidence and visible must-cite overrides** + +When constructing `RawPaper`, map: + +```python +source=list(result.retrieval_sources or ([result.source] if result.source else [])), +retrievalRoles=list(result.retrieval_roles), +matchedQueries=list(result.matched_queries), +evidenceTier=result.evidence_tier, +decisiveAnchors=list(result.decisive_anchors), +relevanceComponents=dict(result.relevance_components), +rejectionReason=result.rejection_reason, +mustCiteOverride=result.must_cite_override, +relevanceScore=min(1.0, max(0.0, result.relevance_score)), +``` + +Add these outputs: + +```python +"topicIntentProfile": profile.to_dict(), +"resultCountBeforeDedup": len(all_results), +"uniqueResultCount": unique_count, +"duplicateMergeCount": duplicate_merge_count, +"evidenceTierCounts": { + "direct": sum(item.evidence_tier == "direct" for item in unique_results), + "transferable": sum(item.evidence_tier == "transferable" for item in unique_results), + "rejected": len(rejected_results), +}, +"rejectionReasonCounts": dict(Counter(item.rejection_reason for item in rejected_results)), +``` + +- [ ] **Step 5: Add an early recoverable error at the correct resume point** + +Define: + +```python +class AwaitingLiteratureEvidenceError(RecoverableIdeaError): + waiting_status = IdeaSessionStatus.AWAITING_EVIDENCE + resume_from = "literatureSearch" +``` + +After one anchored repair round, if the raw quality gate still fails, raise this +error with the complete search diagnostics. Add a test asserting: + +```python +paused = service.run_pipeline(session.id) +assert paused.status is IdeaSessionStatus.AWAITING_EVIDENCE +assert paused.qualityLoopSummary["resumeFrom"] == "literatureSearch" +assert service.structured_storage.list_by_session(session.id) == [] +``` + +- [ ] **Step 6: Run the search and waiting regressions** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py backend/tests/test_idea_evidence_gate.py backend/tests/test_idea_final_candidates.py -k "literature or evidence or waiting or resume or completed" +``` + +Expected: all selected tests pass; weak raw pools never call DeepReader. + +- [ ] **Step 7: Commit the early gate** + +```powershell +git add backend/app/modules/idea/service.py backend/tests/test_idea_final_candidates.py +git add -f backend/tests/test_idea_evidence_gate.py +git diff --cached --check +git commit -m "feat: reject weak idea evidence before deep reading" +``` + +## Task 5: Make Role Coverage Semantic and Paper-Type Aware + +**Files:** +- Modify: `backend/app/modules/idea/evidence_relevance.py` +- Modify: `backend/app/modules/idea/service.py` +- Modify: `backend/tests/test_idea_evidence_gate.py` + +- [ ] **Step 1: Write failing paper-type and role tests** + +Add tests with eligible `RawPaper` fixtures: + +```python +from app.models.idea import RawPaper + + +def raw(title: str, *, tier: str, roles: list[str], index: int = 0) -> RawPaper: + return RawPaper( + id=f"raw_gate_{index}", + sessionId="idea_gate", + title=title, + abstract=title, + source=["openalex"], + retrievalRoles=roles, + evidenceTier=tier, + relevanceScore=0.9, + ) + + +def test_role_coverage_does_not_pass_with_rejected_query_hits(): + papers = [ + raw("Clinical forecasting", tier="rejected", roles=["task"], index=1), + raw("Chemical language-model evaluation", tier="rejected", roles=["evaluation"], index=2), + raw("Generic framework", tier="rejected", roles=["method"], index=3), + raw("Generic survey", tier="rejected", roles=["domain"], index=4), + ] + gate = _evaluate_paper_quality_gate( + seed="citation-faithful medical RAG", + domain="medical QA", + papers=papers, + stage="test", + paper_type="system", + ) + assert gate["roleCoverage"]["passed"] is False + assert gate["alignedPaperCount"] == 0 + + +def test_survey_does_not_require_method_query_role(): + papers = [ + raw("RAG safety survey and open gaps", tier="direct", roles=["domain"], index=5), + raw("Retrieval augmented generation safety limitations", tier="direct", roles=["task"], index=6), + raw("RAG safety claims and synthesis", tier="direct", roles=["domain"], index=7), + raw("Open questions in safe RAG", tier="direct", roles=["task"], index=8), + ] + gate = _evaluate_paper_quality_gate( + seed="RAG safety survey", + domain="retrieval augmented generation", + papers=papers, + stage="test", + paper_type="survey", + ) + assert gate["roleCoverage"]["requirements"]["method"] == 0 + assert gate["roleCoverage"]["passed"] is True + + +def test_transferable_paper_cannot_fill_domain_role(): + paper = raw( + "Narrative completion method", + tier="transferable", + roles=["domain", "method"], + index=9, + ) + gate = _evaluate_paper_quality_gate( + seed="Dream of the Red Chamber ending", + domain="", + papers=[paper], + stage="test", + paper_type="system", + ) + assert gate["roleCoverage"]["counts"]["domain"] == 0 + assert gate["roleCoverage"]["counts"]["method"] == 1 +``` + +- [ ] **Step 2: Run the new gate tests and verify RED** + +Run the three tests directly. Expected: failures because the existing gate has +universal role requirements and treats retrieval roles as semantic proof. + +- [ ] **Step 3: Implement paper-type role requirements** + +Add to `evidence_relevance.py`: + +```python +def role_requirements_for_paper_type(paper_type: str) -> dict[str, int]: + normalized = (paper_type or "algorithm").lower() + if normalized in {"survey", "position", "theory"}: + return {"domainOrTask": 2, "method": 0, "evaluation": 0} + if normalized in {"benchmark", "evaluation", "reproducibility"}: + return {"domainOrTask": 2, "method": 0, "evaluation": 2} + return {"domainOrTask": 2, "method": 1, "evaluation": 1} + + +def semantically_eligible_roles(tier: str, roles: Sequence[str]) -> tuple[str, ...]: + if tier == EvidenceTier.DIRECT.value: + return _unique(roles) + if tier == EvidenceTier.TRANSFERABLE.value: + return _unique(role for role in roles if role in {"method", "evaluation"}) + if tier == "unclassified": + return _unique(roles) + return () + + +def evidence_tier_allows_dimension(tier: str, dimension: str) -> bool: + if tier in {EvidenceTier.DIRECT.value, "unclassified"}: + return True + if tier == EvidenceTier.TRANSFERABLE.value: + return dimension in { + "method", "evaluation", "dataset", "metric", "baseline", "limitation" + } + return False +``` + +- [ ] **Step 4: Make the quality gate read both model shapes and never bypass alignment** + +Change `_evaluate_paper_quality_gate` to accept `paper_type: str = "algorithm"`. +Resolve metadata consistently: + +```python +roles = ( + (paper_roles or {}).get(paper_id, []) + or getattr(paper, "retrievalRoles", None) + or getattr(paper, "retrieval_roles", []) + or [] +) +tier = ( + getattr(paper, "evidenceTier", None) + or getattr(paper, "evidence_tier", "direct") +) +eligible_roles = semantically_eligible_roles(tier, roles) +``` + +Always enforce `min_aligned` and `avg_top_score`; remove conditions that skip +those errors when role coverage passes. Pass `paper_type` at every call site. +Set the displayed/alignment score to `0.0` when `tier == "rejected"`, so a +provider score or generic word cannot increment `alignedPaperCount`. +Keep `_paper_type_coverage_requirements` as the structured-content requirement +and make its categories match `role_requirements_for_paper_type`. + +When validating the LLM Evidence Coverage report, resolve every +`supportingPaperId` to its RawPaper tier and remove IDs for which +`evidence_tier_allows_dimension(tier, dimension)` is false. If a required +dimension has no allowed supporting IDs after verification, mark it missing. +Include `evidenceTier` in each paper summary sent to the reviewer so the LLM can +distinguish direct background/GAP evidence from transferable method evidence. + +Implement the verifier in `service.py`: + +```python +def _verify_coverage_dimension_support( + *, + dimension: str, + supporting_paper_ids: List[str], + paper_tiers: Dict[str, str], +) -> List[str]: + return [ + paper_id + for paper_id in dict.fromkeys(supporting_paper_ids) + if paper_id in paper_tiers + and evidence_tier_allows_dimension(paper_tiers[paper_id], dimension) + ] +``` + +Add this regression: + +```python +def test_transferable_paper_cannot_be_the_only_gap_support(): + verified = _verify_coverage_dimension_support( + dimension="gap", + supporting_paper_ids=["raw_transfer"], + paper_tiers={"raw_transfer": "transferable"}, + ) + assert verified == [] + + +def test_transferable_paper_can_support_method_dimension(): + verified = _verify_coverage_dimension_support( + dimension="method", + supporting_paper_ids=["raw_transfer"], + paper_tiers={"raw_transfer": "transferable"}, + ) + assert verified == ["raw_transfer"] +``` + +- [ ] **Step 5: Run all evidence-gate regressions** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_gate.py backend/tests/test_idea_final_candidates.py -k "gate or role or evidence or survey or benchmark" +``` + +Expected: all selected tests pass; role counts and structured coverage agree by +paper type. + +- [ ] **Step 6: Commit semantic role coverage** + +```powershell +git add backend/app/modules/idea/evidence_relevance.py backend/app/modules/idea/service.py +git add -f backend/tests/test_idea_evidence_gate.py +git diff --cached --check +git commit -m "feat: align idea evidence roles with paper type" +``` + +## Task 6: Bound Deep Reading and Make Repair/Resume Idempotent + +**Files:** +- Modify: `backend/app/modules/idea/service.py` +- Modify: `backend/app/storage/idea_storage.py` +- Modify: `backend/tests/test_idea_final_candidates.py` +- Modify: `backend/tests/test_idea_evidence_relevance.py` + +- [ ] **Step 1: Write failing deep-read quota and resume tests** + +Add tests asserting: + +```python +def identity_set(papers): + return { + paper.doi + or paper.arxivId + or paper.semanticScholarId + or paper.openalexId + or paper.normalizedTitleHash + for paper in papers + } + + +monkeypatch.setenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "6") +_, outputs, _ = service._step_novelty_check(session) +assert outputs["deepReadRequestedCount"] <= 6 +assert len(outputs["selectedPaperIds"]) <= 6 + len(session.config.mustCiteList or []) + +before = identity_set(service.raw_paper_storage.list_by_session(session.id)) +service.resume_session(session.id) +resumed = service.run_pipeline(session.id) +after = identity_set(service.raw_paper_storage.list_by_session(session.id)) +assert before <= after +assert len(after) == len(service.raw_paper_storage.list_by_session(session.id)) +assert resumed.trace.totalSteps == len(resumed.trace.steps) +``` + +Also simulate a repair result matching an existing DOI and assert the existing +paper gains the new role/query instead of creating a second RawPaper. + +- [ ] **Step 2: Run the quota/resume tests and verify RED** + +Expected: selection still reaches 40 papers and repair skips existing papers +without merging provenance. + +- [ ] **Step 3: Bound selection with a validated environment setting** + +Add: + +```python +def _deep_read_max_papers() -> int: + try: + configured = int(os.getenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "24")) + except ValueError: + configured = 24 + return max(4, min(40, configured)) +``` + +Set `num_select = min(_deep_read_max_papers(), max(4, len(raw_papers)))` and +reserve no more than one third of selected slots for transferable evidence. +Direct evidence fills the remaining slots first. Must-cite papers may exceed the +cap but must retain their actual evidence tier in telemetry. + +- [ ] **Step 4: Upsert repair papers by stable identity** + +Refactor `_persist_repair_search_results` to run the same +deduplicate/classify pipeline as initial search. Build an identity map for +existing RawPaper records. When a match exists: + +```python +updated = existing.model_copy(update={ + "retrievalRoles": list(dict.fromkeys([*existing.retrievalRoles, *result.retrieval_roles])), + "matchedQueries": list(dict.fromkeys([*existing.matchedQueries, *result.matched_queries])), + "source": list(dict.fromkeys([*existing.source, *result.retrieval_sources])), + "relevanceScore": max(existing.relevanceScore, result.relevance_score), + "evidenceTier": better_evidence_tier(existing.evidenceTier, result.evidence_tier), + "decisiveAnchors": list(dict.fromkeys([*existing.decisiveAnchors, *result.decisive_anchors])), +}) +self.raw_paper_storage.update(updated) +``` + +Return `updatedRawPaperIds`, `duplicateMergeCount`, tier counts, and rejection +counts in the repair report. Do not deep-read rejected repair results. + +- [ ] **Step 5: Keep resume trace counters internally consistent** + +Add a `_record_step_result` helper used by both success and failure paths: + +```python +def _record_step_result(trace: PipelineTrace, result: StepResult) -> None: + trace.steps.append(result) + trace.totalSteps = len(trace.steps) + trace.successfulSteps = sum(step.status == "ok" for step in trace.steps) + trace.failedSteps = sum(step.status == "failed" for step in trace.steps) +``` + +Repeated step names represent visible attempts, while artifacts and raw-paper +identities remain deduplicated. + +- [ ] **Step 6: Run resume, cache, and completion tests** + +Run: + +```powershell +pytest -q backend/tests/test_idea_evidence_relevance.py backend/tests/test_idea_final_candidates.py backend/tests/test_structured_paper_cache.py -k "resume or repair or cache or final or complete or deep_read" +``` + +Expected: all selected tests pass. + +- [ ] **Step 7: Commit bounded, idempotent evidence processing** + +```powershell +git add backend/app/modules/idea/service.py backend/app/storage/idea_storage.py backend/tests/test_idea_final_candidates.py +git add -f backend/tests/test_idea_evidence_relevance.py +git diff --cached --check +git commit -m "perf: bound idea deep reading and reuse evidence" +``` + +## Task 7: Extend Evaluation Telemetry and Run the Freeze Suite + +**Files:** +- Modify: `backend/scripts/run_idea_plan_eval.py` +- Test: all Idea/Plan boundary tests listed below +- Runtime output: `backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/idea-closure-report.json` + +- [ ] **Step 1: Add evaluator assertions and summaries** + +Extend the per-seed result with: + +```python +status_value = getattr(session.status, "value", str(session.status)) +is_negative_stress = bool(spec.get("negativeStress", False)) +literature_out = steps.get("literatureSearch", {}).get("outputs", {}) +raw_gate = literature_out.get("paperQualityGate", {}) +item["retrievalQuality"] = { + "resultCountBeforeDedup": literature_out.get("resultCountBeforeDedup", 0), + "uniqueResultCount": literature_out.get("uniqueResultCount", 0), + "duplicateMergeCount": literature_out.get("duplicateMergeCount", 0), + "evidenceTierCounts": literature_out.get("evidenceTierCounts", {}), + "rejectionReasonCounts": literature_out.get("rejectionReasonCounts", {}), + "retrievalRoleCounts": literature_out.get("retrievalRoleCounts", {}), + "topicIntentProfile": literature_out.get("topicIntentProfile", {}), +} +item["freezeChecks"] = { + "positiveSeedCompleted": is_negative_stress or status_value == "completed", + "completionHasTwoIdeas": status_value != "completed" + or len(session.finalCandidateIds or []) >= 2, + "waitingStateIsRecoverable": status_value in { + "completed", "awaiting_evidence", "awaiting_ideas" + }, + "rawRoleCoverageEnabled": bool( + (raw_gate.get("roleCoverage") or {}).get("enabled", False) + ), + "structuredRoleCoverageEnabled": ( + is_negative_stress and status_value == "awaiting_evidence" + ) or bool( + (evidence_gate.get("roleCoverage") or {}).get("enabled", False) + ), + "deepReadBounded": int(novelty_out.get("deepReadRequestedCount", 0) or 0) + <= int(os.getenv("FAROS_IDEA_DEEP_READ_MAX_PAPERS", "24")), +} +``` + +Add the negative stress case to `SEEDS` without changing the three positive +technical cases: + +```python +{ + "label": "D_cjk_named_work_negative_stress", + "seed": "预测红楼梦可能结局", + "domain": "", + "paperType": "system", + "maxCandidates": 3, + "baselineSeconds": 975, + "baselineNote": "negative pollution stress; awaiting_evidence is valid", + "negativeStress": True, +}, +``` + +Set the evaluator process exit code to nonzero only when a hard freeze check is +false. A valid `awaiting_evidence` result is not a failure for the CJK negative +stress seed. + +- [ ] **Step 2: Run the complete local regression suite** + +Run: + +```powershell +pytest -q backend/tests/test_idea_final_candidates.py backend/tests/test_idea_evidence_gate.py backend/tests/test_idea_evidence_relevance.py backend/tests/test_search_service.py backend/tests/test_structured_paper_cache.py backend/tests/test_plan_package_llm_schema.py backend/tests/test_plan_package_quality_regressions.py backend/tests/test_plan_package_service_timeout.py backend/tests/test_llm_task_scheduler.py +python -m py_compile backend/app/models/idea.py backend/app/modules/idea/evidence_relevance.py backend/app/modules/idea/ideas_api.py backend/app/modules/idea/service.py backend/app/services/search_service.py backend/app/storage/idea_storage.py backend/scripts/run_idea_plan_eval.py +git diff --check +``` + +Expected: all tests pass, compilation succeeds, and diff check produces no +errors. + +- [ ] **Step 3: Run the three technical black-box seeds** + +Use a fresh run ID and output directory: + +```powershell +$env:FAROS_EVAL_RUN_ID = "idea-closure-$(Get-Date -Format yyyyMMdd-HHmmss)" +$env:FAROS_EVAL_SUMMARY = "backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/summary.json" +$env:FAROS_IDEA_DEEP_READ_MAX_PAPERS = "24" +$env:FAROS_EVAL_ONLY = "A_llm_agents_scientific_discovery,B_citation_faithful_medical_rag,C_reliable_multi_agent_research_automation" +python backend/scripts/run_idea_plan_eval.py +Remove-Item Env:FAROS_EVAL_ONLY +``` + +Expected for every successful session: + +- status is `completed`; +- at least two final candidate IDs; +- real LLM reviewer telemetry when the configured provider is available; +- no duplicate final candidates or unrequested application drift; +- no clearly unrelated paper among the top ten evidence records; +- role coverage enabled at raw and structured stages; +- deep-read request count at most 24, excluding visible must-cite overrides. + +A provider outage may produce `awaiting_evidence`; record it as an environmental +result and do not weaken gates to force completion. + +- [ ] **Step 4: Run the CJK negative pollution stress seed** + +Run: + +```powershell +$env:FAROS_EVAL_ONLY = "D_cjk_named_work_negative_stress" +$env:FAROS_EVAL_RUN_ID = "idea-cjk-stress-$(Get-Date -Format yyyyMMdd-HHmmss)" +$env:FAROS_EVAL_SUMMARY = "backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/summary.json" +python backend/scripts/run_idea_plan_eval.py +Remove-Item Env:FAROS_EVAL_ONLY +``` + +Expected: + +- no clinical forecasting or chemical evaluation paper is classified direct or + transferable; +- the session either produces two genuinely supported candidates or reaches + `awaiting_evidence` before bulk deep reading; +- it never completes with zero or one candidate. + +- [ ] **Step 5: Repeat one technical seed to verify stable cache identity** + +Run only `B_citation_faithful_medical_rag` again: + +```powershell +$env:FAROS_EVAL_ONLY = "B_citation_faithful_medical_rag" +$env:FAROS_EVAL_RUN_ID = "idea-cache-$(Get-Date -Format yyyyMMdd-HHmmss)" +$env:FAROS_EVAL_SUMMARY = "backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/summary.json" +python backend/scripts/run_idea_plan_eval.py +Remove-Item Env:FAROS_EVAL_ONLY +``` + +Expected: `structuredGlobalCacheHitCount > 0` when stable paper identities recur. + +- [ ] **Step 6: Write the closure report from measured JSON** + +Write `backend/data/eval_runs/$env:FAROS_EVAL_RUN_ID/idea-closure-report.json` +from the measured session summaries with this construction: + +```python +hard_checks = [ + check + for seed in summary["seeds"] + for check in seed.get("freezeChecks", {}).values() +] +report = { + "decision": "freeze" if hard_checks and all(hard_checks) else "continue_closure", + "technicalSeeds": [ + { + "label": seed["label"], + "sessionId": seed.get("sessionId"), + "status": seed.get("status"), + "finalCandidateIds": seed.get("finalCandidateIds", []), + "performance": seed.get("performance", {}), + "retrievalQuality": seed.get("retrievalQuality", {}), + "freezeChecks": seed.get("freezeChecks", {}), + } + for seed in summary["seeds"] + ], + "externalProviderIncidents": [ + seed.get("sessionError") + for seed in summary["seeds"] + if seed.get("sessionError") + and "provider" in seed.get("sessionError", "").lower() + ], + "remainingRisks": [ + f"{seed['label']}:{name}" + for seed in summary["seeds"] + for name, passed in seed.get("freezeChecks", {}).items() + if not passed + ], +} +``` + +Set `decision` to `freeze` only when every criterion in design section 14 is +satisfied. Otherwise use `continue_closure` and list the failing criterion with +its session ID and measured evidence. Generated evaluation data stays ignored. + +- [ ] **Step 7: Commit evaluator telemetry and the final code state** + +```powershell +git add backend/scripts/run_idea_plan_eval.py +git diff --cached --check +git commit -m "test: add idea closure acceptance telemetry" +git status --short --branch +``` + +Expected: source and test changes are committed; ignored runtime evaluation +artifacts are not added. Any pre-existing unrelated user changes remain visible +and untouched. + +## Final Completion Gate + +Do not declare Idea frozen until all conditions below are evidenced by commands +or black-box JSON: + +- All focused tests pass. +- Python compilation, frontend verification for the waiting UI, and + `git diff --check` pass. +- Completed technical sessions expose at least two approved, diverse ideas. +- Weak evidence uses `awaiting_evidence`; weak candidates use `awaiting_ideas`. +- Initial and structured role coverage are enabled when role queries exist. +- Generic provider hits cannot enter the evidence pool. +- Duplicate identities merge roles and matched queries. +- Evidence-rejected sessions stop before bulk DeepReader work. +- Repeated stable identities produce observable StructuredPaper cache hits. +- No other team module is modified. diff --git a/docs/superpowers/specs/2026-07-12-idea-module-closure-design.md b/docs/superpowers/specs/2026-07-12-idea-module-closure-design.md new file mode 100644 index 0000000..f2d4b25 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-idea-module-closure-design.md @@ -0,0 +1,357 @@ +# FAROS Idea Module Closure Design + +## 1. Objective + +Complete one bounded closure round for the Idea module, then freeze its primary +pipeline. The closure must improve generic evidence relevance, make retrieval +role telemetry trustworthy, preserve the two-final-candidate contract, and +reduce the cost of rejecting low-quality evidence pools. + +This work does not add a literature-specific, Chinese-specific, or other +domain-specific research module. It applies the same evidence rules to all +seeds and paper types. + +## 2. Current Evidence + +The black-box session `idea_61873f0261c9` used the seed +`预测红楼梦可能结局` and reached `awaiting_evidence`, which is the correct +lifecycle outcome for insufficient evidence. It did not fail because of a +backend exception or a complete external-search outage. + +The run still exposed general pipeline defects: + +- 120 papers survived into the stored raw pool, despite 72 results being + filtered out. +- High-ranked papers included unrelated clinical forecasting and chemical + evaluation work. +- The initial paper gate reported `roleCoverage.enabled=false` because it + evaluated `SearchResult` objects with snake-case role fields through logic + written for camel-case `RawPaper` fields. +- Later role coverage passed even though the LLM evidence reviewer correctly + found that method, evaluation, and limitation evidence was not directly + applicable to the seed. +- Duplicate results were discarded without merging their retrieval roles and + matched queries. +- The run spent about 975 seconds before pausing: about 481 seconds in deep + reading and 325 seconds in the evidence gate and repair path. + +These are generic retrieval and evidence-governance problems. They should be +fixed without teaching FAROS how to conduct literary research. + +## 3. Scope + +### In Scope + +- Extract stable core topic anchors from the seed, domain, and translated query + families. +- Separate discriminative topic signals from generic academic or task words. +- Classify retrieved papers as direct, transferable, or rejected evidence. +- Merge retrieval provenance during deduplication. +- Make role coverage depend on semantic eligibility and paper type. +- Reject weak evidence pools before expensive deep reading. +- Preserve resumable waiting states and the two-final-candidate completion + contract. +- Add unit, regression, and real black-box verification for these behaviors. +- Add telemetry needed to explain why papers were accepted, transferred, or + rejected. + +### Out of Scope + +- A literature, humanities, Chinese, or named-work research subsystem. +- CNKI or WanFang integration. +- A new vector database, embedding provider, or cross-encoder service. +- Changes to code, experiment, paper, review, or other team-owned modules. +- PlanPackage optimization. +- Rewriting the entire Idea service or storage architecture. + +## 4. System Contract + +The Idea session lifecycle is the external contract: + +1. When evidence and reviewer quality are sufficient, the session may become + `completed` only with at least two approved, diverse final candidates. +2. When topic-grounded evidence is insufficient, the session becomes + `awaiting_evidence` and retains its completed work for resume. +3. When evidence is sufficient but fewer than two candidates survive review, + the session becomes `awaiting_ideas` and retains the evidence pool for + regeneration. +4. A session must never become `completed` with zero or one final candidate. +5. The system must not manufacture weak candidates merely to satisfy the count + requirement. + +## 5. Topic Intent Profile + +Query expansion will expose a normalized topic intent profile in its step +outputs. The profile is diagnostic data and can remain a structured dictionary; +it does not require a new persisted API model in this closure. + +The profile contains: + +- `coreAnchors`: named works, named entities, domain-specific phrases, acronyms, + and other terms that identify the research object. +- `taskAnchors`: phrases describing the requested research operation or outcome. +- `methodAnchors`: method families explicitly requested by the user or produced + by role-specific query expansion. +- `evaluationAnchors`: evaluation concepts explicitly requested or needed for + the selected paper type. +- `genericTerms`: query words that are too common to establish relevance by + themselves, such as `analysis`, `method`, `prediction`, `potential`, + `outcome`, `model`, and `evaluation`. + +For a CJK seed, the original CJK text and its English role queries are analyzed +together. A translated named work such as `Dream of the Red Chamber` remains a +core anchor rather than being split into independently decisive words such as +`dream`, `red`, and `chamber`. + +Anchor extraction must be deterministic after query expansion. LLM output may +suggest anchors, but rule normalization determines the final profile so that +tests and resumption remain stable. + +## 6. Evidence Relevance Tiers + +Every search result receives one of three tiers before it becomes a `RawPaper`. + +### Direct Evidence + +A paper is direct evidence when it visibly matches the research object or a +strong equivalent phrase and also matches at least one task, method, or +evaluation signal. Direct evidence may support background, GAP, principle, +novelty, method, and evaluation claims when its structured contents permit it. + +Examples include a paper specifically about `Dream of the Red Chamber`, or a +paper directly about citation-faithful medical RAG for the corresponding seed. + +### Transferable Evidence + +A paper is transferable evidence when it does not study the exact research +object but strongly matches a requested task-method or task-evaluation +combination. It may support method choices, baselines, metrics, limitations, or +evaluation design, but it must not be presented as direct evidence of the seed's +domain-specific GAP. + +For example, computational detection of novel endings may be transferable to a +specific novel-ending task. Generic clinical time-series forecasting is not +transferable merely because the query contains `predicting`. + +### Rejected Evidence + +A paper is rejected when it matches only generic terms, provider relevance, or +one weak token. Rejected papers do not become raw evidence, do not participate +in role coverage, and are not sent to DeepReader. + +### Eligibility Rules + +Provider relevance and citation count may reorder eligible papers but cannot +make an ineligible paper eligible. The low external relevance threshold remains +a fallback signal, not a replacement for core or transferable relevance. + +Role-query provenance also cannot make a paper eligible. Being returned for an +`evaluation` query means only that the provider returned it for that query. + +## 7. Deduplication and Provenance Merge + +Deduplication retains the existing identity priority: + +1. DOI +2. arXiv ID +3. Semantic Scholar ID +4. normalized title hash + +When a duplicate is found, the retained result must merge: + +- all unique `retrieval_roles`; +- all unique `matched_queries`; +- available external identifiers; +- sources when the model supports multiple sources, or the preferred source + plus source telemetry when it does not; +- the strongest provider relevance score; +- the richer non-empty abstract and metadata fields. + +The merged result is then scored once. This prevents a paper found by both task +and method queries from retaining only the role of its first occurrence. + +## 8. Role-Aware Evidence Gate + +Role coverage is calculated only from direct or transferable papers whose +semantic score passes the corresponding tier threshold. + +Role requirements are aligned with the existing paper-type evidence coverage: + +- `algorithm` and `system`: domain or task evidence, method evidence, evaluation + evidence, and limitation signals. +- `benchmark`, `evaluation`, and `reproducibility`: task or domain evidence, + dataset/evaluation evidence, baseline or metric evidence, and limitations. +- `survey`, `position`, and `theory`: domain evidence, claims or synthesis, + limitations, and an explicit GAP. A method-query hit is not mandatory. +- Other paper types use the conservative algorithm/system requirements. + +Role coverage complements topic alignment; it does not bypass minimum aligned +paper counts or minimum top alignment. A passing role count with weak semantic +alignment must still fail. + +The rule gate and LLM Evidence Coverage reviewer have separate responsibilities: + +- The rule gate verifies identities, anchors, relevance tiers, counts, sources, + paper-type requirements, and evidence IDs. +- The LLM reviewer judges whether the eligible evidence substantively supports + the required dimensions. +- The final gate fails when either hard rule requirements or required LLM + dimensions fail. + +This removes the current contradiction where role coverage passes while the LLM +correctly reports that the evidence is unrelated. + +## 9. Early Cost Gate + +The pipeline performs an inexpensive relevance and coverage check before deep +reading: + +1. Expand and translate role-specific queries. +2. Search all configured providers. +3. Merge duplicates and build the topic intent profile. +4. Classify results into direct, transferable, and rejected tiers. +5. Run the raw evidence gate on eligible results. +6. If insufficient, run one anchored literature repair round. +7. If still insufficient, persist diagnostics and enter `awaiting_evidence`. +8. Only then select the strongest eligible papers for DeepReader. +9. Run structured Evidence Gate 2.1 before brainstorming. + +Deep-reading selection should remain configurable and bounded. The default +closure target is at most 24 papers, prioritizing direct evidence and reserving +a smaller quota for transferable method and evaluation evidence. Existing +must-cite requirements remain exempt from normal ranking but must be visibly +marked when they do not pass topic relevance. + +## 10. Repair and Resume + +Evidence repair queries must preserve at least one core anchor and add exactly +one missing dimension at a time. Generic suffixes such as `method evidence` +must not replace the topic anchor. + +Repair results pass through the same deduplication, provenance merge, and tier +classification as initial results. A repair query does not grant eligibility. + +Resuming `awaiting_evidence` reruns from the stored `resume_from` step without +duplicating RawPaper, StructuredPaper, literature-map, or trace artifacts. +Resuming `awaiting_ideas` reuses the accepted evidence pool and starts from idea +brainstorming or regeneration. + +## 11. Candidate Completion Contract + +Existing multi-candidate review remains the final authority after evidence +passes. Closure verification will ensure: + +- all final candidates pass the mixed rule and LLM review path; +- final candidates directly answer the seed and retain core topic anchors; +- candidate evidence references resolve to eligible papers; +- the selected two candidates are not near-duplicates; +- failed, hidden, or unreviewed candidates cannot fill the final count; +- insufficient final candidates produce `awaiting_ideas`, never `completed`. + +No new candidate-generation feature is introduced unless black-box testing +shows that relevant evidence passes but the existing repair and regeneration +loop systematically fails to produce two candidates. + +## 12. Telemetry + +Literature-search and evidence-gate step outputs will expose: + +- result count before deduplication; +- unique result count after deduplication; +- duplicate merge count; +- direct, transferable, and rejected counts; +- rejection counts grouped by reason; +- eligible paper count by retrieval role; +- role coverage requirements and observed counts; +- deep-read requested count and cache-hit count; +- initial and repaired gate decisions; +- time spent in search, relevance filtering, deep reading, gate review, repair, + brainstorming, and candidate review. + +Top-paper diagnostics include title, source, tier, semantic score components, +retrieval roles, matched queries, and the decisive anchors. This telemetry must +not expose hidden model reasoning. + +## 13. Testing Strategy + +### Unit and Regression Tests + +Tests will cover: + +- English and CJK tokenization without treating a full CJK sentence as one + decisive token. +- Preservation of translated named-work and multi-word anchors. +- Rejection of clinical forecasting and chemical evaluation results for an + unrelated narrative seed. +- Acceptance of direct named-object papers. +- Acceptance of strongly transferable task-method papers. +- Rejection of papers matching only generic query terms. +- Deduplication merging roles and matched queries across providers. +- Initial `SearchResult` and later `RawPaper` gates reading role metadata + consistently. +- Paper-type-specific role and coverage requirements. +- Role coverage never bypassing weak semantic alignment. +- Evidence repair retaining core anchors. +- `completed` requiring two approved candidates. +- Evidence shortage entering `awaiting_evidence`. +- Candidate shortage entering `awaiting_ideas`. +- Resume avoiding duplicate artifacts. + +### Real Black-Box Regression + +The existing evaluator's three technical seeds remain the positive suite: + +1. `LLM agents for scientific discovery` +2. `citation-faithful medical RAG for high-risk clinical question answering` +3. `reliable multi-agent research automation with evidence-grounded planning and self-review` + +The CJK named-work seed remains a negative stress test for generic topic +pollution, not a requirement to add literary capability. Its acceptable outcome +is either two genuinely supported candidates or `awaiting_evidence`; it must not +complete with fabricated support. + +## 14. Freeze Acceptance Criteria + +Idea is ready to freeze when all of the following hold: + +1. All targeted Idea unit and regression tests pass. +2. Python compilation and `git diff --check` pass. +3. Each successful technical black-box session has at least two final candidate + IDs, and each exposed candidate has passed the configured review gate. +4. No completed session has fewer than two final candidates. +5. Evidence-insufficient sessions use a waiting state rather than failed or + completed. +6. The top ten evidence papers in each technical run contain no clearly + unrelated application-domain pollution on manual inspection. +7. Duplicate papers preserve the union of their roles and matched queries. +8. Role coverage is enabled at both raw and structured gate stages when role + queries exist. +9. Reviewer telemetry confirms real LLM usage when the provider is available, + while rule fallback remains explicit when it is not. +10. Candidate pairs are not near-duplicates and show no unrequested application + drift. +11. The CJK stress run no longer ranks generic clinical or chemical papers as + strong evidence. +12. The closure run records a nonzero structured-paper cache hit when repeated + stable paper identities are present. +13. Median Idea runtime across the three technical seeds improves relative to + their recorded baselines, and no evidence-rejected run performs bulk deep + reading before pausing. + +Runtime is an optimization criterion rather than a reason to weaken evidence +quality. Correctness and evidence integrity remain the release gate. + +## 15. Delivery Boundaries + +Implementation will be split into reviewable commits: + +1. Topic intent profile and evidence-tier tests. +2. Deduplication provenance merge. +3. Tier-aware ranking and early raw gate. +4. Paper-type-aware role coverage and structured gate consistency. +5. Resume and completion-contract regressions. +6. Evaluation telemetry and black-box closure report. + +After these commits pass the freeze criteria, no further Idea feature work is +planned. Remaining improvements move to the Plan module unless a production +regression violates the lifecycle or evidence-integrity contract above. diff --git a/frontend/src/components/ideas/IdeaGenerationPanel.tsx b/frontend/src/components/ideas/IdeaGenerationPanel.tsx index 04287ef..61c668d 100644 --- a/frontend/src/components/ideas/IdeaGenerationPanel.tsx +++ b/frontend/src/components/ideas/IdeaGenerationPanel.tsx @@ -44,6 +44,9 @@ interface IdeaSession { hiddenCandidateCount?: number rejectedCandidateCount?: number warnings?: string[] + blockingReason?: string + resumeFrom?: string + qualityStatus?: string } selectedCandidateId?: string errorMessage?: string @@ -494,7 +497,7 @@ export function IdeaGenerationPanel({ const litResponse = await fetch(`${API_BASE}/api/v1/ideas/sessions/${session.id}/literature`) const litData = await litResponse.json() setLiterature(litData.items || []) - if (sessionData.status === 'completed' || sessionData.status === 'failed') { + if (['completed', 'failed', 'awaiting_evidence', 'awaiting_ideas'].includes(sessionData.status)) { setIsPolling(false) if (sessionData.status === 'completed') { const candResponse = await fetch(`${API_BASE}/api/v1/ideas/sessions/${session.id}/candidates`) @@ -506,6 +509,27 @@ export function IdeaGenerationPanel({ } catch (err) { console.error('Polling error:', err) } }, [session?.id, isPolling]) + const resumeSession = async () => { + if (!session?.id) return + setError(null) + setIsLoading(true) + try { + const response = await fetch(`${API_BASE}/api/v1/ideas/sessions/${session.id}/resume`, { + method: 'POST', + }) + if (!response.ok) { + const data = await response.json().catch(() => ({})) + throw new Error(data.detail || `Failed to resume: ${response.status}`) + } + setSession(await response.json()) + setIsPolling(true) + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to resume session') + } finally { + setIsLoading(false) + } + } + useEffect(() => { if (!isPolling) return const interval = setInterval(pollSession, 2000) @@ -539,6 +563,8 @@ export function IdeaGenerationPanel({ case 'completed': return 'bg-emerald-700 text-white' case 'running': return 'bg-blue-700 text-white' case 'failed': return 'bg-red-700 text-white' + case 'awaiting_evidence': return 'bg-amber-700 text-white' + case 'awaiting_ideas': return 'bg-amber-700 text-white' case 'pending': return 'bg-amber-600 text-white' default: return 'bg-slate-600 text-white' } @@ -709,6 +735,24 @@ export function IdeaGenerationPanel({ )} {isPolling && (
+ {session.status === 'awaiting_evidence' + ? 'More relevant evidence is required' + : 'Two approved ideas are required'} +
++ {session.qualityLoopSummary?.blockingReason || session.errorMessage} +
+