feat: Idea Pipeline性能优化 + 强制多方向探索

.gitignore

@@ -7,6 +7,7 @@ backend/tests/
 
 # Temp docs, backups, work logs
 docs/
+.worktrees/
 *.bak
 stop_dev.sh
 

backend/app/models/idea.py

@@ -23,6 +23,8 @@ class IdeaSessionStatus(str, Enum):
     """Idea session lifecycle states."""
     PENDING = "pending"
     RUNNING = "running"
+    AWAITING_EVIDENCE = "awaiting_evidence"
+    AWAITING_IDEAS = "awaiting_ideas"
     COMPLETED = "completed"
     FAILED = "failed"
     CANCELLED = "cancelled"
@@ -81,7 +83,7 @@ class IdeaSessionConfig(BaseModel):
         description="Optional search budget for BFTS; defaults to maxPapers if unset"
     )
     maxReviewIterations: int = Field(
-        default=2,
+        default=3,
         ge=1,
         le=5,
         description="Maximum internal idea reviewer repair iterations before final handoff"
@@ -419,6 +421,16 @@ class RawPaper(BaseModel):
     citationCount: int = 0
     abstract: str = ""
     source: List[str] = Field(default_factory=list, description="List of sources: semantic_scholar, arxiv, local, openalex, crossref")
+    retrievalRoles: List[str] = Field(default_factory=list)
+    matchedQueries: List[str] = Field(default_factory=list)
+    evidenceTier: str = Field(
+        default="unclassified",
+        description="direct, transferable, rejected, unclassified",
+    )
+    decisiveAnchors: List[str] = Field(default_factory=list)
+    relevanceComponents: Dict[str, float] = Field(default_factory=dict)
+    rejectionReason: str = ""
+    mustCiteOverride: bool = False
     normalizedTitleHash: str = Field(default="", description="SHA256 of normalized title for dedup")
     references: List[str] = Field(default_factory=list, description="Paper IDs cited by this paper")
     citedBy: List[str] = Field(default_factory=list, description="Paper IDs citing this paper")

backend/app/modules/idea/bfts_search.py

@@ -14,6 +14,7 @@
 import heapq
 import logging
 import re
+import math
 from collections import defaultdict
 from typing import List, Optional, Set, Tuple, Dict, Any
 from datetime import UTC, datetime
@@ -53,6 +54,61 @@
     "graphGrounding": 0.10,
 }
 
+# --- P0: Semantic token-overlap helpers (replaces keyword heuristics) ---
+
+_STOPWORDS: Set[str] = {
+    "the", "a", "an", "and", "or", "of", "in", "to", "for", "with",
+    "on", "at", "by", "from", "that", "this", "it", "is", "are", "be",
+    "we", "our", "can", "has", "have", "not", "but", "as", "its",
+    "using", "based", "approach", "method", "model", "proposed",
+    "novel", "new", "improve", "via", "towards", "beyond",
+}
+
+def _token_ngrams(text: str, n: int = 2) -> Set[str]:
+    """Extract overlapping n-gram token sets from text for semantic comparison."""
+    tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", text.lower())
+    tokens = [t for t in tokens if t not in _STOPWORDS]
+    if not tokens:
+        return set()
+    ngrams: Set[str] = set()
+    for i in range(len(tokens) - n + 1):
+        ngrams.add(" ".join(tokens[i:i + n]))
+    # Also include unigrams for fallback coverage
+    if n == 2:
+        ngrams.update(tokens)
+    return ngrams
+
+
+def _semantic_overlap(text_a: str, text_b: str) -> float:
+    """Compute semantic overlap score (0-1) using 2-gram token Jaccard.
+
+    This is a fast, deterministic proxy for semantic similarity that works
+    across languages (CJK + English) and requires no embedding model.
+
+    Returns a float in [0, 1].
+    """
+    if not text_a or not text_b:
+        return 0.0
+    ng_a = _token_ngrams(text_a)
+    ng_b = _token_ngrams(text_b)
+    if not ng_a or not ng_b:
+        return 0.0
+    intersection = len(ng_a & ng_b)
+    union = len(ng_a | ng_b)
+    return intersection / union if union > 0 else 0.0
+
+
+def _normalized_hypothesis(node: Any) -> str:
+    """Extract a normalized text representation of the idea for scoring."""
+    parts = []
+    if node.title:
+        parts.append(node.title)
+    if node.hypothesis:
+        parts.append(node.hypothesis)
+    if node.abstract:
+        parts.append(node.abstract[:300])
+    return " ".join(parts)
+
 
 def _build_literature_context(structured_papers: List[StructuredPaper], limit: int = 8) -> str:
     """Format selected papers as context string for LLM prompts."""
@@ -466,6 +522,13 @@ def run(self) -> List[IdeaCandidate]:
 
                 child = self._expand_node(parent, max_reflection)
                 if child:
+                    # P0: dedup pruning — skip near-duplicate ideas
+                    if self._is_duplicate(child):
+                        child.status = "pruned"
+                        self.nodes.append(child)  # keep for tracking
+                        parent.isExpanded = True
+                        continue
+
                     self._add_child(parent, child)
                     expansion_count += 1
 
@@ -560,6 +623,31 @@ def _add_child(self, parent: IdeaNode, child: IdeaNode) -> None:
             self._beam, (-child.combinedScore, child.nodeId, child.depth)
         )
 
+    def _is_duplicate(self, new_node: IdeaNode) -> bool:
+        """P0: Check if new_node is a near-duplicate of any existing node.
+
+        Uses n-gram Jaccard similarity on title+hypothesis. Skips the node
+        if similarity exceeds pruneDuplicateThreshold (default 0.82).
+        """
+        threshold = getattr(self.config, 'pruneDuplicateThreshold', 0.82) or 0.82
+        new_text = _normalized_hypothesis(new_node)
+        if not new_text:
+            return False
+
+        existing_nodes = [n for n in self.nodes if n.nodeId != new_node.nodeId]
+        for existing in existing_nodes:
+            existing_text = _normalized_hypothesis(existing)
+            if not existing_text:
+                continue
+            sim = _semantic_overlap(new_text, existing_text)
+            if sim > threshold:
+                logger.info(
+                    f"BFTS: pruning duplicate node (sim={sim:.3f} > "
+                    f"threshold={threshold}): '{new_node.title[:50] if new_node.title else '?'}...'"
+                )
+                return True
+        return False
+
     def _probe_topic_terms(self, node: IdeaNode) -> List[str]:
         """Extract topic-bearing terms for targeted literature probes."""
         text = " ".join([
@@ -884,38 +972,69 @@ def _score_node(self, node: IdeaNode) -> None:
         ) * 10.0  # Scale back to 0-10
 
     def _estimate_novelty(self, node: IdeaNode, prior: float) -> float:
-        """Estimate novelty: heuristic based on hypothesis text."""
-        import re
-        # Check for novel-sounding keywords
-        novel_kw = ["novel", "new", "unexplored", "first", "towards", "beyond"]
-        hyp = (node.hypothesis or "").lower()
-        title = (node.title or "").lower()
-        kw_score = sum(1 for kw in novel_kw if kw in hyp or kw in title)
-        text_score = min(1.0, kw_score / 3.0)
-        return min(10.0, max(0.0, (prior + text_score) * 5.0))
+        """Estimate novelty: semantic overlap with seed_query (lower overlap = more novel).
+
+        P0 improvement: replaces keyword matching ("novel", "new", ...) with
+        n-gram semantic overlap against the seed. Lower overlap means the idea
+        spans vocabulary not already in the query — a proxy for genuine novelty.
+        """
+        seed_text = self.seed_query or ""
+        idea_text = _normalized_hypothesis(node)
+        overlap = _semantic_overlap(seed_text, idea_text)
+
+        # Score inversion: high overlap → low novelty, low overlap → high novelty
+        # Map [0, 1] overlap → [2, 10] novelty (anchored at 6 for moderate overlap ~0.4)
+        novelty_from_overlap = max(2.0, min(10.0, 10.0 - overlap * 8.0))
+
+        # Blend with path-seed prior (30% prior, 70% overlap-based)
+        return 0.3 * (prior * 10.0) + 0.7 * novelty_from_overlap
 
     def _estimate_feasibility(self, node: IdeaNode, prior: float) -> float:
-        """Estimate feasibility: based on experiment count and concreteness."""
+        """Estimate feasibility: experiment concreteness + literature grounding.
+
+        P0 improvement: adds literature probe evidence (actual paper support)
+        alongside experiment count.
+        """
         exp_count = len(node.experiments or [])
-        has_concrete_experiments = exp_count >= 1
         score = prior * 10.0
-        if has_concrete_experiments:
-            score += min(2.0, exp_count * 0.5)
+        if exp_count >= 2:
+            score += 2.5
+        elif exp_count >= 1:
+            score += 1.5
+        # Bonus for literature-probe backing (evidence from actual papers)
+        if node.literatureProbeIds:
+            score += min(1.5, len(node.literatureProbeIds) * 0.3)
         return min(10.0, max(0.0, score))
 
     def _estimate_impact(self, node: IdeaNode) -> float:
-        """Estimate impact: based on hypothesis ambition."""
-        hyp = (node.hypothesis or "").lower()
-        impact_kw = ["improving", "outperforms", "state-of-the-art", "SOTA", "significant"]
-        kw_score = sum(1 for kw in impact_kw if kw in hyp)
-        return min(10.0, 5.0 + kw_score * 1.5)
+        """Estimate impact: hypothesis specificity and reference depth.
+
+        P0 improvement: replaces keyword matching ("improving", "SOTA", ...)
+        with a composite of semantic clarity and citation anchoring.
+        """
+        hyp = (node.hypothesis or "")
+        hyp_len = len(hyp.split())
+        # Longer, more detailed hypotheses tend to be more impactful
+        length_score = min(1.0, hyp_len / 60.0) * 3.0
+        # References indicate domain grounding
+        ref_count = len(node.references or []) if hasattr(node, 'references') else 0
+        ref_score = min(2.0, ref_count * 0.4)
+        # Base
+        return min(10.0, 5.0 + length_score + ref_score)
 
     def _estimate_specificity(self, node: IdeaNode) -> float:
-        """Estimate specificity: 0-1, based on title concreteness."""
+        """Estimate specificity: concept density in title and hypothesis.
+
+        P0 improvement: replaces capital-letter heuristic with n-gram
+        concept density — counts unique semantic tokens normalized by length.
+        """
         title = node.title or ""
-        # Count specific nouns / named entities (heuristic)
-        words = re.findall(r'\b[A-Z][a-z]{2,}\b', title)
-        return min(1.0, len(words) / 5.0)
+        hyp = node.hypothesis or ""
+        combined = f"{title} {hyp}"
+        tokens = re.findall(r"[a-zA-Z][a-zA-Z0-9]{2,}|[\u4e00-\u9fff]+", combined.lower())
+        unique_meaningful = {t for t in tokens if t not in _STOPWORDS}
+        density = len(unique_meaningful) / max(1, len(tokens)) if tokens else 0
+        return min(1.0, density * 1.5)
 
     def _estimate_evidence_support(self, node: IdeaNode) -> float:
         """Estimate evidence support from path seed prior plus reflection/probe evidence."""