Merge pull request #205 from PredicateSystems/fix_ext2

Honor successful search submission verification

Author: rcholic

src/agents/planner-executor/planner-executor-agent.ts

@@ -892,6 +892,7 @@ export class PlannerExecutorAgent {
           }
         }
 
+        plannerAction = this.promoteVisibleResultClick(task, ctx, plannerAction);
         this.composableHeuristics.setStepHints(plannerAction.heuristicHints || []);
         this.emitPlannerAction(stepNum, plannerAction, plannerActionSource);
 
@@ -1182,6 +1183,20 @@ export class PlannerExecutorAgent {
         };
       }
 
+      if (this.isCopiedPlaceholderNavigation(plannerAction.target, currentUrl, task)) {
+        return {
+          stepId: stepNum,
+          goal: stepGoal,
+          status: StepStatus.SKIPPED,
+          actionTaken: 'SKIPPED(placeholder_navigation)',
+          verificationPassed: true,
+          usedVision: false,
+          durationMs: Date.now() - stepStart,
+          urlBefore: currentUrl,
+          urlAfter: currentUrl,
+        };
+      }
+
       try {
         await runtime.goto(plannerAction.target);
         const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
@@ -1414,10 +1429,29 @@ export class PlannerExecutorAgent {
       const elementId = parsed.args[0] as number;
 
       if (parsed.action === 'CLICK') {
+        const targetElement =
+          activeCtx.snapshot?.elements.find(element => element.id === elementId) || null;
         await runtime.click(elementId);
         await this.handlePostClickEffects(runtime, plannerAction, activeCtx);
-        const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
         const urlAfter = await runtime.getCurrentUrl();
+        const hasUrlVerification = (plannerAction.verify || []).some(
+          predicate =>
+            predicate.predicate === 'url_contains' ||
+            predicate.predicate === 'url_equals' ||
+            predicate.predicate === 'url_matches'
+        );
+        const relevantUrlChange = isUrlChangeRelevantToIntent(
+          currentUrl,
+          urlAfter,
+          plannerAction,
+          targetElement
+        );
+        const navigationSatisfied =
+          relevantUrlChange &&
+          (!hasUrlVerification ||
+            this.clickedHrefMatchesNavigation(currentUrl, urlAfter, targetElement));
+        const verificationPassed =
+          navigationSatisfied || (await this.verifyStepOutcome(runtime, plannerAction));
         return {
           stepId: stepNum,
           goal: plannerAction.intent || 'Click element',
@@ -1437,6 +1471,7 @@ export class PlannerExecutorAgent {
         const elements = activeCtx.snapshot?.elements || [];
         const inputElement = elements.find(element => element.id === elementId) || null;
         const isSearchLike = isSearchLikeTypeAndSubmit(plannerAction, inputElement);
+        let submissionSatisfied = false;
 
         // Submit with Enter key for TYPE_AND_SUBMIT, plus planner TYPE actions that clearly target search.
         if (
@@ -1448,7 +1483,6 @@ export class PlannerExecutorAgent {
           const hasRetryBudget = this.config.retry.executorRepairAttempts > 0;
 
           let changedUrl: string | null = null;
-          let submissionSatisfied = false;
 
           const checkSubmissionSatisfied = async (): Promise<boolean> => {
             if (
@@ -1535,7 +1569,8 @@ export class PlannerExecutorAgent {
           }
         }
 
-        const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
+        const verificationPassed =
+          submissionSatisfied || (await this.verifyStepOutcome(runtime, plannerAction));
         const urlAfter = await runtime.getCurrentUrl();
 
         return {
@@ -2055,6 +2090,127 @@ export class PlannerExecutorAgent {
     };
   }
 
+  private isCopiedPlaceholderNavigation(
+    targetUrl: string,
+    currentUrl: string,
+    task: string
+  ): boolean {
+    if (!this.isExampleDotComUrl(targetUrl)) {
+      return false;
+    }
+
+    if (this.isExampleDotComUrl(currentUrl) || /\bexample\.com\b/i.test(task)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private isExampleDotComUrl(url: string): boolean {
+    try {
+      const hostname = new URL(url).hostname.toLowerCase();
+      return hostname === 'example.com' || hostname.endsWith('.example.com');
+    } catch {
+      return false;
+    }
+  }
+
+  private promoteVisibleResultClick(
+    task: string,
+    ctx: SnapshotContext,
+    plannerAction: StepwisePlannerResponse
+  ): StepwisePlannerResponse {
+    if (plannerAction.action !== 'SCROLL' && plannerAction.action !== 'WAIT') {
+      return plannerAction;
+    }
+
+    const candidate = this.findVisibleResultLink(task, ctx.snapshot);
+    if (!candidate) {
+      return plannerAction;
+    }
+
+    const label = this.elementLabel(candidate);
+    const hrefVerify = this.hrefVerificationSignal(candidate.href || '', ctx.snapshot?.url || '');
+
+    return {
+      ...plannerAction,
+      action: 'CLICK',
+      goal: plannerAction.goal || 'Open visible result link',
+      intent: 'visible result link',
+      input: label || plannerAction.input,
+      verify: hrefVerify ? [{ predicate: 'url_contains', args: [hrefVerify] }] : [],
+      heuristicHints: [
+        {
+          intent_pattern: 'visible_result_link',
+          text_patterns: label ? [label] : [],
+          role_filter: ['link'],
+          priority: 20,
+        },
+      ],
+      reasoning:
+        plannerAction.reasoning ||
+        'Visible result link matched the task goal; clicking it is more direct than scrolling.',
+    };
+  }
+
+  private findVisibleResultLink(
+    task: string,
+    snapshot: Snapshot | null | undefined
+  ): SnapshotElement | null {
+    const elements = snapshot?.elements || [];
+    if (elements.length === 0 || !this.taskWantsResultNavigation(task)) {
+      return null;
+    }
+
+    const candidates = elements
+      .filter(element => this.isResultNavigationLink(element))
+      .sort((left, right) => (right.importance || 0) - (left.importance || 0));
+
+    return candidates[0] || null;
+  }
+
+  private taskWantsResultNavigation(task: string): boolean {
+    const normalized = task.toLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ');
+    const actionCue = /\b(click|open|pick|choose|select|go to|visit)\b/.test(normalized);
+    const targetCue = /\b(product|result|item|listing|detail page|details page)\b/.test(normalized);
+    return actionCue && targetCue;
+  }
+
+  private isResultNavigationLink(element: SnapshotElement): boolean {
+    const role = (element.role || '').toLowerCase();
+    const href = (element.href || '').toLowerCase();
+    if (role !== 'link' || !href) {
+      return false;
+    }
+
+    if (/\/(?:dp|gp\/product|product|products|item|items|p)\//.test(href)) {
+      return true;
+    }
+
+    const label = this.elementLabel(element);
+    return Boolean(element.inDominantGroup && label.length >= 15);
+  }
+
+  private elementLabel(element: SnapshotElement): string {
+    return (element.text || element.ariaLabel || element.name || '').trim();
+  }
+
+  private hrefVerificationSignal(href: string, baseUrl: string): string | null {
+    if (!href.trim()) {
+      return null;
+    }
+
+    try {
+      const parsed = new URL(href, baseUrl || undefined);
+      if (parsed.pathname && parsed.pathname !== '/') {
+        return parsed.pathname;
+      }
+      return parsed.hostname || null;
+    } catch {
+      return href.startsWith('/') ? href : null;
+    }
+  }
+
   private summarizePlannerActionTarget(plannerAction: StepwisePlannerResponse): string | null {
     if (plannerAction.action === 'TYPE' || plannerAction.action === 'TYPE_AND_SUBMIT') {
       return plannerAction.input || plannerAction.intent || plannerAction.target || null;
@@ -2287,6 +2443,44 @@ export class PlannerExecutorAgent {
     return false;
   }
 
+  private clickedHrefMatchesNavigation(
+    previousUrl: string,
+    nextUrl: string,
+    element: SnapshotElement | null
+  ): boolean {
+    const href = element?.href?.trim();
+    if (!href) {
+      return false;
+    }
+
+    try {
+      const expected = new URL(href, previousUrl);
+      const actual = new URL(nextUrl, previousUrl);
+      if (
+        !['http:', 'https:'].includes(expected.protocol) ||
+        !['http:', 'https:'].includes(actual.protocol)
+      ) {
+        return false;
+      }
+
+      expected.hash = '';
+      actual.hash = '';
+      expected.search = '';
+      actual.search = '';
+
+      return (
+        this.normalizeNavigationUrl(expected.toString()) ===
+        this.normalizeNavigationUrl(actual.toString())
+      );
+    } catch {
+      return false;
+    }
+  }
+
+  private normalizeNavigationUrl(url: string): string {
+    return url.trim().replace(/\/+$/, '').toLowerCase();
+  }
+
   private async isCartAdditionTerminal(
     runtime: AgentRuntime,
     task: string,

src/agents/planner-executor/prompts.ts

@@ -79,7 +79,7 @@ CRITICAL RULE FOR ADD TO CART:
 - Set "input" to "Add to Cart" (or exact button text from elements)
 
 Output ONLY valid JSON (no markdown, no \`\`\`):
-{"action":"NAVIGATE","target":"https://example.com/search","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"open the search page"}
+{"action":"NAVIGATE","target":"https://shop.test/search","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"open the known search page"}
 {"action":"TYPE_AND_SUBMIT","intent":"searchbox","input":"wireless headphones","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"search for product"}
 {"action":"CLICK","intent":"product link","input":"Sony WH-1000XM4 Wireless...","verify":[],"required":true,"heuristic_hints":[{"intent_pattern":"product_link","text_patterns":["sony wh-1000xm4"],"role_filter":["link"],"priority":8}],"reasoning":"click first product result"}
 {"action":"CLICK","intent":"add to cart button","input":"Add to Cart","verify":[],"required":true,"heuristic_hints":[{"intent_pattern":"add_to_cart","text_patterns":["add to cart","buy now"],"role_filter":["button"],"priority":10}],"reasoning":"add item to cart"}
@@ -95,7 +95,8 @@ RULES:
 7. "heuristic_hints" entries may use snake_case fields: "intent_pattern", "text_patterns", "role_filter", "attribute_patterns", "priority"
 8. Output ONLY JSON - no <think> tags, no markdown, no prose
 9. Do NOT output <think> or any reasoning
-10. Do NOT return DONE until ALL parts of the goal are complete`;
+10. Do NOT return DONE until ALL parts of the goal are complete
+11. Never copy example URLs from these instructions. Only NAVIGATE to a URL from the user's task, the current page, or a visible element.`;
 
   // NOTE: /no_think MUST be at the START of user message for Qwen3 models
   const user = `/no_think