Add FILL_FORM action, snapshot escalation early-exit, and URL-only verify optimization

- Add FILL_FORM planner action for multi-field forms (login, signup, checkout)
  that fills all fields deterministically from snapshot in a single step
  instead of one TYPE action per field (eliminates per-field LLM calls)
- Add early-exit in snapshot escalation when element count is unchanged
  across iterations, preventing unnecessary limit escalation
- Skip scroll-after-escalation when page has fewer elements than limitBase
  (nothing below the fold)
- Optimize verifyStepOutcome to use getCurrentUrl() instead of full
  snapshot when all predicates are URL-only (url_contains, url_equals,
  url_matches), dramatically reducing snapshot count on form pages
- Add 3 new tests for snapshot escalation fixes
- Update planner prompt with FILL_FORM action, examples, and rules

Author: SentienceDEV

src/agents/planner-executor/planner-executor-agent.ts

@@ -2215,6 +2215,95 @@ export class PlannerExecutorAgent {
       };
     }
 
+    if (plannerAction.action === 'FILL_FORM') {
+      const fields = plannerAction.fields || [];
+      const submitText = plannerAction.submitText || '';
+      const elements = ctx.snapshot?.elements || [];
+      const actions: string[] = [];
+      let filledCount = 0;
+
+      const inputRoles = ['textbox', 'searchbox', 'combobox', 'input', 'password'];
+
+      for (const field of fields) {
+        const label = (field.label || '').toLowerCase();
+        const value = field.value || '';
+        let matched: SnapshotElement | null = null;
+
+        for (const el of elements) {
+          const role = (el.role || '').toLowerCase();
+          if (!inputRoles.some(r => role.includes(r))) continue;
+
+          const elText = (el.text || '').toLowerCase();
+          const elName = (el.name || '').toLowerCase();
+          const elAria = (el.ariaLabel || '').toLowerCase();
+          const elNearby = (el.nearbyText || '').toLowerCase();
+          const allText = `${elText} ${elName} ${elAria} ${elNearby}`;
+
+          if (allText.includes(label) || label.includes(elText) || label.includes(elName)) {
+            matched = el;
+            break;
+          }
+        }
+
+        if (matched) {
+          await runtime.type(matched.id, value);
+          actions.push(`TYPE(${matched.id}, "${value}")`);
+          filledCount++;
+          if (this.config.verbose) {
+            console.log(
+              `[FILL_FORM] Typed "${value}" into element ${matched.id} (${matched.text || matched.role})`
+            );
+          }
+        } else if (this.config.verbose) {
+          console.log(`[FILL_FORM] No match for field label "${label}"`);
+        }
+      }
+
+      if (filledCount === 0) {
+        return {
+          stepId: stepNum,
+          goal: stepGoal,
+          status: StepStatus.FAILED,
+          actionTaken: 'FILL_FORM(0 fields matched)',
+          verificationPassed: false,
+          usedVision: false,
+          durationMs: Date.now() - stepStart,
+          urlBefore: currentUrl,
+          urlAfter: await runtime.getCurrentUrl(),
+          error: 'No form fields matched the provided labels',
+        };
+      }
+
+      let clickedSubmit = false;
+      if (submitText) {
+        const submitEl = this.findSubmitButtonByText(elements, submitText);
+        if (submitEl !== null) {
+          await runtime.click(submitEl);
+          actions.push(`CLICK(${submitEl})`);
+          clickedSubmit = true;
+          if (this.config.verbose) {
+            console.log(`[FILL_FORM] Clicked submit button element ${submitEl}`);
+          }
+        }
+      }
+
+      await new Promise(r => setTimeout(r, clickedSubmit ? 1000 : 300));
+      const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
+      const urlAfter = await runtime.getCurrentUrl();
+
+      return {
+        stepId: stepNum,
+        goal: stepGoal,
+        status: verificationPassed || clickedSubmit ? StepStatus.SUCCESS : StepStatus.FAILED,
+        actionTaken: `FILL_FORM(${actions.join(' -> ')})`,
+        verificationPassed,
+        usedVision: false,
+        durationMs: Date.now() - stepStart,
+        urlBefore: currentUrl,
+        urlAfter,
+      };
+    }
+
     // Handle EXTRACT action — read page content and extract data via LLM
     if (plannerAction.action === 'EXTRACT') {
       const extractQuery =
@@ -3063,6 +3152,7 @@ export class PlannerExecutorAgent {
     let visionReason: string | null = null;
     let pruningCategory: string | null = null;
     let prunedNodeCount = 0;
+    let previousElementCount = -1;
 
     // Phase 1: Limit escalation loop
     while (currentLimit <= maxLimit) {
@@ -3087,10 +3177,23 @@ export class PlannerExecutorAgent {
           currentLimit
         ).length;
 
+        const elementCount = snap.elements?.length || 0;
+
+        // Early exit: widening the limit didn't discover new elements
+        if (elementCount > 0 && elementCount === previousElementCount) {
+          if (this.config.verbose) {
+            console.log(
+              `[ESCALATION] Element count unchanged (${elementCount}), stopping escalation`
+            );
+          }
+          break;
+        }
+        previousElementCount = elementCount;
+
         const visionResult = detectSnapshotFailure(snap);
         if (this.config.verbose) {
           console.log(
-            `[VISION CHECK] elements=${(snap.elements || []).length}, shouldUseVision=${visionResult.shouldUseVision}, reason=${visionResult.reason}, screenshot=${screenshotBase64 ? 'yes' : 'no'}, executor.supportsVision=${this.executor.supportsVision()}`
+            `[VISION CHECK] elements=${elementCount}, shouldUseVision=${visionResult.shouldUseVision}, reason=${visionResult.reason}, screenshot=${screenshotBase64 ? 'yes' : 'no'}, executor.supportsVision=${this.executor.supportsVision()}`
           );
         }
         if (visionResult.shouldUseVision) {
@@ -3125,7 +3228,6 @@ export class PlannerExecutorAgent {
         if (!cfg.enabled) break;
 
         // Check element count - if sufficient, no need to escalate
-        const elementCount = snap.elements?.length || 0;
         const hasEnoughContext = pruningCategory
           ? actionableContextCount >= cfg.pruningMinElements
           : elementCount >= 10;
@@ -3153,13 +3255,16 @@ export class PlannerExecutorAgent {
 
     // Phase 2: Scroll-after-escalation
     // Only trigger for CLICK actions with specific intents
+    // Skip entirely if the page has fewer elements than limitBase (nothing below the fold)
+    const pageElementCount = lastSnapshot?.elements?.length ?? 0;
     const shouldTryScroll =
       cfg.scrollAfterEscalation &&
       step !== undefined &&
       lastSnapshot !== null &&
       !requiresVision &&
       step.action === 'CLICK' &&
-      step.intent;
+      step.intent &&
+      pageElementCount >= cfg.limitBase;
 
     if (shouldTryScroll && lastSnapshot) {
       // Check if we can find the target element using intent heuristics
@@ -3528,6 +3633,10 @@ export class PlannerExecutorAgent {
         ? (step.heuristicHints as Array<Record<string, unknown>>)
         : [],
       reasoning: typeof step.reasoning === 'string' ? step.reasoning : undefined,
+      fields: Array.isArray(step.fields)
+        ? (step.fields as Array<{ label: string; value: string }>)
+        : undefined,
+      submitText: typeof step.submitText === 'string' ? step.submitText : undefined,
     };
   }
 
@@ -3657,6 +3766,10 @@ export class PlannerExecutorAgent {
   }
 
   private summarizePlannerActionTarget(plannerAction: StepwisePlannerResponse): string | null {
+    if (plannerAction.action === 'FILL_FORM') {
+      const fields = plannerAction.fields || [];
+      return fields.map(f => `${f.label}=${f.value}`).join(', ') || null;
+    }
     if (plannerAction.action === 'TYPE' || plannerAction.action === 'TYPE_AND_SUBMIT') {
       return plannerAction.input || plannerAction.intent || plannerAction.target || null;
     }
@@ -3673,7 +3786,7 @@ export class PlannerExecutorAgent {
     }
 
     const action = plannerAction.action;
-    if (!['TYPE', 'TYPE_AND_SUBMIT', 'CLICK'].includes(action)) {
+    if (!['TYPE', 'TYPE_AND_SUBMIT', 'CLICK', 'FILL_FORM'].includes(action)) {
       return false;
     }
 
@@ -3987,15 +4100,30 @@ export class PlannerExecutorAgent {
     const pollMs = Math.max(1, this.config.retry.verifyPollMs);
     const start = Date.now();
 
+    const allUrlPredicates = plannerAction.verify.every(
+      p =>
+        p.predicate === 'url_contains' ||
+        p.predicate === 'url_equals' ||
+        p.predicate === 'url_matches'
+    );
+
     while (Date.now() - start <= timeoutMs) {
       try {
-        const snap = await runtime.snapshot({
-          limit: this.config.snapshot.limitBase,
-          screenshot: false,
-          goal: plannerAction.intent || plannerAction.action,
-        });
-        if (snap && evaluatePredicates(plannerAction.verify, snap)) {
-          return true;
+        if (allUrlPredicates) {
+          const url = await runtime.getCurrentUrl();
+          const pseudoSnap: Snapshot = { url, title: '', elements: [] };
+          if (evaluatePredicates(plannerAction.verify, pseudoSnap)) {
+            return true;
+          }
+        } else {
+          const snap = await runtime.snapshot({
+            limit: this.config.snapshot.limitBase,
+            screenshot: false,
+            goal: plannerAction.intent || plannerAction.action,
+          });
+          if (snap && evaluatePredicates(plannerAction.verify, snap)) {
+            return true;
+          }
         }
       } catch {
         // Keep polling until timeout.
@@ -4613,6 +4741,26 @@ export class PlannerExecutorAgent {
     return candidates[0].id;
   }
 
+  private findSubmitButtonByText(elements: SnapshotElement[], targetText: string): number | null {
+    const lower = targetText.toLowerCase().trim();
+    for (const el of elements) {
+      const role = (el.role || '').toLowerCase();
+      if (!['button', 'link'].includes(role)) continue;
+      if (el.clickable === false) continue;
+      const text = (el.text || '').toLowerCase().trim();
+      const ariaLabel = (el.ariaLabel || '').toLowerCase().trim();
+      if (
+        text === lower ||
+        text.includes(lower) ||
+        lower.includes(text) ||
+        ariaLabel.includes(lower)
+      ) {
+        return el.id;
+      }
+    }
+    return null;
+  }
+
   private inferSameUrlWizardProgressAfterNext(
     plannerAction: StepwisePlannerResponse,
     beforeSnapshot: Snapshot | null,

src/agents/planner-executor/prompts.ts

@@ -59,7 +59,8 @@ export function buildStepwisePlannerPrompt(
 Actions:
 - NAVIGATE: Go directly to a URL when the next destination is known. Set "target" to the URL.
 - CLICK: Click an element. Set "intent" to describe the SPECIFIC element (include label, placeholder, or nearby text, e.g. "email textbox", "display name field", "Next button", NOT just "textbox" or "button"). Set "input" to EXACT text from elements list.
-- TYPE: Type text into a form field (not a search box). Set "input" to the VALUE from the goal. Set "intent" to describe the field (e.g., "email field", "name field").
+- FILL_FORM: Fill ALL visible form fields and submit. Use for login, signup, checkout, or any multi-field form. Set "fields" to an array of {label, value} pairs. Set "submitText" to the submit button text. Set "verify" to check navigation after submit.
+- TYPE: Type text into a SINGLE form field. Prefer FILL_FORM for forms with multiple fields.
 - TYPE_AND_SUBMIT: Type text into a search box and submit. Set "input" to the SEARCH QUERY from the goal (NOT the element label).
 - SCROLL: Scroll page. Set "direction" to "up" or "down".
 - WAIT: Wait for content to appear when a follow-up verification is needed.
@@ -71,8 +72,16 @@ WHEN TO USE DONE:
 - "Add to Cart" task: DONE only AFTER clicking the Add to Cart button
 - "Search and click product" task: DONE only AFTER clicking a product link
 - "Search only" task: DONE after search results appear
+- "Log in" task: DONE only AFTER the page navigates away from /login
 - If goal has multiple steps, complete ALL steps before returning DONE
 
+CRITICAL RULE FOR FILL_FORM (PREFERRED for login/signup/checkout):
+- Use FILL_FORM when the goal provides values for 2+ form fields (e.g. "username: X, password: Y")
+- "fields" is an array of {label, value} where label matches the field's visible text/placeholder
+- "submitText" is the text on the submit button (e.g. "Sign in", "Log in", "Submit", "Next")
+- The system will find and fill each field by matching label to element text/role
+- This is MUCH faster than TYPE one field at a time
+
 CRITICAL RULE FOR TYPE_AND_SUBMIT:
 - "input" must be the SEARCH QUERY you want to type (e.g., "wireless headphones")
 - "input" is NOT the element label (e.g., NOT "Search Amazon")
@@ -89,6 +98,8 @@ CRITICAL RULE FOR ADD TO CART:
 
 Output ONLY valid JSON (no markdown, no \`\`\`):
 {"action":"NAVIGATE","target":"https://shop.test/search","verify":[{"predicate":"url_contains","args":["search"]}]}
+{"action":"FILL_FORM","fields":[{"label":"username","value":"john@example.com"},{"label":"password","value":"secret123"}],"submitText":"Sign in","verify":[{"predicate":"url_contains","args":["dashboard"]}]}
+{"action":"FILL_FORM","fields":[{"label":"email","value":"test@test.com"},{"label":"password","value":"pass123"},{"label":"confirm password","value":"pass123"}],"submitText":"Create Account","verify":[]}
 {"action":"TYPE_AND_SUBMIT","intent":"searchbox","input":"wireless headphones","verify":[{"predicate":"url_contains","args":["search"]}]}
 {"action":"CLICK","intent":"product link","input":"Sony WH-1000XM4 Wireless...","verify":[]}
 {"action":"CLICK","intent":"add to cart button","input":"Add to Cart","verify":[]}
@@ -102,11 +113,11 @@ RULES:
 5. Include "verify" when you know a simple URL or element predicate that proves success; otherwise use []
 6. Include planner metadata when useful: "target", "required", "stop_if_true", "optional_substeps", "heuristic_hints"
 7. "heuristic_hints" entries may use snake_case fields: "intent_pattern", "text_patterns", "role_filter", "attribute_patterns", "priority"
-8. Output ONLY JSON - no <think> tags, no markdown, no prose
-9. Do NOT output <think> or any reasoning
+8. Output ONLY JSON - no 时光网 tags, no markdown, no prose
+9. Do NOT output 时光网 or any reasoning
 10. Do NOT return DONE until ALL parts of the goal are complete
 11. Never copy example URLs from these instructions. Only NAVIGATE to a URL from the user's task, the current page, or a visible element.
-12. For multi-step forms: TYPE into each field (action: TYPE) BEFORE clicking Next. Never click Next without filling required fields first.
+12. PREFER FILL_FORM for login/signup/checkout forms with 2+ fields. Do NOT use multiple TYPE actions when FILL_FORM can do it in one step.
 13. "intent" must be SPECIFIC: describe the element with its label or context (e.g., "email field", "plan dropdown", "Next button on step 2")
 14. Treat history results "success", "skipped", and "vision_fallback" as already satisfied. Do not repeat those steps; choose the next incomplete part of the goal.`;
 
@@ -329,6 +340,7 @@ export interface StepwisePlannerResponse {
     | 'CLICK'
     | 'TYPE'
     | 'TYPE_AND_SUBMIT'
+    | 'FILL_FORM'
     | 'SCROLL'
     | 'PRESS'
     | 'WAIT'
@@ -339,6 +351,8 @@ export interface StepwisePlannerResponse {
   intent?: string;
   input?: string;
   direction?: 'up' | 'down';
+  fields?: Array<{ label: string; value: string }>;
+  submitText?: string;
   verify?: Array<{ predicate: string; args: unknown[] }>;
   required?: boolean;
   stopIfTrue?: boolean;