From 7fc8e06e315045d0eda04b721a873f36dd09b4cd Mon Sep 17 00:00:00 2001 From: nickiovets Date: Tue, 10 Feb 2026 19:46:25 +0100 Subject: [PATCH] wire performInteractiveExploration into --deep mode Connect the unused performInteractiveExploration() to --deep research flow, producing a deterministic element inventory alongside AI analysis. Changes: - Add --deep flag to TUI /research command and CLI - Call performInteractiveExploration after performDeepAnalysis in deep mode - Expand CLICKABLE_ROLES with checkbox, radio, slider, textbox, treeitem - Include unnamed buttons in ARIA collection (mark with unnamed flag) - Remove 30-char link name length filter - Handle unnamed elements with role-only click fallback - Accept maxElements override param, use 50 in deep mode Co-authored-by: Cursor --- bin/explorbot-cli.ts | 2 ++ src/ai/researcher.ts | 36 +++++++++++++++++--------------- src/commands/research-command.ts | 10 +++++---- src/utils/aria.ts | 6 ++---- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 474ca17..74c1ec6 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -367,6 +367,7 @@ program .option('-s, --show', 'Show browser window') .option('--headless', 'Run browser in headless mode') .option('--data', 'Include data extraction in research') + .option('--deep', 'Enable deep analysis (expand hidden elements)') .action(async (url, options) => { try { const mainOptions: ExplorBotOptions = { @@ -390,6 +391,7 @@ program screenshot: true, force: true, data: options.data || false, + deep: options.deep || false, }); await explorBot.stop(); diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 7d6c9a8..73dcee1 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -37,7 +37,7 @@ const POSSIBLE_SECTIONS = { const DEFAULT_STOP_WORDS = ['close', 'cancel', 'dismiss', 'exit', 'back', 'cookie', 'consent', 'gdpr', 'privacy', 'accept all', 'decline all', 'reject all', 'share', 'print', 'download']; -const CLICKABLE_ROLES = new Set(['button', 'link', 'menuitem', 'tab', 'option', 'combobox', 'switch']); +const CLICKABLE_ROLES = new Set(['button', 'link', 'menuitem', 'tab', 'option', 'combobox', 'switch', 'checkbox', 'radio', 'slider', 'textbox', 'treeitem']); export class Researcher implements Agent { emoji = '🔍'; @@ -131,13 +131,16 @@ export class Researcher implements Agent { this.hasScreenshotToAnalyze = screenshot && this.provider.hasVision() && isOnCurrentState; - const prompt = await this.buildResearchPrompt(); - const conversation = this.provider.startConversation(this.getSystemMessage(), 'researcher'); - conversation.addUserText(prompt); if (this.hasScreenshotToAnalyze) { this.actionResult = await this.explorer.createAction().caputrePageWithScreenshot(); + } + + const prompt = await this.buildResearchPrompt(); + conversation.addUserText(prompt); + + if (this.hasScreenshotToAnalyze) { const screenshotAnalysis = await this.analyzeScreenshotForUIElements(); if (screenshotAnalysis) { this.addScreenshotPrompt(conversation, screenshotAnalysis); @@ -153,6 +156,8 @@ export class Researcher implements Agent { if (deep) { researchText += await this.performDeepAnalysis(conversation, state, state.html ?? ''); + researchText += '\n\n## Interactive Elements Exploration\n\n'; + researchText += await this.performInteractiveExploration(state, { maxElements: 50 }); } if (data) { @@ -950,12 +955,12 @@ export class Researcher implements Agent { } } - async performInteractiveExploration(state: WebPageState): Promise { + async performInteractiveExploration(state: WebPageState, opts: { maxElements?: number } = {}): Promise { const config = this.getResearcherConfig(); const stopWords = config?.stopWords ?? DEFAULT_STOP_WORDS; const excludeSelectors = config?.excludeSelectors || []; const includeSelectors = config?.includeSelectors || []; - const maxElements = config?.maxElementsToExplore ?? 10; + const maxElements = opts.maxElements ?? config?.maxElementsToExplore ?? 10; const interactiveNodes = collectInteractiveNodes(state.ariaSnapshot || ''); const originalUrl = state.url; @@ -969,12 +974,7 @@ export class Researcher implements Agent { return false; } - if (!name) { - debugLog(`Skipping unnamed ${role} element`); - return false; - } - - if (name.length > 50) { + if (name.length > 80) { debugLog(`Skipping "${name.slice(0, 30)}..." - name too long`); return false; } @@ -1011,22 +1011,24 @@ export class Researcher implements Agent { const role = String(node.role || ''); const name = String(node.name || '').trim(); - tag('substep').log(`[${i + 1}/${targets.length}] Exploring: "${name}" (${role})`); + const label = name || `unnamed ${role}`; + tag('substep').log(`[${i + 1}/${targets.length}] Exploring: "${label}" (${role})`); const action = this.explorer.createAction(); const beforeState = await action.capturePageState({}); try { - await action.execute(`I.click({ role: '${role}', text: '${name.replace(/'/g, "\\'")}' })`); + const clickCommand = name ? `I.click({ role: '${role}', text: '${name.replace(/'/g, "\\'")}' })` : `I.click({ role: '${role}' })`; + await action.execute(clickCommand); const afterState = await action.capturePageState({}); const resultDescription = this.detectChangeResult(beforeState, afterState, originalUrl); - results.push({ element: name, role, result: resultDescription }); + results.push({ element: label, role, result: resultDescription }); await this.restoreState(afterState, originalUrl); } catch (error) { - debugLog(`Failed to explore ${name}:`, error); - results.push({ element: name, role, result: 'click failed' }); + debugLog(`Failed to explore ${label}:`, error); + results.push({ element: label, role, result: 'click failed' }); } } diff --git a/src/commands/research-command.ts b/src/commands/research-command.ts index 16fed0e..a81147b 100644 --- a/src/commands/research-command.ts +++ b/src/commands/research-command.ts @@ -2,15 +2,16 @@ import { BaseCommand } from './base-command.js'; export class ResearchCommand extends BaseCommand { name = 'research'; - description = 'Research current page or navigate to URI and research'; - suggestions = ['/navigate - to go to another page', '/plan - to plan testing']; + description = 'Research current page or navigate to URI and research. Use --deep to explore interactive elements by clicking them. Use --data to include page data.'; + suggestions = ['/research --deep - explore by clicking buttons', '/navigate - to go to another page', '/plan - to plan testing']; async execute(args: string): Promise { const includeData = args.includes('--data'); - const target = args.replace('--data', '').trim(); + const enableDeep = args.includes('--deep'); + const target = args.replace('--data', '').replace('--deep', '').trim(); if (target) { - await this.explorBot.getExplorer().visit(target); + await this.explorBot.agentNavigator().visit(target); } const state = this.explorBot.getExplorer().getStateManager().getCurrentState(); @@ -22,6 +23,7 @@ export class ResearchCommand extends BaseCommand { screenshot: true, force: true, data: includeData, + deep: enableDeep, }); } } diff --git a/src/utils/aria.ts b/src/utils/aria.ts index 0c86069..f67d49a 100644 --- a/src/utils/aria.ts +++ b/src/utils/aria.ts @@ -79,10 +79,8 @@ const buildInteractiveEntry = (node: AriaNode): Record | null = shouldInclude = true; } if (isButtonOrLink && !entryName && !hasValue) { - shouldInclude = false; - } - if (node.role === 'link' && entryName && entryName.length > 30) { - shouldInclude = false; + entry.unnamed = true; + shouldInclude = true; } if (!shouldInclude) { return null;