Skip to content

Commit 1a8a7ee

Browse files
cameroncookecodex
andcommitted
fix(ui-automation): Use cells as fallback scroll targets
Allow swipe-capable cells to appear as scroll next-step candidates when the runtime snapshot lacks a stronger list or scroll-view target. Rank cells below real scroll containers but above whole-window fallbacks. Co-Authored-By: Codex <noreply@openai.com>
1 parent abde598 commit 1a8a7ee

2 files changed

Lines changed: 51 additions & 2 deletions

File tree

src/mcp/tools/ui-automation/__tests__/runtime-next-steps.test.ts

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,52 @@ describe('runtime snapshot next steps', () => {
158158
});
159159
});
160160

161+
it('uses a cell as a fallback scroll surface before whole-application scrolling', () => {
162+
const snapshot: RuntimeSnapshotV1 = {
163+
type: 'runtime-snapshot',
164+
protocol: 'rs/1',
165+
simulatorId,
166+
screenHash: 'cell-scroll-fallback',
167+
seq: 1,
168+
capturedAtMs: Date.now(),
169+
expiresAtMs: Date.now() + 60_000,
170+
actions: [],
171+
elements: [
172+
{
173+
ref: 'e1',
174+
role: 'application',
175+
label: 'Example',
176+
frame: { x: 0, y: 0, width: 390, height: 844 },
177+
actions: ['swipeWithin'],
178+
},
179+
{
180+
ref: 'e2',
181+
role: 'cell',
182+
label: 'Visible row',
183+
frame: { x: 20, y: 420, width: 350, height: 80 },
184+
actions: ['tap', 'swipeWithin'],
185+
},
186+
],
187+
};
188+
189+
const steps = createRuntimeSnapshotNextSteps({
190+
simulatorId,
191+
runtimeSnapshot: snapshot,
192+
includeRefreshAndWait: false,
193+
});
194+
195+
expect(steps).toContainEqual({
196+
label: 'Scroll visible content',
197+
tool: 'swipe',
198+
params: {
199+
simulatorId,
200+
withinElementRef: 'e2',
201+
direction: 'up',
202+
distance: 0.5,
203+
},
204+
});
205+
});
206+
161207
it('prioritizes real scrolling over low-information chrome taps', () => {
162208
const snapshot: RuntimeSnapshotV1 = {
163209
type: 'runtime-snapshot',

src/mcp/tools/ui-automation/shared/runtime-next-steps.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ function isScrollableNextStepElement(element: {
174174
element.actions.includes('swipeWithin') &&
175175
(element.role === 'scroll-view' ||
176176
element.role === 'list' ||
177+
element.role === 'cell' ||
177178
element.role === 'application' ||
178179
element.role === 'window' ||
179180
(element.role === 'other' && hasScrollSemanticIdentity(element)))
@@ -187,11 +188,13 @@ function getScrollRolePriority(element: RuntimeElementV1): number {
187188
return 0;
188189
case 'other':
189190
return 1;
191+
case 'cell':
192+
return 2;
190193
case 'application':
191194
case 'window':
192-
return 2;
193-
default:
194195
return 3;
196+
default:
197+
return 4;
195198
}
196199
}
197200

0 commit comments

Comments
 (0)