From dad16273929c8c2ecdd9d6dcd46fcd0caf09cf31 Mon Sep 17 00:00:00 2001 From: maciej Date: Fri, 8 May 2026 12:09:09 -0500 Subject: [PATCH 1/2] Add goal-driven mobile replay harness Generic harness for driving any iOS app through Lidar live tools or Appium, then validating the matching FullStory replay. Navigation, target screen, scroll counts, and replay heuristics all come from a goal JSON manifest. No app-specific defaults in committed code. Scripts: run-lidar-live-ios, run-goal, run-local-lidar-ios, validate-goal-artifacts, validate-replay-observations, fetch-subtext-review-evidence, capture-subtext-review-observations, capture-replay-observations-from-snapshot, prepare-subtext-review. --- scripts/mobile/.gitignore | 12 + scripts/mobile/ARCHITECTURE.md | 84 ++++ scripts/mobile/CUSTOMER_INTAKE.md | 55 +++ scripts/mobile/README.md | 190 ++++++++ scripts/mobile/appium-layer.mjs | 454 ++++++++++++++++++ ...ture-replay-observations-from-snapshot.mjs | 158 ++++++ .../capture-subtext-review-observations.mjs | 175 +++++++ scripts/mobile/device-e2e-common.mjs | 171 +++++++ .../mobile/fetch-subtext-review-evidence.mjs | 247 ++++++++++ scripts/mobile/goals/example.json | 83 ++++ scripts/mobile/mobile.env.example | 40 ++ scripts/mobile/prepare-subtext-review.mjs | 98 ++++ scripts/mobile/run-goal.mjs | 132 +++++ scripts/mobile/run-lidar-live-ios.mjs | 395 +++++++++++++++ scripts/mobile/run-local-lidar-ios.mjs | 376 +++++++++++++++ scripts/mobile/validate-goal-artifacts.mjs | 81 ++++ .../mobile/validate-replay-observations.mjs | 227 +++++++++ 17 files changed, 2978 insertions(+) create mode 100644 scripts/mobile/.gitignore create mode 100644 scripts/mobile/ARCHITECTURE.md create mode 100644 scripts/mobile/CUSTOMER_INTAKE.md create mode 100644 scripts/mobile/README.md create mode 100644 scripts/mobile/appium-layer.mjs create mode 100644 scripts/mobile/capture-replay-observations-from-snapshot.mjs create mode 100644 scripts/mobile/capture-subtext-review-observations.mjs create mode 100644 scripts/mobile/device-e2e-common.mjs create mode 100644 scripts/mobile/fetch-subtext-review-evidence.mjs create mode 100644 scripts/mobile/goals/example.json create mode 100644 scripts/mobile/mobile.env.example create mode 100644 scripts/mobile/prepare-subtext-review.mjs create mode 100644 scripts/mobile/run-goal.mjs create mode 100644 scripts/mobile/run-lidar-live-ios.mjs create mode 100644 scripts/mobile/run-local-lidar-ios.mjs create mode 100644 scripts/mobile/validate-goal-artifacts.mjs create mode 100644 scripts/mobile/validate-replay-observations.mjs diff --git a/scripts/mobile/.gitignore b/scripts/mobile/.gitignore new file mode 100644 index 00000000..558bc36a --- /dev/null +++ b/scripts/mobile/.gitignore @@ -0,0 +1,12 @@ +*.local.json +.env.local +tmp/ + +# Goal files are app-specific. Only example.json is committed. +goals/* +!goals/example.json + +# Fixture-specific scripts that stay local for internal testing. +images-goal.mjs +run-images-*.mjs +validate-images-*.mjs diff --git a/scripts/mobile/ARCHITECTURE.md b/scripts/mobile/ARCHITECTURE.md new file mode 100644 index 00000000..140a6680 --- /dev/null +++ b/scripts/mobile/ARCHITECTURE.md @@ -0,0 +1,84 @@ +# Mobile Harness Architecture + +## Ownership Split + +The flow has four separate pieces: + +1. **Goal runner:** `run-lidar-live-ios.mjs` or `run-goal.mjs` drives any app through a goal manifest. Navigation steps, target screen, and scroll counts come from the goal JSON, not hardcoded values. +2. **Mobile driver:** Lidar owns generic device control through `live-*` tools: connect, snapshot, tap, fill, scroll, handle system alerts, and apply sightmap annotations. +3. **Session discovery:** the runner or a future Lidar/mobile bridge maps a device run to a FullStory session URL, trace ID, user ID, or session ID. +4. **Replay diagnosis:** Subtext `review-open`, `review-view`, and `review-diff` inspect the FullStory replay and compare it to the goal. + +## Flow + +``` +Goal JSON + .env.local + | | + v v + run-lidar-live-ios.mjs (or run-goal.mjs for Appium) + | + |-- live-connect (platform: "ios") + |-- live-view-snapshot + |-- live-act-click / live-act-drag + |-- live-disconnect + | + v + Snapshots + report --> validate-goal-artifacts.mjs + | + v + fetch-subtext-review-evidence.mjs + | + v + capture-subtext-review-observations.mjs + | + v + validate-replay-observations.mjs +``` + +## Scripts + +| Script | Purpose | +| --- | --- | +| `run-lidar-live-ios.mjs` | Drive an app through Lidar MCP live tools using a goal manifest | +| `run-goal.mjs` | Drive an app through Appium/WebDriverIO using a goal manifest | +| `run-local-lidar-ios.mjs` | Build and start a local Lidar, start Appium, run the Lidar goal | +| `validate-goal-artifacts.mjs` | Validate device artifacts against goal expectations | +| `prepare-subtext-review.mjs` | Generate a replay review request from goal and session URL | +| `fetch-subtext-review-evidence.mjs` | Fetch replay evidence from Subtext MCP | +| `capture-subtext-review-observations.mjs` | Extract observations from Subtext review evidence | +| `capture-replay-observations-from-snapshot.mjs` | Extract observations from browser replay snapshots | +| `validate-replay-observations.mjs` | Validate replay observations against goal expectations | +| `appium-layer.mjs` | Appium/WebDriverIO connection, capabilities, and device primitives | +| `device-e2e-common.mjs` | Shared env loading, log capture, session URL extraction | + +## Goal Manifest + +All app-specific behavior comes from the goal JSON. The runner does not assume any screen names, labels, or navigation paths. See `goals/example.json` for the documented schema. + +Key fields: + +- `run.navigation[]`: steps to reach the target screen. Each step has an `action` (`tap`, `scrollToLabel`, `screenshot`, `source`, `dismissAlert`) and relevant parameters. +- `run.targetScreen`: the expected active screen after navigation. +- `run.scrollDownCount` / `run.scrollUpCount`: how many times to scroll. +- `replayChecks.observationHeuristics`: keyword lists used by the observation scripts to detect events, screen presence, image content, and scroll activity in replay evidence. +- `sensitiveRegions[]`: privacy expectations for UI regions. + +## Lidar iOS Integration + +The Lidar live tools provide a unified surface for iOS. `live-connect` with `platform: "ios"` creates an Appium session through the Lidar iOS backend. All subsequent `live-view-snapshot`, `live-act-click`, `live-act-drag`, and `live-disconnect` calls route to the iOS driver based on the connection type. + +The local wrapper `run-local-lidar-ios.mjs` builds Lidar from source, starts it on free ports, starts Appium if needed, generates MCP caps, and delegates to `run-lidar-live-ios.mjs`. + +## Replay Sampling + +Replay validation inspects multiple timestamps after the target screen opens. The fetcher samples a short timeline, and callers can override it: + +```bash +MOBILE_SUBTEXT_VIEW_TIMESTAMPS=12000,22000,37000 node scripts/mobile/fetch-subtext-review-evidence.mjs +``` + +## Current Limits + +- The runner supports tap, scroll, and simple navigation. Login, deep links, text input, multi-screen flows, and manual checkpoints need a richer step format. +- Session discovery depends on SDK logs or a supplied session identifier. +- Build and install of the customer app is outside the runner. The customer must provide an installed app or clear install steps. diff --git a/scripts/mobile/CUSTOMER_INTAKE.md b/scripts/mobile/CUSTOMER_INTAKE.md new file mode 100644 index 00000000..151b6dbb --- /dev/null +++ b/scripts/mobile/CUSTOMER_INTAKE.md @@ -0,0 +1,55 @@ +# Mobile Debugging Intake + +Use this template when asking what Subtext needs for a mobile replay debugging run. + +## App Access + +- App format: installed app, `.ipa`, `.app`, simulator build, or source repo. +- Bundle ID: +- FullStory org and environment: +- Is the app already instrumented with FullStory? +- If source/build is required, exact build and install commands: + +## Device Target + +- Device type: physical iOS device or simulator. +- Device name: +- UDID: +- iOS version: +- Signing or WebDriverAgent requirements: + +## Auth Path + +- Starting state: logged out, logged in, fresh install, or existing app state. +- Test account or login method: +- MFA, passkey, captcha, deep link, or manual step requirements: +- Any system prompts expected on first launch: + +## Goal + +- User task to perform: +- Screen or flow that should appear: +- Interactions required: taps, typing, scrolls, waits, gestures. +- Expected replay behavior: +- Things that should be considered failure: + +## Session Discovery + +Provide at least one way to identify the matching FullStory session: + +- FullStory session URL from SDK logs. +- User ID and approximate run time. +- Session ID from logs. +- Trace ID. +- A user-provided replay URL. +- A deterministic test account that can be searched in FullStory. + +## Evidence Output + +The run should produce: + +- Device snapshots/screenshots for the performed goal. +- FullStory session URL or equivalent session identifier. +- Subtext `review-open` evidence. +- Multiple `review-view` samples around the target flow. +- A final pass/warn/fail report against the rubric. diff --git a/scripts/mobile/README.md b/scripts/mobile/README.md new file mode 100644 index 00000000..f7060fbd --- /dev/null +++ b/scripts/mobile/README.md @@ -0,0 +1,190 @@ +# Mobile Replay Harness + +Drives any iOS app on a physical device, then checks whether the FullStory replay of that session looks right. Everything the runner needs to know about your app comes from two places: environment variables and a goal JSON file. Nothing is hardcoded for any particular app. + +There are two ways to drive the app: + +- **Lidar live tools** (`run-lidar-live-ios.mjs`) -- talks to a Lidar MCP server over HTTP. This is the primary path. +- **Appium direct** (`run-goal.mjs`) -- talks to a local Appium server through WebDriverIO. Useful when you don't have a Lidar server. + +After driving, a separate set of scripts fetches the FullStory replay, pulls out observations, and compares them to what the goal says should have happened. + +## Setup + +1. Copy the env example and fill in your values: + +``` +cp scripts/mobile/mobile.env.example scripts/mobile/.env.local +``` + +2. Write a goal JSON file for your app. Copy `goals/example.json` and edit it. The goal says which screen to navigate to, what taps to perform, how many times to scroll, and what the replay should contain. + +3. Make sure you have: + - A physical iOS device connected (or a simulator). + - The app already installed on the device. + - The device UDID (run `xcrun devicectl list devices` to find it). + - A FullStory API key for the org that instruments the app. + +## Environment Variables + +Put these in `scripts/mobile/.env.local`. That file is gitignored. + +**Always required:** + +| Variable | What it is | +| --- | --- | +| `FULLSTORY_API_KEY` | API key for MCP auth and replay fetches | +| `MOBILE_BUNDLE_ID` | Bundle ID of the app installed on the device | +| `MOBILE_UDID` | Device UDID | +| `MOBILE_DEVICE_NAME` | Device name (e.g. "My iPhone") | +| `MOBILE_GOAL_EXPECTATIONS` | Path to the goal JSON file | +| `MOBILE_OUT_DIR` | Where to write output artifacts | + +**For Lidar runs:** + +| Variable | What it is | +| --- | --- | +| `LIDAR_IOS_MCP_URL` | URL of the Lidar MCP endpoint | + +**For local Lidar development** (when you want to build and run Lidar from source): + +| Variable | What it is | +| --- | --- | +| `LOCAL_MCP_ORG_ID` | FullStory org ID for generating local MCP caps | +| `LOCAL_MCP_EMAIL` | Email for the fake signed session | + +**For Appium runs:** + +| Variable | What it is | +| --- | --- | +| `MOBILE_CAPABILITIES_PATH` | Path to a capabilities JSON file, if env vars aren't enough | +| `MOBILE_CONSOLE_LAUNCH_PATTERN` | Regex to detect app launch in console output | + +## Goal Files + +A goal file tells the runner what to do and what to check. Here's what goes in it: + +- `name` -- human name for the goal. +- `run.targetScreen` -- the screen you're navigating to. +- `run.slug` -- short name used in filenames. +- `run.navigation` -- array of steps to get to the target screen. Each step has an `action` (`tap`, `scrollToLabel`) and a `label`. A tap can set `"ifVisible": true` to skip if the element isn't there. +- `run.scrollDownCount` / `run.scrollUpCount` -- how many times to scroll once you're on the target screen. +- `replayChecks` -- what the replay validation scripts look for. +- `replayChecks.observationHeuristics` -- keyword lists that the observation scripts use to detect events, screen presence, image content, and scroll activity in replay evidence. +- `sensitiveRegions` -- privacy expectations for specific parts of the UI. + +If `navigation` is missing, the runner just connects, snapshots, scrolls, and disconnects. No assumptions about how your app's navigation works. + +See `goals/example.json` for the full shape with comments. + +## Running + +### Against a remote Lidar server + +Set `LIDAR_IOS_MCP_URL` to the server, set your goal and output dir, and run: + +``` +node scripts/mobile/run-lidar-live-ios.mjs +``` + +### Against a local Lidar (build from source) + +This builds Lidar from your local Go checkout, starts Appium if it's not running, starts Lidar on free ports, generates MCP caps, and runs the goal. All you need in `.env.local` is device info, API key, org ID, and email. + +``` +node scripts/mobile/run-local-lidar-ios.mjs +``` + +Set `MOBILE_LIDAR_START=0` and `LIDAR_IOS_MCP_URL` to skip the build and reuse an existing Lidar. + +### Through Appium directly (no Lidar) + +Start Appium first: + +``` +pnpm exec appium --address 127.0.0.1 --port 4723 --base-path / +``` + +Then run: + +``` +node scripts/mobile/run-goal.mjs +``` + +## Replay Validation + +After the device run finishes, there are separate scripts to check the replay. You run them in order: + +1. **Validate device artifacts** -- checks that the expected files exist and contain what the goal says. + ``` + node scripts/mobile/validate-goal-artifacts.mjs + ``` + +2. **Fetch replay evidence** -- calls Subtext MCP to open the session and grab snapshots at several timestamps. + ``` + node scripts/mobile/fetch-subtext-review-evidence.mjs + ``` + +3. **Extract observations** -- reads the raw replay evidence and pulls out structured observations (which screen, which events, whether content was visible, etc). + ``` + node scripts/mobile/capture-subtext-review-observations.mjs + ``` + +4. **Validate observations** -- compares observations to the goal's expected checks and writes a pass/warn/fail report. + ``` + node scripts/mobile/validate-replay-observations.mjs + ``` + +You can also use `prepare-subtext-review.mjs` to generate a markdown review request instead of fetching evidence directly. + +## Output + +Everything goes in whatever you set `MOBILE_OUT_DIR` to. Typical files: + +- `live-ios-*.json` / `live-ios-*.txt` -- snapshots from Lidar. +- `live-ios-*.png.base64` -- screenshot data. +- `*-source.xml` / `*.png` -- Appium source and screenshots. +- `fullstory-session-url.txt` -- the FullStory replay URL. +- `replay-observations.json` -- structured observations from replay. +- `*-validation-report.md` -- final report. + +## What's Not Committed + +The `.gitignore` keeps out anything app-specific: + +- `.env.local` -- your personal config. +- `goals/*` except `example.json` -- your app-specific goals. +- `tmp/` -- all run output. +- `capabilities.local.json` -- your device capabilities. +- Any `run-images-*` or `images-goal.mjs` scripts from internal testing. + +## Device Config + +For Appium runs, you can provide full capabilities as a JSON file: + +``` +MOBILE_CAPABILITIES_PATH=./scripts/mobile/capabilities.local.json node scripts/mobile/run-goal.mjs +``` + +Or as inline JSON: + +``` +MOBILE_CAPABILITIES_JSON='{"platformName":"iOS","appium:automationName":"XCUITest","appium:udid":"device-udid"}' node scripts/mobile/run-goal.mjs +``` + +## Replay Validation Details + +Replay checks are semantic, not pixel-perfect. The goal is to catch real problems: + +- Wrong screen in replay. +- Missing events (taps, page properties). +- Blank or frozen content where there should be images or text. +- Privacy violations -- content visible when it should be masked, or masked when it should be visible. + +Privacy states you can set in goal manifests: + +- `unmasked` -- content should be visible. +- `masked` -- content should be obscured. +- `excluded` -- content should be blocked entirely. +- `omitted` -- the element should not appear at all. +- `config_dependent` -- depends on the org's privacy rules. diff --git a/scripts/mobile/appium-layer.mjs b/scripts/mobile/appium-layer.mjs new file mode 100644 index 00000000..fc76609c --- /dev/null +++ b/scripts/mobile/appium-layer.mjs @@ -0,0 +1,454 @@ +import fs from "node:fs/promises"; +import { execFile, spawn } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const DEFAULT_OUT_DIR = fileURLToPath(new URL("./tmp/appium-poc", import.meta.url)); +export const MOBILE_FAST = process.env.MOBILE_FAST === "1"; +export const TAP_PAUSE_MS = MOBILE_FAST ? 250 : 1000; +export const SCROLL_PAUSE_MS = MOBILE_FAST ? 250 : 800; +const LOCAL_CAPABILITIES_URL = new URL("./capabilities.local.json", import.meta.url); + +function formatSimulatorLogDate(date) { + const pad = (value) => String(value).padStart(2, "0"); + return [ + date.getFullYear(), + pad(date.getMonth() + 1), + pad(date.getDate()), + ].join("-") + " " + [ + pad(date.getHours()), + pad(date.getMinutes()), + pad(date.getSeconds()), + ].join(":"); +} + +function simulatorLogProcessName(capabilities) { + return ( + process.env.MOBILE_LOG_PROCESS_NAME ?? + capabilities["appium:processName"] ?? + capabilities["appium:bundleId"]?.split(".").at(-1) ?? + "App" + ); +} + +function execFileText(command, args) { + return new Promise((resolve, reject) => { + execFile(command, args, { maxBuffer: 25 * 1024 * 1024 }, (err, stdout, stderr) => { + if (err) { + err.stdout = stdout; + err.stderr = stderr; + reject(err); + return; + } + resolve(`${stdout}${stderr}`); + }); + }); +} + +function withoutEmptyValues(caps) { + return Object.fromEntries( + Object.entries(caps).filter(([, value]) => value !== undefined && value !== ""), + ); +} + +export function defaultIosCapabilities() { + return withoutEmptyValues({ + platformName: "iOS", + "appium:automationName": "XCUITest", + "appium:deviceName": process.env.MOBILE_DEVICE_NAME ?? "iPhone", + "appium:udid": process.env.MOBILE_UDID, + "appium:platformVersion": process.env.MOBILE_PLATFORM_VERSION, + "appium:bundleId": process.env.MOBILE_BUNDLE_ID, + "appium:xcodeOrgId": process.env.MOBILE_XCODE_ORG_ID, + "appium:xcodeSigningId": process.env.MOBILE_XCODE_SIGNING_ID ?? "Apple Development", + "appium:updatedWDABundleId": process.env.MOBILE_WDA_BUNDLE_ID, + "appium:useNewWDA": false, + "appium:newCommandTimeout": 120, + "appium:noReset": true, + "appium:fullReset": false, + "appium:autoLaunch": true, + "appium:autoAcceptAlerts": process.env.MOBILE_AUTO_ACCEPT_ALERTS === "1", + "appium:autoDismissAlerts": process.env.MOBILE_AUTO_DISMISS_ALERTS !== "0", + "appium:showXcodeLog": true, + "appium:wdaLaunchTimeout": 180000, + "appium:wdaConnectionTimeout": 180000, + "appium:wdaStartupRetries": 3, + "appium:wdaStartupRetryInterval": 20000, + }); +} + +export async function mobileCapabilities() { + const capsPath = process.env.MOBILE_CAPABILITIES_PATH; + const capsJson = process.env.MOBILE_CAPABILITIES_JSON; + + if (capsPath && capsJson) { + throw new Error("Set MOBILE_CAPABILITIES_PATH or MOBILE_CAPABILITIES_JSON, not both"); + } + + if (capsPath) { + return JSON.parse(await fs.readFile(capsPath, "utf8")); + } + + if (capsJson) { + return JSON.parse(capsJson); + } + + try { + await fs.access(LOCAL_CAPABILITIES_URL); + return JSON.parse(await fs.readFile(LOCAL_CAPABILITIES_URL, "utf8")); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + return defaultIosCapabilities(); +} + +export const FULLSTORY_SESSION_URL_REGEX = + /https:\/\/app(?:\.staging)?\.fullstory\.com\/ui\/[^/]+\/(?:client-)?session\/[0-9a-zA-Z%:-]+/; + +export async function mobileStatus(appiumUrl = "http://127.0.0.1:4723") { + const response = await fetch(`${appiumUrl}/status`); + if (!response.ok) { + throw new Error(`Appium status failed: HTTP ${response.status}`); + } + return response.json(); +} + +export async function mobileConnect({ + appiumUrl = "http://127.0.0.1:4723", + capabilities = defaultIosCapabilities(), +} = {}) { + const webdriverioModule = process.env.MOBILE_WEBDRIVERIO_MODULE ?? "webdriverio"; + let remote; + try { + ({ remote } = await import(webdriverioModule)); + } catch (err) { + if (err.code !== "ERR_MODULE_NOT_FOUND") { + throw err; + } + console.warn(`Could not load ${webdriverioModule}; using direct Appium HTTP client`); + } + const url = new URL(appiumUrl); + const status = await mobileStatus(appiumUrl); + console.log("appium status:", JSON.stringify(status.value ?? status, null, 2)); + + if (!remote) { + const driver = await directAppiumRemote({ appiumUrl, capabilities }); + console.log("session id:", driver.sessionId); + console.log("contexts:", await driver.getContexts()); + return driver; + } + + const driver = await remote({ + hostname: url.hostname, + port: Number(url.port || 4723), + path: url.pathname === "/" ? "/" : url.pathname, + logLevel: "warn", + connectionRetryTimeout: 600000, + connectionRetryCount: 0, + capabilities, + }); + + console.log("session id:", driver.sessionId); + console.log("contexts:", await driver.getContexts()); + return driver; +} + +async function directAppiumRemote({ appiumUrl, capabilities }) { + const session = await appiumRequest(appiumUrl, "/session", { + method: "POST", + body: { + capabilities: { + alwaysMatch: capabilities, + }, + }, + }); + const sessionId = session.sessionId; + if (!sessionId) { + throw new Error("Appium did not return a session id"); + } + + const elementId = (value = {}) => + value["element-6066-11e4-a52e-4f735466cecf"] ?? value.ELEMENT; + + const makeElement = (id) => ({ + id, + async isExisting() { + return Boolean(id); + }, + async isDisplayed() { + const displayed = await appiumRequest(appiumUrl, `/session/${sessionId}/element/${id}/displayed`); + return Boolean(displayed); + }, + async click() { + await appiumRequest(appiumUrl, `/session/${sessionId}/element/${id}/click`, { + method: "POST", + body: {}, + }); + }, + }); + + async function findElements(using, value) { + const elements = await appiumRequest(appiumUrl, `/session/${sessionId}/elements`, { + method: "POST", + body: { using, value }, + }); + return elements + .map((element) => makeElement(elementId(element))) + .filter((element) => element.id !== null); + } + + return { + sessionId, + async getContexts() { + return appiumRequest(appiumUrl, `/session/${sessionId}/contexts`).catch(() => ["NATIVE_APP"]); + }, + async dismissAlert() { + await appiumRequest(appiumUrl, `/session/${sessionId}/alert/dismiss`, { + method: "POST", + body: {}, + }); + }, + async acceptAlert() { + await appiumRequest(appiumUrl, `/session/${sessionId}/alert/accept`, { + method: "POST", + body: {}, + }); + }, + async takeScreenshot() { + return appiumRequest(appiumUrl, `/session/${sessionId}/screenshot`); + }, + async getPageSource() { + return appiumRequest(appiumUrl, `/session/${sessionId}/source?format=xml`); + }, + async execute(script, args) { + return appiumRequest(appiumUrl, `/session/${sessionId}/execute/sync`, { + method: "POST", + body: { + script, + args: [args], + }, + }); + }, + async pause(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); + }, + async deleteSession() { + await appiumRequest(appiumUrl, `/session/${sessionId}`, { + method: "DELETE", + }); + }, + async $(selector) { + if (selector.startsWith("-ios predicate string:")) { + const value = selector.slice("-ios predicate string:".length); + const elements = await findElements("-ios predicate string", value); + return elements[0] ?? makeElement(null); + } + const elements = await findElements("class name", selector); + return elements[0] ?? makeElement(null); + }, + async $$(selector) { + return findElements("class name", selector); + }, + }; +} + +async function appiumRequest(appiumUrl, requestPath, { method = "GET", body } = {}) { + const response = await fetch(new URL(requestPath, appiumUrl), { + method, + headers: body === undefined ? undefined : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + const parsed = text ? JSON.parse(text) : {}; + if (!response.ok) { + throw new Error(`Appium ${method} ${requestPath} failed: HTTP ${response.status}\n${text}`); + } + return parsed.value; +} + +export async function mobileScreenshot(driver, outDir, name) { + const pngBase64 = await driver.takeScreenshot(); + const file = path.join(outDir, name); + await fs.writeFile(file, Buffer.from(pngBase64, "base64")); + console.log(`wrote ${file}`); + return file; +} + +export async function mobileSource(driver, outDir, name) { + const source = await driver.getPageSource(); + const file = path.join(outDir, name); + await fs.writeFile(file, source); + console.log(`wrote ${file}`); + return file; +} + +export async function mobileScroll(driver, { fromX = 195, fromY, toX = 195, toY, duration = 0.5 }) { + await driver.execute("mobile: dragFromToForDuration", { + duration, + fromX, + fromY, + toX, + toY, + }); + await driver.pause(SCROLL_PAUSE_MS); +} + +export async function mobileTapByName(driver, name, { maxScrolls = 0 } = {}) { + const selector = `name == "${name}"`; + + for (let attempt = 0; attempt <= maxScrolls; attempt += 1) { + const element = await driver.$(`-ios predicate string:${selector}`); + if (await element.isExisting()) { + const displayed = await element.isDisplayed().catch(() => false); + if (displayed) { + console.log(`tapping "${name}"`); + await element.click(); + await driver.pause(TAP_PAUSE_MS); + return true; + } + } + + if (attempt < maxScrolls) { + await mobileScroll(driver, { fromY: 720, toY: 240 }); + } + } + + throw new Error(`Could not find visible element named "${name}"`); +} + +export async function mobileTapIfVisible(driver, name) { + const element = await driver.$(`-ios predicate string:name == "${name}"`); + if (!(await element.isExisting())) { + return false; + } + + const displayed = await element.isDisplayed().catch(() => false); + if (!displayed) { + return false; + } + + console.log(`tapping "${name}"`); + await element.click(); + await driver.pause(TAP_PAUSE_MS); + return true; +} + +export async function mobileDismissSystemAlert(driver) { + if (typeof driver.dismissAlert === "function") { + try { + await driver.dismissAlert(); + await driver.pause(TAP_PAUSE_MS); + console.log("dismissed system alert"); + return true; + } catch { + // No native alert is present. Fall through to visible button labels. + } + } + + return ( + (await mobileTapIfVisible(driver, "Don’t Allow")) || + (await mobileTapIfVisible(driver, "Don't Allow")) || + (await mobileTapIfVisible(driver, "Not Now")) || + (await mobileTapIfVisible(driver, "Cancel")) || + false + ); +} + +export function startFullStorySessionLogCapture({ capabilities, outDir }) { + const udid = capabilities["appium:udid"]; + if (!udid) { + throw new Error("Missing appium:udid. Set MOBILE_UDID or provide it in capabilities."); + } + + const isSimulator = capabilities["appium:isSimulator"] === true; + const processName = simulatorLogProcessName(capabilities); + const simulatorStart = new Date(Date.now() - 5000); + const simulatorPredicate = `process == "${processName}"`; + const lines = []; + const proc = isSimulator + ? spawn( + "xcrun", + [ + "simctl", + "spawn", + udid, + "log", + "stream", + "--style", + "compact", + "--info", + "--debug", + "--predicate", + simulatorPredicate, + ], + { + stdio: ["ignore", "pipe", "pipe"], + }, + ) + : spawn("idevicesyslog", ["-u", udid], { + stdio: ["ignore", "pipe", "pipe"], + }); + const closed = new Promise((resolve) => proc.once("close", resolve)); + + const collect = (chunk) => { + lines.push(chunk.toString()); + }; + + proc.stdout.on("data", collect); + proc.stderr.on("data", collect); + + return { + async stop() { + if (!proc.killed) { + proc.kill("SIGTERM"); + } + await closed; + + const logText = lines.join(""); + const logPath = path.join(outDir, "fullstory-device.log"); + let combinedLogText = logText; + + if (isSimulator) { + const recentSimulatorLogText = await execFileText("xcrun", [ + "simctl", + "spawn", + udid, + "log", + "show", + "--start", + formatSimulatorLogDate(simulatorStart), + "--style", + "compact", + "--info", + "--debug", + "--predicate", + simulatorPredicate, + ]).catch((err) => { + console.warn(`Could not read simulator log history: ${err.message}`); + return ""; + }); + + combinedLogText = `${logText}\n${recentSimulatorLogText}`; + } + + await fs.writeFile(logPath, combinedLogText); + console.log(`wrote ${logPath}`); + + const match = combinedLogText.match(FULLSTORY_SESSION_URL_REGEX); + if (!match) { + console.log("FullStory session URL not found in device logs"); + return null; + } + + const sessionUrl = match[0]; + const sessionUrlPath = path.join(outDir, "fullstory-session-url.txt"); + await fs.writeFile(sessionUrlPath, `${sessionUrl}\n`); + console.log(`FullStory session URL: ${sessionUrl}`); + console.log(`wrote ${sessionUrlPath}`); + return sessionUrl; + }, + }; +} diff --git a/scripts/mobile/capture-replay-observations-from-snapshot.mjs b/scripts/mobile/capture-replay-observations-from-snapshot.mjs new file mode 100644 index 00000000..30ecad64 --- /dev/null +++ b/scripts/mobile/capture-replay-observations-from-snapshot.mjs @@ -0,0 +1,158 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const sessionUrlPath = process.env.MOBILE_SESSION_URL_PATH ?? path.join(outDir, "fullstory-session-url.txt"); +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} + +function includesAny(text, values) { + return values.some((value) => text.toLowerCase().includes(value.toLowerCase())); +} + +function eventIfPresent(snapshot, text, normalized = text) { + return snapshot.includes(text) ? [normalized] : []; +} + +function defaultUnmaskedTerms(region) { + return [region.deviceContains, region.label, region.id].filter(Boolean); +} + +function defaultMaskedTerms(region) { + return [`MASKED_STATE_OBSERVED:${region.id}`, `masked:${region.id}`]; +} + +async function readReplaySnapshots() { + if (process.env.MOBILE_REPLAY_SNAPSHOT_PATH) { + const file = process.env.MOBILE_REPLAY_SNAPSHOT_PATH; + return [{ file, text: await fs.readFile(file, "utf8") }]; + } + + const files = (await fs.readdir(outDir)) + .filter((file) => /^replay-snapshot.*\.txt$/.test(file)) + .sort(); + + if (files.length === 0) { + throw new Error(`No replay snapshot files found in ${outDir}`); + } + + return Promise.all( + files.map(async (file) => ({ + file: path.join(outDir, file), + text: await fs.readFile(path.join(outDir, file), "utf8"), + })), + ); +} + +function privacyEvidenceForRegions(snapshots, regions = []) { + return regions.map((region) => { + const unmaskedTerms = region.replayEvidence?.unmaskedContains ?? defaultUnmaskedTerms(region); + const maskedTerms = region.replayEvidence?.maskedContains ?? defaultMaskedTerms(region); + const unmaskedSnapshots = snapshots + .filter(({ text }) => includesAny(text, unmaskedTerms)) + .map(({ file }) => file); + const maskedSnapshots = snapshots + .filter(({ text }) => includesAny(text, maskedTerms)) + .map(({ file }) => file); + const replayState = + unmaskedSnapshots.length > 0 && maskedSnapshots.length > 0 + ? "mixed" + : unmaskedSnapshots.length > 0 + ? "unmasked" + : maskedSnapshots.length > 0 + ? "masked" + : "not_observed"; + + return { + regionId: region.id, + label: region.label, + expectedPrivacyState: region.expectedPrivacyState ?? "config_dependent", + expectedPrivacySource: region.expectedPrivacySource ?? "goal manifest", + bounds: region.bounds, + replay: { + state: replayState, + unmaskedStateObserved: unmaskedSnapshots.length > 0, + maskedStateObserved: maskedSnapshots.length > 0, + unmaskedSnapshots, + maskedSnapshots, + evidenceSource: "browser replay snapshot text", + }, + engine: { + state: "not_available", + maskedFlagObserved: null, + blockedFlagObserved: null, + evidenceSource: "not collected", + }, + sources: ["goal manifest", "browser replay snapshot text"], + }; + }); +} + +async function main() { + const snapshots = await readReplaySnapshots(); + const snapshot = snapshots.map(({ text }) => text).join("\n\n--- replay snapshot boundary ---\n\n"); + const sessionUrl = (await fs.readFile(sessionUrlPath, "utf8")).trim(); + const expectations = JSON.parse(await fs.readFile(expectationsPath, "utf8")); + const screen = expectations.replayChecks?.screen ?? expectations.name; + + const heuristics = expectations.replayChecks?.observationHeuristics ?? {}; + + const eventHeuristics = heuristics.events ?? [ + { terms: [`Set Page Properties: ${screen}`], normalized: `Set Page Properties: ${screen}` }, + ]; + const eventStream = []; + for (const rule of eventHeuristics) { + for (const term of rule.terms) { + if (snapshot.includes(term)) { + eventStream.push(rule.normalized); + break; + } + } + } + + const screenTerms = heuristics.reachedScreen ?? [screen, `Set Page Properties: ${screen}`]; + const reachedScreen = includesAny(snapshot, screenTerms); + + const imageTerms = heuristics.imageContent ?? [ + "VISUAL_IMAGE_CONTENT_OBSERVED", + `Set Page Properties: ${screen}`, + ]; + + const scrollTerms = heuristics.scrollPlayback ?? [ + "SCROLL_PLAYBACK_OBSERVED", + ]; + + const privacyEvidence = privacyEvidenceForRegions(snapshots, expectations.sensitiveRegions); + const observations = { + sessionUrl, + snapshotFiles: snapshots.map(({ file }) => file), + replayScreen: reachedScreen ? screen : "unknown", + visibleText: [ + ...(snapshot.includes(screen) ? [screen] : []), + ...(snapshot.includes("iPhone") ? ["iPhone"] : []), + ...(snapshot.includes("Mobile · iOS") ? ["Mobile · iOS"] : []), + ], + eventStream, + imageContentVisible: includesAny(snapshot, imageTerms), + privacyScreenVisible: reachedScreen, + privacyEvidence, + scrollPlaybackLooksCorrect: includesAny(snapshot, scrollTerms), + notes: [ + "Generated from browser replay snapshot text.", + "This is semantic replay evidence, not pixel-perfect image comparison.", + "Privacy checks compare declared expectations against available replay evidence. Native flags are not collected yet.", + ], + }; + + const observationsPath = path.join(outDir, "replay-observations.json"); + await fs.writeFile(observationsPath, `${JSON.stringify(observations, null, 2)}\n`); + console.log(`wrote ${observationsPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/capture-subtext-review-observations.mjs b/scripts/mobile/capture-subtext-review-observations.mjs new file mode 100644 index 00000000..fc786519 --- /dev/null +++ b/scripts/mobile/capture-subtext-review-observations.mjs @@ -0,0 +1,175 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const sessionUrlPath = + process.env.MOBILE_SESSION_URL_PATH ?? + path.join(outDir, "fullstory-session-url.txt"); +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} + +function includesAny(text, values) { + return values.some((value) => text.toLowerCase().includes(value.toLowerCase())); +} + +function eventIfPresent(text, terms, normalized) { + return includesAny(text, terms) ? [normalized] : []; +} + +function hasMultipleViewTimestamps(evidenceFiles) { + const timestamps = new Set(); + for (const { text } of evidenceFiles) { + const match = text.match(/^View timestamp:\s*(\d+)ms/m); + if (match) { + timestamps.add(match[1]); + } + } + return timestamps.size >= 2; +} + +function defaultUnmaskedTerms(region) { + return [region.deviceContains, region.label, region.id].filter(Boolean); +} + +function defaultMaskedTerms(region) { + return [`MASKED_STATE_OBSERVED:${region.id}`, `masked:${region.id}`]; +} + +async function readSubtextReviewEvidence() { + if (process.env.MOBILE_SUBTEXT_REVIEW_EVIDENCE_PATH) { + const file = process.env.MOBILE_SUBTEXT_REVIEW_EVIDENCE_PATH; + return [{ file, text: await fs.readFile(file, "utf8") }]; + } + + const files = (await fs.readdir(outDir)) + .filter((file) => /^replay-view-subtext.*\.txt$/.test(file) || /^review-(open|view).*\.txt$/.test(file)) + .sort(); + + if (files.length === 0) { + throw new Error( + `No Subtext review evidence files found in ${outDir}. Save review-open/review-view output as replay-view-subtext-*.txt or set MOBILE_SUBTEXT_REVIEW_EVIDENCE_PATH.`, + ); + } + + return Promise.all( + files.map(async (file) => ({ + file: path.join(outDir, file), + text: await fs.readFile(path.join(outDir, file), "utf8"), + })), + ); +} + +function privacyEvidenceForRegions(evidenceFiles, regions = []) { + return regions.map((region) => { + const unmaskedTerms = region.replayEvidence?.unmaskedContains ?? defaultUnmaskedTerms(region); + const maskedTerms = region.replayEvidence?.maskedContains ?? defaultMaskedTerms(region); + const unmaskedSnapshots = evidenceFiles + .filter(({ text }) => includesAny(text, unmaskedTerms)) + .map(({ file }) => file); + const maskedSnapshots = evidenceFiles + .filter(({ text }) => includesAny(text, maskedTerms)) + .map(({ file }) => file); + const replayState = + unmaskedSnapshots.length > 0 && maskedSnapshots.length > 0 + ? "mixed" + : unmaskedSnapshots.length > 0 + ? "unmasked" + : maskedSnapshots.length > 0 + ? "masked" + : "not_observed"; + + return { + regionId: region.id, + label: region.label, + expectedPrivacyState: region.expectedPrivacyState ?? "config_dependent", + expectedPrivacySource: region.expectedPrivacySource ?? "goal manifest", + bounds: region.bounds, + replay: { + state: replayState, + unmaskedStateObserved: unmaskedSnapshots.length > 0, + maskedStateObserved: maskedSnapshots.length > 0, + unmaskedSnapshots, + maskedSnapshots, + evidenceSource: "Subtext review evidence", + }, + engine: { + state: "not_available", + maskedFlagObserved: null, + blockedFlagObserved: null, + evidenceSource: "not collected", + }, + sources: ["goal manifest", "Subtext review evidence"], + }; + }); +} + +async function main() { + const evidenceFiles = await readSubtextReviewEvidence(); + const evidence = evidenceFiles.map(({ text }) => text).join("\n\n--- subtext review boundary ---\n\n"); + const sessionUrl = (await fs.readFile(sessionUrlPath, "utf8")).trim(); + const expectations = JSON.parse(await fs.readFile(expectationsPath, "utf8")); + const screen = expectations.replayChecks?.screen ?? expectations.name; + + const heuristics = expectations.replayChecks?.observationHeuristics ?? {}; + + const eventHeuristics = heuristics.events ?? [ + { terms: [`Set Page Properties: ${screen}`, "page-properties", "page-view"], normalized: `Set Page Properties: ${screen}` }, + ]; + const eventStream = []; + for (const rule of eventHeuristics) { + eventStream.push(...eventIfPresent(evidence, rule.terms, rule.normalized)); + } + + const screenTerms = heuristics.reachedScreen ?? [screen, `navigation text "${screen}"`]; + const reachedScreen = includesAny(evidence, screenTerms); + + const imageTerms = heuristics.imageContent ?? [ + "VISUAL_IMAGE_CONTENT_OBSERVED", + "fsvisualtestview", + " img", + ]; + const imageContentVisible = includesAny(evidence, imageTerms); + + const scrollTerms = heuristics.scrollPlayback ?? [ + "SCROLL_PLAYBACK_OBSERVED", + "lower scrolled", + ]; + const scrollPlaybackLooksCorrect = includesAny(evidence, scrollTerms) + || (hasMultipleViewTimestamps(evidenceFiles) && imageContentVisible && reachedScreen); + + const observations = { + sessionUrl, + snapshotFiles: evidenceFiles.map(({ file }) => file), + replayScreen: reachedScreen ? screen : "unknown", + visibleText: [ + ...(reachedScreen ? [screen] : []), + ...(evidence.includes("iPhone") ? ["iPhone"] : []), + ...(evidence.includes("Mobile · iOS") ? ["Mobile · iOS"] : []), + ], + eventStream, + imageContentVisible, + privacyScreenVisible: reachedScreen, + privacyEvidence: privacyEvidenceForRegions(evidenceFiles, expectations.sensitiveRegions), + scrollPlaybackLooksCorrect, + notes: [ + "Generated from Subtext review-open/review-view evidence.", + "This is semantic replay evidence, not pixel-perfect image comparison.", + "Native privacy flags are not collected yet.", + ], + }; + + const observationsPath = path.join(outDir, "replay-observations.json"); + await fs.writeFile(observationsPath, `${JSON.stringify(observations, null, 2)}\n`); + console.log(`wrote ${observationsPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/device-e2e-common.mjs b/scripts/mobile/device-e2e-common.mjs new file mode 100644 index 00000000..2883f451 --- /dev/null +++ b/scripts/mobile/device-e2e-common.mjs @@ -0,0 +1,171 @@ +import { createWriteStream } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { FULLSTORY_SESSION_URL_REGEX } from "./appium-layer.mjs"; + +export async function loadLocalEnv(localEnvUrl = new URL(".env.local", import.meta.url)) { + let text; + try { + text = await fs.readFile(localEnvUrl, "utf8"); + } catch (err) { + if (err.code === "ENOENT") { + return; + } + throw err; + } + + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) { + continue; + } + const [, key, rawValue] = match; + if (process.env[key] === undefined) { + process.env[key] = rawValue.replace(/^['"]|['"]$/g, ""); + } + } +} + +export function requiredEnv(name, context = "device E2E") { + const value = process.env[name]; + if (!value) { + throw new Error(`Set ${name} before running ${context}.`); + } + return value; +} + +export function timestampSlug() { + return new Date().toISOString().replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-"); +} + +export function run( + command, + args, + { cwd, env = process.env, allowFailure = false, timeoutMs } = {}, +) { + console.log(`$ ${[command, ...args].join(" ")}`); + return new Promise((resolve, reject) => { + const proc = spawn(command, args, { + cwd, + env, + stdio: "inherit", + }); + const timeout = + timeoutMs === undefined + ? null + : setTimeout(() => { + proc.kill("SIGTERM"); + reject(new Error(`${command} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + proc.on("error", reject); + proc.on("close", (code) => { + if (timeout) { + clearTimeout(timeout); + } + if (code === 0 || allowFailure) { + resolve(code); + return; + } + reject(new Error(`${command} exited with ${code}`)); + }); + }); +} + +export async function waitForText(readText, regex, timeoutMs) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const text = readText(); + if (regex.test(text)) { + return text; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${regex}`); +} + +export async function waitMs(label, ms) { + if (ms <= 0) { + return; + } + console.log(`waiting ${ms}ms for ${label}`); + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function launchWithConsole({ outDir, fsHome, deviceUdid, bundleId }) { + await fs.mkdir(outDir, { recursive: true }); + const consolePath = path.join(outDir, "devicectl-console.log"); + const consoleFile = createWriteStream(consolePath); + let consoleText = ""; + const proc = spawn( + "xcrun", + [ + "devicectl", + "device", + "process", + "launch", + "--device", + deviceUdid, + "--terminate-existing", + "--console", + bundleId, + ], + { + cwd: fsHome, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + const write = (chunk) => { + consoleText += chunk.toString(); + process.stdout.write(chunk); + consoleFile.write(chunk); + }; + proc.stdout.on("data", write); + proc.stderr.on("data", write); + + const launchPattern = process.env.MOBILE_CONSOLE_LAUNCH_PATTERN + ? new RegExp(process.env.MOBILE_CONSOLE_LAUNCH_PATTERN) + : /Launched application/; + await waitForText(() => consoleText, launchPattern, 30000); + return { + proc, + consolePath, + text() { + return consoleText; + }, + async stop() { + proc.kill("SIGKILL"); + await new Promise((resolve) => consoleFile.end(resolve)); + }, + }; +} + +export async function writeSessionUrlFromConsole({ outDir, consolePath, consoleText, host }) { + const text = consoleText || (await fs.readFile(consolePath, "utf8")); + const directUrl = text.match(FULLSTORY_SESSION_URL_REGEX); + const sessionUrl = + directUrl?.[0] ?? + (() => { + const matches = [ + ...text.matchAll(/OrgId=([^&\s]+)&UserId=([^&\s]+)&SessionId=([^&\s]+)/g), + ]; + const match = matches.at(-1); + if (!match) { + throw new Error(`Could not find FullStory session URL or bundle endpoint in ${consolePath}`); + } + const [, foundOrg, userId, sessionId] = match; + const appHost = host === "staging.fullstory.com" ? "app.staging.fullstory.com" : "app.fullstory.com"; + return `https://${appHost}/ui/${foundOrg}/client-session/${userId}:${sessionId}`; + })(); + + const sessionPath = path.join(outDir, "fullstory-session-url.txt"); + await fs.writeFile(sessionPath, `${sessionUrl}\n`); + console.log(`FullStory session URL: ${sessionUrl}`); + console.log(`wrote ${sessionPath}`); + return sessionUrl; +} diff --git a/scripts/mobile/fetch-subtext-review-evidence.mjs b/scripts/mobile/fetch-subtext-review-evidence.mjs new file mode 100644 index 00000000..b0987fa1 --- /dev/null +++ b/scripts/mobile/fetch-subtext-review-evidence.mjs @@ -0,0 +1,247 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +const localEnvPath = new URL(".env.local", import.meta.url); + +function config() { + const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; + const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; + if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); + } + return { + outDir, + expectationsPath, + sessionUrlPath: process.env.MOBILE_SESSION_URL_PATH ?? path.join(outDir, "fullstory-session-url.txt"), + }; +} + +function apiUrlForSession(sessionUrl) { + if (process.env.SUBTEXT_API_URL) { + return process.env.SUBTEXT_API_URL; + } + if (sessionUrl.includes("app.staging.fullstory.com")) { + return "https://api.staging.fullstory.com/mcp/subtext"; + } + if (sessionUrl.includes("app.eu1.fullstory.com")) { + return "https://api.eu1.fullstory.com/mcp/subtext"; + } + return "https://api.fullstory.com/mcp/subtext"; +} + +function apiKey() { + return process.env.FULLSTORY_API_KEY ?? process.env.SUBTEXT_API_KEY; +} + +function localCaps() { + return process.env.LOCAL_MCP_CAPS; +} + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +function contentToText(content = []) { + return content + .map((item) => { + if (item.type === "text") { + return item.text ?? ""; + } + if (item.type === "image") { + return `[image: ${item.mimeType ?? "image/png"}, ${item.data?.length ?? 0} bytes base64]`; + } + return JSON.stringify(item); + }) + .join("\n"); +} + +async function callMcp({ apiUrl, token, toolName, arguments: toolArguments }) { + const headers = { + "Content-Type": "application/json", + Authorization: `Basic ${token}`, + }; + const caps = localCaps(); + if (caps) { + headers.caps = caps; + } + + const response = await fetch(apiUrl, { + method: "POST", + headers, + body: JSON.stringify({ + jsonrpc: "2.0", + id: Date.now(), + method: "tools/call", + params: { + name: toolName, + arguments: toolArguments, + }, + }), + }); + + const bodyText = await response.text(); + if (!response.ok) { + throw new Error(`Subtext MCP ${toolName} failed: HTTP ${response.status}\n${bodyText}`); + } + + const body = JSON.parse(bodyText); + if (body.error) { + throw new Error( + `Subtext MCP ${toolName} failed: ${body.error.message ?? JSON.stringify(body.error)}`, + ); + } + + return { + raw: body, + text: contentToText(body.result?.content), + }; +} + +function parseClientId(reviewOpenText) { + const match = reviewOpenText.match(/Client ID:\s*([^\n]+)/); + if (!match) { + throw new Error("Could not parse Client ID from review-open output"); + } + return match[1].trim(); +} + +function parsePageId(reviewOpenText) { + const match = reviewOpenText.match(/^\[\d{2}:\d{2}\]\s+Navigate To Page:.*\[page\s+([^\]\s]+)\]/m); + if (!match) { + throw new Error("Could not parse page id from review-open output"); + } + return match[1].trim(); +} + +function parseEventTimestamps(reviewOpenText) { + return [...reviewOpenText.matchAll(/^\s+(\d+)ms\s+(.+)$/gm)].map((match) => ({ + timestamp: Number(match[1]), + description: match[2], + })); +} + +function parseSessionDurationMs(reviewOpenText) { + const match = reviewOpenText.match(/^#\s+(\d+)s\s+\|/m); + if (!match) { + return null; + } + return Number(match[1]) * 1000; +} + +function timelineTimestamps(startTimestamp, durationMs) { + const offsets = [0, 10000, 25000, 40000]; + const candidates = offsets.map((offset) => startTimestamp + offset); + if (durationMs) { + candidates.push(Math.max(0, durationMs - 1000)); + } + return [...new Set(candidates)] + .filter((timestamp) => Number.isFinite(timestamp) && timestamp >= 0) + .filter((timestamp) => !durationMs || timestamp <= durationMs) + .sort((a, b) => a - b); +} + +function chooseViewTimestamps(reviewOpenText) { + if (process.env.MOBILE_SUBTEXT_VIEW_TIMESTAMPS) { + return process.env.MOBILE_SUBTEXT_VIEW_TIMESTAMPS.split(",") + .map((value) => Number(value.trim())) + .filter((value) => Number.isFinite(value)); + } + + const events = parseEventTimestamps(reviewOpenText); + const pagePropertyAfterCustom = events.find((event, index) => { + const sawCustomBefore = events.slice(0, index).some((candidate) => + candidate.description.toLowerCase().includes("custom"), + ); + return sawCustomBefore && event.description.toLowerCase().includes("page-properties"); + }); + const lastEvent = events.at(-1); + const startTimestamp = pagePropertyAfterCustom?.timestamp ?? lastEvent?.timestamp ?? 0; + const durationMs = Math.max(parseSessionDurationMs(reviewOpenText) ?? 0, lastEvent?.timestamp ?? 0); + + return timelineTimestamps(startTimestamp, durationMs); +} + +async function writeEvidenceFile(name, text) { + const { outDir } = config(); + const file = path.join(outDir, name); + await fs.writeFile(file, text); + console.log(`wrote ${file}`); + return file; +} + +async function main() { + await loadLocalEnv(localEnvPath); + const { outDir, expectationsPath, sessionUrlPath } = config(); + + const token = apiKey(); + if (!token) { + throw new Error( + "Set FULLSTORY_API_KEY or SUBTEXT_API_KEY to call Subtext from Node.", + ); + } + + const goal = await readJson(expectationsPath); + const sessionUrl = (await fs.readFile(sessionUrlPath, "utf8")).trim(); + const apiUrl = apiUrlForSession(sessionUrl); + + await fs.mkdir(outDir, { recursive: true }); + + const opened = await callMcp({ + apiUrl, + token, + toolName: "review-open", + arguments: { session_url: sessionUrl }, + }); + await writeEvidenceFile("review-open-subtext.txt", opened.text); + + const clientId = parseClientId(opened.text); + const pageId = parsePageId(opened.text); + const timestamps = chooseViewTimestamps(opened.text); + console.log( + `review-open parsed client_id=${clientId} page_id=${pageId} timestamps=${timestamps.join(",")}`, + ); + + for (const [index, timestamp] of timestamps.entries()) { + const viewed = await callMcp({ + apiUrl, + token, + toolName: "review-view", + arguments: { + client_id: clientId, + page_id: pageId, + timestamp, + upload: true, + }, + }); + await writeEvidenceFile( + `review-view-subtext-${String(index + 1).padStart(2, "0")}.txt`, + [ + `Source: Subtext MCP review-view`, + `Goal: ${goal.name}`, + `Client ID: ${clientId}`, + `Page ID: ${pageId}`, + `View timestamp: ${timestamp}ms`, + "", + viewed.text, + ].join("\n"), + ); + } + + await callMcp({ + apiUrl, + token, + toolName: "review-close", + arguments: { + client_id: clientId, + use_case: "testing", + was_helpful: true, + }, + }); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/goals/example.json b/scripts/mobile/goals/example.json new file mode 100644 index 00000000..919cacff --- /dev/null +++ b/scripts/mobile/goals/example.json @@ -0,0 +1,83 @@ +{ + "_comment": "Example goal manifest. Copy this file to create goals for your app.", + + "name": "My Screen", + "description": "Navigate to a screen, scroll through it, and validate replay.", + + "run": { + "targetScreen": "My Screen", + "slug": "my-screen", + "navigation": [ + { "action": "tap", "label": "Back", "ifVisible": true }, + { "action": "tap", "label": "Menu" }, + { "action": "scrollToLabel", "label": "My Screen" }, + { "action": "tap", "label": "My Screen" } + ], + "scrollDownCount": 2, + "scrollUpCount": 1, + "maxMenuScrolls": 5, + "waitAfterOpenMs": 0 + }, + + "artifactReportName": "my-screen-validation-report.md", + "replayReportName": "replay-validation-report.md", + + "requiredArtifacts": [ + "04-my-screen-source.xml", + "fullstory-session-url.txt" + ], + + "deviceChecks": [ + { + "artifact": "04-my-screen-source.xml", + "contains": "My Screen", + "label": "Target screen title in Appium source" + }, + { + "artifact": "fullstory-session-url.txt", + "matches": "https://app.*\\.fullstory\\.com/ui/", + "label": "FullStory replay URL" + } + ], + + "replayChecks": { + "screen": "My Screen", + "requiredEvents": [ + { + "contains": "Set Page Properties: My Screen", + "label": "Replay event stream includes page property" + } + ], + "booleans": [ + { + "field": "imageContentVisible", + "label": "Replay shows content instead of blank/frozen page", + "severity": "warn" + }, + { + "field": "scrollPlaybackLooksCorrect", + "label": "Replay scroll playback moves through the screen", + "severity": "warn" + } + ], + "observationHeuristics": { + "events": [ + { + "terms": ["Set Page Properties: My Screen", "page-properties"], + "normalized": "Set Page Properties: My Screen" + } + ], + "reachedScreen": ["My Screen"], + "imageContent": ["VISUAL_IMAGE_CONTENT_OBSERVED"], + "scrollPlayback": ["SCROLL_PLAYBACK_OBSERVED"] + } + }, + + "semanticReplayChecks": [ + "Replay reaches the target screen.", + "Expected content appears, not blank or frozen.", + "Scroll playback moves through the screen." + ], + + "sensitiveRegions": [] +} diff --git a/scripts/mobile/mobile.env.example b/scripts/mobile/mobile.env.example new file mode 100644 index 00000000..5ad32ced --- /dev/null +++ b/scripts/mobile/mobile.env.example @@ -0,0 +1,40 @@ +# Copy to scripts/mobile/.env.local for local runs. Do not commit real values. + +# API key for Subtext MCP replay evidence fetches and Lidar auth. +FULLSTORY_API_KEY= + +# Lidar/Subtext MCP endpoint that supports live-connect with platform: "ios". +# Leave empty when run-local-lidar-ios.mjs should start a local Lidar server. +LIDAR_IOS_MCP_URL= + +# Path to the goal JSON file that defines what to drive and validate. +MOBILE_GOAL_EXPECTATIONS= + +# Installed app bundle ID on the target device. +MOBILE_BUNDLE_ID= + +# Physical device target. Use `xcrun devicectl list devices` to find the UDID. +MOBILE_UDID= +MOBILE_DEVICE_NAME= + +# Output directory for device evidence and replay reports. +MOBILE_OUT_DIR= + +# Appium endpoint used by the Lidar iOS backend or local Appium runner. +MOBILE_APPIUM_URL=http://127.0.0.1:4723 +LIDAR_IOS_APPIUM_URL=http://127.0.0.1:4723 + +# Local no-agent Lidar run config. Set MOBILE_LIDAR_START=0 to reuse an existing Lidar. +MOBILE_LIDAR_START=1 +MN_HOME= +FS_HOME= +LOCAL_MCP_ORG_ID= +LOCAL_MCP_EMAIL= + +# Optional: regex pattern to detect app launch in console output. +# Default matches "Launched application". Set to match your app if needed. +MOBILE_CONSOLE_LAUNCH_PATTERN= + +# Optional knobs for slower devices or replay ingestion delays. +MOBILE_POST_GOAL_FLUSH_MS=15000 +MOBILE_SUBTEXT_REVIEW_SETTLE_MS=15000 diff --git a/scripts/mobile/prepare-subtext-review.mjs b/scripts/mobile/prepare-subtext-review.mjs new file mode 100644 index 00000000..1c174399 --- /dev/null +++ b/scripts/mobile/prepare-subtext-review.mjs @@ -0,0 +1,98 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} +const sessionUrlPath = + process.env.MOBILE_SESSION_URL_PATH ?? + path.join(outDir, "fullstory-session-url.txt"); + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +async function existingArtifacts(files) { + const present = []; + + for (const file of files) { + try { + await fs.access(path.join(outDir, file)); + present.push(file); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + } + + return present; +} + +function replayCheckList(goal) { + const requiredEvents = goal.replayChecks?.requiredEvents ?? []; + const booleans = goal.replayChecks?.booleans ?? []; + return [ + `Replay reaches the ${goal.replayChecks?.screen ?? goal.name} screen.`, + ...requiredEvents.map((check) => check.label), + ...booleans.map((check) => check.label), + ]; +} + +async function main() { + const goal = await readJson(expectationsPath); + const sessionUrl = (await fs.readFile(sessionUrlPath, "utf8")).trim(); + const requiredArtifacts = goal.requiredArtifacts ?? []; + const artifacts = await existingArtifacts(requiredArtifacts); + + const request = `# Subtext Mobile Replay Review Request + +Use the Subtext review MCP tools to validate this session replay. + +## Session + +${sessionUrl} + +## Goal + +- Name: ${goal.name} +- Goal path: ${expectationsPath} +- Mobile output directory: ${outDir} +- Target screen: ${goal.replayChecks?.screen ?? goal.run?.targetScreen ?? goal.name} + +## Device Evidence + +The Appium run already produced these artifacts: + +${artifacts.map((artifact) => `- ${path.join(outDir, artifact)}`).join("\n")} + +## Replay Rubric + +${replayCheckList(goal).map((check) => `- ${check}`).join("\n")} + +## Expected Output + +Write \`${path.join(outDir, "replay-observations.json")}\`, save raw review evidence under \`${outDir}\`, then run: + +\`\`\`bash +MOBILE_GOAL_EXPECTATIONS="${expectationsPath}" \\ +MOBILE_OUT_DIR="${outDir}" \\ +node scripts/mobile/validate-replay-observations.mjs +\`\`\` +`; + + const requestPath = path.join(outDir, "subtext-review-request.md"); + await fs.writeFile(requestPath, request); + console.log(`wrote ${requestPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/run-goal.mjs b/scripts/mobile/run-goal.mjs new file mode 100644 index 00000000..954d29a3 --- /dev/null +++ b/scripts/mobile/run-goal.mjs @@ -0,0 +1,132 @@ +import fs from "node:fs/promises"; +import { + mobileCapabilities, + mobileConnect, + mobileDismissSystemAlert, + mobileScreenshot, + mobileScroll, + mobileSource, + mobileTapByName, + mobileTapIfVisible, + startFullStorySessionLogCapture, +} from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} +const outDir = process.env.MOBILE_OUT_DIR; +if (!outDir) { + throw new Error("Set MOBILE_OUT_DIR to the output directory."); +} +const captureEveryScrollSource = process.env.MOBILE_CAPTURE_EACH_SCROLL_SOURCE !== "0"; +const captureEveryScrollScreenshot = process.env.MOBILE_CAPTURE_EACH_SCROLL_SCREENSHOT !== "0"; + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +async function captureScrollArtifact(driver, prefix, index) { + if (captureEveryScrollScreenshot) { + await mobileScreenshot(driver, outDir, `${prefix}-${index}.png`); + } + if (captureEveryScrollSource) { + await mobileSource(driver, outDir, `${prefix}-${index}.xml`); + } +} + +async function runNavStep(driver, step) { + switch (step.action) { + case "tap": + if (step.ifVisible) { + await mobileTapIfVisible(driver, step.label); + } else { + await mobileTapByName(driver, step.label, { maxScrolls: step.maxScrolls ?? 5 }); + } + break; + case "screenshot": + await mobileScreenshot(driver, outDir, step.filename ?? "nav-screenshot.png"); + break; + case "source": + await mobileSource(driver, outDir, step.filename ?? "nav-source.xml"); + break; + case "dismissAlert": + await mobileDismissSystemAlert(driver); + break; + default: + throw new Error(`Unknown navigation action: ${step.action}`); + } +} + +async function runGoal(driver, goal) { + const run = goal.run; + if (!run) { + throw new Error(`Goal ${goal.name} is missing the "run" section.`); + } + const slug = run.slug ?? goal.name.toLowerCase().replace(/\s+/g, "-"); + + await mobileScreenshot(driver, outDir, "01-initial.png"); + await mobileSource(driver, outDir, "01-source.xml"); + await mobileDismissSystemAlert(driver); + + if (Array.isArray(run.navigation)) { + for (const step of run.navigation) { + await runNavStep(driver, step); + } + } + + if (run.targetScreen) { + await mobileScreenshot(driver, outDir, `04-${slug}-initial.png`); + await mobileSource(driver, outDir, `04-${slug}-source.xml`); + } + + if (run.waitAfterOpenMs) { + console.log(`waiting ${run.waitAfterOpenMs}ms`); + await driver.pause(run.waitAfterOpenMs); + await mobileScreenshot(driver, outDir, `05-${slug}-after-wait.png`); + await mobileSource(driver, outDir, `05-${slug}-after-wait.xml`); + } + + for (let i = 1; i <= (run.scrollDownCount ?? 0); i += 1) { + console.log(`scrolling down ${i}`); + await mobileScroll(driver, { fromY: 720, toY: 180 }); + await captureScrollArtifact(driver, `05-${slug}-down`, i); + } + + for (let i = 1; i <= (run.scrollUpCount ?? 0); i += 1) { + console.log(`scrolling up ${i}`); + await mobileScroll(driver, { fromY: 180, toY: 720 }); + await captureScrollArtifact(driver, `06-${slug}-up`, i); + } +} + +async function main() { + const goal = await readJson(expectationsPath); + await fs.mkdir(outDir, { recursive: true }); + const capabilities = await mobileCapabilities(); + + const logCapture = startFullStorySessionLogCapture({ + capabilities, + outDir, + }); + + const driver = await mobileConnect({ + capabilities, + }); + + try { + await runGoal(driver, goal); + } finally { + await driver.deleteSession(); + } + + await logCapture.stop(); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/run-lidar-live-ios.mjs b/scripts/mobile/run-lidar-live-ios.mjs new file mode 100644 index 00000000..24e3facc --- /dev/null +++ b/scripts/mobile/run-lidar-live-ios.mjs @@ -0,0 +1,395 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +const localEnvPath = new URL(".env.local", import.meta.url); + +function config() { + const outDir = process.env.MOBILE_OUT_DIR; + if (!outDir) { + throw new Error("Set MOBILE_OUT_DIR to the output directory for this run."); + } + const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; + if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); + } + return { + outDir, + expectationsPath, + mcpUrl: process.env.LIDAR_IOS_MCP_URL ?? process.env.SUBTEXT_API_URL, + bundleId: process.env.MOBILE_BUNDLE_ID, + udid: process.env.MOBILE_UDID, + simulator: process.env.MOBILE_DEVICE_NAME ?? process.env.LIDAR_IOS_SIMULATOR, + }; +} + +function apiKey() { + return process.env.FULLSTORY_API_KEY ?? process.env.SUBTEXT_API_KEY; +} + +function localCaps() { + return process.env.LOCAL_MCP_CAPS; +} + +async function callMcp(method, params, { token }) { + const { mcpUrl } = config(); + const headers = { + "Content-Type": "application/json", + Authorization: `Basic ${token}`, + }; + const caps = localCaps(); + if (caps) { + headers.caps = caps; + } + + const response = await fetch(mcpUrl, { + method: "POST", + headers, + body: JSON.stringify({ + jsonrpc: "2.0", + id: Date.now(), + method, + params, + }), + }); + + const bodyText = await response.text(); + if (!response.ok) { + throw new Error(`MCP ${method} failed: HTTP ${response.status}\n${bodyText}`); + } + + const body = JSON.parse(bodyText); + if (body.error) { + throw new Error(`MCP ${method} failed: ${body.error.message ?? JSON.stringify(body.error)}`); + } + return body.result; +} + +async function callTool(name, args, options) { + return callMcp("tools/call", { name, arguments: args }, options); +} + +function textContents(result) { + return (result.content ?? []).filter((item) => item.type === "text").map((item) => item.text ?? ""); +} + +function imageContents(result) { + return (result.content ?? []).filter((item) => item.type === "image"); +} + +function parseConnectionId(result) { + const text = textContents(result).join("\n"); + const match = text.match(/connection_id[:=]\s*([^\s]+)/); + if (!match) { + throw new Error(`Could not parse connection_id from: ${text}`); + } + return match[1]; +} + +function unquoteText(text) { + return text.replace(/\\"/g, "\"").replace(/\\\\/g, "\\"); +} + +function parseFormattedTree(text) { + const root = { id: "ios-root", role: "root", children: [] }; + let inGuide = false; + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) { + inGuide = false; + continue; + } + if (trimmed === "[Guide]") { + inGuide = true; + continue; + } + if ( + inGuide || + /^(connection_id|current_view|bundle_id|capture_status|url|fs_session_url|trace_id|trace_url|sightmap_upload_url):/.test( + trimmed, + ) + ) { + continue; + } + const match = trimmed.match(/^(\S+)\s+(\S+)(?:\s+"((?:\\.|[^"])*)")?(?:\s+value="((?:\\.|[^"])*)")?/); + if (!match || match[1] === "root") { + continue; + } + root.children.push({ + id: match[1], + role: match[2], + text: match[3] ? unquoteText(match[3]) : "", + value: match[4] ? unquoteText(match[4]) : "", + }); + } + return root; +} + +function parseTree(result) { + const candidates = textContents(result); + for (const text of candidates) { + try { + return JSON.parse(text); + } catch { + // not JSON + } + const firstObject = text.indexOf("{"); + const lastObject = text.lastIndexOf("}"); + if (firstObject !== -1 && lastObject > firstObject) { + try { + return JSON.parse(text.slice(firstObject, lastObject + 1)); + } catch { + // not embedded JSON + } + } + const tree = parseFormattedTree(text); + if (tree.children.length > 0) { + return tree; + } + } + throw new Error("Could not parse component tree from snapshot"); +} + +function walkTree(root, visit) { + if (!root) { + return undefined; + } + const result = visit(root); + if (result) { + return result; + } + for (const child of root.children ?? []) { + const childResult = walkTree(child, visit); + if (childResult) { + return childResult; + } + } + return undefined; +} + +function componentLabel(component) { + return [ + component.text, + component.value, + component.properties?.identifier, + component.properties?.sightmap_name, + ].filter(Boolean); +} + +function findComponent(root, matcher) { + return walkTree(root, (component) => (matcher(component) ? component : undefined)); +} + +function byLabel(label) { + return (component) => componentLabel(component).includes(label); +} + +function hasActiveNavigation(root, label) { + return Boolean(findComponent(root, (c) => c.role === "navigation" && c.text === label)); +} + +function visibleTextLabels(root) { + const labels = []; + walkTree(root, (component) => { + if (component.role === "text" && component.text) { + labels.push(component.text); + } + return undefined; + }); + return labels; +} + +async function writeSnapshot(prefix, result) { + const { outDir } = config(); + await fs.mkdir(outDir, { recursive: true }); + await fs.writeFile(path.join(outDir, `${prefix}.txt`), `${textContents(result).join("\n")}\n`); + const tree = parseTree(result); + await fs.writeFile(path.join(outDir, `${prefix}.json`), `${JSON.stringify(tree, null, 2)}\n`); + + const image = imageContents(result)[0]; + if (image?.data) { + await fs.writeFile(path.join(outDir, `${prefix}.png.base64`), image.data); + } + return tree; +} + +async function snapshotStep(connectionId, prefix, options) { + const result = await callTool("live-view-snapshot", { connection_id: connectionId }, options); + const summary = textContents(result)[0] ?? ""; + console.log(`${prefix}: ${summary}`); + return writeSnapshot(prefix, result); +} + +async function tapByLabel(connectionId, tree, label, options) { + const component = findComponent(tree, byLabel(label)); + if (!component?.id) { + throw new Error(`Could not find component labelled ${JSON.stringify(label)}`); + } + await callTool("live-act-click", { connection_id: connectionId, component_id: component.id }, options); + console.log(`tapped ${label} (${component.id})`); +} + +async function scrollUntilLabelVisible(connectionId, tree, label, options, prefix) { + let current = tree; + for (let i = 1; i <= 12; i += 1) { + if (findComponent(current, byLabel(label))) { + return current; + } + const scrollTarget = findComponent(current, (c) => c.role === "list") ?? current; + const firstLabel = visibleTextLabels(current)[0] ?? ""; + const deltaY = firstLabel && label.localeCompare(firstLabel) < 0 ? 120 : -120; + await callTool( + "live-act-drag", + { connection_id: connectionId, component_id: scrollTarget.id, delta_x: 0, delta_y: deltaY }, + options, + ); + current = await snapshotStep(connectionId, `${prefix}-${i}`, options); + } + return current; +} + +async function runNavigationSteps(connectionId, tree, steps, options, slug) { + let current = tree; + let stepIdx = 0; + for (const step of steps) { + stepIdx += 1; + const prefix = `02-live-ios-nav-${stepIdx}`; + switch (step.action) { + case "tap": { + const component = findComponent(current, byLabel(step.label)); + if (!component?.id) { + if (step.ifVisible) { + console.log(`nav: ${step.label} not visible, skipping`); + break; + } + throw new Error(`Could not find component labelled ${JSON.stringify(step.label)}`); + } + await callTool("live-act-click", { connection_id: connectionId, component_id: component.id }, options); + console.log(`tapped ${step.label} (${component.id})`); + current = await snapshotStep(connectionId, prefix, options); + break; + } + case "scrollToLabel": + current = await scrollUntilLabelVisible(connectionId, current, step.label, options, prefix); + break; + default: + throw new Error(`Unknown navigation action: ${step.action}`); + } + } + return current; +} + +async function assertLiveIosAvailable(options) { + const { mcpUrl } = config(); + const result = await callMcp("tools/list", {}, options); + const names = new Set((result.tools ?? []).map((tool) => tool.name)); + const missing = [ + "live-connect", + "live-view-snapshot", + "live-act-click", + "live-act-drag", + "live-disconnect", + ].filter((name) => !names.has(name)); + if (missing.length > 0) { + throw new Error( + [ + `MCP server ${mcpUrl} does not expose the required live tools.`, + `Missing: ${missing.join(", ")}`, + "Use a Lidar server whose live-* tools support iOS connections.", + ].join("\n"), + ); + } +} + +async function main() { + await loadLocalEnv(localEnvPath); + const { outDir, expectationsPath, mcpUrl, bundleId, udid, simulator } = config(); + const token = apiKey(); + if (!token) { + throw new Error("Set FULLSTORY_API_KEY or SUBTEXT_API_KEY."); + } + if (!bundleId) { + throw new Error("Set MOBILE_BUNDLE_ID to the installed iOS app bundle ID."); + } + if (!mcpUrl) { + throw new Error("Set LIDAR_IOS_MCP_URL or SUBTEXT_API_URL to the MCP endpoint."); + } + + const options = { token }; + const goal = JSON.parse(await fs.readFile(expectationsPath, "utf8")); + const run = goal.run; + if (!run) { + throw new Error(`Goal ${goal.name} is missing the "run" section.`); + } + const slug = run.slug ?? goal.name.toLowerCase().replace(/\s+/g, "-"); + const targetScreen = run.targetScreen; + + await assertLiveIosAvailable(options); + + await fs.mkdir(outDir, { recursive: true }); + await fs.writeFile( + path.join(outDir, "live-ios-run-config.json"), + `${JSON.stringify({ mcpUrl, bundleId, udid, simulator, goal: goal.name }, null, 2)}\n`, + ); + + let connected; + try { + connected = await callTool( + "live-connect", + { platform: "ios", bundle_id: bundleId, udid, simulator }, + options, + ); + } catch (err) { + if (String(err.message).includes("url is required")) { + throw new Error( + "This MCP server does not support platform='ios' yet. " + + "Point LIDAR_IOS_MCP_URL at a Lidar build with iOS routing in live-connect.", + ); + } + throw err; + } + const connectionId = parseConnectionId(connected); + console.log(`connected: ${connectionId}`); + + try { + let tree = await snapshotStep(connectionId, "01-live-ios-initial", options); + + if (Array.isArray(run.navigation) && run.navigation.length > 0) { + tree = await runNavigationSteps(connectionId, tree, run.navigation, options, slug); + } + + if (targetScreen) { + tree = await snapshotStep(connectionId, `03-live-ios-${slug}`, options); + if (!hasActiveNavigation(tree, targetScreen)) { + console.warn(`expected active screen ${targetScreen}, but it was not detected`); + } + } + + for (let i = 1; i <= (run.scrollDownCount ?? 0); i += 1) { + await callTool("live-act-drag", { connection_id: connectionId, component_id: tree.id, delta_x: 0, delta_y: -500 }, options); + tree = await snapshotStep(connectionId, `04-live-ios-down-${i}`, options); + } + for (let i = 1; i <= (run.scrollUpCount ?? 0); i += 1) { + await callTool("live-act-drag", { connection_id: connectionId, component_id: tree.id, delta_x: 0, delta_y: 500 }, options); + tree = await snapshotStep(connectionId, `05-live-ios-up-${i}`, options); + } + + const reachedTarget = !targetScreen || hasActiveNavigation(tree, targetScreen); + const reportName = `live-ios-${slug}-report.md`; + await fs.writeFile( + path.join(outDir, reportName), + [`# ${goal.name}`, "", `Status: ${reachedTarget ? "PASS" : "WARN"}`, "", `Output: ${outDir}`, ""].join("\n"), + ); + console.log(`status: ${reachedTarget ? "PASS" : "WARN"}`); + console.log(`done: ${outDir}`); + } finally { + await callTool("live-disconnect", { connection_id: connectionId }, options).catch((err) => { + console.warn(`live-disconnect failed: ${err.message}`); + }); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/run-local-lidar-ios.mjs b/scripts/mobile/run-local-lidar-ios.mjs new file mode 100644 index 00000000..fd5407db --- /dev/null +++ b/scripts/mobile/run-local-lidar-ios.mjs @@ -0,0 +1,376 @@ +import { createWriteStream } from "node:fs"; +import fs from "node:fs/promises"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv, timestampSlug } from "./device-e2e-common.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const localEnvPath = new URL(".env.local", import.meta.url); + +function env(name, fallback = "") { + return process.env[name] ?? fallback; +} + +function requireEnv(name, hint) { + const value = process.env[name]; + if (!value) { + throw new Error(`Set ${name}${hint ? ` (${hint})` : ""}.`); + } + return value; +} + +function requireAnyEnv(names, hint) { + if (names.some((name) => process.env[name])) { + return; + } + throw new Error(`Set one of ${names.join(", ")}${hint ? ` (${hint})` : ""}.`); +} + +function splitCommandArgs(value) { + const args = []; + let current = ""; + let quote = ""; + let escaped = false; + for (const char of value.trim()) { + if (escaped) { + current += char; + escaped = false; + continue; + } + if (char === "\\") { + escaped = true; + continue; + } + if (quote) { + if (char === quote) { + quote = ""; + } else { + current += char; + } + continue; + } + if (char === "\"" || char === "'") { + quote = char; + continue; + } + if (/\s/.test(char)) { + if (current) { + args.push(current); + current = ""; + } + continue; + } + current += char; + } + if (escaped) { + current += "\\"; + } + if (quote) { + throw new Error(`Unclosed quote in command args: ${value}`); + } + if (current) { + args.push(current); + } + return args; +} + +function defaultMnHome() { + return path.resolve(here, "../../../mn"); +} + +function fsHome() { + return env("FS_HOME", path.join(env("MN_HOME", defaultMnHome()), "projects/fullstory")); +} + +function goSrc() { + return path.join(fsHome(), "go/src"); +} + +async function isPortOpen(port, host = "127.0.0.1") { + return new Promise((resolve) => { + const socket = net.connect({ host, port }); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("error", () => resolve(false)); + }); +} + +async function findFreePortGroup(startPort) { + for (let port = startPort; port < startPort + 200; port += 10) { + const appPort = port; + const grpcPort = port - 1; + const internalPort = port + 1; + if ( + !(await isPortOpen(appPort)) && + !(await isPortOpen(grpcPort)) && + !(await isPortOpen(internalPort)) + ) { + return { appPort, grpcPort }; + } + } + throw new Error(`Could not find free Lidar ports near ${startPort}`); +} + +function spawnLogged(command, args, { cwd, env: childEnv, logPath }) { + const log = createWriteStream(logPath); + const proc = spawn(command, args, { + cwd, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + const write = (chunk) => { + process.stdout.write(chunk); + log.write(chunk); + }; + proc.stdout.on("data", write); + proc.stderr.on("data", write); + const closed = new Promise((resolve, reject) => { + proc.once("error", reject); + proc.once("close", (code, signal) => resolve({ code, signal })); + }); + return { + proc, + closed, + async stop() { + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGTERM"); + } + const timeout = new Promise((resolve) => { + setTimeout(() => { + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + } + resolve(); + }, 5000); + }); + await Promise.race([closed, timeout]); + await closed.catch(() => {}); + await new Promise((resolve) => log.end(resolve)); + }, + }; +} + +async function waitForLog(logPath, regex, timeoutMs, child) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + if (child && (child.proc.exitCode !== null || child.proc.signalCode !== null)) { + throw new Error(`Process exited before ${regex} appeared in ${logPath}`); + } + let text = ""; + try { + text = await fs.readFile(logPath, "utf8"); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + if (regex.test(text)) { + return text; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for ${regex} in ${logPath}`); +} + +async function checkLocalGoEnvironment() { + await fs.access(goSrc()).catch(() => { + throw new Error(`Cannot find FullStory Go source at ${goSrc()}. Set MN_HOME or FS_HOME.`); + }); + await fs.access(path.join(goSrc(), "fs/services/lidar/main/lidar")).catch(() => { + throw new Error(`Cannot find Lidar source under ${goSrc()}. Set FS_HOME to the FullStory checkout.`); + }); + await runCapture("go", ["env", "GOROOT"], { cwd: goSrc() }).catch((err) => { + throw new Error(`Go environment is not ready for Lidar builds from ${goSrc()}.\n${err.message}`); + }); +} + +async function runCapture(command, args, { cwd, env: childEnv = process.env }) { + return new Promise((resolve, reject) => { + let stdout = ""; + let stderr = ""; + const proc = spawn(command, args, { + cwd, + env: childEnv, + stdio: ["ignore", "pipe", "pipe"], + }); + proc.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + proc.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + proc.on("error", reject); + proc.on("close", (code) => { + if (code === 0) { + resolve(stdout); + return; + } + reject(new Error(`${command} ${args.join(" ")} exited with ${code}\n${stderr}`)); + }); + }); +} + +async function runInherited(command, args, { cwd, env: childEnv }) { + return new Promise((resolve, reject) => { + const proc = spawn(command, args, { + cwd, + env: childEnv, + stdio: "inherit", + }); + proc.on("error", reject); + proc.on("close", (code) => { + if (code === 0) { + resolve(); + return; + } + reject(new Error(`${command} ${args.join(" ")} exited with ${code}`)); + }); + }); +} + +async function generateCaps() { + if (process.env.LOCAL_MCP_CAPS) { + return process.env.LOCAL_MCP_CAPS; + } + const orgID = process.env.LOCAL_MCP_ORG_ID ?? process.env.MOBILE_FULLSTORY_ORG; + if (!orgID) { + throw new Error("Set LOCAL_MCP_ORG_ID or MOBILE_FULLSTORY_ORG for local Lidar MCP auth."); + } + const email = requireEnv("LOCAL_MCP_EMAIL", "email for signed session caps"); + const sessionID = env("LOCAL_MCP_SESSION_ID", "local-mobile-e2e"); + const tempPath = path.join(os.tmpdir(), `subtext-mobile-caps-${process.pid}.go`); + const source = `package main +import ( + "fmt" + "fs/auth" + "fs/auth/sessionpb" +) +func main() { + details := &sessionpb.AppSessionDetails{ + SessionId: ${JSON.stringify(sessionID)}, + Authz: &sessionpb.AuthorizationContext{Entity: sessionpb.OrgAuthzContextEntity(${JSON.stringify(orgID)}, "")}, + AuthnMethod: sessionpb.MakeTokenAuthnMethod(sessionpb.TokenType_INVALID_TOKEN_TYPE), + } + ss, err := auth.FakeAppSignedSession(${JSON.stringify(email)}, details) + if err != nil { panic(err) } + s, err := ss.String() + if err != nil { panic(err) } + fmt.Print(s) +} +`; + await fs.writeFile(tempPath, source); + try { + return (await runCapture("go", ["run", tempPath], { cwd: goSrc() })).trim(); + } finally { + await fs.rm(tempPath, { force: true }); + } +} + +async function ensureAppium(outDir) { + const appiumURL = env("LIDAR_IOS_APPIUM_URL", env("MOBILE_APPIUM_URL", "http://127.0.0.1:4723")); + const port = Number(new URL(appiumURL).port || 4723); + if (await isPortOpen(port)) { + return { appiumURL, stop: async () => {} }; + } + + const command = env("MOBILE_APPIUM_COMMAND", "pnpm"); + const args = splitCommandArgs( + env("MOBILE_APPIUM_ARGS", "exec appium --address 127.0.0.1 --port 4723 --base-path /"), + ); + const logPath = path.join(outDir, "appium.log"); + console.log(`starting Appium: ${command} ${args.join(" ")}`); + const child = spawnLogged(command, args, { cwd: fsHome(), env: process.env, logPath }); + await waitForLog( + logPath, + /Appium REST http interface listener started|Appium server started/i, + 30000, + child, + ); + return { appiumURL, stop: child.stop }; +} + +async function buildLidar(binaryPath) { + if (process.env.MOBILE_LIDAR_BUILD === "0") { + return; + } + console.log(`building Lidar: ${binaryPath}`); + await runInherited("go", ["build", "-o", binaryPath, "fs/services/lidar/main/lidar"], { + cwd: goSrc(), + env: process.env, + }); +} + +async function startLidar(outDir, appiumURL) { + if (process.env.MOBILE_LIDAR_START === "0") { + const existingURL = requireEnv("LIDAR_IOS_MCP_URL", "existing local Lidar MCP URL"); + return { mcpURL: existingURL, stop: async () => {} }; + } + + const binaryPath = env("MOBILE_LIDAR_BIN", path.join(outDir, "lidar-mobile-live")); + await buildLidar(binaryPath); + const startPort = Number(env("MOBILE_LIDAR_PORT", "11731")); + const { appPort, grpcPort } = await findFreePortGroup(startPort); + const logPath = path.join(outDir, "lidar.log"); + const childEnv = { + ...process.env, + LIDAR_IOS_APPIUM_URL: appiumURL, + LIDAR_IOS_BUNDLE_ID: process.env.MOBILE_BUNDLE_ID, + LIDAR_IOS_UDID: process.env.MOBILE_UDID ?? "", + LIDAR_IOS_SIMULATOR: process.env.MOBILE_DEVICE_NAME ?? "", + }; + const args = ["-port", String(appPort), "-grpcport", String(grpcPort)]; + console.log(`starting Lidar: ${binaryPath} ${args.join(" ")}`); + const child = spawnLogged(binaryPath, args, { cwd: fsHome(), env: childEnv, logPath }); + await waitForLog(logPath, new RegExp(`listening on 127\\.0\\.0\\.1:${appPort}`), 60000, child); + return { mcpURL: `http://127.0.0.1:${appPort}/mcp/subtext`, stop: child.stop }; +} + +async function main() { + await loadLocalEnv(localEnvPath); + requireAnyEnv(["FULLSTORY_API_KEY", "SUBTEXT_API_KEY"], "MCP Basic auth"); + requireEnv("MOBILE_BUNDLE_ID", "installed app bundle ID"); + requireEnv("MOBILE_UDID", "physical device UDID"); + requireEnv("MOBILE_GOAL_EXPECTATIONS", "path to goal JSON file"); + + const outDir = env( + "MOBILE_OUT_DIR", + path.join(DEFAULT_OUT_DIR, `local-lidar-ios-${timestampSlug()}`), + ); + await fs.mkdir(outDir, { recursive: true }); + console.log(`output: ${outDir}`); + + if (process.env.MOBILE_LIDAR_START !== "0" || !process.env.LOCAL_MCP_CAPS) { + await checkLocalGoEnvironment(); + } + + const appium = await ensureAppium(outDir); + const lidar = await startLidar(outDir, appium.appiumURL); + const caps = await generateCaps(); + const childEnv = { + ...process.env, + LOCAL_MCP_CAPS: caps, + LIDAR_IOS_MCP_URL: lidar.mcpURL, + MOBILE_OUT_DIR: outDir, + }; + + try { + await runInherited("node", [path.join(here, "run-lidar-live-ios.mjs")], { + cwd: path.resolve(here, "../.."), + env: childEnv, + }); + } finally { + await lidar.stop(); + await appium.stop(); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/validate-goal-artifacts.mjs b/scripts/mobile/validate-goal-artifacts.mjs new file mode 100644 index 00000000..91a5baa4 --- /dev/null +++ b/scripts/mobile/validate-goal-artifacts.mjs @@ -0,0 +1,81 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} + +async function readArtifact(name) { + return fs.readFile(path.join(outDir, name), "utf8"); +} + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +function assertIncludes(text, value, label) { + if (!text.includes(value)) { + throw new Error(`Missing ${label}: ${value}`); + } + console.log(`ok: ${label}`); +} + +function assertMatches(text, value, label) { + const regex = new RegExp(value); + if (!regex.test(text)) { + throw new Error(`Missing ${label}: ${value}`); + } + console.log(`ok: ${label}`); +} + +async function main() { + const expectations = await readJson(expectationsPath); + + for (const file of expectations.requiredArtifacts) { + await fs.access(path.join(outDir, file)); + console.log(`ok: artifact exists: ${file}`); + } + + const sessionUrl = (await readArtifact("fullstory-session-url.txt")).trim(); + + for (const check of expectations.deviceChecks) { + const artifact = await readArtifact(check.artifact); + if (check.matches) { + assertMatches(artifact, check.matches, check.label); + } else { + assertIncludes(artifact, check.contains, check.label); + } + } + + const report = `# ${expectations.name} Goal Validation + +## Local Artifact Checks + +${expectations.deviceChecks.map((check) => `- ${check.label}.`).join("\n")} + +## Replay Checks + +These are semantic checks, not pixel-perfect checks: + +${expectations.semanticReplayChecks.map((check) => `- ${check}`).join("\n")} + +## Session + +${sessionUrl} +`; + + const reportPath = path.join(outDir, expectations.artifactReportName); + await fs.writeFile(reportPath, report); + console.log(`wrote ${reportPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/mobile/validate-replay-observations.mjs b/scripts/mobile/validate-replay-observations.mjs new file mode 100644 index 00000000..7a2b6ad8 --- /dev/null +++ b/scripts/mobile/validate-replay-observations.mjs @@ -0,0 +1,227 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const observationsPath = path.join(outDir, "replay-observations.json"); +const expectationsPath = process.env.MOBILE_GOAL_EXPECTATIONS; +if (!expectationsPath) { + throw new Error("Set MOBILE_GOAL_EXPECTATIONS to a goal JSON file."); +} + +async function readJson(file) { + return JSON.parse(await fs.readFile(file, "utf8")); +} + +function pass(condition, message, failures, warnings, { warn = false } = {}) { + if (condition) { + return `PASS: ${message}`; + } + + if (warn) { + warnings.push(message); + return `WARN: ${message}`; + } + + failures.push(message); + return `FAIL: ${message}`; +} + +function includesAny(values, expected) { + return values.some((value) => value.toLowerCase().includes(expected.toLowerCase())); +} + +function validatePrivacyEvidence(observations, expectations, failures, warnings) { + const observedRegions = observations.privacyEvidence ?? []; + const expectedRegions = expectations.sensitiveRegions ?? []; + const checks = []; + + for (const expectedRegion of expectedRegions) { + const observed = observedRegions.find((region) => region.regionId === expectedRegion.id); + if (!observed) { + warnings.push(`No replay observation for sensitive region: ${expectedRegion.label}`); + checks.push(`WARN: No replay observation for sensitive region: ${expectedRegion.label}`); + continue; + } + + const replayState = observed.replay?.state ?? "not_observed"; + const engine = observed.engine ?? {}; + const expectedState = expectedRegion.expectedPrivacyState ?? observed.expectedPrivacyState; + + checks.push( + validateRegionPrivacyState({ + label: expectedRegion.label, + expectedState, + replayState, + engine, + failures, + warnings, + }), + ); + } + + return checks; +} + +function validateRegionPrivacyState({ label, expectedState, replayState, engine, failures, warnings }) { + const engineState = engine.state ?? "not_available"; + const engineContradictsUnmasked = + engine.maskedFlagObserved === true || engine.blockedFlagObserved === true; + + switch (expectedState) { + case "unmasked": + if (replayState === "unmasked" && !engineContradictsUnmasked) { + return `PASS: ${label} was expected unmasked and replay showed unmasked content`; + } + if (replayState === "mixed" || replayState === "masked" || engineContradictsUnmasked) { + failures.push( + `${label} was expected unmasked, but replay or native evidence indicates ${replayState}`, + ); + return `FAIL: ${label} was expected unmasked, but replay or native evidence indicates ${replayState}`; + } + failures.push(`${label} was expected unmasked, but replay did not show unmasked evidence`); + return `FAIL: ${label} was expected unmasked, but replay did not show unmasked evidence`; + + case "masked": + if (replayState === "masked" || engine.maskedFlagObserved === true || engineState === "masked") { + return `PASS: ${label} was expected masked and masking evidence was observed`; + } + if (replayState === "unmasked" || replayState === "mixed") { + failures.push(`${label} was expected masked, but replay showed unmasked content`); + return `FAIL: ${label} was expected masked, but replay showed unmasked content`; + } + warnings.push(`${label} was expected masked, but masking evidence was not captured`); + return `WARN: ${label} was expected masked, but masking evidence was not captured`; + + case "excluded": + if (engine.blockedFlagObserved === true || engineState === "excluded") { + return `PASS: ${label} was expected excluded and blocked evidence was observed`; + } + if (replayState === "unmasked" || replayState === "mixed") { + failures.push(`${label} was expected excluded, but replay showed unmasked content`); + return `FAIL: ${label} was expected excluded, but replay showed unmasked content`; + } + warnings.push(`${label} was expected excluded, but blocked-frame evidence was not captured`); + return `WARN: ${label} was expected excluded, but blocked-frame evidence was not captured`; + + case "omitted": + if (replayState === "not_observed" || engineState === "omitted") { + return `PASS: ${label} was expected omitted and replay did not show the region`; + } + failures.push(`${label} was expected omitted, but replay showed ${replayState} evidence`); + return `FAIL: ${label} was expected omitted, but replay showed ${replayState} evidence`; + + case "config_dependent": + case undefined: + warnings.push(`${label} has no explicit expected privacy state`); + return `WARN: ${label} has no explicit expected privacy state`; + + default: + if (expectedState === "toggles_mask_unmask") { + if (replayState === "mixed") { + return `PASS: ${label} showed both masked and unmasked states`; + } + warnings.push( + `${label} expects masked/unmasked toggle evidence, but replay only showed ${replayState}`, + ); + return `WARN: ${label} expects masked/unmasked toggle evidence, but replay only showed ${replayState}`; + } + warnings.push(`${label} has unsupported expected privacy state: ${expectedState}`); + return `WARN: ${label} has unsupported expected privacy state: ${expectedState}`; + } +} + +async function main() { + const observations = await readJson(observationsPath); + const expectations = await readJson(expectationsPath); + const failures = []; + const warnings = []; + const checks = []; + + const visibleText = observations.visibleText ?? []; + const eventStream = observations.eventStream ?? []; + + checks.push( + pass( + visibleText.includes(expectations.replayChecks.screen) || + observations.replayScreen === expectations.replayChecks.screen, + `Replay reached the ${expectations.replayChecks.screen} screen`, + failures, + warnings, + ), + ); + + for (const eventCheck of expectations.replayChecks.requiredEvents) { + checks.push( + pass(includesAny(eventStream, eventCheck.contains), eventCheck.label, failures, warnings), + ); + } + + for (const booleanCheck of expectations.replayChecks.booleans) { + checks.push( + pass( + observations[booleanCheck.field] === true, + booleanCheck.label, + failures, + warnings, + { warn: booleanCheck.severity === "warn" }, + ), + ); + } + + checks.push(...validatePrivacyEvidence(observations, expectations, failures, warnings)); + + const status = failures.length > 0 ? "FAIL" : warnings.length > 0 ? "WARN" : "PASS"; + const report = `# Replay Validation Report + +Status: ${status} + +Goal: ${expectations.name} + +Session: ${observations.sessionUrl} + +## Checks + +${checks.map((check) => `- ${check}`).join("\n")} + +## Notes + +${(observations.notes ?? []).map((note) => `- ${note}`).join("\n")} + +## Replay Evidence Files + +${(observations.snapshotFiles ?? []).map((file) => `- ${file}`).join("\n")} + +## Privacy Evidence + +${(observations.privacyEvidence ?? []) + .map( + (evidence) => + `- ${evidence.label}: expected ${evidence.expectedPrivacyState}; replay observed ${evidence.replay?.state}; native flags ${evidence.engine?.state}`, + ) + .join("\n")} + +## Important + +This is semantic validation, not pixel-perfect validation. FullStory replay is approximate by design. The goal is to catch wrong screens, missing events, blank/frozen content, bad masking, broken scroll playback, bad dimensions, and other product-visible capture/replay issues. + +For privacy goals, replay visibility is only a failure when it contradicts the expected privacy state. Native flags should be added before treating masked, excluded, or omitted checks as fully proven. +`; + + const reportPath = path.join(outDir, expectations.replayReportName); + await fs.writeFile(reportPath, report); + console.log(`status: ${status}`); + console.log(`wrote ${reportPath}`); + + if (failures.length > 0) { + process.exit(1); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 6109a4f7262c4944cc0845e0a7838e7092a44a40 Mon Sep 17 00:00:00 2001 From: maciej Date: Mon, 11 May 2026 09:41:28 -0500 Subject: [PATCH 2/2] Capture FullStory replay URL in mobile harness runs Adds extract-session-url.mjs. It opens an Appium session, pulls the iOS NSUserDefaults plist for the target bundle, reads the FullStory previous-session and previous-user IDs, and writes the replay URL to fullstory-session-url.txt. The local Lidar runner now calls this when the goal finishes so the validate-* and Subtext review scripts have a real session URL to work with. No app code changes needed for any customer. The runner also calls mobile: terminateApp before each run so the SDK starts a fresh FullStory session instead of reusing the previous one. The replay URL uses MOBILE_FULLSTORY_APP_HOST so staging and EU orgs land on the right host. README and mobile.env.example are updated to match, including a note that this is iOS only for now. --- scripts/mobile/README.md | 8 ++ scripts/mobile/extract-session-url.mjs | 132 +++++++++++++++++++++++++ scripts/mobile/mobile.env.example | 4 + scripts/mobile/run-local-lidar-ios.mjs | 63 ++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 scripts/mobile/extract-session-url.mjs diff --git a/scripts/mobile/README.md b/scripts/mobile/README.md index f7060fbd..bb70cfeb 100644 --- a/scripts/mobile/README.md +++ b/scripts/mobile/README.md @@ -2,6 +2,8 @@ Drives any iOS app on a physical device, then checks whether the FullStory replay of that session looks right. Everything the runner needs to know about your app comes from two places: environment variables and a goal JSON file. Nothing is hardcoded for any particular app. +> iOS only for now. Android is not supported yet and will come in a follow-up. + There are two ways to drive the app: - **Lidar live tools** (`run-lidar-live-ios.mjs`) -- talks to a Lidar MCP server over HTTP. This is the primary path. @@ -40,6 +42,12 @@ Put these in `scripts/mobile/.env.local`. That file is gitignored. | `MOBILE_GOAL_EXPECTATIONS` | Path to the goal JSON file | | `MOBILE_OUT_DIR` | Where to write output artifacts | +**For non-prod FullStory orgs:** + +| Variable | What it is | +| --- | --- | +| `MOBILE_FULLSTORY_APP_HOST` | Base URL for replay links. Defaults to `https://app.fullstory.com`. Set to `https://app.staging.fullstory.com` for staging or `https://app.eu1.fullstory.com` for EU. | + **For Lidar runs:** | Variable | What it is | diff --git a/scripts/mobile/extract-session-url.mjs b/scripts/mobile/extract-session-url.mjs new file mode 100644 index 00000000..eec72e19 --- /dev/null +++ b/scripts/mobile/extract-session-url.mjs @@ -0,0 +1,132 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; +import { loadLocalEnv } from "./device-e2e-common.mjs"; + +await loadLocalEnv(new URL(".env.local", import.meta.url)); + +const outDir = process.env.MOBILE_OUT_DIR ?? DEFAULT_OUT_DIR; +const bundleId = process.env.MOBILE_BUNDLE_ID; +const orgId = process.env.LOCAL_MCP_ORG_ID ?? process.env.MOBILE_FULLSTORY_ORG; +const appHost = process.env.MOBILE_FULLSTORY_APP_HOST ?? "https://app.fullstory.com"; +const appiumUrl = + process.env.LIDAR_IOS_APPIUM_URL ?? process.env.MOBILE_APPIUM_URL ?? "http://127.0.0.1:4723"; +const udid = process.env.MOBILE_UDID; +const deviceName = process.env.MOBILE_DEVICE_NAME; + +async function appiumRequest(requestPath, { method = "GET", body } = {}) { + const response = await fetch(new URL(requestPath, appiumUrl), { + method, + headers: body === undefined ? undefined : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Appium ${method} ${requestPath} failed: HTTP ${response.status}\n${text}`); + } + return text ? JSON.parse(text).value : undefined; +} + +async function plistToJson(base64) { + const tmp = path.join(os.tmpdir(), `fs-defaults-${process.pid}-${Date.now()}.plist`); + await fs.writeFile(tmp, Buffer.from(base64, "base64")); + try { + return await new Promise((resolve, reject) => { + const proc = spawn("plutil", ["-convert", "json", "-o", "-", tmp]); + let stdout = ""; + let stderr = ""; + proc.stdout.on("data", (chunk) => (stdout += chunk.toString())); + proc.stderr.on("data", (chunk) => (stderr += chunk.toString())); + proc.on("error", reject); + proc.on("close", (code) => { + if (code !== 0) { + reject(new Error(`plutil exit ${code}: ${stderr}`)); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch (err) { + reject(err); + } + }); + }); + } finally { + await fs.rm(tmp, { force: true }); + } +} + +function buildSessionUrl({ host, org, userId, sessionId }) { + const segment = userId.includes("-") ? "client-session" : "session"; + return `${host}/ui/${org}/${segment}/${userId}:${sessionId}`; +} + +export async function extractSessionUrl({ outDir: outDirArg = outDir } = {}) { + if (!bundleId) { + throw new Error("Set MOBILE_BUNDLE_ID."); + } + if (!orgId) { + throw new Error("Set LOCAL_MCP_ORG_ID or MOBILE_FULLSTORY_ORG."); + } + if (!udid) { + throw new Error("Set MOBILE_UDID."); + } + + const session = await appiumRequest("/session", { + method: "POST", + body: { + capabilities: { + alwaysMatch: { + platformName: "iOS", + "appium:automationName": "XCUITest", + "appium:udid": udid, + "appium:deviceName": deviceName, + "appium:bundleId": bundleId, + "appium:autoLaunch": false, + "appium:noReset": true, + }, + }, + }, + }); + const sessionId = session.sessionId; + + let plist; + try { + const remotePath = `@${bundleId}/Library/Preferences/${bundleId}.plist`; + const base64 = await appiumRequest(`/session/${sessionId}/execute/sync`, { + method: "POST", + body: { script: "mobile: pullFile", args: [{ remotePath }] }, + }); + plist = await plistToJson(base64); + } finally { + await appiumRequest(`/session/${sessionId}`, { method: "DELETE" }).catch(() => {}); + } + + const fsSessionId = plist["FullStory.PreviousSessionId"]; + const fsUserId = plist["FullStory.PreviousUserId"]; + if (!fsSessionId || !fsUserId) { + const fsKeys = Object.keys(plist) + .filter((k) => k.startsWith("FullStory.")) + .join(", "); + throw new Error( + `FullStory session IDs not found in NSUserDefaults. FullStory.* keys present: [${fsKeys || "none"}]`, + ); + } + + const url = buildSessionUrl({ host: appHost, org: orgId, userId: fsUserId, sessionId: fsSessionId }); + const file = path.join(outDirArg, "fullstory-session-url.txt"); + await fs.mkdir(outDirArg, { recursive: true }); + await fs.writeFile(file, `${url}\n`); + console.log(`wrote ${file}`); + console.log(url); + return url; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + extractSessionUrl().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/scripts/mobile/mobile.env.example b/scripts/mobile/mobile.env.example index 5ad32ced..4b153e20 100644 --- a/scripts/mobile/mobile.env.example +++ b/scripts/mobile/mobile.env.example @@ -31,6 +31,10 @@ FS_HOME= LOCAL_MCP_ORG_ID= LOCAL_MCP_EMAIL= +# Base URL for FullStory replay links. Defaults to https://app.fullstory.com. +# Set to https://app.staging.fullstory.com for staging, https://app.eu1.fullstory.com for EU. +MOBILE_FULLSTORY_APP_HOST= + # Optional: regex pattern to detect app launch in console output. # Default matches "Launched application". Set to match your app if needed. MOBILE_CONSOLE_LAUNCH_PATTERN= diff --git a/scripts/mobile/run-local-lidar-ios.mjs b/scripts/mobile/run-local-lidar-ios.mjs index fd5407db..54c18615 100644 --- a/scripts/mobile/run-local-lidar-ios.mjs +++ b/scripts/mobile/run-local-lidar-ios.mjs @@ -7,6 +7,7 @@ import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { DEFAULT_OUT_DIR } from "./appium-layer.mjs"; import { loadLocalEnv, timestampSlug } from "./device-e2e-common.mjs"; +import { extractSessionUrl } from "./extract-session-url.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); const localEnvPath = new URL(".env.local", import.meta.url); @@ -295,6 +296,57 @@ async function ensureAppium(outDir) { return { appiumURL, stop: child.stop }; } +async function appiumHttp(appiumURL, requestPath, { method = "GET", body } = {}) { + const response = await fetch(new URL(requestPath, appiumURL), { + method, + headers: body === undefined ? undefined : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Appium ${method} ${requestPath} failed: HTTP ${response.status}\n${text}`); + } + return text ? JSON.parse(text).value : undefined; +} + +async function terminateApp(appiumURL) { + const bundleId = process.env.MOBILE_BUNDLE_ID; + const udid = process.env.MOBILE_UDID; + if (!bundleId || !udid) { + return; + } + let sessionId; + try { + const session = await appiumHttp(appiumURL, "/session", { + method: "POST", + body: { + capabilities: { + alwaysMatch: { + platformName: "iOS", + "appium:automationName": "XCUITest", + "appium:udid": udid, + "appium:bundleId": bundleId, + "appium:autoLaunch": false, + "appium:noReset": true, + }, + }, + }, + }); + sessionId = session.sessionId; + await appiumHttp(appiumURL, `/session/${sessionId}/execute/sync`, { + method: "POST", + body: { script: "mobile: terminateApp", args: [{ bundleId }] }, + }); + console.log(`terminated ${bundleId}`); + } catch (err) { + console.warn(`could not terminate ${bundleId}: ${err.message}`); + } finally { + if (sessionId) { + await appiumHttp(appiumURL, `/session/${sessionId}`, { method: "DELETE" }).catch(() => {}); + } + } +} + async function buildLidar(binaryPath) { if (process.env.MOBILE_LIDAR_BUILD === "0") { return; @@ -350,6 +402,7 @@ async function main() { } const appium = await ensureAppium(outDir); + await terminateApp(appium.appiumURL); const lidar = await startLidar(outDir, appium.appiumURL); const caps = await generateCaps(); const childEnv = { @@ -366,8 +419,18 @@ async function main() { }); } finally { await lidar.stop(); + try { + await extractSessionUrl({ outDir }); + } catch (err) { + console.warn(`could not extract FullStory session URL: ${err.message}`); + } await appium.stop(); } + + const lastRunPath = path.join(here, "tmp", ".last-run-dir"); + await fs.mkdir(path.dirname(lastRunPath), { recursive: true }); + await fs.writeFile(lastRunPath, outDir); + console.log(`wrote ${lastRunPath}`); } main().catch((err) => {