diff --git a/README.md b/README.md index 962efba..4dc306b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

SimDeck is a developer tool built for streamlining mobile app development for coding agents. - Drive Simulator from the CLI using agents, browser, and automated tests on macOS. + Drive iOS Simulators and Android emulators from the CLI using agents, browser, and automated tests on macOS.

@@ -35,8 +35,9 @@ view inside the editor. ## Features -- Local simulator video stream over browser-native WebRTC H.264 -- Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI +- Local iOS Simulator and Android emulator video over browser-native WebRTC H.264 +- Android emulator frames are sourced from emulator gRPC and encoded through macOS VideoToolbox +- Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels - NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live @@ -133,6 +134,7 @@ simdeck boot simdeck shutdown simdeck erase simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck open-url https://example.com simdeck launch com.apple.Preferences @@ -172,6 +174,14 @@ simdeck logs --seconds 30 --limit 200 without launching Simulator.app, then falls back to `xcrun simctl` when private booting is unavailable. +Android emulators appear in `simdeck list` with IDs like +`android:SimDeck_Pixel_8_API_36`. For Android IDs, lifecycle, install, launch, +URL, screenshot, logs, UIAutomator `describe`, tap, swipe, text, key, home, app +switcher, rotation, pasteboard, and browser live view route through the Android +SDK tools (`emulator` and `adb`) plus the emulator gRPC screenshot stream for +live video. `simdeck stream` remains iOS-only because it writes the iOS H.264 +transport stream. + `stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or external tools such as `ffplay`. diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index bc02dae..768a3fe 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -84,6 +84,11 @@ bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_frame_callback _Nullable callback, void * _Nullable user_data); +void * _Nullable xcw_native_h264_encoder_create(xcw_native_frame_callback _Nullable callback, void * _Nullable user_data, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_destroy(void * _Nullable handle); +bool xcw_native_h264_encoder_encode_rgba(void * _Nonnull handle, const uint8_t * _Nonnull rgba, size_t length, uint32_t width, uint32_t height, uint64_t timestamp_us, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_request_keyframe(void * _Nonnull handle); + void xcw_native_free_string(char * _Nullable value); void xcw_native_free_bytes(xcw_native_owned_bytes bytes); void xcw_native_release_shared_bytes(xcw_native_shared_bytes bytes); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index bdbaa45..4f73f96 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -3,11 +3,13 @@ #import "DFPrivateSimulatorDisplayBridge.h" #import "XCWAccessibilityBridge.h" #import "XCWChromeRenderer.h" +#import "XCWH264Encoder.h" #import "XCWNativeSession.h" #import "XCWSimctl.h" #import #import +#import #include #include @@ -63,10 +65,190 @@ static xcw_native_owned_bytes XCWOwnedBytesFromData(NSData *data) { return bytes; } +static xcw_native_shared_bytes XCWSharedBytesFromData(NSData *data) { + if (data.length == 0) { + return (xcw_native_shared_bytes){0}; + } + + CFTypeRef owner = CFRetain((__bridge CFTypeRef)data); + return (xcw_native_shared_bytes){ + .data = data.bytes, + .length = data.length, + .owner = (const void *)owner, + }; +} + static XCWNativeSession *XCWNativeSessionFromHandle(void *handle) { return (__bridge XCWNativeSession *)handle; } +@interface XCWNativeH264Encoder : NSObject + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData; +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error; +- (void)requestKeyFrame; +- (void)invalidate; + +@end + +@implementation XCWNativeH264Encoder { + XCWH264Encoder *_encoder; + xcw_native_frame_callback _callback; + void *_callbackUserData; + uint64_t _frameSequence; +} + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData { + self = [super init]; + if (self == nil) { + return nil; + } + + _callback = callback; + _callbackUserData = userData; + __weak typeof(self) weakSelf = self; + @synchronized (XCWNativeH264Encoder.class) { + const char *previousCodec = getenv("SIMDECK_VIDEO_CODEC"); + char *previousCodecCopy = previousCodec != NULL ? strdup(previousCodec) : NULL; + const char *androidCodec = getenv("SIMDECK_ANDROID_VIDEO_CODEC"); + if (androidCodec == NULL || strlen(androidCodec) == 0) { + androidCodec = "software"; + } + setenv("SIMDECK_VIDEO_CODEC", androidCodec, 1); + _encoder = [[XCWH264Encoder alloc] initWithOutputHandler:^(NSData *sampleData, + uint64_t timestampUs, + BOOL isKeyFrame, + NSString * _Nullable codec, + NSData * _Nullable decoderConfig, + CGSize dimensions) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil || strongSelf->_callback == NULL || sampleData.length == 0) { + return; + } + strongSelf->_frameSequence += 1; + xcw_native_frame frame = { + .frame_sequence = strongSelf->_frameSequence, + .timestamp_us = timestampUs, + .is_keyframe = isKeyFrame, + .width = (uint32_t)llround(dimensions.width), + .height = (uint32_t)llround(dimensions.height), + .codec = codec.UTF8String, + .description = XCWSharedBytesFromData(decoderConfig), + .data = XCWSharedBytesFromData(sampleData), + }; + strongSelf->_callback(&frame, strongSelf->_callbackUserData); + }]; + if (previousCodecCopy != NULL) { + setenv("SIMDECK_VIDEO_CODEC", previousCodecCopy, 1); + free(previousCodecCopy); + } else { + unsetenv("SIMDECK_VIDEO_CODEC"); + } + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error { + if (rgba == NULL || width == 0 || height == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:1 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was empty." }]; + } + return NO; + } + size_t expectedLength = (size_t)width * (size_t)height * 4; + if (length < expectedLength) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:2 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was truncated." }]; + } + return NO; + } + + NSDictionary *attributes = @{ + (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (__bridge NSString *)kCVPixelBufferWidthKey: @(width), + (__bridge NSString *)kCVPixelBufferHeightKey: @(height), + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + CVPixelBufferRef pixelBuffer = NULL; + CVReturn createStatus = CVPixelBufferCreate(kCFAllocatorDefault, + (size_t)width, + (size_t)height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attributes, + &pixelBuffer); + if (createStatus != kCVReturnSuccess || pixelBuffer == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:createStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to allocate a VideoToolbox pixel buffer." }]; + } + return NO; + } + + CVReturn lockStatus = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (lockStatus != kCVReturnSuccess) { + CVPixelBufferRelease(pixelBuffer); + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:lockStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to lock a VideoToolbox pixel buffer." }]; + } + return NO; + } + + uint8_t *dst = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t dstRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + size_t srcRowBytes = (size_t)width * 4; + for (uint32_t y = 0; y < height; y += 1) { + const uint8_t *srcRow = rgba + ((size_t)y * srcRowBytes); + uint8_t *dstRow = dst + ((size_t)y * dstRowBytes); + for (uint32_t x = 0; x < width; x += 1) { + const uint8_t *src = srcRow + ((size_t)x * 4); + uint8_t *pixel = dstRow + ((size_t)x * 4); + pixel[0] = src[2]; + pixel[1] = src[1]; + pixel[2] = src[0]; + pixel[3] = src[3]; + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + [_encoder encodePixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + return YES; +} + +- (void)requestKeyFrame { + [_encoder requestKeyFrame]; +} + +- (void)invalidate { + [_encoder invalidate]; +} + +@end + +static XCWNativeH264Encoder *XCWNativeH264EncoderFromHandle(void *handle) { + return (__bridge XCWNativeH264Encoder *)handle; +} + static BOOL XCWPerformSimctlAction(char **errorMessage, BOOL (^action)(XCWSimctl *simctl, NSError **error)) { XCWSimctl *simctl = [[XCWSimctl alloc] init]; NSError *error = nil; @@ -803,6 +985,58 @@ void xcw_native_session_set_frame_callback(void *handle, xcw_native_frame_callba } } +void *xcw_native_h264_encoder_create(xcw_native_frame_callback callback, void *user_data, char **error_message) { + @autoreleasepool { + XCWNativeH264Encoder *encoder = [[XCWNativeH264Encoder alloc] initWithFrameCallback:callback + userData:user_data]; + if (encoder == nil) { + if (error_message != NULL) { + *error_message = XCWCopyCString(@"Unable to create the native H.264 encoder."); + } + return NULL; + } + return (__bridge_retained void *)encoder; + } +} + +void xcw_native_h264_encoder_destroy(void *handle) { + if (handle == NULL) { + return; + } + @autoreleasepool { + XCWNativeH264Encoder *encoder = CFBridgingRelease(handle); + [encoder invalidate]; + } +} + +bool xcw_native_h264_encoder_encode_rgba(void *handle, + const uint8_t *rgba, + size_t length, + uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)timestamp_us; + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeH264EncoderFromHandle(handle) encodeRGBA:rgba + length:length + width:width + height:height + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + +void xcw_native_h264_encoder_request_keyframe(void *handle) { + @autoreleasepool { + [XCWNativeH264EncoderFromHandle(handle) requestKeyFrame]; + } +} + void xcw_native_free_string(char *value) { if (value != NULL) { free(value); diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 44a65b5..2092319 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -10,11 +10,17 @@ export interface PrivateDisplayInfo { export interface SimulatorMetadata { udid: string; name: string; + platform?: "ios-simulator" | "android-emulator" | string; runtimeName?: string; runtimeIdentifier?: string; deviceTypeName?: string; deviceTypeIdentifier?: string; isBooted: boolean; + android?: { + avdName?: string; + grpcPort?: number; + serial?: string; + }; privateDisplay?: PrivateDisplayInfo; } @@ -43,6 +49,7 @@ export interface ChromeProfile { screenWidth: number; screenHeight: number; cornerRadius: number; + chromeStyle?: "asset" | "css-android" | string; hasScreenMask?: boolean; } diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index b3f9580..56e45df 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -164,6 +164,9 @@ function shouldUseRemoteStreamDefault(apiRoot: string): boolean { } function shouldRenderNativeChrome(simulator: SimulatorMetadata): boolean { + if (simulator.platform === "android-emulator") { + return true; + } const identifier = simulator.deviceTypeIdentifier ?? ""; const name = simulator.name ?? ""; return ( @@ -604,9 +607,12 @@ export function AppShell({ const chromeUrl = selectedSimulator ? buildChromeUrl(selectedSimulator.udid, streamStamp) : ""; + const chromeUsesAsset = Boolean( + viewportChromeProfile && viewportChromeProfile.chromeStyle !== "css-android", + ); const chromeRequired = Boolean( (shouldRenderChrome && !chromeProfileReady) || - (viewportChromeProfile && chromeUrl), + (chromeUsesAsset && chromeUrl), ); const simulatorRotationQuarterTurns = normalizeSimulatorRotationQuarterTurns(selectedSimulator); @@ -1665,6 +1671,7 @@ export function AppShell({ chromeProfile={viewportChromeProfile} chromeRequired={chromeRequired} chromeScreenStyle={viewportScreenStyle} + chromeStyle={viewportChromeProfile?.chromeStyle} chromeUrl={chromeUrl} debugPanel={ debugVisible ? ( diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 6c0f4b2..63cfe2f 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -26,7 +26,7 @@ let activeWebRtcControlChannel: RTCDataChannel | null = null; let activeWebRtcTelemetryChannel: RTCDataChannel | null = null; let activeStreamClient: StreamWorkerClient | null = null; -export type StreamBackend = "webrtc"; +export type StreamBackend = "screenshot" | "webrtc"; export function sendWebRtcControlMessage(encoded: string): boolean { return sendDataChannelMessage(activeWebRtcControlChannel, encoded); diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index d3eccf5..293f093 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -1,6 +1,6 @@ import { useEffect, useRef, useState } from "react"; -import { apiHeaders } from "../../api/client"; +import { accessTokenFromLocation, apiHeaders } from "../../api/client"; import { apiUrl } from "../../api/config"; import type { SimulatorMetadata } from "../../api/types"; import type { Size } from "../viewport/types"; @@ -26,6 +26,10 @@ const CLIENT_TELEMETRY_INTERVAL_MS = 1000; const REMOTE_CLIENT_TELEMETRY_INTERVAL_MS = 5000; const CLIENT_TELEMETRY_ID_STORAGE_KEY = "simdeck.streamClientId"; const VISUAL_ARTIFACT_TELEMETRY_INTERVAL_MS = 1000; +const SCREENSHOT_POLL_INTERVAL_MS = 500; +const ANDROID_GRPC_STREAM_MAX_EDGE = 960; +const ANDROID_GRPC_STREAM_MAX_FPS = 30; +const ANDROID_FRAME_HEADER_BYTES = 32; interface UseLiveStreamOptions { canvasElement: HTMLCanvasElement | null; @@ -88,6 +92,31 @@ function buildClientTelemetryUrl(): string { ).toString(); } +function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { + return simulator?.platform === "android-emulator"; +} + +function clearCanvas(canvasElement: HTMLCanvasElement | null): void { + if (!canvasElement) { + return; + } + const context = canvasElement.getContext("2d"); + if (!context) { + return; + } + context.clearRect(0, 0, canvasElement.width, canvasElement.height); +} + +function buildWebSocketUrl(path: string): string { + const url = new URL(apiUrl(path), window.location.href); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + const token = accessTokenFromLocation(); + if (token) { + url.searchParams.set("simdeckToken", token); + } + return url.toString(); +} + export function useLiveStream({ canvasElement, paused = false, @@ -114,6 +143,8 @@ export function useLiveStream({ const [fps, setFps] = useState(0); const [streamCanvasRevision, setStreamCanvasRevision] = useState(0); const [runtimeInfo] = useState(detectRuntimeInfo); + const androidSimulator = isAndroidSimulator(simulator); + const androidRawFrameStream = false; if (!clientTelemetryIdRef.current) { clientTelemetryIdRef.current = createClientTelemetryId(); @@ -144,6 +175,12 @@ export function useLiveStream({ }, []); useEffect(() => { + if (androidRawFrameStream) { + workerClientRef.current?.destroy(); + workerClientRef.current = null; + return; + } + if (paused || !canvasElement || workerClientRef.current) { return; } @@ -202,7 +239,7 @@ export function useLiveStream({ workerClient.destroy(); workerClientRef.current = null; }; - }, [canvasElement, paused]); + }, [androidRawFrameStream, canvasElement, paused]); useEffect(() => { latestDecodedFramesRef.current = stats.decodedFrames; @@ -255,6 +292,10 @@ export function useLiveStream({ }, [simulator?.udid]); useEffect(() => { + if (androidRawFrameStream) { + return; + } + const workerClient = workerClientRef.current; if (!workerClient) { return; @@ -290,10 +331,264 @@ export function useLiveStream({ return () => { workerClient.disconnect(); }; - }, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]); + }, [ + androidRawFrameStream, + canvasElement, + simulator?.isBooted, + simulator?.udid, + paused, + remote, + ]); + + useEffect(() => { + if (!androidRawFrameStream) { + return; + } + + setDeviceNaturalSize(null); + setStats(createEmptyStreamStats()); + setStatus({ state: "idle" }); + setError(""); + setFps(0); + + if (paused || !canvasElement || !simulator?.isBooted) { + clearCanvas(canvasElement); + return; + } + + const context = canvasElement.getContext("2d", { alpha: false }); + if (!context) { + const message = "Unable to attach the screenshot stream canvas."; + setError(message); + setStatus({ error: message, state: "error" }); + return; + } + + let cancelled = false; + let frameSequence = 0; + let lastFrameRenderedAt = 0; + const controller = new AbortController(); + let fallbackStarted = false; + + const noteRenderedFrame = ( + width: number, + height: number, + renderStartedAt: number, + codec: string, + ) => { + const frameRenderedAt = performance.now(); + const renderMs = frameRenderedAt - renderStartedAt; + const latestFrameGapMs = + lastFrameRenderedAt > 0 ? frameRenderedAt - lastFrameRenderedAt : 0; + lastFrameRenderedAt = frameRenderedAt; + frameSequence += 1; + setDeviceNaturalSize({ height, width }); + setStatus({ state: "streaming" }); + setError(""); + setStats((current) => ({ + ...current, + averageRenderMs: + (current.averageRenderMs * Math.max(0, frameSequence - 1) + + renderMs) / + frameSequence, + codec, + decodedFrames: frameSequence, + frameSequence, + height, + latestFrameGapMs, + latestRenderMs: renderMs, + maxRenderMs: Math.max(current.maxRenderMs, renderMs), + receivedPackets: frameSequence, + renderedFrames: frameSequence, + width, + })); + }; + + const renderRgbaFrame = (buffer: ArrayBuffer) => { + if (buffer.byteLength <= ANDROID_FRAME_HEADER_BYTES) { + throw new Error("Android frame was truncated."); + } + const view = new DataView(buffer); + if ( + view.getUint8(0) !== 0x53 || + view.getUint8(1) !== 0x44 || + view.getUint8(2) !== 0x41 || + view.getUint8(3) !== 0x46 + ) { + throw new Error("Android frame had an invalid header."); + } + const width = view.getUint32(8, true); + const height = view.getUint32(12, true); + const expectedBytes = ANDROID_FRAME_HEADER_BYTES + width * height * 4; + if (width <= 0 || height <= 0 || buffer.byteLength < expectedBytes) { + throw new Error("Android frame dimensions were invalid."); + } + const renderStartedAt = performance.now(); + if (canvasElement.width !== width || canvasElement.height !== height) { + canvasElement.width = width; + canvasElement.height = height; + } + context.putImageData( + new ImageData( + new Uint8ClampedArray( + buffer, + ANDROID_FRAME_HEADER_BYTES, + width * height * 4, + ), + width, + height, + ), + 0, + 0, + ); + noteRenderedFrame(width, height, renderStartedAt, "rgba-grpc"); + }; + + const renderScreenshot = async () => { + const url = new URL( + apiUrl( + `/api/simulators/${encodeURIComponent(simulator.udid)}/screenshot.png`, + ), + window.location.href, + ); + url.searchParams.set("stamp", String(Date.now())); + const response = await fetch(url.toString(), { + cache: "no-store", + headers: apiHeaders({ Accept: "image/png" }), + signal: controller.signal, + }); + if (!response.ok) { + throw new Error(`Screenshot request failed with ${response.status}.`); + } + + const renderStartedAt = performance.now(); + const bitmap = await createImageBitmap(await response.blob()); + try { + if ( + canvasElement.width !== bitmap.width || + canvasElement.height !== bitmap.height + ) { + canvasElement.width = bitmap.width; + canvasElement.height = bitmap.height; + } + context.drawImage(bitmap, 0, 0); + } finally { + bitmap.close?.(); + } + + noteRenderedFrame(canvasElement.width, canvasElement.height, renderStartedAt, "png"); + }; + + const pollScreenshots = async () => { + setStatus({ detail: "Waiting for Android screenshots.", state: "connecting" }); + while (!cancelled) { + const startedAt = performance.now(); + try { + await renderScreenshot(); + } catch (pollError) { + if (cancelled || controller.signal.aborted) { + return; + } + const message = + pollError instanceof Error + ? pollError.message + : "Unable to load Android screenshot."; + setError(message); + setStatus({ error: message, state: "error" }); + } + const elapsedMs = performance.now() - startedAt; + const waitMs = Math.max(80, SCREENSHOT_POLL_INTERVAL_MS - elapsedMs); + await new Promise((resolve) => window.setTimeout(resolve, waitMs)); + } + }; + + const startScreenshotFallback = () => { + if (fallbackStarted || cancelled) { + return; + } + fallbackStarted = true; + void pollScreenshots(); + }; + + const streamUrl = new URL( + buildWebSocketUrl( + `/api/simulators/${encodeURIComponent(simulator.udid)}/android/frames`, + ), + ); + streamUrl.searchParams.set( + "maxEdge", + String(streamConfig?.maxEdge ?? ANDROID_GRPC_STREAM_MAX_EDGE), + ); + streamUrl.searchParams.set( + "maxFps", + String(Math.min(60, streamConfig?.fps ?? ANDROID_GRPC_STREAM_MAX_FPS)), + ); + const frameSocket = new WebSocket(streamUrl.toString()); + frameSocket.binaryType = "arraybuffer"; + frameSocket.onopen = () => { + setStatus({ detail: "Waiting for Android emulator frames.", state: "connecting" }); + }; + frameSocket.onmessage = (event) => { + if (cancelled) { + return; + } + if (typeof event.data === "string") { + try { + const message = JSON.parse(event.data) as { error?: string; type?: string }; + if (message.error) { + throw new Error(message.error); + } + } catch (streamError) { + const message = + streamError instanceof Error + ? streamError.message + : "Android emulator frame stream failed."; + setError(message); + setStatus({ error: message, state: "error" }); + startScreenshotFallback(); + } + return; + } + try { + renderRgbaFrame(event.data as ArrayBuffer); + } catch (streamError) { + const message = + streamError instanceof Error + ? streamError.message + : "Unable to render Android emulator frame."; + setError(message); + setStatus({ error: message, state: "error" }); + startScreenshotFallback(); + } + }; + frameSocket.onerror = () => { + if (frameSequence === 0) { + startScreenshotFallback(); + } + }; + frameSocket.onclose = () => { + if (!cancelled && frameSequence === 0) { + startScreenshotFallback(); + } + }; + return () => { + cancelled = true; + controller.abort(); + frameSocket.close(); + }; + }, [ + androidRawFrameStream, + canvasElement, + simulator?.isBooted, + simulator?.udid, + paused, + streamConfig?.fps, + streamConfig?.maxEdge, + ]); useEffect(() => { if ( + androidRawFrameStream || streamConfigApplyKey <= 0 || paused || !simulator?.isBooted || @@ -310,6 +605,7 @@ export function useLiveStream({ streamConfig?.fps, streamConfig?.maxEdge, streamConfig?.quality, + androidRawFrameStream, ]); useEffect(() => { @@ -378,15 +674,18 @@ export function useLiveStream({ }; }, [remote, simulator?.udid]); + const effectiveRuntimeInfo = runtimeInfo; + const streamBackend: StreamBackend = "webrtc"; + return { deviceNaturalSize, error, fps, hasFrame: status.state === "streaming" || stats.decodedFrames > 0, - runtimeInfo, + runtimeInfo: effectiveRuntimeInfo, stats, status, - streamBackend: "webrtc", - streamCanvasKey: `webrtc-${streamCanvasRevision}`, + streamBackend, + streamCanvasKey: `${streamBackend}-${streamCanvasRevision}`, }; } diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index be333aa..a00a935 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -12,6 +12,7 @@ interface DeviceChromeProps { accessibilityRoots: AccessibilityNode[]; accessibilitySelectedId: string; chromeScreenStyle: CSSProperties | null; + chromeStyle?: string; chromeUrl: string; hasFrame: boolean; isBooted: boolean; @@ -47,6 +48,7 @@ export function DeviceChrome({ accessibilityRoots, accessibilitySelectedId, chromeScreenStyle, + chromeStyle, chromeUrl, hasFrame, isBooted, @@ -76,23 +78,28 @@ export function DeviceChrome({ useChromeProfile, }: DeviceChromeProps) { if (useChromeProfile) { + const useCssAndroidChrome = chromeStyle === "css-android"; return (
- + {useCssAndroidChrome ? ( +