diff --git a/client/package.json b/client/package.json index 64a0401f..6a961ef4 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "my-project", - "version": "0.1.41", + "version": "0.1.42", "private": true, "scripts": { "dev": "vite --host 0.0.0.0 --port 3000", diff --git a/client/src-tauri/Cargo.lock b/client/src-tauri/Cargo.lock index 880cbd21..6c751b16 100644 --- a/client/src-tauri/Cargo.lock +++ b/client/src-tauri/Cargo.lock @@ -11839,7 +11839,7 @@ dependencies = [ [[package]] name = "xero-cli" -version = "0.1.41" +version = "0.1.42" dependencies = [ "crossterm", "flate2", @@ -11860,7 +11860,7 @@ dependencies = [ [[package]] name = "xero-desktop" -version = "0.1.41" +version = "0.1.42" dependencies = [ "arc-swap", "arrow-array", diff --git a/client/src-tauri/Cargo.toml b/client/src-tauri/Cargo.toml index 931721fd..36288f4b 100644 --- a/client/src-tauri/Cargo.toml +++ b/client/src-tauri/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" [package] name = "xero-desktop" -version = "0.1.41" +version = "0.1.42" edition = "2021" default-run = "xero-desktop" description = "Xero desktop host" diff --git a/client/src-tauri/crates/xero-cli/Cargo.toml b/client/src-tauri/crates/xero-cli/Cargo.toml index cc1ad26e..99ff5541 100644 --- a/client/src-tauri/crates/xero-cli/Cargo.toml +++ b/client/src-tauri/crates/xero-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xero-cli" -version = "0.1.41" +version = "0.1.42" edition = "2021" description = "Headless Xero CLI backed by xero-agent-core" diff --git a/client/src-tauri/crates/xero-remote-bridge/src/lib.rs b/client/src-tauri/crates/xero-remote-bridge/src/lib.rs index 64ad2ab4..0bc9c159 100644 --- a/client/src-tauri/crates/xero-remote-bridge/src/lib.rs +++ b/client/src-tauri/crates/xero-remote-bridge/src/lib.rs @@ -1155,6 +1155,11 @@ where } } + pub fn refresh_account_devices(&self) -> BridgeResult> { + self.clear_account_devices_cache()?; + self.list_account_devices() + } + fn list_account_devices_once(&self) -> BridgeResult> { let Some(identity) = self.identity_store.load()? else { return Ok(Vec::new()); @@ -2393,6 +2398,60 @@ mod tests { .is_none()); } + #[test] + fn refresh_account_devices_bypasses_fresh_cache() { + let (relay_url, server) = serve_http_responses(vec![( + 200, + json!({ + "devices": [{ + "id": "web-2", + "account_id": "account-1", + "kind": "web", + "name": "Xero Web", + "user_agent": "browser", + "last_seen": null, + "created_at": "2026-05-31T00:00:00Z", + "revoked_at": null + }] + }) + .to_string(), + )]); + let temp = tempfile_path("device-cache-refresh"); + let identity_store = FileIdentityStore::new(temp.join("identity.json")); + let identity = test_identity(); + identity_store.save(&identity).expect("identity"); + let bridge = RemoteBridge::new( + BridgeConfig { + relay_url, + device_name: Some("Xero Test".into()), + }, + identity_store, + ); + bridge + .store_account_devices_cache( + &identity, + vec![AccountDevice { + id: "web-1".into(), + account_id: "account-1".into(), + kind: "web".into(), + name: Some("Old Web".into()), + user_agent: Some("browser".into()), + last_seen: None, + created_at: "2026-05-30T00:00:00Z".into(), + revoked_at: None, + }], + ) + .expect("store stale devices"); + + let devices = bridge.refresh_account_devices().expect("devices"); + + assert_eq!(devices.len(), 1); + assert_eq!(devices[0].id, "web-2"); + let requests = server.join().expect("fake relay thread"); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET /api/devices ")); + } + #[test] fn list_account_devices_refreshes_expired_token_before_request() { let (relay_url, server) = serve_http_responses(vec![ diff --git a/client/src-tauri/src/commands/remote_bridge.rs b/client/src-tauri/src/commands/remote_bridge.rs index 94890f71..7064db3b 100644 --- a/client/src-tauri/src/commands/remote_bridge.rs +++ b/client/src-tauri/src/commands/remote_bridge.rs @@ -684,14 +684,13 @@ fn handle_inbound_command( } let result = route_inbound_command(app, state, Arc::clone(&bridge), command.clone()); if let Err(error) = &result { - let _ = bridge.forward_control_event( - &response_session, - json!({ - "schema": "xero.remote_command_result.v1", - "ok": false, - "error": error, - }), - ); + let mut payload = json!({ + "schema": "xero.remote_command_result.v1", + "ok": false, + "error": error, + }); + attach_command_context(&mut payload, &command, Some("rejected")); + let _ = bridge.forward_control_event(&response_session, payload); } if let Err(error) = &result { let _ = bridge.forward_control_event( @@ -963,10 +962,12 @@ fn route_inbound_command( fn ensure_known_web_device(bridge: &AppRemoteBridge, device_id: &str) -> CommandResult<()> { validate_non_empty(device_id, "deviceId")?; let devices = bridge.list_account_devices().map_err(map_bridge_error)?; - if devices - .iter() - .any(|device| device.kind == "web" && device.revoked_at.is_none() && device.id == device_id) - { + if account_devices_include_web_device(&devices, device_id) { + return Ok(()); + } + + let devices = bridge.refresh_account_devices().map_err(map_bridge_error)?; + if account_devices_include_web_device(&devices, device_id) { return Ok(()); } @@ -975,6 +976,12 @@ fn ensure_known_web_device(bridge: &AppRemoteBridge, device_id: &str) -> Command )) } +fn account_devices_include_web_device(devices: &[AccountDevice], device_id: &str) -> bool { + devices + .iter() + .any(|device| device.kind == "web" && device.revoked_at.is_none() && device.id == device_id) +} + fn route_authorize_session_join( app: &AppHandle, state: &DesktopState, @@ -4795,6 +4802,46 @@ mod tests { assert!(!command_is_duplicate(&command)); } + #[test] + fn known_web_device_check_ignores_revoked_and_desktop_devices() { + let devices = vec![ + AccountDevice { + id: "desktop-1".into(), + account_id: "account-1".into(), + kind: "desktop".into(), + name: Some("Xero Desktop".into()), + user_agent: None, + last_seen: None, + created_at: "2026-05-31T00:00:00Z".into(), + revoked_at: None, + }, + AccountDevice { + id: "web-1".into(), + account_id: "account-1".into(), + kind: "web".into(), + name: Some("Old Web".into()), + user_agent: Some("browser".into()), + last_seen: None, + created_at: "2026-05-31T00:00:00Z".into(), + revoked_at: Some("2026-05-31T00:05:00Z".into()), + }, + AccountDevice { + id: "web-2".into(), + account_id: "account-1".into(), + kind: "web".into(), + name: Some("Xero Web".into()), + user_agent: Some("browser".into()), + last_seen: None, + created_at: "2026-05-31T00:10:00Z".into(), + revoked_at: None, + }, + ]; + + assert!(!account_devices_include_web_device(&devices, "desktop-1")); + assert!(!account_devices_include_web_device(&devices, "web-1")); + assert!(account_devices_include_web_device(&devices, "web-2")); + } + #[test] fn inbound_pointer_coalescing_keeps_latest_contiguous_move() { let (sender, receiver) = std::sync::mpsc::sync_channel(8); diff --git a/client/src-tauri/tauri.conf.json b/client/src-tauri/tauri.conf.json index 5cde45dd..acfa48d4 100644 --- a/client/src-tauri/tauri.conf.json +++ b/client/src-tauri/tauri.conf.json @@ -2,7 +2,7 @@ "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "Xero", "mainBinaryName": "xero-desktop", - "version": "0.1.41", + "version": "0.1.42", "identifier": "com.hyperpush.xero", "build": { "beforeDevCommand": "pnpm dev", diff --git a/cloud/src/routes/-desktop-click-ripple.test.tsx b/cloud/src/routes/-desktop-click-ripple.test.tsx index dc624452..b666a34c 100644 --- a/cloud/src/routes/-desktop-click-ripple.test.tsx +++ b/cloud/src/routes/-desktop-click-ripple.test.tsx @@ -163,6 +163,73 @@ describe("ComputerUseDesktopViewport click feedback", () => { ).toBeTruthy(); }); + it("surfaces desktop-side stream start failures after the relay accepts Start", async () => { + let frameHandler: ((rawFrame: unknown) => void) | null = null; + const push = vi.fn(); + const channel = { + on: vi.fn((event: string, handler: (rawFrame: unknown) => void) => { + if (event === "frame") frameHandler = handler; + return "frame-ref"; + }), + off: vi.fn(), + push, + } as unknown as Channel; + + render( + , + ); + + const desktop = screen.getByLabelText("Desktop"); + const toolbar = within(desktop).getByRole("toolbar", { + name: "Desktop stream controls", + }); + fireEvent.click(within(toolbar).getByRole("button", { name: /start/i })); + expect(screen.getByText("Connecting stream")).toBeTruthy(); + + act(() => { + frameHandler?.( + relayFrame({ + schema: "xero.remote_command_result.v1", + ok: false, + kind: "computer_use_stream_request", + outcome: "rejected", + error: { + code: "policy_denied", + message: + "Remote command rejected because the web device is not linked or has been revoked.", + }, + }), + ); + }); + + expect(await screen.findByText("Desktop unavailable")).toBeTruthy(); + expect( + screen.getByText( + "Remote command rejected because the web device is not linked or has been revoked.", + ), + ).toBeTruthy(); + expect( + within(toolbar).getByRole("button", { name: /start/i }), + ).toBeTruthy(); + }); + it("retries the desktop stream request when connecting stalls before media arrives", () => { vi.useFakeTimers(); try { diff --git a/cloud/src/routes/sessions.$computerId.$sessionId.tsx b/cloud/src/routes/sessions.$computerId.$sessionId.tsx index ab0f4b90..262c9941 100644 --- a/cloud/src/routes/sessions.$computerId.$sessionId.tsx +++ b/cloud/src/routes/sessions.$computerId.$sessionId.tsx @@ -1085,6 +1085,7 @@ type DesktopViewportState = | "paused" | "manual" | "blocked" + | "failed" | "offline"; type ManualControlState = @@ -1994,6 +1995,9 @@ export function ComputerUseDesktopViewport({ useState("balanced"); const [hasLiveVideo, setHasLiveVideo] = useState(false); const [toolbarPromptPending, setToolbarPromptPending] = useState(false); + const [streamFailureMessage, setStreamFailureMessage] = useState< + string | null + >(null); const [clickRipples, setClickRipples] = useState([]); const [keyboardCaptureState, setKeyboardCaptureState] = useState("inactive"); @@ -2464,6 +2468,7 @@ export function ComputerUseDesktopViewport({ } const peerConnection = ensurePeerConnection({ fresh: true }); if (!peerConnection) return; + setStreamFailureMessage(null); setState("connecting"); await peerConnection.setRemoteDescription(offer); await flushPendingDesktopIceCandidates( @@ -2500,7 +2505,14 @@ export function ComputerUseDesktopViewport({ } } catch { clearDesktopStreamMedia(); - setState(fallbackPreviewUrl ? "degraded" : "waiting"); + if (fallbackPreviewUrl) { + setState("degraded"); + } else { + setStreamFailureMessage( + "The desktop stream could not complete WebRTC negotiation. Try starting it again.", + ); + setState("failed"); + } } }, [ @@ -2515,6 +2527,24 @@ export function ComputerUseDesktopViewport({ streamToken, ], ); + const handleRemoteCommandResult = useCallback( + (payload: RemoteCommandResultPayload) => { + if (!remoteCommandResultFailed(payload)) return false; + const kind = remoteCommandKind(payload); + if (!kind || !kind.startsWith("computer_use_stream_")) return false; + streamStopRequestedRef.current = true; + clearDesktopStreamMedia({ clearPreview: true, clearStreamId: true }); + setStreamFailureMessage(remoteCommandFailureMessage(payload)); + setState( + remoteCommandFailureReason(payload) === + REMOTE_CONTROL_ALREADY_ACTIVE_REASON + ? "blocked" + : "failed", + ); + return true; + }, + [clearDesktopStreamMedia], + ); const applyAdaptiveStreamQuality = useCallback( ( metrics: DesktopStreamMetrics | null, @@ -2611,6 +2641,9 @@ export function ComputerUseDesktopViewport({ const ref = channel.on("frame", (rawFrame: unknown) => { const envelope = decodeRelayFrame(rawFrame); const payload = envelope?.payload; + if (isRemoteCommandResultPayload(payload)) { + if (handleRemoteCommandResult(payload)) return; + } if (!isComputerUseDesktopPayload(payload)) return; const isStreamStopPayload = payload.schema === "xero.computer_use_stream_stop.v1"; @@ -2627,6 +2660,9 @@ export function ComputerUseDesktopViewport({ streamIdRef.current = payload.streamId; setStreamId(payload.streamId); } + if (payload.ok !== false && payload.outcome !== "rejected") { + setStreamFailureMessage(null); + } const nextStreamDetails = desktopStreamDetails(payload); const shouldRecoverFromFallback = shouldRecoverDesktopWebRtcAfterFallback( nextStreamDetails, @@ -2745,6 +2781,7 @@ export function ComputerUseDesktopViewport({ applyAdaptiveStreamQuality, channel, clearDesktopStreamMedia, + handleRemoteCommandResult, handleWebRtcSignal, recoverDesktopWebRtcStream, state, @@ -3119,6 +3156,7 @@ export function ComputerUseDesktopViewport({ adaptiveQualityStableSamplesRef.current = 0; adaptiveQualityLastChangedAtRef.current = Date.now(); clearDesktopStreamMedia({ clearPreview: true, clearStreamId: true }); + setStreamFailureMessage(null); setStreamQuality("balanced"); setState("connecting"); void requestComputerUseStream(channel, { @@ -3145,6 +3183,7 @@ export function ComputerUseDesktopViewport({ const activeManualControlId = manualControlIdRef.current ?? manualControlId; streamStopRequestedRef.current = true; clearDesktopStreamMedia({ clearPreview: true, clearStreamId: true }); + setStreamFailureMessage(null); adaptiveQualityMetricsRef.current = null; adaptiveQualityStableSamplesRef.current = 0; if (state === "manual" || activeManualControlId) { @@ -4481,7 +4520,11 @@ export function ComputerUseDesktopViewport({ draggable={false} /> ) : ( - + )} {clickRipples.length > 0 ? ( @@ -4771,9 +4814,11 @@ export function DesktopToolbarPromptForm({ } function DesktopViewportEmptyState({ + description, state, status, }: { + description?: string | null; state: DesktopViewportState; status: string; }) { @@ -4802,7 +4847,7 @@ function DesktopViewportEmptyState({ {status}

- {desktopViewportEmptyDescription(state)} + {description ?? desktopViewportEmptyDescription(state)}

; + return payload.schema === "xero.remote_command_result.v1"; +} + +function remoteCommandResultFailed( + payload: RemoteCommandResultPayload, +): boolean { + if (payload.schema === "xero.remote_command_result.v1") { + return payload.ok === false; + } + return ( + payload.outcome === "rejected" || + payload.outcome === "rate_limited" || + payload.outcome === "timed_out" || + payload.outcome === "stale" + ); +} + +function remoteCommandKind(payload: RemoteCommandResultPayload): string | null { + return typeof payload.kind === "string" && payload.kind.length > 0 + ? payload.kind + : null; +} + +function remoteCommandFailureReason( + payload: RemoteCommandResultPayload, +): string | null { + if (typeof payload.reason === "string" && payload.reason.length > 0) { + return payload.reason; + } + if ( + payload.schema === "xero.remote_command_result.v1" && + typeof payload.error?.code === "string" && + payload.error.code.length > 0 + ) { + return payload.error.code; + } + return null; +} + +function remoteCommandFailureMessage( + payload: RemoteCommandResultPayload, +): string { + if ( + payload.schema === "xero.remote_command_result.v1" && + typeof payload.error?.message === "string" && + payload.error.message.length > 0 + ) { + return payload.error.message; + } + if (typeof payload.message === "string" && payload.message.length > 0) { + return payload.message; + } + if ( + remoteCommandFailureReason(payload) === REMOTE_CONTROL_ALREADY_ACTIVE_REASON + ) { + return "Stop the running connection in the other cloud app before using it here."; + } + return "The desktop app rejected the stream command. Try starting the desktop stream again."; +} + function remoteControlConnectionAlreadyActive( result: CommandAckResult, ): boolean { @@ -5028,6 +5142,25 @@ interface ComputerUseDesktopPayload { }; } +interface RemoteCommandExecutionResultPayload { + schema: "xero.remote_command_result.v1"; + ok?: boolean; + clientCommandId?: string | null; + clientSeq?: number | null; + kind?: string | null; + outcome?: string | null; + reason?: string | null; + message?: string | null; + error?: { + code?: string | null; + message?: string | null; + } | null; +} + +type RemoteCommandResultPayload = + | CommandAckResult + | RemoteCommandExecutionResultPayload; + interface DesktopStreamSignalPayload { type?: string | null; sdp?: string | null; diff --git a/server/test/xero_web/channels/remote_channel_test.exs b/server/test/xero_web/channels/remote_channel_test.exs index 68ab8406..ede142e3 100644 --- a/server/test/xero_web/channels/remote_channel_test.exs +++ b/server/test/xero_web/channels/remote_channel_test.exs @@ -393,6 +393,145 @@ defmodule XeroWeb.RemoteChannelTest do end) end + test "computer-use stream e2e exchanges WebRTC signaling without a run id", %{conn: conn} do + with_github_env(fn -> + desktop = desktop_login!(conn) + web = web_login!(conn) + session_id = "__computer_use__" + + {:ok, desktop_socket} = + connect(XeroWeb.RemoteDesktopSocket, %{"token" => desktop["desktop_jwt"]}) + + {:ok, _desktop_reply, desktop_socket} = + subscribe_and_join(desktop_socket, "desktop:#{desktop["desktop_device_id"]}", %{}) + + {:ok, _desktop_session_reply, desktop_session} = + subscribe_and_join( + desktop_socket, + "session:#{desktop["desktop_device_id"]}:#{session_id}", + %{} + ) + + {:ok, web_socket} = + connect(XeroWeb.RemoteWebSocket, %{"token" => web["web_jwt"]}) + + join_task = + Task.async(fn -> + subscribe_and_join( + web_socket, + "session:#{desktop["desktop_device_id"]}:#{session_id}", + %{ + "join_ref" => "join-computer-use-stream" + } + ) + end) + + assert_push "session_join_requested", %{ + auth_topic: auth_topic, + join_ref: "join-computer-use-stream", + session_id: "__computer_use__" + } + + ref = + push(desktop_socket, "session_authorized", %{ + "join_ref" => "join-computer-use-stream", + "auth_topic" => auth_topic, + "authorized" => true + }) + + assert_reply ref, :ok + {:ok, web_session_reply, web_session} = Task.await(join_task) + assert is_binary(web_session_reply.stream_token) + refute Map.has_key?(web_session_reply, :stream_run_id) + + start_ref = + push(web_session, "frame", %{ + "kind" => "computer_use_stream_request", + "clientCommandId" => "cmd-computer-use-start", + "payload" => %{ + "quality" => "balanced", + "streamToken" => web_session_reply.stream_token + } + }) + + assert_reply start_ref, :ok + + assert_push "computer_use_command_outcome", %{ + kind: "computer_use_stream_request", + clientCommandId: "cmd-computer-use-start", + outcome: "accepted" + } + + assert_push "frame", %{ + from_kind: "web", + direction: "web_to_desktop", + payload: %{ + "kind" => "computer_use_stream_request", + "payload" => %{ + "quality" => "balanced", + "streamToken" => stream_token + } + } + } + + assert stream_token == web_session_reply.stream_token + + offer_ref = + push(desktop_session, "frame", %{ + "schema" => "xero.computer_use_stream_offer.v1", + "streamId" => "stream-computer-use", + "ok" => true, + "payload" => %{"type" => "offer", "sdp" => "v=0\r\n"} + }) + + assert_reply offer_ref, :ok + + assert_push "frame", %{ + from_kind: "desktop", + direction: "desktop_to_web", + payload: %{ + "schema" => "xero.computer_use_stream_offer.v1", + "streamId" => "stream-computer-use", + "payload" => %{"type" => "offer", "sdp" => "v=0\r\n"} + } + } + + answer_ref = + push(web_session, "frame", %{ + "kind" => "computer_use_stream_answer", + "clientCommandId" => "cmd-computer-use-answer", + "payload" => %{ + "streamId" => "stream-computer-use", + "type" => "answer", + "sdp" => "v=0\r\n", + "streamToken" => web_session_reply.stream_token + } + }) + + assert_reply answer_ref, :ok + + assert_push "computer_use_command_outcome", %{ + kind: "computer_use_stream_answer", + clientCommandId: "cmd-computer-use-answer", + outcome: "accepted" + } + + assert_push "frame", %{ + from_kind: "web", + direction: "web_to_desktop", + payload: %{ + "kind" => "computer_use_stream_answer", + "payload" => %{ + "streamId" => "stream-computer-use", + "type" => "answer", + "sdp" => "v=0\r\n", + "streamToken" => ^stream_token + } + } + } + end) + end + test "computer-use desktop stream connects to only one cloud client at a time", %{conn: conn} do with_github_env(fn -> Process.flag(:trap_exit, true)