From 2b6c938745dc102e6e8dad041ed773d576522f31 Mon Sep 17 00:00:00 2001 From: Brent G Date: Fri, 29 May 2026 12:21:38 -0700 Subject: [PATCH 1/2] feat(remote-control): add agentInput message + remote control panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1 up-lane remote agent control (Astation → Atem): - AstationMessage.agentInput(agentId:kind:text:key:) with encode/decode matching the wire contract (atem_id stays in the relay envelope; the payload carries agentId + kind + text/key, nil fields omitted). - HubManager.sendAgentText / sendAgentKey mirror sendVoiceCommand, routing through routeToFocusedAtem() + sendHandler. - RemoteControlWindowController: text field + Send and a key bar (Enter/Esc/Ctrl-C/↑/↓/y/n). Menu item "🎮 Remote Agent Control". - Voice reuses the existing PTT/hands-free path (unchanged). Also fix a stale RTCJoinOptions test that still expected aes256Gcm2 after the encryption default changed to .none. 🤖 Built with SMT --- Sources/Menubar/AstationHubManager.swift | 26 ++++ Sources/Menubar/AstationMessage.swift | 29 +++- .../RemoteControlWindowController.swift | 133 ++++++++++++++++++ Sources/Menubar/StatusBarController.swift | 15 ++ .../AgentInputMessageTests.swift | 65 +++++++++ Tests/AstationTests/RTCJoinOptionsTests.swift | 4 +- 6 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 Sources/Menubar/RemoteControlWindowController.swift create mode 100644 Tests/AstationTests/AgentInputMessageTests.swift diff --git a/Sources/Menubar/AstationHubManager.swift b/Sources/Menubar/AstationHubManager.swift index de88de1..c636371 100644 --- a/Sources/Menubar/AstationHubManager.swift +++ b/Sources/Menubar/AstationHubManager.swift @@ -802,6 +802,32 @@ class AstationHubManager: ObservableObject { Log.info(" Voice command → \(clientId): \(text)\(isFinal ? " [final]" : "")") } + // MARK: - Remote Agent Control + + /// Send a text instruction to the focused Atem's agent (written to its PTY stdin + Enter). + /// `agentId == nil` targets the Atem's focused/only agent. + func sendAgentText(_ text: String, agentId: String? = nil) { + guard let clientId = routeToFocusedAtem() else { + Log.info("[AgentInput] No Atem connected — text dropped: \(text)") + return + } + let message = AstationMessage.agentInput(agentId: agentId, kind: "text", text: text, key: nil) + sendHandler?(message, clientId) + Log.info("[AgentInput] text → \(clientId): \(text)") + } + + /// Send a control key to the focused Atem's agent (written raw to its PTY). + /// `key` is one of: enter, esc, ctrl-c, up, down, y, n. + func sendAgentKey(_ key: String, agentId: String? = nil) { + guard let clientId = routeToFocusedAtem() else { + Log.info("[AgentInput] No Atem connected — key dropped: \(key)") + return + } + let message = AstationMessage.agentInput(agentId: agentId, kind: "key", text: nil, key: key) + sendHandler?(message, clientId) + Log.info("[AgentInput] key → \(clientId): \(key)") + } + // MARK: - Mark Task Routing private func handleMarkTaskNotify(taskId: String, status: String, description: String) { diff --git a/Sources/Menubar/AstationMessage.swift b/Sources/Menubar/AstationMessage.swift index c418cc1..690ef56 100644 --- a/Sources/Menubar/AstationMessage.swift +++ b/Sources/Menubar/AstationMessage.swift @@ -51,7 +51,10 @@ enum AstationMessage: Codable { astationId: String, saveCredentials: Bool ) - + + // Remote agent control (Astation → Atem): text or key input to the agent PTY. + case agentInput(agentId: String?, kind: String, text: String?, key: String?) + // Custom encoding/decoding to handle the enum cases private enum CodingKeys: String, CodingKey { case type @@ -87,6 +90,7 @@ enum AstationMessage: Codable { case agentListRequest case agentListResponse case credentialSync + case agentInput } func encode(to encoder: Encoder) throws { @@ -257,6 +261,14 @@ enum AstationMessage: Codable { try dc.encodeIfPresent(lid, forKey: .loginId) try dc.encode(aid, forKey: .astationId) try dc.encode(save, forKey: .saveCredentials) + + case .agentInput(let agentId, let kind, let text, let key): + try container.encode(MessageType.agentInput, forKey: .type) + var dc = container.nestedContainer(keyedBy: AgentInputKeys.self, forKey: .data) + try dc.encodeIfPresent(agentId, forKey: .agentId) + try dc.encode(kind, forKey: .kind) + try dc.encodeIfPresent(text, forKey: .text) + try dc.encodeIfPresent(key, forKey: .key) } } @@ -268,6 +280,13 @@ enum AstationMessage: Codable { case astationId = "astation_id" case saveCredentials = "save_credentials" } + + private enum AgentInputKeys: String, CodingKey { + case agentId + case kind + case text + case key + } init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -445,6 +464,14 @@ enum AstationMessage: Codable { let save = try dc.decodeIfPresent(Bool.self, forKey: .saveCredentials) ?? false self = .credentialSync(accessToken: at, refreshToken: rt, expiresAt: exp, loginId: lid, astationId: aid, saveCredentials: save) + + case .agentInput: + let dc = try container.nestedContainer(keyedBy: AgentInputKeys.self, forKey: .data) + let agentId = try dc.decodeIfPresent(String.self, forKey: .agentId) + let kind = try dc.decode(String.self, forKey: .kind) + let text = try dc.decodeIfPresent(String.self, forKey: .text) + let key = try dc.decodeIfPresent(String.self, forKey: .key) + self = .agentInput(agentId: agentId, kind: kind, text: text, key: key) } } } diff --git a/Sources/Menubar/RemoteControlWindowController.swift b/Sources/Menubar/RemoteControlWindowController.swift new file mode 100644 index 0000000..efb6383 --- /dev/null +++ b/Sources/Menubar/RemoteControlWindowController.swift @@ -0,0 +1,133 @@ +import Cocoa +import Foundation + +/// Remote control panel for the v1 "up lane": send text + control keys to the +/// focused Atem's agent. Voice stays on the existing PTT/hands-free path. +class RemoteControlWindowController: NSObject, NSWindowDelegate, NSTextFieldDelegate { + private var window: NSWindow? + private let hubManager: AstationHubManager + private var targetLabel: NSTextField! + private var inputField: NSTextField! + private var sendButton: NSButton! + + /// Control keys exposed in the key bar: (button title, wire key name). + private let keys: [(String, String)] = [ + ("⏎ Enter", "enter"), + ("Esc", "esc"), + ("Ctrl-C", "ctrl-c"), + ("↑", "up"), + ("↓", "down"), + ("y", "y"), + ("n", "n"), + ] + + init(hubManager: AstationHubManager) { + self.hubManager = hubManager + super.init() + } + + func showWindow() { + if let existingWindow = window { + existingWindow.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + refreshTarget() + return + } + + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 460, height: 200), + styleMask: [.titled, .closable, .miniaturizable], + backing: .buffered, + defer: false + ) + window.title = "Remote Agent Control" + window.center() + window.delegate = self + window.isReleasedWhenClosed = false + + let contentView = NSView(frame: window.contentView!.bounds) + contentView.autoresizingMask = [.width, .height] + + // Target Atem label + targetLabel = NSTextField(labelWithString: "") + targetLabel.font = NSFont.systemFont(ofSize: 11) + targetLabel.textColor = .secondaryLabelColor + targetLabel.frame = NSRect(x: 20, y: 165, width: 420, height: 18) + contentView.addSubview(targetLabel) + + // Text input + Send + inputField = NSTextField(frame: NSRect(x: 20, y: 120, width: 320, height: 26)) + inputField.placeholderString = "Type an instruction, press Enter or Send" + inputField.delegate = self + contentView.addSubview(inputField) + + sendButton = NSButton(title: "Send", target: self, action: #selector(sendText)) + sendButton.bezelStyle = .rounded + sendButton.frame = NSRect(x: 350, y: 118, width: 90, height: 30) + sendButton.keyEquivalent = "\r" + contentView.addSubview(sendButton) + + // Key bar + let keyBarLabel = NSTextField(labelWithString: "Keys:") + keyBarLabel.font = NSFont.systemFont(ofSize: 11) + keyBarLabel.frame = NSRect(x: 20, y: 78, width: 40, height: 18) + contentView.addSubview(keyBarLabel) + + var x: CGFloat = 20 + let y: CGFloat = 40 + for (title, keyName) in keys { + let btn = NSButton(title: title, target: self, action: #selector(sendKey(_:))) + btn.bezelStyle = .rounded + btn.font = NSFont.systemFont(ofSize: 11) + let width: CGFloat = title.count > 3 ? 70 : 44 + btn.frame = NSRect(x: x, y: y, width: width, height: 28) + btn.identifier = NSUserInterfaceItemIdentifier(keyName) + contentView.addSubview(btn) + x += width + 6 + } + + window.contentView = contentView + window.makeFirstResponder(inputField) + window.makeKeyAndOrderFront(nil) + NSApp.activate(ignoringOtherApps: true) + self.window = window + refreshTarget() + } + + private func refreshTarget() { + if let clientId = hubManager.routeToFocusedAtem() { + targetLabel.stringValue = "Target: \(clientId)" + targetLabel.textColor = .secondaryLabelColor + } else { + targetLabel.stringValue = "⚠ No Atem connected — input will be dropped" + targetLabel.textColor = .systemOrange + } + } + + @objc private func sendText() { + let text = inputField.stringValue.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + hubManager.sendAgentText(text) + inputField.stringValue = "" + refreshTarget() + } + + @objc private func sendKey(_ sender: NSButton) { + guard let keyName = sender.identifier?.rawValue else { return } + hubManager.sendAgentKey(keyName) + refreshTarget() + } + + // Enter in the text field sends. + func control(_ control: NSControl, textView: NSTextView, doCommandBy selector: Selector) -> Bool { + if selector == #selector(NSResponder.insertNewline(_:)) { + sendText() + return true + } + return false + } + + func windowWillClose(_ notification: Notification) { + window = nil + } +} diff --git a/Sources/Menubar/StatusBarController.swift b/Sources/Menubar/StatusBarController.swift index 1881aec..04ac18a 100644 --- a/Sources/Menubar/StatusBarController.swift +++ b/Sources/Menubar/StatusBarController.swift @@ -11,6 +11,7 @@ class StatusBarController: NSObject, NSMenuDelegate { private lazy var projectsWindowController = ProjectsWindowController(hubManager: hubManager) private lazy var joinChannelWindowController = JoinChannelWindowController(hubManager: hubManager) private lazy var connectionsWindowController = ConnectionsWindowController(hubManager: hubManager) + private lazy var remoteControlWindowController = RemoteControlWindowController(hubManager: hubManager) var hotkeyManager: HotkeyManager? private var headerTapCount = 0 private var lastHeaderTapTime: Date? @@ -375,6 +376,15 @@ class StatusBarController: NSObject, NSMenuDelegate { showProjectsItem.target = self statusMenu.addItem(showProjectsItem) + // Remote Agent Control + let remoteControlItem = NSMenuItem( + title: "🎮 Remote Agent Control", + action: #selector(showRemoteControl), + keyEquivalent: "r" + ) + remoteControlItem.target = self + statusMenu.addItem(remoteControlItem) + // Show Clients & Agents let onlineCount = hubManager.connectedClients.filter { $0.clientType == "Atem" }.count let clientsTitle = onlineCount > 0 @@ -473,6 +483,11 @@ class StatusBarController: NSObject, NSMenuDelegate { projectsWindowController.showWindow() } + @objc private func showRemoteControl() { + Log.info(" Remote agent control requested from status bar") + remoteControlWindowController.showWindow() + } + @objc private func showClientsAndAgents() { Log.info(" Show clients & agents requested from status bar") connectionsWindowController.showAndFocus() diff --git a/Tests/AstationTests/AgentInputMessageTests.swift b/Tests/AstationTests/AgentInputMessageTests.swift new file mode 100644 index 0000000..a30029a --- /dev/null +++ b/Tests/AstationTests/AgentInputMessageTests.swift @@ -0,0 +1,65 @@ +import XCTest +@testable import Menubar + +final class AgentInputMessageTests: XCTestCase { + func testEncodeTextOmitsKeyAndNilAgentId() throws { + let msg = AstationMessage.agentInput(agentId: nil, kind: "text", + text: "refactor the auth module", key: nil) + let data = try JSONEncoder().encode(msg) + let s = String(data: data, encoding: .utf8) ?? "" + XCTAssertTrue(s.contains(#""type":"agentInput""#)) + XCTAssertTrue(s.contains(#""kind":"text""#)) + XCTAssertTrue(s.contains(#""text":"refactor the auth module""#)) + XCTAssertFalse(s.contains(#""key""#)) + XCTAssertFalse(s.contains(#""agentId""#)) + } + + func testEncodeKeyOmitsText() throws { + let msg = AstationMessage.agentInput(agentId: "agent-1", kind: "key", text: nil, key: "ctrl-c") + let data = try JSONEncoder().encode(msg) + let s = String(data: data, encoding: .utf8) ?? "" + XCTAssertTrue(s.contains(#""kind":"key""#)) + XCTAssertTrue(s.contains(#""key":"ctrl-c""#)) + XCTAssertTrue(s.contains(#""agentId":"agent-1""#)) + XCTAssertFalse(s.contains(#""text""#)) + } + + func testDecodeText() throws { + let json = """ + {"type":"agentInput","data":{"kind":"text","text":"print working directory"}} + """.data(using: .utf8)! + let msg = try JSONDecoder().decode(AstationMessage.self, from: json) + guard case let .agentInput(agentId, kind, text, key) = msg else { + return XCTFail("expected agentInput") + } + XCTAssertNil(agentId) + XCTAssertEqual(kind, "text") + XCTAssertEqual(text, "print working directory") + XCTAssertNil(key) + } + + func testDecodeKeyWithAgentId() throws { + let json = """ + {"type":"agentInput","data":{"agentId":"a2","kind":"key","key":"enter"}} + """.data(using: .utf8)! + let msg = try JSONDecoder().decode(AstationMessage.self, from: json) + guard case let .agentInput(agentId, kind, text, key) = msg else { + return XCTFail("expected agentInput") + } + XCTAssertEqual(agentId, "a2") + XCTAssertEqual(kind, "key") + XCTAssertNil(text) + XCTAssertEqual(key, "enter") + } + + func testRoundTrip() throws { + let original = AstationMessage.agentInput(agentId: nil, kind: "text", text: "hello", key: nil) + let data = try JSONEncoder().encode(original) + let decoded = try JSONDecoder().decode(AstationMessage.self, from: data) + guard case let .agentInput(_, kind, text, _) = decoded else { + return XCTFail("expected agentInput") + } + XCTAssertEqual(kind, "text") + XCTAssertEqual(text, "hello") + } +} diff --git a/Tests/AstationTests/RTCJoinOptionsTests.swift b/Tests/AstationTests/RTCJoinOptionsTests.swift index feb354b..b2fd915 100644 --- a/Tests/AstationTests/RTCJoinOptionsTests.swift +++ b/Tests/AstationTests/RTCJoinOptionsTests.swift @@ -3,8 +3,8 @@ import XCTest @testable import Menubar final class RTCJoinOptionsTests: XCTestCase { - func testManualJoinDefaultsUseAes256Gcm2AndNoFence() { - XCTAssertEqual(RTCEncryptionMode.manualJoinDefault, .aes256Gcm2) + func testManualJoinDefaultsUseNoneEncryptionAndNoFence() { + XCTAssertEqual(RTCEncryptionMode.manualJoinDefault, .none) XCTAssertEqual(RTCGeoFence.manualJoinDefault, .noFence) } From b70d9feb2a58bd63b84780eecb6ede7dfa853c8c Mon Sep 17 00:00:00 2001 From: Brent G Date: Fri, 29 May 2026 13:29:04 -0700 Subject: [PATCH 2/2] test(remote-control): add agentInput routing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover HubManager.sendAgentText / sendAgentKey: routes to the connected Atem, emits the correct agentInput payload (text vs key), honors an explicit agentId, drops when no Atem is connected, and prefers the pinned client. 124 Swift tests pass (was 119). 🤖 Built with SMT --- .../AgentInputRoutingTests.swift | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 Tests/AstationTests/AgentInputRoutingTests.swift diff --git a/Tests/AstationTests/AgentInputRoutingTests.swift b/Tests/AstationTests/AgentInputRoutingTests.swift new file mode 100644 index 0000000..22d7344 --- /dev/null +++ b/Tests/AstationTests/AgentInputRoutingTests.swift @@ -0,0 +1,97 @@ +import XCTest +@testable import Menubar + +/// Tests for HubManager.sendAgentText / sendAgentKey routing + the agentInput +/// payload they emit. Uses a stubbed sendHandler to capture messages. +final class AgentInputRoutingTests: XCTestCase { + private func makeHub(withClient: Bool) -> (AstationHubManager, [(AstationMessage, String)]) { + let hub = AstationHubManager(skipProjectLoad: true) + if withClient { + hub.connectedClients = [ + ConnectedClient(id: "atem-1", clientType: "Atem", connectedAt: Date()) + ] + } + return (hub, []) + } + + func testSendAgentTextRoutesToConnectedAtem() { + let hub = AstationHubManager(skipProjectLoad: true) + hub.connectedClients = [ConnectedClient(id: "atem-1", clientType: "Atem", connectedAt: Date())] + + var captured: [(AstationMessage, String)] = [] + hub.sendHandler = { msg, clientId in captured.append((msg, clientId)) } + + hub.sendAgentText("print working directory") + + XCTAssertEqual(captured.count, 1) + XCTAssertEqual(captured[0].1, "atem-1") + guard case let .agentInput(agentId, kind, text, key) = captured[0].0 else { + return XCTFail("expected agentInput") + } + XCTAssertNil(agentId) + XCTAssertEqual(kind, "text") + XCTAssertEqual(text, "print working directory") + XCTAssertNil(key) + } + + func testSendAgentKeyRoutesToConnectedAtem() { + let hub = AstationHubManager(skipProjectLoad: true) + hub.connectedClients = [ConnectedClient(id: "atem-1", clientType: "Atem", connectedAt: Date())] + + var captured: [(AstationMessage, String)] = [] + hub.sendHandler = { msg, clientId in captured.append((msg, clientId)) } + + hub.sendAgentKey("ctrl-c") + + XCTAssertEqual(captured.count, 1) + guard case let .agentInput(_, kind, text, key) = captured[0].0 else { + return XCTFail("expected agentInput") + } + XCTAssertEqual(kind, "key") + XCTAssertNil(text) + XCTAssertEqual(key, "ctrl-c") + } + + func testSendAgentTextWithExplicitAgentId() { + let hub = AstationHubManager(skipProjectLoad: true) + hub.connectedClients = [ConnectedClient(id: "atem-1", clientType: "Atem", connectedAt: Date())] + + var captured: [(AstationMessage, String)] = [] + hub.sendHandler = { msg, clientId in captured.append((msg, clientId)) } + + hub.sendAgentText("hi", agentId: "agent-9") + + guard case let .agentInput(agentId, _, _, _) = captured[0].0 else { + return XCTFail("expected agentInput") + } + XCTAssertEqual(agentId, "agent-9") + } + + func testSendAgentTextDroppedWhenNoAtem() { + let hub = AstationHubManager(skipProjectLoad: true) + hub.connectedClients = [] // no Atems + + var captured: [(AstationMessage, String)] = [] + hub.sendHandler = { msg, clientId in captured.append((msg, clientId)) } + + hub.sendAgentText("nobody home") + hub.sendAgentKey("enter") + + XCTAssertTrue(captured.isEmpty, "messages should be dropped when no Atem is connected") + } + + func testRoutingPrefersPinnedClient() { + let hub = AstationHubManager(skipProjectLoad: true) + hub.connectedClients = [ + ConnectedClient(id: "atem-1", clientType: "Atem", connectedAt: Date()), + ConnectedClient(id: "atem-2", clientType: "Atem", connectedAt: Date()), + ] + hub.pinnedClientId = "atem-2" + + var captured: [(AstationMessage, String)] = [] + hub.sendHandler = { msg, clientId in captured.append((msg, clientId)) } + + hub.sendAgentText("route me") + XCTAssertEqual(captured.first?.1, "atem-2", "pinned client should win") + } +}