From ccf8c551ec54e36828b71309837d44547b8df59d Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Sat, 17 Oct 2026 23:03:33 -0700 Subject: [PATCH] hands test --- docs/operations.md | 4 +- hands/Sources/byre-hands/AX.swift | 94 +++++++++++++++++++------- hands/Sources/byre-hands/Act.swift | 38 +++++------ hands/Sources/byre-hands/Browser.swift | 26 +++---- hands/Sources/byre-hands/Server.swift | 24 ++++--- 5 files changed, 115 insertions(+), 71 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index bf5fa5e..94a1a7b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -132,9 +132,9 @@ Implementer sessions get `byre-hands` as a stdio MCP server named `hands` (`hand - `act`: a list of steps in one window: accessibility actions on refs (press, set value, focus, confirm, any named action), then real mouse and keyboard events posted with `CGEvent` (click, double and right click, drag, scroll, typing, key combinations), Before each of those steps it reads the window's current frame, for `[x, y]`, and makes sure the window is in front: its app frontmost and the app's focused window (through accessibility, so the app's own popups don't count against it). When it isn't, it activates the app and raises the window, and stops with an error, sending nothing, if the window still isn't in front. Every session's `byre-hands` drives the same desktop, so a call that can change what is on screen or focused holds an exclusive `flock` on `byre-hands-input.lock` in the user's temporary directory (`$TMPDIR`): an `act` call with steps and every `browser` call except a `tabs` without capture, each for the whole call, its waiting, tree and capture included, so what it returns shows its own work. Another such call waits up to 30s for it, then fails without doing anything. The lock is released when the steps end or the process exits. Typing posts each ASCII character as its key code, with shift where needed, since Orion ignores key events that carry only a Unicode string. Afterwards it can wait (`wait_ms`, then `wait_for` text in the window's tree) and return the tree and a capture, so one call clicks, waits and looks. - `browser`: Orion through AppleScript: list tabs, navigate (then poll `document.readyState`), run JavaScript in a tab (`do JavaScript`), select and close tabs, with the same `wait_for` (page text) and capture. Capturing a call that names a tab selects that tab first, since only the selected tab is on screen, and waits 300ms for it to draw. The Orion window is found by its AppleScript `id`, which is its window server window id. Durations (`wait_ms`, `timeout_ms`, `act`'s `wait` steps) must be 0 to 600000 ms. `do JavaScript` needs Orion's Develop → "Allow JavaScript from Apple Events". -AppleScript needs the main thread with a run loop, so `byre-hands` reads stdin on a background thread and runs each request on the main actor while the main thread runs `RunLoop.main`; `NSApplication.shared` connects it to the window server at startup, without which ScreenCaptureKit asserts. +`byre-hands` reads stdin on a background thread and starts each request as its own detached task as soon as it arrives, so one call stuck on an app doesn't hold up the others; responses are written whole, one line each, under a lock, in the order they finish. Ref names come from one counter for the life of the process, so no two trees ever share a name, even trees built at the same time; the ref table is swapped in under a lock once a tree is complete, so a ref from any tree but the last published one fails as unknown instead of naming another tree's element. Each `act` call takes the table as it stands when the call arrives and resolves every step's refs from that copy, so a tree finishing during its waits can't point a ref at a different element. AppleScript needs the main thread with a run loop, so every script runs there (`osa` in `Browser.swift`, `DispatchQueue.main.sync`, while the main thread runs `RunLoop.main`), one at a time, wrapped in `with timeout of 10 seconds`; `NSApplication.shared` connects the process to the window server at startup, without which ScreenCaptureKit asserts. An app that is in a menu-tracking loop or otherwise busy answers neither Apple events nor accessibility requests: a script fails with AppleScript's timeout error after 10s, and accessibility requests time out after 3s (the messaging timeout set on the system-wide element at startup), at which point a tree walk or `wait_for` stops with an error saying the app isn't answering instead of timing out on every element. -**Permissions.** Agents' processes, `byre-hands` included, run as byre's descendants, so macOS privacy controls (TCC) hold byre responsible for them: the Screen Recording, Accessibility and Automation grants in System Settings → Privacy & Security are byre's, made by clicking at the Mac's screen, and each app scripted for the first time asks for its own Automation grant there. `codesign` gives byre the identifier `run.fc.byre` and the Apple Development certificate, which TCC keys grants on, so they carry over from build to build. `codesign -dv ~/bin/byre` shows the signature the running binary has. +**Permissions.** Agents' processes, `byre-hands` included, run as byre's descendants, so macOS privacy controls (TCC) hold byre responsible for them: the Screen Recording, Accessibility and Automation grants in System Settings → Privacy & Security are byre's, made by clicking at the Mac's screen, and each app scripted for the first time asks for its own Automation grant there. `ax_tree` and `act` check `AXIsProcessTrusted` first and fail saying byre lacks Accessibility when it isn't granted, since an untrusted process otherwise just sees no windows and no menu bar. Accessibility can list byre switched off; only its switch being on counts. `codesign` gives byre the identifier `run.fc.byre` and the Apple Development certificate, which TCC keys grants on, so they carry over from build to build. `codesign -dv ~/bin/byre` shows the signature the running binary has. ## Previews diff --git a/hands/Sources/byre-hands/AX.swift b/hands/Sources/byre-hands/AX.swift index 8f9ff4b..af4afda 100644 --- a/hands/Sources/byre-hands/AX.swift +++ b/hands/Sources/byre-hands/AX.swift @@ -9,8 +9,25 @@ struct RefSource { let window: Int? } -var refs: [String: AXUIElement] = [:] -var refSource = RefSource(pid: 0, window: nil) +struct RefTable { + let elements: [String: AXUIElement] + let source: RefSource +} + +var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil)) +var refCount = 0 +let refsLock = NSLock() + +func currentRefs() -> RefTable { + return refsLock.withLock { latestRefs } +} + +func newRef() -> String { + return refsLock.withLock { + refCount += 1 + return "e\(refCount)" + } +} let shownActions: [String: String] = [ kAXPressAction: "press", @@ -52,12 +69,25 @@ func children(_ e: AXUIElement) -> [AXUIElement] { } func appElement(_ pid: pid_t) -> AXUIElement { - let app = AXUIElementCreateApplication(pid) - AXUIElementSetMessagingTimeout(app, 3) - return app + return AXUIElementCreateApplication(pid) +} + +func requireTrust() throws { + if !AXIsProcessTrusted() { + throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility") + } +} + +func axRole(_ e: AXUIElement) throws -> String { + var v: CFTypeRef? + if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete { + throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)") + } + return v as? String ?? "" } func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) { + try requireTrust() let info = try window(id) for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] { var wid: CGWindowID = 0 @@ -68,11 +98,11 @@ func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) { throw ToolError("window \(id) has no accessibility element; the app may not expose it") } -func element(_ ref: String, in info: WindowInfo) throws -> AXUIElement { - guard let e = refs[ref] else { - throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act) and are replaced by each new tree") +func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement { + let s = table.source + guard let e = table.elements[ref] else { + throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers") } - let s = refSource if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) { let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)" throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first") @@ -107,21 +137,20 @@ struct TreeOptions { var origin = CGPoint.zero } -func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> String { - refs = [:] - refSource = source +func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String { + var found: [String: AXUIElement] = [:] var lines: [String] = [] var visits = 0 var truncated = false let q = opts.query?.lowercased() - func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) { + func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws { visits += 1 if visits > maxVisits || lines.count >= opts.maxLines { truncated = true return } - let role = text(e, kAXRoleAttribute) + let role = try axRole(e) var parts: [String] = [] for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] { let s = text(e, name) @@ -148,8 +177,8 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St let matches = q == nil || parts.contains { $0.lowercased().contains(q!) } var next = indent if shown && matches { - let ref = "e\(refs.count + 1)" - refs[ref] = e + let ref = newRef() + found[ref] = e var line = String(repeating: " ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased() for p in parts { line += " " + clip(p, 200).debugDescription @@ -167,11 +196,14 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St return } for c in children(e) { - walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts) + try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts) } } - walk(root, 0, 0, []) + try walk(root, 0, 0, []) + refsLock.withLock { + latestRefs = RefTable(elements: found, source: source) + } if truncated { lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines") } @@ -190,30 +222,42 @@ func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions { func axTreeTool(_ args: [String: Any]) async throws -> [Content] { if let pid = args.int("menu_bar_pid") { - guard let p = pid_t(exactly: pid), p > 0, let bar = attr(appElement(p), kAXMenuBarAttribute) else { + try requireTrust() + guard let p = pid_t(exactly: pid), p > 0 else { throw ToolError("process \(pid) has no menu bar") } - return [.text(axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))] + let app = appElement(p) + _ = try axRole(app) + guard let bar = attr(app, kAXMenuBarAttribute) else { + throw ToolError("process \(pid) has no menu bar") + } + return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))] } let id = try args.require("window", args.int) let (info, w) = try axWindow(id) - return [.text(axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))] + return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))] } -func windowText(_ root: AXUIElement, contains needle: String) -> Bool { +func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool { let n = needle.lowercased() var visits = 0 - func walk(_ e: AXUIElement) -> Bool { + func walk(_ e: AXUIElement) throws -> Bool { visits += 1 if visits > maxVisits { return false } + _ = try axRole(e) for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] { if text(e, name).lowercased().contains(n) { return true } } - return children(e).contains { walk($0) } + for c in children(e) { + if try walk(c) { + return true + } + } + return false } - return walk(root) + return try walk(root) } diff --git a/hands/Sources/byre-hands/Act.swift b/hands/Sources/byre-hands/Act.swift index 62ee935..9f9de4f 100644 --- a/hands/Sources/byre-hands/Act.swift +++ b/hands/Sources/byre-hands/Act.swift @@ -1,9 +1,9 @@ import AppKit import ApplicationServices -func point(_ v: Any?, _ info: WindowInfo) throws -> CGPoint { +func point(_ v: Any?, _ info: WindowInfo, _ table: RefTable) throws -> CGPoint { if let ref = v as? String { - guard let f = frame(try element(ref, in: info)) else { + guard let f = frame(try element(ref, in: info, table)) else { throw ToolError("\(ref) has no position on screen") } return CGPoint(x: f.midX, y: f.midY) @@ -26,48 +26,48 @@ func isInput(_ step: [String: Any]) -> Bool { return step.keys.contains(where: inputKeys.contains) } -func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws -> String { +func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> String { var info = try window(id) if isInput(step) { info = try await focusWindow(id, w) } if let ref = step.string("press") { - try check(AXUIElementPerformAction(try element(ref, in: info), kAXPressAction as CFString), "press \(ref)") + try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXPressAction as CFString), "press \(ref)") return "pressed \(ref)" } if let ref = step.string("confirm") { - try check(AXUIElementPerformAction(try element(ref, in: info), kAXConfirmAction as CFString), "confirm \(ref)") + try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXConfirmAction as CFString), "confirm \(ref)") return "confirmed \(ref)" } if let ref = step.string("focus") { - try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)") + try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)") return "focused \(ref)" } if let ref = step.string("set") { let value = try step.require("value", step.string) - try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXValueAttribute as CFString, value as CFString), "set \(ref)") + try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXValueAttribute as CFString, value as CFString), "set \(ref)") return "set \(ref)" } if let name = step.string("action") { let ref = try step.require("ref", step.string) - try check(AXUIElementPerformAction(try element(ref, in: info), name as CFString), "\(name) \(ref)") + try check(AXUIElementPerformAction(try element(ref, in: info, table), name as CFString), "\(name) \(ref)") return "\(name) \(ref)" } for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] { if step[k] != nil { - let p = try point(step[k], info) + let p = try point(step[k], info, table) await click(p, right: right, count: count) return "\(k) at screen \(Int(p.x)),\(Int(p.y))" } } if step["drag"] != nil { - let from = try point(step["drag"], info) - let to = try point(step["to"], info) + let from = try point(step["drag"], info, table) + let to = try point(step["to"], info, table) await drag(from, to) return "dragged" } if step["scroll"] != nil { - let p = try point(step["scroll"], info) + let p = try point(step["scroll"], info, table) let dx = step.int("dx") ?? 0 let dy = step.int("dy") ?? 0 guard (-1000...1000).contains(dx), (-1000...1000).contains(dy) else { @@ -91,8 +91,7 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws -> throw ToolError("unknown step \(step.keys.sorted())") } -@MainActor -func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String) async throws -> Bool) async throws -> String? { +func waitFor(_ args: [String: Any], acted: Bool, _ present: (String) async throws -> Bool) async throws -> String? { let wait = try args.ms("wait_ms") let timeout = Double(try args.ms("timeout_ms") ?? 10000) / 1000 if let wait { @@ -115,11 +114,11 @@ func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String) } } -func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throws -> [String] { +func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> [String] { var log: [String] = [] for (i, step) in steps.enumerated() { do { - log.append("\(i + 1). " + (try await runStep(step, id, w))) + log.append("\(i + 1). " + (try await runStep(step, id, w, table))) await pause(50) } catch let e as ToolError { log.append("\(i + 1). " + e.message + "; stopped here") @@ -130,6 +129,7 @@ func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throw } func actTool(_ args: [String: Any]) async throws -> [Content] { + let table = currentRefs() let id = try args.require("window", args.int) let (_, w) = try axWindow(id) let steps = args["steps"] as? [[String: Any]] ?? [] @@ -139,14 +139,14 @@ func actTool(_ args: [String: Any]) async throws -> [Content] { close(lock) } } - var log = try await runSteps(steps, id, w) - if let note = try await waitFor(args, acted: !steps.isEmpty, { windowText(w, contains: $0) }) { + var log = try await runSteps(steps, id, w, table) + if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowText(w, contains: $0) }) { log.append(note) } var out: [Content] = [] if args.bool("tree") { let current = try window(id) - log.append("\n" + axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin))) + log.append("\n" + (try axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin)))) } if args.bool("capture") { let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568) diff --git a/hands/Sources/byre-hands/Browser.swift b/hands/Sources/byre-hands/Browser.swift index 8dce90b..a701879 100644 --- a/hands/Sources/byre-hands/Browser.swift +++ b/hands/Sources/byre-hands/Browser.swift @@ -1,16 +1,17 @@ import AppKit -@MainActor func osa(_ src: String) throws -> NSAppleEventDescriptor { - guard let script = NSAppleScript(source: src) else { - throw ToolError("AppleScript did not compile") + return try DispatchQueue.main.sync { + guard let script = NSAppleScript(source: "with timeout of 10 seconds\n\(src)\nend timeout") else { + throw ToolError("AppleScript did not compile") + } + var err: NSDictionary? + let r = script.executeAndReturnError(&err) + if let err { + throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)") + } + return r } - var err: NSDictionary? - let r = script.executeAndReturnError(&err) - if let err { - throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)") - } - return r } func quote(_ s: String) -> String { @@ -34,18 +35,15 @@ struct TabTarget { } } -@MainActor func js(_ code: String, _ t: TabTarget) throws -> String { let r = try osa("tell application \"Orion\" to do JavaScript \(quote(code)) in \(t.spec)") return r.stringValue ?? "undefined" } -@MainActor func orionWindow(_ index: Int) throws -> Int { return Int(try osa("tell application \"Orion\" to get id of window \(index)").int32Value) } -@MainActor func tabs() throws -> String { let src = """ tell application "Orion" @@ -65,7 +63,6 @@ func tabs() throws -> String { return "window tab title | url\n" + (try osa(src).stringValue ?? "") } -@MainActor func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String { let start = Date() await pause(300) @@ -78,7 +75,6 @@ func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String { return String(format: "still loading after %.1fs", timeout) } -@MainActor func browserTool(_ args: [String: Any]) async throws -> [Content] { let op = try args.require("op", args.string) let win = args.int("window") ?? 1 @@ -118,7 +114,7 @@ func browserTool(_ args: [String: Any]) async throws -> [Content] { default: throw ToolError("unknown op \(op.debugDescription)") } - let present: @MainActor (String) async throws -> Bool = { needle in + let present: (String) async throws -> Bool = { needle in (try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true" } if let note = try await waitFor(args, acted: false, present) { diff --git a/hands/Sources/byre-hands/Server.swift b/hands/Sources/byre-hands/Server.swift index bafd44f..d93de66 100644 --- a/hands/Sources/byre-hands/Server.swift +++ b/hands/Sources/byre-hands/Server.swift @@ -26,28 +26,31 @@ struct Tool { let name: String let description: String let schema: String - let run: @MainActor ([String: Any]) async throws -> [Content] + let run: ([String: Any]) async throws -> [Content] } +let stdoutLock = NSLock() + @main struct Server { static func main() { _ = NSApplication.shared + AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), 3) Thread.detachNewThread { + let inFlight = DispatchGroup() while let line = readLine() { - let done = DispatchSemaphore(value: 0) - Task { @MainActor in + inFlight.enter() + Task.detached { await handle(line) - done.signal() + inFlight.leave() } - done.wait() } + inFlight.wait() exit(0) } RunLoop.main.run() } - @MainActor static func handle(_ line: String) async { guard let data = line.data(using: .utf8), let req = try? JSONSerialization.jsonObject(with: data) as? [String: Any], @@ -75,12 +78,13 @@ struct Server { default: resp["error"] = ["code": -32601, "message": "method not found: \(req["method"] ?? "")"] } - let out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes) - FileHandle.standardOutput.write(out) - FileHandle.standardOutput.write(Data("\n".utf8)) + var out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes) + out.append(Data("\n".utf8)) + stdoutLock.withLock { + FileHandle.standardOutput.write(out) + } } - @MainActor static func call(_ params: [String: Any]) async -> [String: Any] { let name = params["name"] as? String ?? "" let args = params["arguments"] as? [String: Any] ?? [:]