From 39ca42316fba00285ac0e57c825bf9482bc81110 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Sat, 17 Oct 2026 23:11:31 -0700 Subject: [PATCH] hands test 2 --- docs/operations.md | 4 ++-- hands/Sources/byre-hands/AX.swift | 21 +++++++++------------ hands/Sources/byre-hands/Act.swift | 6 +++++- hands/Sources/byre-hands/Browser.swift | 2 +- hands/Sources/byre-hands/Tools.swift | 6 ++++-- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 94a1a7b..2bf0306 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -129,8 +129,8 @@ Implementer sessions get `byre-hands` as a stdio MCP server named `hands` (`hand - `screen_list`: running apps, on-screen windows (front to back) and displays. - `capture`: one window, or a display, through ScreenCaptureKit. A window is captured as its rectangle of its display with only its own app's windows included, so other apps covering it don't hide it and its menus, popups and sheets show; capturing the window alone fails while a popup is attached to it. The image is scaled to `max_edge` (default 1568, as much as a model sees; 16 to 8192) and the reply gives the pixel-to-point scale. - `ax_tree`: a window's or an app's menu bar's accessibility tree, one line per element with text or actions, each with a ref (`e12`) that `act` takes. Refs live in the process and each tree replaces them. They remember the window, or the app whose menu bar, they came from, and `act` refuses a ref that isn't from its window or its app's menu bar. Unlabelled containers are left out with their children shown in their place, and static text already contained in its labelled ancestor's text is dropped. -- `act`: a list of steps in one window: accessibility actions on refs (press, set value, focus, confirm, any named action), then real mouse and keyboard events posted with `CGEvent` (click, double and right click, drag, scroll, typing, key combinations), Before each of those steps it reads the window's current frame, for `[x, y]`, and makes sure the window is in front: its app frontmost and the app's focused window (through accessibility, so the app's own popups don't count against it). When it isn't, it activates the app and raises the window, and stops with an error, sending nothing, if the window still isn't in front. Every session's `byre-hands` drives the same desktop, so a call that can change what is on screen or focused holds an exclusive `flock` on `byre-hands-input.lock` in the user's temporary directory (`$TMPDIR`): an `act` call with steps and every `browser` call except a `tabs` without capture, each for the whole call, its waiting, tree and capture included, so what it returns shows its own work. Another such call waits up to 30s for it, then fails without doing anything. The lock is released when the steps end or the process exits. Typing posts each ASCII character as its key code, with shift where needed, since Orion ignores key events that carry only a Unicode string. Afterwards it can wait (`wait_ms`, then `wait_for` text in the window's tree) and return the tree and a capture, so one call clicks, waits and looks. -- `browser`: Orion through AppleScript: list tabs, navigate (then poll `document.readyState`), run JavaScript in a tab (`do JavaScript`), select and close tabs, with the same `wait_for` (page text) and capture. Capturing a call that names a tab selects that tab first, since only the selected tab is on screen, and waits 300ms for it to draw. The Orion window is found by its AppleScript `id`, which is its window server window id. Durations (`wait_ms`, `timeout_ms`, `act`'s `wait` steps) must be 0 to 600000 ms. `do JavaScript` needs Orion's Develop → "Allow JavaScript from Apple Events". +- `act`: a list of steps in one window: accessibility actions on refs (press, set value, focus, confirm, any named action), then real mouse and keyboard events posted with `CGEvent` (click, double and right click, drag, scroll, typing, key combinations), Before each of those steps it reads the window's current frame, for `[x, y]`, and makes sure the window is in front: its app frontmost and the app's focused window (through accessibility, so the app's own popups don't count against it). When it isn't, it activates the app and raises the window, and stops with an error, sending nothing, if the window still isn't in front. Every session's `byre-hands` drives the same desktop, so a call that can change what is on screen or focused holds an exclusive `flock` on `byre-hands-input.lock` in the user's temporary directory (`$TMPDIR`): an `act` call with steps and every `browser` call except a `tabs` without capture, each for the whole call, its waiting, tree and capture included, so what it returns shows its own work. Another such call waits up to 30s for it, then fails without doing anything. The lock is released when the steps end or the process exits. Typing posts each ASCII character as its key code, with shift where needed, since Orion ignores key events that carry only a Unicode string. Afterwards it can wait (`wait_ms`, then `wait_for`) and return the tree and a capture, so one call clicks, waits and looks. `wait_for` counts the window's elements whose title, description or value contains the text, ignoring case, once before the steps and then every 200ms, and ends when the count rises above the first one (above zero with no steps), so text already on screen, such as another tab's title, doesn't end the wait. Some apps list `confirm` but ignore it, Orion's address bar among them; the tool description says to send Return instead. +- `browser`: Orion through AppleScript: list tabs, navigate (then poll `document.readyState`), run JavaScript in a tab (`do JavaScript`), select and close tabs, with `wait_for` (the tab's `document.title` and `document.body.innerText`, ignoring case) and capture. Capturing a call that names a tab selects that tab first, since only the selected tab is on screen, and waits 300ms for it to draw. The Orion window is found by its AppleScript `id`, which is its window server window id. Durations (`wait_ms`, `timeout_ms`, `act`'s `wait` steps) must be 0 to 600000 ms. `do JavaScript` needs Orion's Develop → "Allow JavaScript from Apple Events". `byre-hands` reads stdin on a background thread and starts each request as its own detached task as soon as it arrives, so one call stuck on an app doesn't hold up the others; responses are written whole, one line each, under a lock, in the order they finish. Ref names come from one counter for the life of the process, so no two trees ever share a name, even trees built at the same time; the ref table is swapped in under a lock once a tree is complete, so a ref from any tree but the last published one fails as unknown instead of naming another tree's element. Each `act` call takes the table as it stands when the call arrives and resolves every step's refs from that copy, so a tree finishing during its waits can't point a ref at a different element. AppleScript needs the main thread with a run loop, so every script runs there (`osa` in `Browser.swift`, `DispatchQueue.main.sync`, while the main thread runs `RunLoop.main`), one at a time, wrapped in `with timeout of 10 seconds`; `NSApplication.shared` connects the process to the window server at startup, without which ScreenCaptureKit asserts. An app that is in a menu-tracking loop or otherwise busy answers neither Apple events nor accessibility requests: a script fails with AppleScript's timeout error after 10s, and accessibility requests time out after 3s (the messaging timeout set on the system-wide element at startup), at which point a tree walk or `wait_for` stops with an error saying the app isn't answering instead of timing out on every element. diff --git a/hands/Sources/byre-hands/AX.swift b/hands/Sources/byre-hands/AX.swift index af4afda..6f51583 100644 --- a/hands/Sources/byre-hands/AX.swift +++ b/hands/Sources/byre-hands/AX.swift @@ -238,26 +238,23 @@ func axTreeTool(_ args: [String: Any]) async throws -> [Content] { return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))] } -func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool { +func windowMatches(_ root: AXUIElement, _ needle: String) throws -> Int { let n = needle.lowercased() var visits = 0 - func walk(_ e: AXUIElement) throws -> Bool { + var count = 0 + func walk(_ e: AXUIElement) throws { visits += 1 if visits > maxVisits { - return false + return } _ = try axRole(e) - for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] { - if text(e, name).lowercased().contains(n) { - return true - } + if [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { text(e, $0).lowercased().contains(n) }) { + count += 1 } for c in children(e) { - if try walk(c) { - return true - } + try walk(c) } - return false } - return try walk(root) + try walk(root) + return count } diff --git a/hands/Sources/byre-hands/Act.swift b/hands/Sources/byre-hands/Act.swift index 9f9de4f..3fa6a58 100644 --- a/hands/Sources/byre-hands/Act.swift +++ b/hands/Sources/byre-hands/Act.swift @@ -139,8 +139,12 @@ func actTool(_ args: [String: Any]) async throws -> [Content] { close(lock) } } + var before = 0 + if !steps.isEmpty, let needle = args.string("wait_for") { + before = try windowMatches(w, needle) + } var log = try await runSteps(steps, id, w, table) - if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowText(w, contains: $0) }) { + if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowMatches(w, $0) > before }) { log.append(note) } var out: [Content] = [] diff --git a/hands/Sources/byre-hands/Browser.swift b/hands/Sources/byre-hands/Browser.swift index a701879..c18a279 100644 --- a/hands/Sources/byre-hands/Browser.swift +++ b/hands/Sources/byre-hands/Browser.swift @@ -115,7 +115,7 @@ func browserTool(_ args: [String: Any]) async throws -> [Content] { throw ToolError("unknown op \(op.debugDescription)") } let present: (String) async throws -> Bool = { needle in - (try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true" + (try? js("String((document.title + '\\n' + (document.body ? document.body.innerText : '')).toLowerCase().includes(\(jsString(needle.lowercased()))))", target)) == "true" } if let note = try await waitFor(args, acted: false, present) { log.append(note) diff --git a/hands/Sources/byre-hands/Tools.swift b/hands/Sources/byre-hands/Tools.swift index bbaee6f..e8b3ea1 100644 --- a/hands/Sources/byre-hands/Tools.swift +++ b/hands/Sources/byre-hands/Tools.swift @@ -7,7 +7,6 @@ let treeProps = """ let thenProps = """ "wait_ms": {"type": "integer", "description": "Wait this long after the steps, before wait_for, tree and capture (default 300 when there were steps)."}, - "wait_for": {"type": "string", "description": "Then wait until this text appears, polling every 200ms."}, "timeout_ms": {"type": "integer", "description": "How long wait_for waits before giving up (default 10000). Giving up is reported, not an error."}, "capture": {"type": "boolean", "description": "Finish with a capture of the window."}, "crop": {"type": "object", "description": "Capture only this rectangle, in window points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}}, @@ -57,6 +56,7 @@ let tools: [Tool] = [ description: "Run steps in one window, in order, then optionally wait, read the tree and capture: one call for click, wait and look. " + "Each step is an object with one of these keys. Through accessibility, which needs no focus and leaves the mouse alone (prefer these): " + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. " + + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. " + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): " + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), " + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). " @@ -67,6 +67,7 @@ let tools: [Tool] = [ "window": {"type": "integer", "description": "Window id from screen_list."}, "steps": {"type": "array", "items": {"type": "object"}, "description": "The steps, in order."}, "tree": {"type": "boolean", "description": "Finish with the window's accessibility tree, which replaces the refs."}, + "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in more of the window's elements than before the steps, polling every 200ms. It searches the whole window, tab titles and toolbars included, so pick text only the expected result shows."}, \(thenProps), \(treeProps) }, "required": ["window"]} @@ -79,7 +80,7 @@ let tools: [Tool] = [ + "ops: tabs (list every window's tabs: window, tab, title, url), navigate (url, optionally new_tab; waits for the page to load), " + "js (run code in the tab and return its value: the value of the last expression, synchronously, so return strings, JSON.stringify objects, and don't return promises), select (tab), close (tab). " + "window is Orion's window index (1 is the front one) and tab the tab index from tabs; without tab, the window's current tab. " - + "wait_for waits for text in the page, and capture captures the Orion window, selecting the tab first when one is given.", + + "capture captures the Orion window, selecting the tab first when one is given.", schema: """ {"type": "object", "properties": { "op": {"type": "string", "enum": ["tabs", "navigate", "js", "select", "close"]}, @@ -88,6 +89,7 @@ let tools: [Tool] = [ "url": {"type": "string", "description": "For navigate."}, "new_tab": {"type": "boolean", "description": "For navigate: open the url in a new tab and select it."}, "code": {"type": "string", "description": "For js."}, + "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in the tab's title or page text, polling every 200ms."}, \(thenProps) }, "required": ["op"]} """,