ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

hands test 2

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-17 23:11:31 -07:00
parent ccf8c551ec
commit 39ca42316f
5 changed files with 21 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hands/Sources/byre-hands/Tools.swift
+4 -2
View File
@@ -7,7 +7,6 @@ let treeProps = """
let thenProps = """
"wait_ms": {"type": "integer", "description": "Wait this long after the steps, before wait_for, tree and capture (default 300 when there were steps)."},
"wait_for": {"type": "string", "description": "Then wait until this text appears, polling every 200ms."},
"timeout_ms": {"type": "integer", "description": "How long wait_for waits before giving up (default 10000). Giving up is reported, not an error."},
"capture": {"type": "boolean", "description": "Finish with a capture of the window."},
"crop": {"type": "object", "description": "Capture only this rectangle, in window points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}},
@@ -57,6 +56,7 @@ let tools: [Tool] = [
description: "Run steps in one window, in order, then optionally wait, read the tree and capture: one call for click, wait and look. "
+ "Each step is an object with one of these keys. Through accessibility, which needs no focus and leaves the mouse alone (prefer these): "
+ "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. "
+ "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. "
+ "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): "
+ "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
+ "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). "
@@ -67,6 +67,7 @@ let tools: [Tool] = [
"window": {"type": "integer", "description": "Window id from screen_list."},
"steps": {"type": "array", "items": {"type": "object"}, "description": "The steps, in order."},
"tree": {"type": "boolean", "description": "Finish with the window's accessibility tree, which replaces the refs."},
"wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in more of the window's elements than before the steps, polling every 200ms. It searches the whole window, tab titles and toolbars included, so pick text only the expected result shows."},
\(thenProps),
\(treeProps)
}, "required": ["window"]}
@@ -79,7 +80,7 @@ let tools: [Tool] = [
+ "ops: tabs (list every window's tabs: window, tab, title, url), navigate (url, optionally new_tab; waits for the page to load), "
+ "js (run code in the tab and return its value: the value of the last expression, synchronously, so return strings, JSON.stringify objects, and don't return promises), select (tab), close (tab). "
+ "window is Orion's window index (1 is the front one) and tab the tab index from tabs; without tab, the window's current tab. "
+ "wait_for waits for text in the page, and capture captures the Orion window, selecting the tab first when one is given.",
+ "capture captures the Orion window, selecting the tab first when one is given.",
schema: """
{"type": "object", "properties": {
"op": {"type": "string", "enum": ["tabs", "navigate", "js", "select", "close"]},
@@ -88,6 +89,7 @@ let tools: [Tool] = [
"url": {"type": "string", "description": "For navigate."},
"new_tab": {"type": "boolean", "description": "For navigate: open the url in a new tab and select it."},
"code": {"type": "string", "description": "For js."},
"wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in the tab's title or page text, polling every 200ms."},
\(thenProps)
}, "required": ["op"]}
""",