let treeProps = """ "query": {"type": "string", "description": "Only list elements whose title, description or value contains this text (case-insensitive), flat, without their parents."}, "depth": {"type": "integer", "description": "Stop descending after this many levels (default 80)."}, "max_lines": {"type": "integer", "description": "Stop after this many elements (default 400)."}, "frames": {"type": "boolean", "description": "Add each element's position and size, in window points: @x,y WxH."} """ let thenProps = """ "wait_ms": {"type": "integer", "description": "Wait this long after the steps, before wait_for, tree and capture (default 300 when there were steps)."}, "timeout_ms": {"type": "integer", "description": "How long wait_for waits before giving up (default 10000). Giving up is reported, not an error."}, "capture": {"type": "boolean", "description": "Finish with a capture of the window."}, "crop": {"type": "object", "description": "Capture only this rectangle, in window points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}}, "max_edge": {"type": "number", "description": "Longest edge of the captured image in pixels, 16 to 8192 (default 1568, which is as much as you can see; captures are never scaled up)."} """ let tools: [Tool] = [ Tool( name: "screen_list", description: "List the running apps (pid, name, bundle id), the on-screen windows front to back (id, app, title, size and position in screen points) and the displays of this Mac, the machine byre runs on. " + "Window ids are what capture, ax_tree and act take.", schema: #"{"type": "object", "properties": {}}"#, run: screenList ), Tool( name: "capture", description: "Capture one window or a whole display, and see it. A window's capture shows its app's windows only, so other apps covering it don't hide it, and its menus, popups and sheets show; it must be on screen (not minimized). " + "Prefer a window: a display is 5120 points wide here and shrinks until text is unreadable; crop a region to read small text. " + "The reply says how image pixels map to window points, which are what act's [x, y] takes.", schema: """ {"type": "object", "properties": { "window": {"type": "integer", "description": "Window id from screen_list."}, "display": {"type": "integer", "description": "Display index from screen_list, when no window is given (default 0)."}, "crop": {"type": "object", "description": "Only this rectangle, in window (or display) points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}}, "max_edge": {"type": "number", "description": "Longest edge of the image in pixels, 16 to 8192 (default 1568)."} }} """, run: captureTool ), Tool( name: "ax_tree", description: "Read a window's accessibility tree: one line per element that has text or actions, indented under its parent, e.g. `e12 button \"New Tab\" [press]`. " + "Web pages in Orion appear as text, links and form fields. The refs (e12) are for act and stay valid until the next tree, which replaces them. " + "Bracketed: the actions an element takes (press, confirm, menu, increment…), set when its value can be set, focused, disabled. " + "With menu_bar_pid instead of window, read an app's menu bar; pressing a menu item's ref runs it without opening the menu.", schema: """ {"type": "object", "properties": { "window": {"type": "integer", "description": "Window id from screen_list."}, "menu_bar_pid": {"type": "integer", "description": "Read this app's menu bar instead of a window."}, \(treeProps) }} """, run: axTreeTool ), Tool( name: "act", description: "Run steps in one window, in order, then optionally wait, read the tree and capture: one call for click, wait and look. " + "Each step is an object with one of these keys. Through accessibility, which needs no focus and leaves the mouse alone (prefer these): " + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. " + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. " + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): " + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), " + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). " + "And {\"wait\": 500} (ms). [x, y] are window points, from the top left of the window. A failing step stops the rest and says which. " + "Other agents share this Mac's apps, mouse and keyboard: a call with steps (and every browser call but tabs) holds them for the whole call, waiting and capture included, so keep such calls short, and waits up to 30s for another agent's call to finish.", schema: """ {"type": "object", "properties": { "window": {"type": "integer", "description": "Window id from screen_list."}, "steps": {"type": "array", "items": {"type": "object"}, "description": "The steps, in order."}, "tree": {"type": "boolean", "description": "Finish with the window's accessibility tree, which replaces the refs."}, "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in more of the window's elements than before the steps, polling every 200ms. It searches the whole window, tab titles and toolbars included, so pick text only the expected result shows."}, \(thenProps), \(treeProps) }, "required": ["window"]} """, run: actTool ), Tool( name: "browser", description: "Drive Orion, the logged-in browser on this Mac, through AppleScript; for web work, prefer this to clicking. " + "ops: tabs (list every window's tabs: window, tab, title, url), navigate (url, optionally new_tab; waits for the page to load), " + "js (run code in the tab and return its value: the value of the last expression, synchronously, so return strings, JSON.stringify objects, and don't return promises), select (tab), close (tab). " + "window is Orion's window index (1 is the front one) and tab the tab index from tabs; without tab, the window's current tab. " + "capture captures the Orion window, selecting the tab first when one is given.", schema: """ {"type": "object", "properties": { "op": {"type": "string", "enum": ["tabs", "navigate", "js", "select", "close"]}, "window": {"type": "integer", "description": "Orion window index (default 1, the front window)."}, "tab": {"type": "integer", "description": "Tab index within the window (default: its current tab)."}, "url": {"type": "string", "description": "For navigate."}, "new_tab": {"type": "boolean", "description": "For navigate: open the url in a new tab and select it."}, "code": {"type": "string", "description": "For js."}, "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in the tab's title or page text, polling every 200ms."}, \(thenProps) }, "required": ["op"]} """, run: browserTool ), ]