hands test 2

hands/Sources/byre-hands/AX.swift

@@ -238,26 +238,23 @@ func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
     return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
 }
 
-func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool {
+func windowMatches(_ root: AXUIElement, _ needle: String) throws -> Int {
     let n = needle.lowercased()
     var visits = 0
-    func walk(_ e: AXUIElement) throws -> Bool {
+    var count = 0
+    func walk(_ e: AXUIElement) throws {
         visits += 1
         if visits > maxVisits {
-            return false
+            return
         }
         _ = try axRole(e)
-        for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] {
-            if text(e, name).lowercased().contains(n) {
-                return true
-            }
+        if [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { text(e, $0).lowercased().contains(n) }) {
+            count += 1
         }
         for c in children(e) {
-            if try walk(c) {
-                return true
-            }
+            try walk(c)
         }
-        return false
     }
-    return try walk(root)
+    try walk(root)
+    return count
 }

hands/Sources/byre-hands/Act.swift

@@ -139,8 +139,12 @@ func actTool(_ args: [String: Any]) async throws -> [Content] {
             close(lock)
         }
     }
+    var before = 0
+    if !steps.isEmpty, let needle = args.string("wait_for") {
+        before = try windowMatches(w, needle)
+    }
     var log = try await runSteps(steps, id, w, table)
-    if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowText(w, contains: $0) }) {
+    if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowMatches(w, $0) > before }) {
         log.append(note)
     }
     var out: [Content] = []

hands/Sources/byre-hands/Browser.swift

@@ -115,7 +115,7 @@ func browserTool(_ args: [String: Any]) async throws -> [Content] {
         throw ToolError("unknown op \(op.debugDescription)")
     }
     let present: (String) async throws -> Bool = { needle in
-        (try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true"
+        (try? js("String((document.title + '\\n' + (document.body ? document.body.innerText : '')).toLowerCase().includes(\(jsString(needle.lowercased()))))", target)) == "true"
     }
     if let note = try await waitFor(args, acted: false, present) {
         log.append(note)

hands/Sources/byre-hands/Tools.swift

@@ -7,7 +7,6 @@ let treeProps = """
 
 let thenProps = """
     "wait_ms": {"type": "integer", "description": "Wait this long after the steps, before wait_for, tree and capture (default 300 when there were steps)."},
-    "wait_for": {"type": "string", "description": "Then wait until this text appears, polling every 200ms."},
     "timeout_ms": {"type": "integer", "description": "How long wait_for waits before giving up (default 10000). Giving up is reported, not an error."},
     "capture": {"type": "boolean", "description": "Finish with a capture of the window."},
     "crop": {"type": "object", "description": "Capture only this rectangle, in window points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}},
@@ -57,6 +56,7 @@ let tools: [Tool] = [
         description: "Run steps in one window, in order, then optionally wait, read the tree and capture: one call for click, wait and look. "
             + "Each step is an object with one of these keys. Through accessibility, which needs no focus and leaves the mouse alone (prefer these): "
             + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. "
+            + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. "
             + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): "
             + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
             + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). "
@@ -67,6 +67,7 @@ let tools: [Tool] = [
             "window": {"type": "integer", "description": "Window id from screen_list."},
             "steps": {"type": "array", "items": {"type": "object"}, "description": "The steps, in order."},
             "tree": {"type": "boolean", "description": "Finish with the window's accessibility tree, which replaces the refs."},
+            "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in more of the window's elements than before the steps, polling every 200ms. It searches the whole window, tab titles and toolbars included, so pick text only the expected result shows."},
             \(thenProps),
             \(treeProps)
         }, "required": ["window"]}
@@ -79,7 +80,7 @@ let tools: [Tool] = [
             + "ops: tabs (list every window's tabs: window, tab, title, url), navigate (url, optionally new_tab; waits for the page to load), "
             + "js (run code in the tab and return its value: the value of the last expression, synchronously, so return strings, JSON.stringify objects, and don't return promises), select (tab), close (tab). "
             + "window is Orion's window index (1 is the front one) and tab the tab index from tabs; without tab, the window's current tab. "
-            + "wait_for waits for text in the page, and capture captures the Orion window, selecting the tab first when one is given.",
+            + "capture captures the Orion window, selecting the tab first when one is given.",
         schema: """
         {"type": "object", "properties": {
             "op": {"type": "string", "enum": ["tabs", "navigate", "js", "select", "close"]},
@@ -88,6 +89,7 @@ let tools: [Tool] = [
             "url": {"type": "string", "description": "For navigate."},
             "new_tab": {"type": "boolean", "description": "For navigate: open the url in a new tab and select it."},
             "code": {"type": "string", "description": "For js."},
+            "wait_for": {"type": "string", "description": "Then wait until this text (case-insensitive) is in the tab's title or page text, polling every 200ms."},
             \(thenProps)
         }, "required": ["op"]}
         """,