hands test

2026-10-17 23:03:33 -07:00
parent bb908e493f
commit ccf8c551ec
5 changed files with 115 additions and 71 deletions
@@ -9,8 +9,25 @@ struct RefSource {
    let window: Int?
 }

-var refs: [String: AXUIElement] = [:]
-var refSource = RefSource(pid: 0, window: nil)
+struct RefTable {
+    let elements: [String: AXUIElement]
+    let source: RefSource
+}
+
+var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil))
+var refCount = 0
+let refsLock = NSLock()
+
+func currentRefs() -> RefTable {
+    return refsLock.withLock { latestRefs }
+}
+
+func newRef() -> String {
+    return refsLock.withLock {
+        refCount += 1
+        return "e\(refCount)"
+    }
+}

 let shownActions: [String: String] = [
    kAXPressAction: "press",
@@ -52,12 +69,25 @@ func children(_ e: AXUIElement) -> [AXUIElement] {
 }

 func appElement(_ pid: pid_t) -> AXUIElement {
-    let app = AXUIElementCreateApplication(pid)
-    AXUIElementSetMessagingTimeout(app, 3)
-    return app
+    return AXUIElementCreateApplication(pid)
+}
+
+func requireTrust() throws {
+    if !AXIsProcessTrusted() {
+        throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility")
+    }
+}
+
+func axRole(_ e: AXUIElement) throws -> String {
+    var v: CFTypeRef?
+    if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete {
+        throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")
+    }
+    return v as? String ?? ""
 }

 func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
+    try requireTrust()
    let info = try window(id)
    for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
        var wid: CGWindowID = 0
@@ -68,11 +98,11 @@ func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
    throw ToolError("window \(id) has no accessibility element; the app may not expose it")
 }

-func element(_ ref: String, in info: WindowInfo) throws -> AXUIElement {
-    guard let e = refs[ref] else {
-        throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act) and are replaced by each new tree")
+func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement {
+    let s = table.source
+    guard let e = table.elements[ref] else {
+        throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers")
    }
-    let s = refSource
    if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) {
        let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)"
        throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first")
@@ -107,21 +137,20 @@ struct TreeOptions {
    var origin = CGPoint.zero
 }

-func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> String {
-    refs = [:]
-    refSource = source
+func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String {
+    var found: [String: AXUIElement] = [:]
    var lines: [String] = []
    var visits = 0
    var truncated = false
    let q = opts.query?.lowercased()

-    func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) {
+    func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws {
        visits += 1
        if visits > maxVisits || lines.count >= opts.maxLines {
            truncated = true
            return
        }
-        let role = text(e, kAXRoleAttribute)
+        let role = try axRole(e)
        var parts: [String] = []
        for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
            let s = text(e, name)
@@ -148,8 +177,8 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St
        let matches = q == nil || parts.contains { $0.lowercased().contains(q!) }
        var next = indent
        if shown && matches {
-            let ref = "e\(refs.count + 1)"
-            refs[ref] = e
+            let ref = newRef()
+            found[ref] = e
            var line = String(repeating: "  ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased()
            for p in parts {
                line += " " + clip(p, 200).debugDescription
@@ -167,11 +196,14 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St
            return
        }
        for c in children(e) {
-            walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
+            try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
        }
    }

-    walk(root, 0, 0, [])
+    try walk(root, 0, 0, [])
+    refsLock.withLock {
+        latestRefs = RefTable(elements: found, source: source)
+    }
    if truncated {
        lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines")
    }
@@ -190,30 +222,42 @@ func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions {

 func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
    if let pid = args.int("menu_bar_pid") {
-        guard let p = pid_t(exactly: pid), p > 0, let bar = attr(appElement(p), kAXMenuBarAttribute) else {
+        try requireTrust()
+        guard let p = pid_t(exactly: pid), p > 0 else {
            throw ToolError("process \(pid) has no menu bar")
        }
-        return [.text(axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
+        let app = appElement(p)
+        _ = try axRole(app)
+        guard let bar = attr(app, kAXMenuBarAttribute) else {
+            throw ToolError("process \(pid) has no menu bar")
+        }
+        return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
    }
    let id = try args.require("window", args.int)
    let (info, w) = try axWindow(id)
-    return [.text(axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
+    return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
 }

-func windowText(_ root: AXUIElement, contains needle: String) -> Bool {
+func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool {
    let n = needle.lowercased()
    var visits = 0
-    func walk(_ e: AXUIElement) -> Bool {
+    func walk(_ e: AXUIElement) throws -> Bool {
        visits += 1
        if visits > maxVisits {
            return false
        }
+        _ = try axRole(e)
        for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] {
            if text(e, name).lowercased().contains(n) {
                return true
            }
        }
-        return children(e).contains { walk($0) }
+        for c in children(e) {
+            if try walk(c) {
+                return true
+            }
+        }
+        return false
    }
-    return walk(root)
+    return try walk(root)
 }