byre/hands/Sources/byre-hands/AX.swift

import AppKit
import ApplicationServices

@_silgen_name("_AXUIElementGetWindow")
func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError

struct RefSource {
    let pid: pid_t
    let window: Int?
}

struct RefTable {
    let elements: [String: AXUIElement]
    let source: RefSource
}

var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil))
var refCount = 0
let refsLock = NSLock()

func currentRefs() -> RefTable {
    return refsLock.withLock { latestRefs }
}

func newRef() -> String {
    return refsLock.withLock {
        refCount += 1
        return "e\(refCount)"
    }
}

let shownActions: [String: String] = [
    kAXPressAction: "press",
    kAXConfirmAction: "confirm",
    kAXIncrementAction: "increment",
    kAXDecrementAction: "decrement",
    kAXPickAction: "pick",
    kAXCancelAction: "cancel",
]

let maxVisits = 30000

let notAnswering = ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")

func attr(_ e: AXUIElement, _ name: String) throws -> CFTypeRef? {
    var v: CFTypeRef?
    let err = AXUIElementCopyAttributeValue(e, name as CFString, &v)
    if err == .cannotComplete {
        throw notAnswering
    }
    if err != .success {
        return nil
    }
    return v
}

func text(_ e: AXUIElement, _ name: String) throws -> String {
    guard let v = try attr(e, name) else {
        return ""
    }
    if let s = v as? String {
        return s
    }
    if CFGetTypeID(v) == CFBooleanGetTypeID() {
        return ""
    }
    if let n = v as? NSNumber {
        return n.stringValue
    }
    return ""
}

func children(_ e: AXUIElement) throws -> [AXUIElement] {
    return (try attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? []
}

func appElement(_ pid: pid_t) -> AXUIElement {
    return AXUIElementCreateApplication(pid)
}

func requireTrust() throws {
    if !AXIsProcessTrusted() {
        throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility")
    }
}

func axRole(_ e: AXUIElement) throws -> String {
    return try attr(e, kAXRoleAttribute) as? String ?? ""
}

func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
    try requireTrust()
    let info = try window(id)
    for w in (try attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
        var wid: CGWindowID = 0
        if _AXUIElementGetWindow(w, &wid) == .success && Int(wid) == id {
            return (info, w)
        }
    }
    throw ToolError("window \(id) has no accessibility element; the app may not expose it")
}

func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement {
    let s = table.source
    guard let e = table.elements[ref] else {
        throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers")
    }
    if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) {
        let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)"
        throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first")
    }
    return e
}

func frame(_ e: AXUIElement) throws -> CGRect? {
    guard let p = try attr(e, kAXPositionAttribute), let s = try attr(e, kAXSizeAttribute) else {
        return nil
    }
    var pt = CGPoint.zero
    var sz = CGSize.zero
    AXValueGetValue(p as! AXValue, .cgPoint, &pt)
    AXValueGetValue(s as! AXValue, .cgSize, &sz)
    return CGRect(origin: pt, size: sz)
}

func clip(_ s: String, _ n: Int) -> String {
    let flat = s.replacingOccurrences(of: "\n", with: " ")
    if flat.count <= n {
        return flat
    }
    return String(flat.prefix(n)) + "…"
}

struct TreeOptions {
    var query: String?
    var depth = 80
    var maxLines = 400
    var frames = false
    var origin = CGPoint.zero
}

func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String {
    var found: [String: AXUIElement] = [:]
    var lines: [String] = []
    var visits = 0
    var truncated = false
    let q = opts.query?.lowercased()

    func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws {
        visits += 1
        if visits > maxVisits || lines.count >= opts.maxLines {
            truncated = true
            return
        }
        let role = try axRole(e)
        var parts: [String] = []
        for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
            let s = try text(e, name)
            if !s.isEmpty && !parts.contains(s) {
                parts.append(s)
            }
        }
        var names: CFArray?
        if AXUIElementCopyActionNames(e, &names) == .cannotComplete {
            throw notAnswering
        }
        var acts = ((names as? [String]) ?? []).compactMap { shownActions[$0] }
        var settable: DarwinBoolean = false
        if AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable) == .cannotComplete {
            throw notAnswering
        }
        if settable.boolValue {
            acts.append("set")
        }
        if (try attr(e, kAXFocusedAttribute) as? Bool) == true {
            acts.append("focused")
        }
        if (try attr(e, kAXEnabledAttribute) as? Bool) == false {
            acts.append("disabled")
        }
        let echo = role == kAXStaticTextRole && acts.isEmpty && parts.allSatisfy { p in parentParts.contains { $0.contains(p) } }
        let shown = (!parts.isEmpty || !acts.isEmpty) && !echo
        let matches = q == nil || parts.contains { $0.lowercased().contains(q!) }
        var next = indent
        if shown && matches {
            let ref = newRef()
            found[ref] = e
            var line = String(repeating: "  ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased()
            for p in parts {
                line += " " + clip(p, 200).debugDescription
            }
            if !acts.isEmpty {
                line += " [" + acts.joined(separator: " ") + "]"
            }
            if opts.frames, let f = try frame(e) {
                line += " @\(Int(f.minX - opts.origin.x)),\(Int(f.minY - opts.origin.y)) \(Int(f.width))x\(Int(f.height))"
            }
            lines.append(line)
            next = indent + 1
        }
        if depth >= opts.depth {
            return
        }
        for c in try children(e) {
            try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
        }
    }

    try walk(root, 0, 0, [])
    refsLock.withLock {
        latestRefs = RefTable(elements: found, source: source)
    }
    if truncated {
        lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines")
    }
    return lines.joined(separator: "\n")
}

func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions {
    var o = TreeOptions()
    o.query = args.string("query")
    o.depth = args.int("depth") ?? o.depth
    o.maxLines = args.int("max_lines") ?? o.maxLines
    o.frames = args.bool("frames")
    o.origin = origin
    return o
}

func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
    if let pid = args.int("menu_bar_pid") {
        try requireTrust()
        guard let p = pid_t(exactly: pid), p > 0 else {
            throw ToolError("process \(pid) has no menu bar")
        }
        let app = appElement(p)
        _ = try axRole(app)
        guard let bar = try attr(app, kAXMenuBarAttribute) else {
            throw ToolError("process \(pid) has no menu bar")
        }
        return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
    }
    let id = try args.require("window", args.int)
    let (info, w) = try axWindow(id)
    return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
}

func windowMatches(_ root: AXUIElement, _ needle: String) throws -> Int {
    let n = needle.lowercased()
    var visits = 0
    var count = 0
    func walk(_ e: AXUIElement) throws {
        visits += 1
        if visits > maxVisits {
            return
        }
        _ = try axRole(e)
        if try [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { try text(e, $0).lowercased().contains(n) }) {
            count += 1
        }
        for c in try children(e) {
            try walk(c)
        }
    }
    try walk(root)
    return count
}