import AppKit import ApplicationServices @_silgen_name("_AXUIElementGetWindow") func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer) -> AXError struct RefSource { let pid: pid_t let window: Int? } struct RefTable { let elements: [String: AXUIElement] let source: RefSource } var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil)) var refCount = 0 let refsLock = NSLock() func currentRefs() -> RefTable { return refsLock.withLock { latestRefs } } func newRef() -> String { return refsLock.withLock { refCount += 1 return "e\(refCount)" } } let shownActions: [String: String] = [ kAXPressAction: "press", kAXConfirmAction: "confirm", kAXIncrementAction: "increment", kAXDecrementAction: "decrement", kAXPickAction: "pick", kAXCancelAction: "cancel", ] let maxVisits = 30000 func attr(_ e: AXUIElement, _ name: String) -> CFTypeRef? { var v: CFTypeRef? if AXUIElementCopyAttributeValue(e, name as CFString, &v) != .success { return nil } return v } func text(_ e: AXUIElement, _ name: String) -> String { guard let v = attr(e, name) else { return "" } if let s = v as? String { return s } if CFGetTypeID(v) == CFBooleanGetTypeID() { return "" } if let n = v as? NSNumber { return n.stringValue } return "" } func children(_ e: AXUIElement) -> [AXUIElement] { return (attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? [] } func appElement(_ pid: pid_t) -> AXUIElement { return AXUIElementCreateApplication(pid) } func requireTrust() throws { if !AXIsProcessTrusted() { throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility") } } func axRole(_ e: AXUIElement) throws -> String { var v: CFTypeRef? if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete { throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)") } return v as? String ?? "" } func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) { try requireTrust() let info = try window(id) for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] { var wid: CGWindowID = 0 if _AXUIElementGetWindow(w, &wid) == .success && Int(wid) == id { return (info, w) } } throw ToolError("window \(id) has no accessibility element; the app may not expose it") } func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement { let s = table.source guard let e = table.elements[ref] else { throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers") } if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) { let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)" throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first") } return e } func frame(_ e: AXUIElement) -> CGRect? { guard let p = attr(e, kAXPositionAttribute), let s = attr(e, kAXSizeAttribute) else { return nil } var pt = CGPoint.zero var sz = CGSize.zero AXValueGetValue(p as! AXValue, .cgPoint, &pt) AXValueGetValue(s as! AXValue, .cgSize, &sz) return CGRect(origin: pt, size: sz) } func clip(_ s: String, _ n: Int) -> String { let flat = s.replacingOccurrences(of: "\n", with: " ") if flat.count <= n { return flat } return String(flat.prefix(n)) + "…" } struct TreeOptions { var query: String? var depth = 80 var maxLines = 400 var frames = false var origin = CGPoint.zero } func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String { var found: [String: AXUIElement] = [:] var lines: [String] = [] var visits = 0 var truncated = false let q = opts.query?.lowercased() func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws { visits += 1 if visits > maxVisits || lines.count >= opts.maxLines { truncated = true return } let role = try axRole(e) var parts: [String] = [] for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] { let s = text(e, name) if !s.isEmpty && !parts.contains(s) { parts.append(s) } } var names: CFArray? AXUIElementCopyActionNames(e, &names) var acts = ((names as? [String]) ?? []).compactMap { shownActions[$0] } var settable: DarwinBoolean = false AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable) if settable.boolValue { acts.append("set") } if (attr(e, kAXFocusedAttribute) as? Bool) == true { acts.append("focused") } if (attr(e, kAXEnabledAttribute) as? Bool) == false { acts.append("disabled") } let echo = role == kAXStaticTextRole && acts.isEmpty && parts.allSatisfy { p in parentParts.contains { $0.contains(p) } } let shown = (!parts.isEmpty || !acts.isEmpty) && !echo let matches = q == nil || parts.contains { $0.lowercased().contains(q!) } var next = indent if shown && matches { let ref = newRef() found[ref] = e var line = String(repeating: " ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased() for p in parts { line += " " + clip(p, 200).debugDescription } if !acts.isEmpty { line += " [" + acts.joined(separator: " ") + "]" } if opts.frames, let f = frame(e) { line += " @\(Int(f.minX - opts.origin.x)),\(Int(f.minY - opts.origin.y)) \(Int(f.width))x\(Int(f.height))" } lines.append(line) next = indent + 1 } if depth >= opts.depth { return } for c in children(e) { try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts) } } try walk(root, 0, 0, []) refsLock.withLock { latestRefs = RefTable(elements: found, source: source) } if truncated { lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines") } return lines.joined(separator: "\n") } func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions { var o = TreeOptions() o.query = args.string("query") o.depth = args.int("depth") ?? o.depth o.maxLines = args.int("max_lines") ?? o.maxLines o.frames = args.bool("frames") o.origin = origin return o } func axTreeTool(_ args: [String: Any]) async throws -> [Content] { if let pid = args.int("menu_bar_pid") { try requireTrust() guard let p = pid_t(exactly: pid), p > 0 else { throw ToolError("process \(pid) has no menu bar") } let app = appElement(p) _ = try axRole(app) guard let bar = attr(app, kAXMenuBarAttribute) else { throw ToolError("process \(pid) has no menu bar") } return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))] } let id = try args.require("window", args.int) let (info, w) = try axWindow(id) return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))] } func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool { let n = needle.lowercased() var visits = 0 func walk(_ e: AXUIElement) throws -> Bool { visits += 1 if visits > maxVisits { return false } _ = try axRole(e) for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] { if text(e, name).lowercased().contains(n) { return true } } for c in children(e) { if try walk(c) { return true } } return false } return try walk(root) }