261 lines
8.4 KiB
Swift
261 lines
8.4 KiB
Swift
import AppKit
|
|
import ApplicationServices
|
|
|
|
@_silgen_name("_AXUIElementGetWindow")
|
|
func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
|
|
|
|
struct RefSource {
|
|
let pid: pid_t
|
|
let window: Int?
|
|
}
|
|
|
|
struct RefTable {
|
|
let elements: [String: AXUIElement]
|
|
let source: RefSource
|
|
}
|
|
|
|
var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil))
|
|
var refCount = 0
|
|
let refsLock = NSLock()
|
|
|
|
func currentRefs() -> RefTable {
|
|
return refsLock.withLock { latestRefs }
|
|
}
|
|
|
|
func newRef() -> String {
|
|
return refsLock.withLock {
|
|
refCount += 1
|
|
return "e\(refCount)"
|
|
}
|
|
}
|
|
|
|
let shownActions: [String: String] = [
|
|
kAXPressAction: "press",
|
|
kAXConfirmAction: "confirm",
|
|
kAXIncrementAction: "increment",
|
|
kAXDecrementAction: "decrement",
|
|
kAXPickAction: "pick",
|
|
kAXCancelAction: "cancel",
|
|
]
|
|
|
|
let maxVisits = 30000
|
|
|
|
func attr(_ e: AXUIElement, _ name: String) -> CFTypeRef? {
|
|
var v: CFTypeRef?
|
|
if AXUIElementCopyAttributeValue(e, name as CFString, &v) != .success {
|
|
return nil
|
|
}
|
|
return v
|
|
}
|
|
|
|
func text(_ e: AXUIElement, _ name: String) -> String {
|
|
guard let v = attr(e, name) else {
|
|
return ""
|
|
}
|
|
if let s = v as? String {
|
|
return s
|
|
}
|
|
if CFGetTypeID(v) == CFBooleanGetTypeID() {
|
|
return ""
|
|
}
|
|
if let n = v as? NSNumber {
|
|
return n.stringValue
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func children(_ e: AXUIElement) -> [AXUIElement] {
|
|
return (attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? []
|
|
}
|
|
|
|
func appElement(_ pid: pid_t) -> AXUIElement {
|
|
return AXUIElementCreateApplication(pid)
|
|
}
|
|
|
|
func requireTrust() throws {
|
|
if !AXIsProcessTrusted() {
|
|
throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility")
|
|
}
|
|
}
|
|
|
|
func axRole(_ e: AXUIElement) throws -> String {
|
|
var v: CFTypeRef?
|
|
if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete {
|
|
throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")
|
|
}
|
|
return v as? String ?? ""
|
|
}
|
|
|
|
func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
|
|
try requireTrust()
|
|
let info = try window(id)
|
|
for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
|
|
var wid: CGWindowID = 0
|
|
if _AXUIElementGetWindow(w, &wid) == .success && Int(wid) == id {
|
|
return (info, w)
|
|
}
|
|
}
|
|
throw ToolError("window \(id) has no accessibility element; the app may not expose it")
|
|
}
|
|
|
|
func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement {
|
|
let s = table.source
|
|
guard let e = table.elements[ref] else {
|
|
throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers")
|
|
}
|
|
if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) {
|
|
let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)"
|
|
throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first")
|
|
}
|
|
return e
|
|
}
|
|
|
|
func frame(_ e: AXUIElement) -> CGRect? {
|
|
guard let p = attr(e, kAXPositionAttribute), let s = attr(e, kAXSizeAttribute) else {
|
|
return nil
|
|
}
|
|
var pt = CGPoint.zero
|
|
var sz = CGSize.zero
|
|
AXValueGetValue(p as! AXValue, .cgPoint, &pt)
|
|
AXValueGetValue(s as! AXValue, .cgSize, &sz)
|
|
return CGRect(origin: pt, size: sz)
|
|
}
|
|
|
|
func clip(_ s: String, _ n: Int) -> String {
|
|
let flat = s.replacingOccurrences(of: "\n", with: " ")
|
|
if flat.count <= n {
|
|
return flat
|
|
}
|
|
return String(flat.prefix(n)) + "…"
|
|
}
|
|
|
|
struct TreeOptions {
|
|
var query: String?
|
|
var depth = 80
|
|
var maxLines = 400
|
|
var frames = false
|
|
var origin = CGPoint.zero
|
|
}
|
|
|
|
func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String {
|
|
var found: [String: AXUIElement] = [:]
|
|
var lines: [String] = []
|
|
var visits = 0
|
|
var truncated = false
|
|
let q = opts.query?.lowercased()
|
|
|
|
func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws {
|
|
visits += 1
|
|
if visits > maxVisits || lines.count >= opts.maxLines {
|
|
truncated = true
|
|
return
|
|
}
|
|
let role = try axRole(e)
|
|
var parts: [String] = []
|
|
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
|
|
let s = text(e, name)
|
|
if !s.isEmpty && !parts.contains(s) {
|
|
parts.append(s)
|
|
}
|
|
}
|
|
var names: CFArray?
|
|
AXUIElementCopyActionNames(e, &names)
|
|
var acts = ((names as? [String]) ?? []).compactMap { shownActions[$0] }
|
|
var settable: DarwinBoolean = false
|
|
AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable)
|
|
if settable.boolValue {
|
|
acts.append("set")
|
|
}
|
|
if (attr(e, kAXFocusedAttribute) as? Bool) == true {
|
|
acts.append("focused")
|
|
}
|
|
if (attr(e, kAXEnabledAttribute) as? Bool) == false {
|
|
acts.append("disabled")
|
|
}
|
|
let echo = role == kAXStaticTextRole && acts.isEmpty && parts.allSatisfy { p in parentParts.contains { $0.contains(p) } }
|
|
let shown = (!parts.isEmpty || !acts.isEmpty) && !echo
|
|
let matches = q == nil || parts.contains { $0.lowercased().contains(q!) }
|
|
var next = indent
|
|
if shown && matches {
|
|
let ref = newRef()
|
|
found[ref] = e
|
|
var line = String(repeating: " ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased()
|
|
for p in parts {
|
|
line += " " + clip(p, 200).debugDescription
|
|
}
|
|
if !acts.isEmpty {
|
|
line += " [" + acts.joined(separator: " ") + "]"
|
|
}
|
|
if opts.frames, let f = frame(e) {
|
|
line += " @\(Int(f.minX - opts.origin.x)),\(Int(f.minY - opts.origin.y)) \(Int(f.width))x\(Int(f.height))"
|
|
}
|
|
lines.append(line)
|
|
next = indent + 1
|
|
}
|
|
if depth >= opts.depth {
|
|
return
|
|
}
|
|
for c in children(e) {
|
|
try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
|
|
}
|
|
}
|
|
|
|
try walk(root, 0, 0, [])
|
|
refsLock.withLock {
|
|
latestRefs = RefTable(elements: found, source: source)
|
|
}
|
|
if truncated {
|
|
lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines")
|
|
}
|
|
return lines.joined(separator: "\n")
|
|
}
|
|
|
|
func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions {
|
|
var o = TreeOptions()
|
|
o.query = args.string("query")
|
|
o.depth = args.int("depth") ?? o.depth
|
|
o.maxLines = args.int("max_lines") ?? o.maxLines
|
|
o.frames = args.bool("frames")
|
|
o.origin = origin
|
|
return o
|
|
}
|
|
|
|
func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
|
|
if let pid = args.int("menu_bar_pid") {
|
|
try requireTrust()
|
|
guard let p = pid_t(exactly: pid), p > 0 else {
|
|
throw ToolError("process \(pid) has no menu bar")
|
|
}
|
|
let app = appElement(p)
|
|
_ = try axRole(app)
|
|
guard let bar = attr(app, kAXMenuBarAttribute) else {
|
|
throw ToolError("process \(pid) has no menu bar")
|
|
}
|
|
return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
|
|
}
|
|
let id = try args.require("window", args.int)
|
|
let (info, w) = try axWindow(id)
|
|
return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
|
|
}
|
|
|
|
func windowMatches(_ root: AXUIElement, _ needle: String) throws -> Int {
|
|
let n = needle.lowercased()
|
|
var visits = 0
|
|
var count = 0
|
|
func walk(_ e: AXUIElement) throws {
|
|
visits += 1
|
|
if visits > maxVisits {
|
|
return
|
|
}
|
|
_ = try axRole(e)
|
|
if [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { text(e, $0).lowercased().contains(n) }) {
|
|
count += 1
|
|
}
|
|
for c in children(e) {
|
|
try walk(c)
|
|
}
|
|
}
|
|
try walk(root)
|
|
return count
|
|
}
|