ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Fail hands accessibility reads fast when an app stops answering

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-18 08:43:52 -07:00
parent 1020d7eb15
commit 7f62df19d2
4 changed files with 36 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hands/Sources/byre-hands/AX.swift
+30 -24
View File
@@ -40,16 +40,22 @@ let shownActions: [String: String] = [
let maxVisits = 30000
func attr(_ e: AXUIElement, _ name: String) -> CFTypeRef? {
let notAnswering = ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")
func attr(_ e: AXUIElement, _ name: String) throws -> CFTypeRef? {
var v: CFTypeRef?
if AXUIElementCopyAttributeValue(e, name as CFString, &v) != .success {
let err = AXUIElementCopyAttributeValue(e, name as CFString, &v)
if err == .cannotComplete {
throw notAnswering
}
if err != .success {
return nil
}
return v
}
func text(_ e: AXUIElement, _ name: String) -> String {
guard let v = attr(e, name) else {
func text(_ e: AXUIElement, _ name: String) throws -> String {
guard let v = try attr(e, name) else {
return ""
}
if let s = v as? String {
@@ -64,8 +70,8 @@ func text(_ e: AXUIElement, _ name: String) -> String {
return ""
}
func children(_ e: AXUIElement) -> [AXUIElement] {
return (attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? []
func children(_ e: AXUIElement) throws -> [AXUIElement] {
return (try attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? []
}
func appElement(_ pid: pid_t) -> AXUIElement {
@@ -79,17 +85,13 @@ func requireTrust() throws {
}
func axRole(_ e: AXUIElement) throws -> String {
var v: CFTypeRef?
if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete {
throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")
}
return v as? String ?? ""
return try attr(e, kAXRoleAttribute) as? String ?? ""
}
func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
try requireTrust()
let info = try window(id)
for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
for w in (try attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
var wid: CGWindowID = 0
if _AXUIElementGetWindow(w, &wid) == .success && Int(wid) == id {
return (info, w)
@@ -110,8 +112,8 @@ func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AX
return e
}
func frame(_ e: AXUIElement) -> CGRect? {
guard let p = attr(e, kAXPositionAttribute), let s = attr(e, kAXSizeAttribute) else {
func frame(_ e: AXUIElement) throws -> CGRect? {
guard let p = try attr(e, kAXPositionAttribute), let s = try attr(e, kAXSizeAttribute) else {
return nil
}
var pt = CGPoint.zero
@@ -153,23 +155,27 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throw
let role = try axRole(e)
var parts: [String] = []
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
let s = text(e, name)
let s = try text(e, name)
if !s.isEmpty && !parts.contains(s) {
parts.append(s)
}
}
var names: CFArray?
AXUIElementCopyActionNames(e, &names)
if AXUIElementCopyActionNames(e, &names) == .cannotComplete {
throw notAnswering
}
var acts = ((names as? [String]) ?? []).compactMap { shownActions[$0] }
var settable: DarwinBoolean = false
AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable)
if AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable) == .cannotComplete {
throw notAnswering
}
if settable.boolValue {
acts.append("set")
}
if (attr(e, kAXFocusedAttribute) as? Bool) == true {
if (try attr(e, kAXFocusedAttribute) as? Bool) == true {
acts.append("focused")
}
if (attr(e, kAXEnabledAttribute) as? Bool) == false {
if (try attr(e, kAXEnabledAttribute) as? Bool) == false {
acts.append("disabled")
}
let echo = role == kAXStaticTextRole && acts.isEmpty && parts.allSatisfy { p in parentParts.contains { $0.contains(p) } }
@@ -186,7 +192,7 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throw
if !acts.isEmpty {
line += " [" + acts.joined(separator: " ") + "]"
}
if opts.frames, let f = frame(e) {
if opts.frames, let f = try frame(e) {
line += " @\(Int(f.minX - opts.origin.x)),\(Int(f.minY - opts.origin.y)) \(Int(f.width))x\(Int(f.height))"
}
lines.append(line)
@@ -195,7 +201,7 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throw
if depth >= opts.depth {
return
}
for c in children(e) {
for c in try children(e) {
try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
}
}
@@ -228,7 +234,7 @@ func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
}
let app = appElement(p)
_ = try axRole(app)
guard let bar = attr(app, kAXMenuBarAttribute) else {
guard let bar = try attr(app, kAXMenuBarAttribute) else {
throw ToolError("process \(pid) has no menu bar")
}
return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
@@ -248,10 +254,10 @@ func windowMatches(_ root: AXUIElement, _ needle: String) throws -> Int {
return
}
_ = try axRole(e)
if [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { text(e, $0).lowercased().contains(n) }) {
if try [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute].contains(where: { try text(e, $0).lowercased().contains(n) }) {
count += 1
}
for c in children(e) {
for c in try children(e) {
try walk(c)
}
}
hands/Sources/byre-hands/Act.swift
+1 -1
View File
@@ -3,7 +3,7 @@ import ApplicationServices
func point(_ v: Any?, _ info: WindowInfo, _ table: RefTable) throws -> CGPoint {
if let ref = v as? String {
guard let f = frame(try element(ref, in: info, table)) else {
guard let f = try frame(try element(ref, in: info, table)) else {
throw ToolError("\(ref) has no position on screen")
}
return CGPoint(x: f.midX, y: f.midY)
hands/Sources/byre-hands/Input.swift
+4 -4
View File
@@ -65,9 +65,9 @@ func post(_ e: CGEvent?) {
e?.post(tap: .cghidEventTap)
}
func isFront(_ info: WindowInfo) -> Bool {
func isFront(_ info: WindowInfo) throws -> Bool {
guard NSWorkspace.shared.frontmostApplication?.processIdentifier == info.pid,
let focused = attr(appElement(info.pid), kAXFocusedWindowAttribute)
let focused = try attr(appElement(info.pid), kAXFocusedWindowAttribute)
else {
return false
}
@@ -77,14 +77,14 @@ func isFront(_ info: WindowInfo) -> Bool {
func focusWindow(_ id: Int, _ w: AXUIElement) async throws -> WindowInfo {
let info = try window(id)
if isFront(info) {
if try isFront(info) {
return info
}
NSRunningApplication(processIdentifier: info.pid)?.activate()
AXUIElementPerformAction(w, kAXRaiseAction as CFString)
await pause(150)
let now = try window(id)
if !isFront(now) {
if try !isFront(now) {
throw ToolError("could not bring window \(id) to the front (\(NSWorkspace.shared.frontmostApplication?.localizedName ?? "nothing") is frontmost), so no input was sent")
}
return now