ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

hands test

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-17 23:03:33 -07:00
parent bb908e493f
commit ccf8c551ec
5 changed files with 115 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hands/Sources/byre-hands/AX.swift
+69 -25
View File
@@ -9,8 +9,25 @@ struct RefSource {
let window: Int?
}
var refs: [String: AXUIElement] = [:]
var refSource = RefSource(pid: 0, window: nil)
struct RefTable {
let elements: [String: AXUIElement]
let source: RefSource
}
var latestRefs = RefTable(elements: [:], source: RefSource(pid: 0, window: nil))
var refCount = 0
let refsLock = NSLock()
func currentRefs() -> RefTable {
return refsLock.withLock { latestRefs }
}
func newRef() -> String {
return refsLock.withLock {
refCount += 1
return "e\(refCount)"
}
}
let shownActions: [String: String] = [
kAXPressAction: "press",
@@ -52,12 +69,25 @@ func children(_ e: AXUIElement) -> [AXUIElement] {
}
func appElement(_ pid: pid_t) -> AXUIElement {
let app = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(app, 3)
return app
return AXUIElementCreateApplication(pid)
}
func requireTrust() throws {
if !AXIsProcessTrusted() {
throw ToolError("byre lacks Accessibility, so no app's tree can be read or acted on; the human must switch byre on in System Settings → Privacy & Security → Accessibility")
}
}
func axRole(_ e: AXUIElement) throws -> String {
var v: CFTypeRef?
if AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &v) == .cannotComplete {
throw ToolError("the app isn't answering accessibility requests; it may be busy or have a menu open (capture it to see, and escape closes a menu)")
}
return v as? String ?? ""
}
func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
try requireTrust()
let info = try window(id)
for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
var wid: CGWindowID = 0
@@ -68,11 +98,11 @@ func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
throw ToolError("window \(id) has no accessibility element; the app may not expose it")
}
func element(_ ref: String, in info: WindowInfo) throws -> AXUIElement {
guard let e = refs[ref] else {
throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act) and are replaced by each new tree")
func element(_ ref: String, in info: WindowInfo, _ table: RefTable) throws -> AXUIElement {
let s = table.source
guard let e = table.elements[ref] else {
throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act), and each new tree replaces them with new numbers")
}
let s = refSource
if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) {
let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)"
throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first")
@@ -107,21 +137,20 @@ struct TreeOptions {
var origin = CGPoint.zero
}
func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> String {
refs = [:]
refSource = source
func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) throws -> String {
var found: [String: AXUIElement] = [:]
var lines: [String] = []
var visits = 0
var truncated = false
let q = opts.query?.lowercased()
func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) {
func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) throws {
visits += 1
if visits > maxVisits || lines.count >= opts.maxLines {
truncated = true
return
}
let role = text(e, kAXRoleAttribute)
let role = try axRole(e)
var parts: [String] = []
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
let s = text(e, name)
@@ -148,8 +177,8 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St
let matches = q == nil || parts.contains { $0.lowercased().contains(q!) }
var next = indent
if shown && matches {
let ref = "e\(refs.count + 1)"
refs[ref] = e
let ref = newRef()
found[ref] = e
var line = String(repeating: " ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased()
for p in parts {
line += " " + clip(p, 200).debugDescription
@@ -167,11 +196,14 @@ func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> St
return
}
for c in children(e) {
walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
try walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
}
}
walk(root, 0, 0, [])
try walk(root, 0, 0, [])
refsLock.withLock {
latestRefs = RefTable(elements: found, source: source)
}
if truncated {
lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines")
}
@@ -190,30 +222,42 @@ func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions {
func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
if let pid = args.int("menu_bar_pid") {
guard let p = pid_t(exactly: pid), p > 0, let bar = attr(appElement(p), kAXMenuBarAttribute) else {
try requireTrust()
guard let p = pid_t(exactly: pid), p > 0 else {
throw ToolError("process \(pid) has no menu bar")
}
return [.text(axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
let app = appElement(p)
_ = try axRole(app)
guard let bar = attr(app, kAXMenuBarAttribute) else {
throw ToolError("process \(pid) has no menu bar")
}
return [.text(try axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
}
let id = try args.require("window", args.int)
let (info, w) = try axWindow(id)
return [.text(axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
return [.text(try axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
}
func windowText(_ root: AXUIElement, contains needle: String) -> Bool {
func windowText(_ root: AXUIElement, contains needle: String) throws -> Bool {
let n = needle.lowercased()
var visits = 0
func walk(_ e: AXUIElement) -> Bool {
func walk(_ e: AXUIElement) throws -> Bool {
visits += 1
if visits > maxVisits {
return false
}
_ = try axRole(e)
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] {
if text(e, name).lowercased().contains(n) {
return true
}
}
return children(e).contains { walk($0) }
for c in children(e) {
if try walk(c) {
return true
}
}
return false
}
return walk(root)
return try walk(root)
}
hands/Sources/byre-hands/Act.swift
+19 -19
View File
@@ -1,9 +1,9 @@
import AppKit
import ApplicationServices
func point(_ v: Any?, _ info: WindowInfo) throws -> CGPoint {
func point(_ v: Any?, _ info: WindowInfo, _ table: RefTable) throws -> CGPoint {
if let ref = v as? String {
guard let f = frame(try element(ref, in: info)) else {
guard let f = frame(try element(ref, in: info, table)) else {
throw ToolError("\(ref) has no position on screen")
}
return CGPoint(x: f.midX, y: f.midY)
@@ -26,48 +26,48 @@ func isInput(_ step: [String: Any]) -> Bool {
return step.keys.contains(where: inputKeys.contains)
}
func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws -> String {
func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> String {
var info = try window(id)
if isInput(step) {
info = try await focusWindow(id, w)
}
if let ref = step.string("press") {
try check(AXUIElementPerformAction(try element(ref, in: info), kAXPressAction as CFString), "press \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXPressAction as CFString), "press \(ref)")
return "pressed \(ref)"
}
if let ref = step.string("confirm") {
try check(AXUIElementPerformAction(try element(ref, in: info), kAXConfirmAction as CFString), "confirm \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXConfirmAction as CFString), "confirm \(ref)")
return "confirmed \(ref)"
}
if let ref = step.string("focus") {
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
return "focused \(ref)"
}
if let ref = step.string("set") {
let value = try step.require("value", step.string)
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
return "set \(ref)"
}
if let name = step.string("action") {
let ref = try step.require("ref", step.string)
try check(AXUIElementPerformAction(try element(ref, in: info), name as CFString), "\(name) \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), name as CFString), "\(name) \(ref)")
return "\(name) \(ref)"
}
for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
if step[k] != nil {
let p = try point(step[k], info)
let p = try point(step[k], info, table)
await click(p, right: right, count: count)
return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
}
}
if step["drag"] != nil {
let from = try point(step["drag"], info)
let to = try point(step["to"], info)
let from = try point(step["drag"], info, table)
let to = try point(step["to"], info, table)
await drag(from, to)
return "dragged"
}
if step["scroll"] != nil {
let p = try point(step["scroll"], info)
let p = try point(step["scroll"], info, table)
let dx = step.int("dx") ?? 0
let dy = step.int("dy") ?? 0
guard (-1000...1000).contains(dx), (-1000...1000).contains(dy) else {
@@ -91,8 +91,7 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws ->
throw ToolError("unknown step \(step.keys.sorted())")
}
@MainActor
func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String) async throws -> Bool) async throws -> String? {
func waitFor(_ args: [String: Any], acted: Bool, _ present: (String) async throws -> Bool) async throws -> String? {
let wait = try args.ms("wait_ms")
let timeout = Double(try args.ms("timeout_ms") ?? 10000) / 1000
if let wait {
@@ -115,11 +114,11 @@ func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String)
}
}
func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throws -> [String] {
func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> [String] {
var log: [String] = []
for (i, step) in steps.enumerated() {
do {
log.append("\(i + 1). " + (try await runStep(step, id, w)))
log.append("\(i + 1). " + (try await runStep(step, id, w, table)))
await pause(50)
} catch let e as ToolError {
log.append("\(i + 1). " + e.message + "; stopped here")
@@ -130,6 +129,7 @@ func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throw
}
func actTool(_ args: [String: Any]) async throws -> [Content] {
let table = currentRefs()
let id = try args.require("window", args.int)
let (_, w) = try axWindow(id)
let steps = args["steps"] as? [[String: Any]] ?? []
@@ -139,14 +139,14 @@ func actTool(_ args: [String: Any]) async throws -> [Content] {
close(lock)
}
}
var log = try await runSteps(steps, id, w)
if let note = try await waitFor(args, acted: !steps.isEmpty, { windowText(w, contains: $0) }) {
var log = try await runSteps(steps, id, w, table)
if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowText(w, contains: $0) }) {
log.append(note)
}
var out: [Content] = []
if args.bool("tree") {
let current = try window(id)
log.append("\n" + axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin)))
log.append("\n" + (try axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin))))
}
if args.bool("capture") {
let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
hands/Sources/byre-hands/Browser.swift
+11 -15
View File
@@ -1,16 +1,17 @@
import AppKit
@MainActor
func osa(_ src: String) throws -> NSAppleEventDescriptor {
guard let script = NSAppleScript(source: src) else {
throw ToolError("AppleScript did not compile")
return try DispatchQueue.main.sync {
guard let script = NSAppleScript(source: "with timeout of 10 seconds\n\(src)\nend timeout") else {
throw ToolError("AppleScript did not compile")
}
var err: NSDictionary?
let r = script.executeAndReturnError(&err)
if let err {
throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)")
}
return r
}
var err: NSDictionary?
let r = script.executeAndReturnError(&err)
if let err {
throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)")
}
return r
}
func quote(_ s: String) -> String {
@@ -34,18 +35,15 @@ struct TabTarget {
}
}
@MainActor
func js(_ code: String, _ t: TabTarget) throws -> String {
let r = try osa("tell application \"Orion\" to do JavaScript \(quote(code)) in \(t.spec)")
return r.stringValue ?? "undefined"
}
@MainActor
func orionWindow(_ index: Int) throws -> Int {
return Int(try osa("tell application \"Orion\" to get id of window \(index)").int32Value)
}
@MainActor
func tabs() throws -> String {
let src = """
tell application "Orion"
@@ -65,7 +63,6 @@ func tabs() throws -> String {
return "window tab title | url\n" + (try osa(src).stringValue ?? "")
}
@MainActor
func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String {
let start = Date()
await pause(300)
@@ -78,7 +75,6 @@ func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String {
return String(format: "still loading after %.1fs", timeout)
}
@MainActor
func browserTool(_ args: [String: Any]) async throws -> [Content] {
let op = try args.require("op", args.string)
let win = args.int("window") ?? 1
@@ -118,7 +114,7 @@ func browserTool(_ args: [String: Any]) async throws -> [Content] {
default:
throw ToolError("unknown op \(op.debugDescription)")
}
let present: @MainActor (String) async throws -> Bool = { needle in
let present: (String) async throws -> Bool = { needle in
(try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true"
}
if let note = try await waitFor(args, acted: false, present) {
hands/Sources/byre-hands/Server.swift
+14 -10
View File
@@ -26,28 +26,31 @@ struct Tool {
let name: String
let description: String
let schema: String
let run: @MainActor ([String: Any]) async throws -> [Content]
let run: ([String: Any]) async throws -> [Content]
}
let stdoutLock = NSLock()
@main
struct Server {
static func main() {
_ = NSApplication.shared
AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), 3)
Thread.detachNewThread {
let inFlight = DispatchGroup()
while let line = readLine() {
let done = DispatchSemaphore(value: 0)
Task { @MainActor in
inFlight.enter()
Task.detached {
await handle(line)
done.signal()
inFlight.leave()
}
done.wait()
}
inFlight.wait()
exit(0)
}
RunLoop.main.run()
}
@MainActor
static func handle(_ line: String) async {
guard let data = line.data(using: .utf8),
let req = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
@@ -75,12 +78,13 @@ struct Server {
default:
resp["error"] = ["code": -32601, "message": "method not found: \(req["method"] ?? "")"]
}
let out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes)
FileHandle.standardOutput.write(out)
FileHandle.standardOutput.write(Data("\n".utf8))
var out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes)
out.append(Data("\n".utf8))
stdoutLock.withLock {
FileHandle.standardOutput.write(out)
}
}
@MainActor
static func call(_ params: [String: Any]) async -> [String: Any] {
let name = params["name"] as? String ?? ""
let args = params["arguments"] as? [String: Any] ?? [:]