ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

hands test

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-17 23:03:33 -07:00
parent bb908e493f
commit ccf8c551ec
5 changed files with 115 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hands/Sources/byre-hands/Act.swift
+19 -19
View File
@@ -1,9 +1,9 @@
import AppKit
import ApplicationServices
func point(_ v: Any?, _ info: WindowInfo) throws -> CGPoint {
func point(_ v: Any?, _ info: WindowInfo, _ table: RefTable) throws -> CGPoint {
if let ref = v as? String {
guard let f = frame(try element(ref, in: info)) else {
guard let f = frame(try element(ref, in: info, table)) else {
throw ToolError("\(ref) has no position on screen")
}
return CGPoint(x: f.midX, y: f.midY)
@@ -26,48 +26,48 @@ func isInput(_ step: [String: Any]) -> Bool {
return step.keys.contains(where: inputKeys.contains)
}
func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws -> String {
func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> String {
var info = try window(id)
if isInput(step) {
info = try await focusWindow(id, w)
}
if let ref = step.string("press") {
try check(AXUIElementPerformAction(try element(ref, in: info), kAXPressAction as CFString), "press \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXPressAction as CFString), "press \(ref)")
return "pressed \(ref)"
}
if let ref = step.string("confirm") {
try check(AXUIElementPerformAction(try element(ref, in: info), kAXConfirmAction as CFString), "confirm \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXConfirmAction as CFString), "confirm \(ref)")
return "confirmed \(ref)"
}
if let ref = step.string("focus") {
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
return "focused \(ref)"
}
if let ref = step.string("set") {
let value = try step.require("value", step.string)
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
return "set \(ref)"
}
if let name = step.string("action") {
let ref = try step.require("ref", step.string)
try check(AXUIElementPerformAction(try element(ref, in: info), name as CFString), "\(name) \(ref)")
try check(AXUIElementPerformAction(try element(ref, in: info, table), name as CFString), "\(name) \(ref)")
return "\(name) \(ref)"
}
for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
if step[k] != nil {
let p = try point(step[k], info)
let p = try point(step[k], info, table)
await click(p, right: right, count: count)
return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
}
}
if step["drag"] != nil {
let from = try point(step["drag"], info)
let to = try point(step["to"], info)
let from = try point(step["drag"], info, table)
let to = try point(step["to"], info, table)
await drag(from, to)
return "dragged"
}
if step["scroll"] != nil {
let p = try point(step["scroll"], info)
let p = try point(step["scroll"], info, table)
let dx = step.int("dx") ?? 0
let dy = step.int("dy") ?? 0
guard (-1000...1000).contains(dx), (-1000...1000).contains(dy) else {
@@ -91,8 +91,7 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws ->
throw ToolError("unknown step \(step.keys.sorted())")
}
@MainActor
func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String) async throws -> Bool) async throws -> String? {
func waitFor(_ args: [String: Any], acted: Bool, _ present: (String) async throws -> Bool) async throws -> String? {
let wait = try args.ms("wait_ms")
let timeout = Double(try args.ms("timeout_ms") ?? 10000) / 1000
if let wait {
@@ -115,11 +114,11 @@ func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String)
}
}
func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throws -> [String] {
func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> [String] {
var log: [String] = []
for (i, step) in steps.enumerated() {
do {
log.append("\(i + 1). " + (try await runStep(step, id, w)))
log.append("\(i + 1). " + (try await runStep(step, id, w, table)))
await pause(50)
} catch let e as ToolError {
log.append("\(i + 1). " + e.message + "; stopped here")
@@ -130,6 +129,7 @@ func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throw
}
func actTool(_ args: [String: Any]) async throws -> [Content] {
let table = currentRefs()
let id = try args.require("window", args.int)
let (_, w) = try axWindow(id)
let steps = args["steps"] as? [[String: Any]] ?? []
@@ -139,14 +139,14 @@ func actTool(_ args: [String: Any]) async throws -> [Content] {
close(lock)
}
}
var log = try await runSteps(steps, id, w)
if let note = try await waitFor(args, acted: !steps.isEmpty, { windowText(w, contains: $0) }) {
var log = try await runSteps(steps, id, w, table)
if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowText(w, contains: $0) }) {
log.append(note)
}
var out: [Content] = []
if args.bool("tree") {
let current = try window(id)
log.append("\n" + axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin)))
log.append("\n" + (try axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin))))
}
if args.bool("capture") {
let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)