Add byre-hands: screen capture and app control tools for agents
This commit is contained in:
Ian Gulliver
@@ -0,0 +1,13 @@
|
||||
// swift-tools-version: 6.2
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "ByreHands",
|
||||
platforms: [.macOS("26.0")],
|
||||
targets: [
|
||||
.executableTarget(
|
||||
name: "byre-hands",
|
||||
swiftSettings: [.swiftLanguageMode(.v5)]
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,219 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
|
||||
@_silgen_name("_AXUIElementGetWindow")
|
||||
func _AXUIElementGetWindow(_ element: AXUIElement, _ id: UnsafeMutablePointer<CGWindowID>) -> AXError
|
||||
|
||||
struct RefSource {
|
||||
let pid: pid_t
|
||||
let window: Int?
|
||||
}
|
||||
|
||||
var refs: [String: AXUIElement] = [:]
|
||||
var refSource = RefSource(pid: 0, window: nil)
|
||||
|
||||
let shownActions: [String: String] = [
|
||||
kAXPressAction: "press",
|
||||
kAXConfirmAction: "confirm",
|
||||
kAXIncrementAction: "increment",
|
||||
kAXDecrementAction: "decrement",
|
||||
kAXPickAction: "pick",
|
||||
kAXCancelAction: "cancel",
|
||||
]
|
||||
|
||||
let maxVisits = 30000
|
||||
|
||||
func attr(_ e: AXUIElement, _ name: String) -> CFTypeRef? {
|
||||
var v: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(e, name as CFString, &v) != .success {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func text(_ e: AXUIElement, _ name: String) -> String {
|
||||
guard let v = attr(e, name) else {
|
||||
return ""
|
||||
}
|
||||
if let s = v as? String {
|
||||
return s
|
||||
}
|
||||
if CFGetTypeID(v) == CFBooleanGetTypeID() {
|
||||
return ""
|
||||
}
|
||||
if let n = v as? NSNumber {
|
||||
return n.stringValue
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func children(_ e: AXUIElement) -> [AXUIElement] {
|
||||
return (attr(e, kAXChildrenAttribute) as? [AXUIElement]) ?? []
|
||||
}
|
||||
|
||||
func appElement(_ pid: pid_t) -> AXUIElement {
|
||||
let app = AXUIElementCreateApplication(pid)
|
||||
AXUIElementSetMessagingTimeout(app, 3)
|
||||
return app
|
||||
}
|
||||
|
||||
func axWindow(_ id: Int) throws -> (WindowInfo, AXUIElement) {
|
||||
let info = try window(id)
|
||||
for w in (attr(appElement(info.pid), kAXWindowsAttribute) as? [AXUIElement]) ?? [] {
|
||||
var wid: CGWindowID = 0
|
||||
if _AXUIElementGetWindow(w, &wid) == .success && Int(wid) == id {
|
||||
return (info, w)
|
||||
}
|
||||
}
|
||||
throw ToolError("window \(id) has no accessibility element; the app may not expose it")
|
||||
}
|
||||
|
||||
func element(_ ref: String, in info: WindowInfo) throws -> AXUIElement {
|
||||
guard let e = refs[ref] else {
|
||||
throw ToolError("unknown ref \(ref); refs come from the latest ax_tree (or a tree returned by act) and are replaced by each new tree")
|
||||
}
|
||||
let s = refSource
|
||||
if s.pid != info.pid || (s.window != nil && s.window != Int(info.id)) {
|
||||
let from = s.window.map { "window \($0)" } ?? "the menu bar of process \(s.pid)"
|
||||
throw ToolError("\(ref) is from \(from), not window \(info.id); read window \(info.id)'s tree (or its app's menu bar) first")
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func frame(_ e: AXUIElement) -> CGRect? {
|
||||
guard let p = attr(e, kAXPositionAttribute), let s = attr(e, kAXSizeAttribute) else {
|
||||
return nil
|
||||
}
|
||||
var pt = CGPoint.zero
|
||||
var sz = CGSize.zero
|
||||
AXValueGetValue(p as! AXValue, .cgPoint, &pt)
|
||||
AXValueGetValue(s as! AXValue, .cgSize, &sz)
|
||||
return CGRect(origin: pt, size: sz)
|
||||
}
|
||||
|
||||
func clip(_ s: String, _ n: Int) -> String {
|
||||
let flat = s.replacingOccurrences(of: "\n", with: " ")
|
||||
if flat.count <= n {
|
||||
return flat
|
||||
}
|
||||
return String(flat.prefix(n)) + "…"
|
||||
}
|
||||
|
||||
struct TreeOptions {
|
||||
var query: String?
|
||||
var depth = 80
|
||||
var maxLines = 400
|
||||
var frames = false
|
||||
var origin = CGPoint.zero
|
||||
}
|
||||
|
||||
func axTree(_ root: AXUIElement, _ source: RefSource, _ opts: TreeOptions) -> String {
|
||||
refs = [:]
|
||||
refSource = source
|
||||
var lines: [String] = []
|
||||
var visits = 0
|
||||
var truncated = false
|
||||
let q = opts.query?.lowercased()
|
||||
|
||||
func walk(_ e: AXUIElement, _ depth: Int, _ indent: Int, _ parentParts: [String]) {
|
||||
visits += 1
|
||||
if visits > maxVisits || lines.count >= opts.maxLines {
|
||||
truncated = true
|
||||
return
|
||||
}
|
||||
let role = text(e, kAXRoleAttribute)
|
||||
var parts: [String] = []
|
||||
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute, kAXPlaceholderValueAttribute] {
|
||||
let s = text(e, name)
|
||||
if !s.isEmpty && !parts.contains(s) {
|
||||
parts.append(s)
|
||||
}
|
||||
}
|
||||
var names: CFArray?
|
||||
AXUIElementCopyActionNames(e, &names)
|
||||
var acts = ((names as? [String]) ?? []).compactMap { shownActions[$0] }
|
||||
var settable: DarwinBoolean = false
|
||||
AXUIElementIsAttributeSettable(e, kAXValueAttribute as CFString, &settable)
|
||||
if settable.boolValue {
|
||||
acts.append("set")
|
||||
}
|
||||
if (attr(e, kAXFocusedAttribute) as? Bool) == true {
|
||||
acts.append("focused")
|
||||
}
|
||||
if (attr(e, kAXEnabledAttribute) as? Bool) == false {
|
||||
acts.append("disabled")
|
||||
}
|
||||
let echo = role == kAXStaticTextRole && acts.isEmpty && parts.allSatisfy { p in parentParts.contains { $0.contains(p) } }
|
||||
let shown = (!parts.isEmpty || !acts.isEmpty) && !echo
|
||||
let matches = q == nil || parts.contains { $0.lowercased().contains(q!) }
|
||||
var next = indent
|
||||
if shown && matches {
|
||||
let ref = "e\(refs.count + 1)"
|
||||
refs[ref] = e
|
||||
var line = String(repeating: " ", count: q == nil ? indent : 0) + ref + " " + role.replacingOccurrences(of: "AX", with: "").lowercased()
|
||||
for p in parts {
|
||||
line += " " + clip(p, 200).debugDescription
|
||||
}
|
||||
if !acts.isEmpty {
|
||||
line += " [" + acts.joined(separator: " ") + "]"
|
||||
}
|
||||
if opts.frames, let f = frame(e) {
|
||||
line += " @\(Int(f.minX - opts.origin.x)),\(Int(f.minY - opts.origin.y)) \(Int(f.width))x\(Int(f.height))"
|
||||
}
|
||||
lines.append(line)
|
||||
next = indent + 1
|
||||
}
|
||||
if depth >= opts.depth {
|
||||
return
|
||||
}
|
||||
for c in children(e) {
|
||||
walk(c, depth + 1, next, parts.isEmpty ? parentParts : parts)
|
||||
}
|
||||
}
|
||||
|
||||
walk(root, 0, 0, [])
|
||||
if truncated {
|
||||
lines.append("… truncated at \(lines.count) lines; narrow it with query or depth, or raise max_lines")
|
||||
}
|
||||
return lines.joined(separator: "\n")
|
||||
}
|
||||
|
||||
func treeOptions(_ args: [String: Any], origin: CGPoint) -> TreeOptions {
|
||||
var o = TreeOptions()
|
||||
o.query = args.string("query")
|
||||
o.depth = args.int("depth") ?? o.depth
|
||||
o.maxLines = args.int("max_lines") ?? o.maxLines
|
||||
o.frames = args.bool("frames")
|
||||
o.origin = origin
|
||||
return o
|
||||
}
|
||||
|
||||
func axTreeTool(_ args: [String: Any]) async throws -> [Content] {
|
||||
if let pid = args.int("menu_bar_pid") {
|
||||
guard let p = pid_t(exactly: pid), p > 0, let bar = attr(appElement(p), kAXMenuBarAttribute) else {
|
||||
throw ToolError("process \(pid) has no menu bar")
|
||||
}
|
||||
return [.text(axTree(bar as! AXUIElement, RefSource(pid: p, window: nil), treeOptions(args, origin: .zero)))]
|
||||
}
|
||||
let id = try args.require("window", args.int)
|
||||
let (info, w) = try axWindow(id)
|
||||
return [.text(axTree(w, RefSource(pid: info.pid, window: id), treeOptions(args, origin: info.frame.origin)))]
|
||||
}
|
||||
|
||||
func windowText(_ root: AXUIElement, contains needle: String) -> Bool {
|
||||
let n = needle.lowercased()
|
||||
var visits = 0
|
||||
func walk(_ e: AXUIElement) -> Bool {
|
||||
visits += 1
|
||||
if visits > maxVisits {
|
||||
return false
|
||||
}
|
||||
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute] {
|
||||
if text(e, name).lowercased().contains(n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return children(e).contains { walk($0) }
|
||||
}
|
||||
return walk(root)
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
|
||||
func point(_ v: Any?, _ info: WindowInfo) throws -> CGPoint {
|
||||
if let ref = v as? String {
|
||||
guard let f = frame(try element(ref, in: info)) else {
|
||||
throw ToolError("\(ref) has no position on screen")
|
||||
}
|
||||
return CGPoint(x: f.midX, y: f.midY)
|
||||
}
|
||||
if let xy = v as? [NSNumber], xy.count == 2 {
|
||||
return CGPoint(x: info.frame.minX + CGFloat(xy[0].doubleValue), y: info.frame.minY + CGFloat(xy[1].doubleValue))
|
||||
}
|
||||
throw ToolError("a point is a ref (\"e5\") or window coordinates ([x, y])")
|
||||
}
|
||||
|
||||
func check(_ err: AXError, _ what: String) throws {
|
||||
if err != .success {
|
||||
throw ToolError("\(what) failed: AXError \(err.rawValue)")
|
||||
}
|
||||
}
|
||||
|
||||
let inputKeys = ["click", "double_click", "right_click", "drag", "scroll", "type", "key"]
|
||||
|
||||
func isInput(_ step: [String: Any]) -> Bool {
|
||||
return step.keys.contains(where: inputKeys.contains)
|
||||
}
|
||||
|
||||
func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement) async throws -> String {
|
||||
var info = try window(id)
|
||||
if isInput(step) {
|
||||
info = try await focusWindow(id, w)
|
||||
}
|
||||
if let ref = step.string("press") {
|
||||
try check(AXUIElementPerformAction(try element(ref, in: info), kAXPressAction as CFString), "press \(ref)")
|
||||
return "pressed \(ref)"
|
||||
}
|
||||
if let ref = step.string("confirm") {
|
||||
try check(AXUIElementPerformAction(try element(ref, in: info), kAXConfirmAction as CFString), "confirm \(ref)")
|
||||
return "confirmed \(ref)"
|
||||
}
|
||||
if let ref = step.string("focus") {
|
||||
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
|
||||
return "focused \(ref)"
|
||||
}
|
||||
if let ref = step.string("set") {
|
||||
let value = try step.require("value", step.string)
|
||||
try check(AXUIElementSetAttributeValue(try element(ref, in: info), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
|
||||
return "set \(ref)"
|
||||
}
|
||||
if let name = step.string("action") {
|
||||
let ref = try step.require("ref", step.string)
|
||||
try check(AXUIElementPerformAction(try element(ref, in: info), name as CFString), "\(name) \(ref)")
|
||||
return "\(name) \(ref)"
|
||||
}
|
||||
for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
|
||||
if step[k] != nil {
|
||||
let p = try point(step[k], info)
|
||||
await click(p, right: right, count: count)
|
||||
return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
|
||||
}
|
||||
}
|
||||
if step["drag"] != nil {
|
||||
let from = try point(step["drag"], info)
|
||||
let to = try point(step["to"], info)
|
||||
await drag(from, to)
|
||||
return "dragged"
|
||||
}
|
||||
if step["scroll"] != nil {
|
||||
let p = try point(step["scroll"], info)
|
||||
let dx = step.int("dx") ?? 0
|
||||
let dy = step.int("dy") ?? 0
|
||||
guard (-1000...1000).contains(dx), (-1000...1000).contains(dy) else {
|
||||
throw ToolError("dx and dy must be between -1000 and 1000 lines")
|
||||
}
|
||||
scroll(p, dx: Int32(dx), dy: Int32(dy))
|
||||
return "scrolled"
|
||||
}
|
||||
if let s = step.string("type") {
|
||||
await type(s)
|
||||
return "typed \(s.count) characters"
|
||||
}
|
||||
if let k = step.string("key") {
|
||||
try key(k)
|
||||
return "key \(k)"
|
||||
}
|
||||
if let ms = try step.ms("wait") {
|
||||
await pause(ms)
|
||||
return "waited \(ms)ms"
|
||||
}
|
||||
throw ToolError("unknown step \(step.keys.sorted())")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func waitFor(_ args: [String: Any], acted: Bool, _ present: @MainActor (String) async throws -> Bool) async throws -> String? {
|
||||
let wait = try args.ms("wait_ms")
|
||||
let timeout = Double(try args.ms("timeout_ms") ?? 10000) / 1000
|
||||
if let wait {
|
||||
await pause(wait)
|
||||
} else if acted {
|
||||
await pause(300)
|
||||
}
|
||||
guard let needle = args.string("wait_for") else {
|
||||
return nil
|
||||
}
|
||||
let start = Date()
|
||||
while true {
|
||||
if try await present(needle) {
|
||||
return String(format: "found %@ after %.1fs", needle.debugDescription, Date().timeIntervalSince(start))
|
||||
}
|
||||
if Date().timeIntervalSince(start) > timeout {
|
||||
return String(format: "gave up waiting for %@ after %.1fs", needle.debugDescription, timeout)
|
||||
}
|
||||
await pause(200)
|
||||
}
|
||||
}
|
||||
|
||||
func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement) async throws -> [String] {
|
||||
var log: [String] = []
|
||||
for (i, step) in steps.enumerated() {
|
||||
do {
|
||||
log.append("\(i + 1). " + (try await runStep(step, id, w)))
|
||||
await pause(50)
|
||||
} catch let e as ToolError {
|
||||
log.append("\(i + 1). " + e.message + "; stopped here")
|
||||
break
|
||||
}
|
||||
}
|
||||
return log
|
||||
}
|
||||
|
||||
func actTool(_ args: [String: Any]) async throws -> [Content] {
|
||||
let id = try args.require("window", args.int)
|
||||
let (_, w) = try axWindow(id)
|
||||
let steps = args["steps"] as? [[String: Any]] ?? []
|
||||
let lock = steps.isEmpty ? nil : try await lockInput()
|
||||
defer {
|
||||
if let lock {
|
||||
close(lock)
|
||||
}
|
||||
}
|
||||
var log = try await runSteps(steps, id, w)
|
||||
if let note = try await waitFor(args, acted: !steps.isEmpty, { windowText(w, contains: $0) }) {
|
||||
log.append(note)
|
||||
}
|
||||
var out: [Content] = []
|
||||
if args.bool("tree") {
|
||||
let current = try window(id)
|
||||
log.append("\n" + axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin)))
|
||||
}
|
||||
if args.bool("capture") {
|
||||
let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
|
||||
log.append(shot.note)
|
||||
out.append(.png(shot.png))
|
||||
}
|
||||
return [.text(log.joined(separator: "\n"))] + out
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import Foundation
|
||||
|
||||
extension Dictionary where Key == String, Value == Any {
|
||||
func int(_ key: String) -> Int? {
|
||||
return (self[key] as? NSNumber)?.intValue
|
||||
}
|
||||
|
||||
func double(_ key: String) -> Double? {
|
||||
return (self[key] as? NSNumber)?.doubleValue
|
||||
}
|
||||
|
||||
func string(_ key: String) -> String? {
|
||||
return self[key] as? String
|
||||
}
|
||||
|
||||
func bool(_ key: String) -> Bool {
|
||||
return (self[key] as? NSNumber)?.boolValue ?? false
|
||||
}
|
||||
|
||||
func dict(_ key: String) -> [String: Any]? {
|
||||
return self[key] as? [String: Any]
|
||||
}
|
||||
|
||||
func ms(_ key: String) throws -> Int? {
|
||||
guard let v = int(key) else {
|
||||
return nil
|
||||
}
|
||||
if v < 0 || v > 600_000 {
|
||||
throw ToolError("\(key) must be between 0 and 600000 ms")
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func require<T>(_ key: String, _ get: (String) -> T?) throws -> T {
|
||||
guard let v = get(key) else {
|
||||
throw ToolError("missing or malformed argument: \(key)")
|
||||
}
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import AppKit
|
||||
|
||||
@MainActor
|
||||
func osa(_ src: String) throws -> NSAppleEventDescriptor {
|
||||
guard let script = NSAppleScript(source: src) else {
|
||||
throw ToolError("AppleScript did not compile")
|
||||
}
|
||||
var err: NSDictionary?
|
||||
let r = script.executeAndReturnError(&err)
|
||||
if let err {
|
||||
throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)")
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func quote(_ s: String) -> String {
|
||||
return "\"" + s.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + "\""
|
||||
}
|
||||
|
||||
func jsString(_ s: String) -> String {
|
||||
let d = try! JSONSerialization.data(withJSONObject: s, options: .fragmentsAllowed)
|
||||
return String(decoding: d, as: UTF8.self)
|
||||
}
|
||||
|
||||
struct TabTarget {
|
||||
let window: Int
|
||||
let tab: Int?
|
||||
|
||||
var spec: String {
|
||||
if let tab {
|
||||
return "tab \(tab) of window \(window)"
|
||||
}
|
||||
return "current tab of window \(window)"
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func js(_ code: String, _ t: TabTarget) throws -> String {
|
||||
let r = try osa("tell application \"Orion\" to do JavaScript \(quote(code)) in \(t.spec)")
|
||||
return r.stringValue ?? "undefined"
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func orionWindow(_ index: Int) throws -> Int {
|
||||
return Int(try osa("tell application \"Orion\" to get id of window \(index)").int32Value)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func tabs() throws -> String {
|
||||
let src = """
|
||||
tell application "Orion"
|
||||
set out to ""
|
||||
repeat with wi from 1 to count of windows
|
||||
try
|
||||
set w to window wi
|
||||
repeat with ti from 1 to count of tabs of w
|
||||
set t to tab ti of w
|
||||
set out to out & wi & " " & ti & " " & (name of t) & " | " & (URL of t) & linefeed
|
||||
end repeat
|
||||
end try
|
||||
end repeat
|
||||
return out
|
||||
end tell
|
||||
"""
|
||||
return "window tab title | url\n" + (try osa(src).stringValue ?? "")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String {
|
||||
let start = Date()
|
||||
await pause(300)
|
||||
while Date().timeIntervalSince(start) < timeout {
|
||||
if (try? js("document.readyState", t)) == "complete" {
|
||||
return String(format: "loaded after %.1fs", Date().timeIntervalSince(start))
|
||||
}
|
||||
await pause(200)
|
||||
}
|
||||
return String(format: "still loading after %.1fs", timeout)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func browserTool(_ args: [String: Any]) async throws -> [Content] {
|
||||
let op = try args.require("op", args.string)
|
||||
let win = args.int("window") ?? 1
|
||||
var target = TabTarget(window: win, tab: args.int("tab"))
|
||||
let lock = op == "tabs" && !args.bool("capture") ? nil : try await lockInput()
|
||||
defer {
|
||||
if let lock {
|
||||
close(lock)
|
||||
}
|
||||
}
|
||||
var log: [String] = []
|
||||
switch op {
|
||||
case "tabs":
|
||||
log.append(try tabs())
|
||||
case "navigate":
|
||||
let url = try args.require("url", args.string)
|
||||
if args.bool("new_tab") {
|
||||
_ = try osa("tell application \"Orion\" to tell window \(win) to set current tab to (make new tab with properties {URL:\(quote(url))})")
|
||||
target = TabTarget(window: win, tab: nil)
|
||||
} else {
|
||||
_ = try osa("tell application \"Orion\" to set URL of \(target.spec) to \(quote(url))")
|
||||
}
|
||||
log.append(try await waitLoaded(target, timeout: Double(try args.ms("timeout_ms") ?? 15000) / 1000))
|
||||
log.append(try js("document.title + ' | ' + location.href", target))
|
||||
case "js":
|
||||
log.append(try js(try args.require("code", args.string), target))
|
||||
case "select":
|
||||
let tab = try args.require("tab", args.int)
|
||||
_ = try osa("tell application \"Orion\" to set current tab of window \(win) to tab \(tab) of window \(win)")
|
||||
log.append("selected tab \(tab)")
|
||||
target = TabTarget(window: win, tab: nil)
|
||||
case "close":
|
||||
let tab = try args.require("tab", args.int)
|
||||
_ = try osa("tell application \"Orion\" to close tab \(tab) of window \(win)")
|
||||
log.append("closed tab \(tab)")
|
||||
target = TabTarget(window: win, tab: nil)
|
||||
default:
|
||||
throw ToolError("unknown op \(op.debugDescription)")
|
||||
}
|
||||
let present: @MainActor (String) async throws -> Bool = { needle in
|
||||
(try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true"
|
||||
}
|
||||
if let note = try await waitFor(args, acted: false, present) {
|
||||
log.append(note)
|
||||
}
|
||||
var out: [Content] = []
|
||||
if args.bool("capture") {
|
||||
if let tab = target.tab {
|
||||
_ = try osa("tell application \"Orion\" to set current tab of window \(win) to tab \(tab) of window \(win)")
|
||||
log.append("selected tab \(tab) to capture it")
|
||||
await pause(300)
|
||||
}
|
||||
let id = try orionWindow(win)
|
||||
let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
|
||||
log.append(shot.note)
|
||||
out.append(.png(shot.png))
|
||||
}
|
||||
return [.text(log.joined(separator: "\n"))] + out
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
|
||||
let keyCodes: [String: CGKeyCode] = [
|
||||
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11, "q": 12,
|
||||
"w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21, "6": 22, "5": 23,
|
||||
"=": 24, "9": 25, "7": 26, "-": 27, "8": 28, "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34,
|
||||
"p": 35, "return": 36, "enter": 36, "l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43,
|
||||
"/": 44, "n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, "`": 50, "delete": 51, "backspace": 51,
|
||||
"escape": 53, "esc": 53, "f1": 122, "f2": 120, "f3": 99, "f4": 118, "f5": 96, "f6": 97, "f7": 98,
|
||||
"f8": 100, "f9": 101, "f10": 109, "f11": 103, "f12": 111, "home": 115, "pageup": 116,
|
||||
"forwarddelete": 117, "end": 119, "pagedown": 121, "left": 123, "right": 124, "down": 125, "up": 126,
|
||||
]
|
||||
|
||||
let modifierFlags: [String: CGEventFlags] = [
|
||||
"cmd": .maskCommand, "command": .maskCommand, "shift": .maskShift, "alt": .maskAlternate,
|
||||
"opt": .maskAlternate, "option": .maskAlternate, "ctrl": .maskControl, "control": .maskControl,
|
||||
"fn": .maskSecondaryFn,
|
||||
]
|
||||
|
||||
func pause(_ ms: Int) async {
|
||||
try? await Task.sleep(nanoseconds: UInt64(ms) * 1_000_000)
|
||||
}
|
||||
|
||||
let shifted: [Character: String] = [
|
||||
"!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", "&": "7", "*": "8", "(": "9", ")": "0",
|
||||
"_": "-", "+": "=", "{": "[", "}": "]", "|": "\\", ":": ";", "\"": "'", "<": ",", ">": ".", "?": "/", "~": "`",
|
||||
]
|
||||
|
||||
let named: [Character: String] = [" ": "space", "\n": "return", "\t": "tab"]
|
||||
|
||||
func keystroke(_ ch: Character) -> (CGKeyCode, CGEventFlags)? {
|
||||
if let name = named[ch], let c = keyCodes[name] {
|
||||
return (c, [])
|
||||
}
|
||||
if let base = shifted[ch], let c = keyCodes[base] {
|
||||
return (c, .maskShift)
|
||||
}
|
||||
if ch.isUppercase, let c = keyCodes[ch.lowercased()] {
|
||||
return (c, .maskShift)
|
||||
}
|
||||
if let c = keyCodes[String(ch)] {
|
||||
return (c, [])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockInput() async throws -> Int32 {
|
||||
let fd = open(NSTemporaryDirectory() + "byre-hands-input.lock", O_CREAT | O_RDWR, 0o600)
|
||||
if fd < 0 {
|
||||
throw ToolError("open the input lock: errno \(errno)")
|
||||
}
|
||||
let start = Date()
|
||||
while flock(fd, LOCK_EX | LOCK_NB) != 0 {
|
||||
if Date().timeIntervalSince(start) > 30 {
|
||||
close(fd)
|
||||
throw ToolError("another agent has been driving this Mac's apps for 30s, so nothing ran; try again")
|
||||
}
|
||||
await pause(100)
|
||||
}
|
||||
return fd
|
||||
}
|
||||
|
||||
func post(_ e: CGEvent?) {
|
||||
e?.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
func isFront(_ info: WindowInfo) -> Bool {
|
||||
guard NSWorkspace.shared.frontmostApplication?.processIdentifier == info.pid,
|
||||
let focused = attr(appElement(info.pid), kAXFocusedWindowAttribute)
|
||||
else {
|
||||
return false
|
||||
}
|
||||
var wid: CGWindowID = 0
|
||||
return _AXUIElementGetWindow(focused as! AXUIElement, &wid) == .success && wid == info.id
|
||||
}
|
||||
|
||||
func focusWindow(_ id: Int, _ w: AXUIElement) async throws -> WindowInfo {
|
||||
let info = try window(id)
|
||||
if isFront(info) {
|
||||
return info
|
||||
}
|
||||
NSRunningApplication(processIdentifier: info.pid)?.activate()
|
||||
AXUIElementPerformAction(w, kAXRaiseAction as CFString)
|
||||
await pause(150)
|
||||
let now = try window(id)
|
||||
if !isFront(now) {
|
||||
throw ToolError("could not bring window \(id) to the front (\(NSWorkspace.shared.frontmostApplication?.localizedName ?? "nothing") is frontmost), so no input was sent")
|
||||
}
|
||||
return now
|
||||
}
|
||||
|
||||
func click(_ p: CGPoint, right: Bool, count: Int) async {
|
||||
let (down, up): (CGEventType, CGEventType) = right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
|
||||
let button: CGMouseButton = right ? .right : .left
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: button))
|
||||
await pause(30)
|
||||
for i in 1...max(1, count) {
|
||||
for t in [down, up] {
|
||||
let e = CGEvent(mouseEventSource: nil, mouseType: t, mouseCursorPosition: p, mouseButton: button)
|
||||
e?.setIntegerValueField(.mouseEventClickState, value: Int64(i))
|
||||
post(e)
|
||||
}
|
||||
await pause(30)
|
||||
}
|
||||
}
|
||||
|
||||
func drag(_ from: CGPoint, _ to: CGPoint) async {
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: from, mouseButton: .left))
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: from, mouseButton: .left))
|
||||
await pause(50)
|
||||
for i in 1...20 {
|
||||
let f = CGFloat(i) / 20
|
||||
let p = CGPoint(x: from.x + (to.x - from.x) * f, y: from.y + (to.y - from.y) * f)
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .leftMouseDragged, mouseCursorPosition: p, mouseButton: .left))
|
||||
await pause(10)
|
||||
}
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: to, mouseButton: .left))
|
||||
}
|
||||
|
||||
func scroll(_ p: CGPoint, dx: Int32, dy: Int32) {
|
||||
post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: .left))
|
||||
post(CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: dy, wheel2: dx, wheel3: 0))
|
||||
}
|
||||
|
||||
func type(_ s: String) async {
|
||||
for ch in s {
|
||||
for down in [true, false] {
|
||||
if let (code, flags) = keystroke(ch) {
|
||||
let e = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: down)
|
||||
e?.flags = flags
|
||||
post(e)
|
||||
continue
|
||||
}
|
||||
let units = Array(String(ch).utf16)
|
||||
let e = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: down)
|
||||
e?.keyboardSetUnicodeString(stringLength: units.count, unicodeString: units)
|
||||
post(e)
|
||||
}
|
||||
await pause(15)
|
||||
}
|
||||
}
|
||||
|
||||
func key(_ combo: String) throws {
|
||||
var flags: CGEventFlags = []
|
||||
var code: CGKeyCode?
|
||||
for part in combo.lowercased().split(separator: "+").map(String.init) {
|
||||
if let f = modifierFlags[part] {
|
||||
flags.insert(f)
|
||||
continue
|
||||
}
|
||||
guard let c = keyCodes[part] else {
|
||||
throw ToolError("unknown key \(part.debugDescription) in \(combo.debugDescription)")
|
||||
}
|
||||
code = c
|
||||
}
|
||||
guard let code else {
|
||||
throw ToolError("no key in \(combo.debugDescription)")
|
||||
}
|
||||
for down in [true, false] {
|
||||
let e = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: down)
|
||||
e?.flags = flags
|
||||
post(e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import ImageIO
|
||||
import ScreenCaptureKit
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
struct WindowInfo {
|
||||
let id: CGWindowID
|
||||
let pid: pid_t
|
||||
let app: String
|
||||
let title: String
|
||||
let frame: CGRect
|
||||
}
|
||||
|
||||
func windowInfos(_ list: [[String: Any]]) -> [WindowInfo] {
|
||||
return list.compactMap { w in
|
||||
guard (w[kCGWindowLayer as String] as? Int) == 0,
|
||||
let id = w[kCGWindowNumber as String] as? Int,
|
||||
let pid = w[kCGWindowOwnerPID as String] as? Int,
|
||||
let b = w[kCGWindowBounds as String] as? NSDictionary,
|
||||
let frame = CGRect(dictionaryRepresentation: b),
|
||||
frame.width > 1, frame.height > 1
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return WindowInfo(id: CGWindowID(id), pid: pid_t(pid), app: w[kCGWindowOwnerName as String] as? String ?? "", title: w[kCGWindowName as String] as? String ?? "", frame: frame)
|
||||
}
|
||||
}
|
||||
|
||||
func onScreenWindows() -> [WindowInfo] {
|
||||
let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
|
||||
return windowInfos(list)
|
||||
}
|
||||
|
||||
func window(_ id: Int) throws -> WindowInfo {
|
||||
guard let wid = CGWindowID(exactly: id) else {
|
||||
throw ToolError("no window \(id); screen_list shows the current ones")
|
||||
}
|
||||
let list = CGWindowListCopyWindowInfo(.optionIncludingWindow, wid) as? [[String: Any]] ?? []
|
||||
guard let w = windowInfos(list).first else {
|
||||
throw ToolError("no window \(id); screen_list shows the current ones")
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func describe(_ w: WindowInfo) -> String {
|
||||
return "\(w.id) \(w.app) \(w.title.debugDescription) \(Int(w.frame.width))x\(Int(w.frame.height)) at \(Int(w.frame.minX)),\(Int(w.frame.minY))"
|
||||
}
|
||||
|
||||
func screenList(_ args: [String: Any]) async throws -> [Content] {
|
||||
var out = "apps (pid name bundle):\n"
|
||||
for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular {
|
||||
out += "\(app.processIdentifier) \(app.localizedName ?? "") \(app.bundleIdentifier ?? "")\(app.isActive ? " active" : "")\(app.isHidden ? " hidden" : "")\n"
|
||||
}
|
||||
out += "\nwindows, front to back (id app title size at x,y):\n"
|
||||
for w in onScreenWindows() {
|
||||
out += describe(w) + "\n"
|
||||
}
|
||||
out += "\ndisplays (index size at x,y):\n"
|
||||
for (i, s) in NSScreen.screens.enumerated() {
|
||||
let id = s.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID ?? 0
|
||||
let b = CGDisplayBounds(id)
|
||||
out += "\(i) \(Int(b.width))x\(Int(b.height)) at \(Int(b.minX)),\(Int(b.minY))\(CGDisplayIsMain(id) != 0 ? " main" : "")\n"
|
||||
}
|
||||
return [.text(out)]
|
||||
}
|
||||
|
||||
struct Shot {
|
||||
let png: Data
|
||||
let note: String
|
||||
}
|
||||
|
||||
func capture(window id: Int?, display: Int?, crop: [String: Any]?, maxEdge: Double) async throws -> Shot {
|
||||
if !maxEdge.isFinite || maxEdge < 16 || maxEdge > 8192 {
|
||||
throw ToolError("max_edge must be between 16 and 8192 pixels")
|
||||
}
|
||||
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
|
||||
let filter: SCContentFilter
|
||||
let size: CGSize
|
||||
let what: String
|
||||
var base = CGPoint.zero
|
||||
if let id {
|
||||
guard let w = content.windows.first(where: { Int($0.windowID) == id }), let app = w.owningApplication else {
|
||||
throw ToolError("no window \(id) to capture; screen_list shows the current ones")
|
||||
}
|
||||
let center = CGPoint(x: w.frame.midX, y: w.frame.midY)
|
||||
guard w.isOnScreen, let d = content.displays.first(where: { $0.frame.contains(center) }) else {
|
||||
throw ToolError("window \(id) is not on screen")
|
||||
}
|
||||
filter = SCContentFilter(display: d, including: [app], exceptingWindows: [])
|
||||
size = w.frame.size
|
||||
base = CGPoint(x: w.frame.minX - d.frame.minX, y: w.frame.minY - d.frame.minY)
|
||||
what = "window \(id)"
|
||||
} else {
|
||||
let screens = NSScreen.screens
|
||||
let i = display ?? 0
|
||||
guard i >= 0, i < screens.count,
|
||||
let did = screens[i].deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID,
|
||||
let d = content.displays.first(where: { $0.displayID == did })
|
||||
else {
|
||||
throw ToolError("no display \(i)")
|
||||
}
|
||||
filter = SCContentFilter(display: d, excludingWindows: [])
|
||||
size = CGSize(width: d.width, height: d.height)
|
||||
what = "display \(i)"
|
||||
}
|
||||
var src = CGRect(origin: .zero, size: size)
|
||||
if let crop {
|
||||
src = CGRect(x: crop.double("x") ?? 0, y: crop.double("y") ?? 0, width: crop.double("width") ?? size.width, height: crop.double("height") ?? size.height).intersection(src)
|
||||
if src.isEmpty {
|
||||
throw ToolError("crop is outside the \(Int(size.width))x\(Int(size.height)) \(what)")
|
||||
}
|
||||
}
|
||||
let k = min(1, maxEdge / max(src.width, src.height))
|
||||
let cfg = SCStreamConfiguration()
|
||||
cfg.sourceRect = src.offsetBy(dx: base.x, dy: base.y)
|
||||
cfg.width = max(1, Int((src.width * k).rounded()))
|
||||
cfg.height = max(1, Int((src.height * k).rounded()))
|
||||
cfg.showsCursor = false
|
||||
let img = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: cfg)
|
||||
let data = NSMutableData()
|
||||
let dest = CGImageDestinationCreateWithData(data, UTType.png.identifier as CFString, 1, nil)!
|
||||
CGImageDestinationAddImage(dest, img, nil)
|
||||
CGImageDestinationFinalize(dest)
|
||||
var note = "\(what), \(Int(size.width))x\(Int(size.height)) points; image \(img.width)x\(img.height)"
|
||||
if k < 1 || src.origin != .zero {
|
||||
note += String(format: "; \(what) point = image pixel / %.4f", k)
|
||||
if src.origin != .zero {
|
||||
note += " + (\(Int(src.minX)), \(Int(src.minY)))"
|
||||
}
|
||||
} else {
|
||||
note += ", 1 pixel = 1 point"
|
||||
}
|
||||
return Shot(png: data as Data, note: note)
|
||||
}
|
||||
|
||||
func captureTool(_ args: [String: Any]) async throws -> [Content] {
|
||||
let shot = try await capture(window: args.int("window"), display: args.int("display"), crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
|
||||
return [.text(shot.note), .png(shot.png)]
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import AppKit
|
||||
|
||||
struct ToolError: Error {
|
||||
let message: String
|
||||
|
||||
init(_ message: String) {
|
||||
self.message = message
|
||||
}
|
||||
}
|
||||
|
||||
enum Content {
|
||||
case text(String)
|
||||
case png(Data)
|
||||
|
||||
var json: [String: Any] {
|
||||
switch self {
|
||||
case .text(let s):
|
||||
return ["type": "text", "text": s]
|
||||
case .png(let d):
|
||||
return ["type": "image", "data": d.base64EncodedString(), "mimeType": "image/png"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Tool {
|
||||
let name: String
|
||||
let description: String
|
||||
let schema: String
|
||||
let run: @MainActor ([String: Any]) async throws -> [Content]
|
||||
}
|
||||
|
||||
@main
|
||||
struct Server {
|
||||
static func main() {
|
||||
_ = NSApplication.shared
|
||||
Thread.detachNewThread {
|
||||
while let line = readLine() {
|
||||
let done = DispatchSemaphore(value: 0)
|
||||
Task { @MainActor in
|
||||
await handle(line)
|
||||
done.signal()
|
||||
}
|
||||
done.wait()
|
||||
}
|
||||
exit(0)
|
||||
}
|
||||
RunLoop.main.run()
|
||||
}
|
||||
|
||||
@MainActor
|
||||
static func handle(_ line: String) async {
|
||||
guard let data = line.data(using: .utf8),
|
||||
let req = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let id = req["id"]
|
||||
else {
|
||||
return
|
||||
}
|
||||
var resp: [String: Any] = ["jsonrpc": "2.0", "id": id]
|
||||
let params = req["params"] as? [String: Any] ?? [:]
|
||||
switch req["method"] as? String ?? "" {
|
||||
case "initialize":
|
||||
resp["result"] = [
|
||||
"protocolVersion": params["protocolVersion"] as? String ?? "2025-06-18",
|
||||
"capabilities": ["tools": [:]],
|
||||
"serverInfo": ["name": "hands", "version": "0.0.0"],
|
||||
]
|
||||
case "ping":
|
||||
resp["result"] = [:] as [String: Any]
|
||||
case "tools/list":
|
||||
resp["result"] = ["tools": tools.map { t in
|
||||
["name": t.name, "description": t.description, "inputSchema": try! JSONSerialization.jsonObject(with: Data(t.schema.utf8))]
|
||||
}]
|
||||
case "tools/call":
|
||||
resp["result"] = await call(params)
|
||||
default:
|
||||
resp["error"] = ["code": -32601, "message": "method not found: \(req["method"] ?? "")"]
|
||||
}
|
||||
let out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes)
|
||||
FileHandle.standardOutput.write(out)
|
||||
FileHandle.standardOutput.write(Data("\n".utf8))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
static func call(_ params: [String: Any]) async -> [String: Any] {
|
||||
let name = params["name"] as? String ?? ""
|
||||
let args = params["arguments"] as? [String: Any] ?? [:]
|
||||
guard let tool = tools.first(where: { $0.name == name }) else {
|
||||
return ["content": [Content.text("unknown tool: \(name)").json], "isError": true]
|
||||
}
|
||||
do {
|
||||
let content = try await tool.run(args)
|
||||
return ["content": content.map { $0.json }]
|
||||
} catch let e as ToolError {
|
||||
return ["content": [Content.text(e.message).json], "isError": true]
|
||||
} catch {
|
||||
return ["content": [Content.text("\(error)").json], "isError": true]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
let treeProps = """
|
||||
"query": {"type": "string", "description": "Only list elements whose title, description or value contains this text (case-insensitive), flat, without their parents."},
|
||||
"depth": {"type": "integer", "description": "Stop descending after this many levels (default 80)."},
|
||||
"max_lines": {"type": "integer", "description": "Stop after this many elements (default 400)."},
|
||||
"frames": {"type": "boolean", "description": "Add each element's position and size, in window points: @x,y WxH."}
|
||||
"""
|
||||
|
||||
let thenProps = """
|
||||
"wait_ms": {"type": "integer", "description": "Wait this long after the steps, before wait_for, tree and capture (default 300 when there were steps)."},
|
||||
"wait_for": {"type": "string", "description": "Then wait until this text appears, polling every 200ms."},
|
||||
"timeout_ms": {"type": "integer", "description": "How long wait_for waits before giving up (default 10000). Giving up is reported, not an error."},
|
||||
"capture": {"type": "boolean", "description": "Finish with a capture of the window."},
|
||||
"crop": {"type": "object", "description": "Capture only this rectangle, in window points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}},
|
||||
"max_edge": {"type": "number", "description": "Longest edge of the captured image in pixels, 16 to 8192 (default 1568, which is as much as you can see; captures are never scaled up)."}
|
||||
"""
|
||||
|
||||
let tools: [Tool] = [
|
||||
Tool(
|
||||
name: "screen_list",
|
||||
description: "List the running apps (pid, name, bundle id), the on-screen windows front to back (id, app, title, size and position in screen points) and the displays of this Mac, the machine byre runs on. "
|
||||
+ "Window ids are what capture, ax_tree and act take.",
|
||||
schema: #"{"type": "object", "properties": {}}"#,
|
||||
run: screenList
|
||||
),
|
||||
Tool(
|
||||
name: "capture",
|
||||
description: "Capture one window or a whole display, and see it. A window's capture shows its app's windows only, so other apps covering it don't hide it, and its menus, popups and sheets show; it must be on screen (not minimized). "
|
||||
+ "Prefer a window: a display is 5120 points wide here and shrinks until text is unreadable; crop a region to read small text. "
|
||||
+ "The reply says how image pixels map to window points, which are what act's [x, y] takes.",
|
||||
schema: """
|
||||
{"type": "object", "properties": {
|
||||
"window": {"type": "integer", "description": "Window id from screen_list."},
|
||||
"display": {"type": "integer", "description": "Display index from screen_list, when no window is given (default 0)."},
|
||||
"crop": {"type": "object", "description": "Only this rectangle, in window (or display) points.", "properties": {"x": {"type": "number"}, "y": {"type": "number"}, "width": {"type": "number"}, "height": {"type": "number"}}},
|
||||
"max_edge": {"type": "number", "description": "Longest edge of the image in pixels, 16 to 8192 (default 1568)."}
|
||||
}}
|
||||
""",
|
||||
run: captureTool
|
||||
),
|
||||
Tool(
|
||||
name: "ax_tree",
|
||||
description: "Read a window's accessibility tree: one line per element that has text or actions, indented under its parent, e.g. `e12 button \"New Tab\" [press]`. "
|
||||
+ "Web pages in Orion appear as text, links and form fields. The refs (e12) are for act and stay valid until the next tree, which replaces them. "
|
||||
+ "Bracketed: the actions an element takes (press, confirm, menu, increment…), set when its value can be set, focused, disabled. "
|
||||
+ "With menu_bar_pid instead of window, read an app's menu bar; pressing a menu item's ref runs it without opening the menu.",
|
||||
schema: """
|
||||
{"type": "object", "properties": {
|
||||
"window": {"type": "integer", "description": "Window id from screen_list."},
|
||||
"menu_bar_pid": {"type": "integer", "description": "Read this app's menu bar instead of a window."},
|
||||
\(treeProps)
|
||||
}}
|
||||
""",
|
||||
run: axTreeTool
|
||||
),
|
||||
Tool(
|
||||
name: "act",
|
||||
description: "Run steps in one window, in order, then optionally wait, read the tree and capture: one call for click, wait and look. "
|
||||
+ "Each step is an object with one of these keys. Through accessibility, which needs no focus and leaves the mouse alone (prefer these): "
|
||||
+ "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. "
|
||||
+ "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): "
|
||||
+ "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
|
||||
+ "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). "
|
||||
+ "And {\"wait\": 500} (ms). [x, y] are window points, from the top left of the window. A failing step stops the rest and says which. "
|
||||
+ "Other agents share this Mac's apps, mouse and keyboard: a call with steps (and every browser call but tabs) holds them for the whole call, waiting and capture included, so keep such calls short, and waits up to 30s for another agent's call to finish.",
|
||||
schema: """
|
||||
{"type": "object", "properties": {
|
||||
"window": {"type": "integer", "description": "Window id from screen_list."},
|
||||
"steps": {"type": "array", "items": {"type": "object"}, "description": "The steps, in order."},
|
||||
"tree": {"type": "boolean", "description": "Finish with the window's accessibility tree, which replaces the refs."},
|
||||
\(thenProps),
|
||||
\(treeProps)
|
||||
}, "required": ["window"]}
|
||||
""",
|
||||
run: actTool
|
||||
),
|
||||
Tool(
|
||||
name: "browser",
|
||||
description: "Drive Orion, the logged-in browser on this Mac, through AppleScript; for web work, prefer this to clicking. "
|
||||
+ "ops: tabs (list every window's tabs: window, tab, title, url), navigate (url, optionally new_tab; waits for the page to load), "
|
||||
+ "js (run code in the tab and return its value: the value of the last expression, synchronously, so return strings, JSON.stringify objects, and don't return promises), select (tab), close (tab). "
|
||||
+ "window is Orion's window index (1 is the front one) and tab the tab index from tabs; without tab, the window's current tab. "
|
||||
+ "wait_for waits for text in the page, and capture captures the Orion window, selecting the tab first when one is given.",
|
||||
schema: """
|
||||
{"type": "object", "properties": {
|
||||
"op": {"type": "string", "enum": ["tabs", "navigate", "js", "select", "close"]},
|
||||
"window": {"type": "integer", "description": "Orion window index (default 1, the front window)."},
|
||||
"tab": {"type": "integer", "description": "Tab index within the window (default: its current tab)."},
|
||||
"url": {"type": "string", "description": "For navigate."},
|
||||
"new_tab": {"type": "boolean", "description": "For navigate: open the url in a new tab and select it."},
|
||||
"code": {"type": "string", "description": "For js."},
|
||||
\(thenProps)
|
||||
}, "required": ["op"]}
|
||||
""",
|
||||
run: browserTool
|
||||
),
|
||||
]
|
||||
Reference in New Issue
Block a user