Add byre-hands: screen capture and app control tools for agents

2026-10-17 22:43:38 -07:00
parent 658f78934c
commit bb908e493f
14 changed files with 1101 additions and 3 deletions
@@ -0,0 +1,140 @@
+import AppKit
+
+@MainActor
+func osa(_ src: String) throws -> NSAppleEventDescriptor {
+    guard let script = NSAppleScript(source: src) else {
+        throw ToolError("AppleScript did not compile")
+    }
+    var err: NSDictionary?
+    let r = script.executeAndReturnError(&err)
+    if let err {
+        throw ToolError("AppleScript: \(err[NSAppleScript.errorMessage] ?? err)")
+    }
+    return r
+}
+
+func quote(_ s: String) -> String {
+    return "\"" + s.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + "\""
+}
+
+func jsString(_ s: String) -> String {
+    let d = try! JSONSerialization.data(withJSONObject: s, options: .fragmentsAllowed)
+    return String(decoding: d, as: UTF8.self)
+}
+
+struct TabTarget {
+    let window: Int
+    let tab: Int?
+
+    var spec: String {
+        if let tab {
+            return "tab \(tab) of window \(window)"
+        }
+        return "current tab of window \(window)"
+    }
+}
+
+@MainActor
+func js(_ code: String, _ t: TabTarget) throws -> String {
+    let r = try osa("tell application \"Orion\" to do JavaScript \(quote(code)) in \(t.spec)")
+    return r.stringValue ?? "undefined"
+}
+
+@MainActor
+func orionWindow(_ index: Int) throws -> Int {
+    return Int(try osa("tell application \"Orion\" to get id of window \(index)").int32Value)
+}
+
+@MainActor
+func tabs() throws -> String {
+    let src = """
+    tell application "Orion"
+        set out to ""
+        repeat with wi from 1 to count of windows
+            try
+                set w to window wi
+                repeat with ti from 1 to count of tabs of w
+                    set t to tab ti of w
+                    set out to out & wi & " " & ti & " " & (name of t) & " | " & (URL of t) & linefeed
+                end repeat
+            end try
+        end repeat
+        return out
+    end tell
+    """
+    return "window tab title | url\n" + (try osa(src).stringValue ?? "")
+}
+
+@MainActor
+func waitLoaded(_ t: TabTarget, timeout: Double) async throws -> String {
+    let start = Date()
+    await pause(300)
+    while Date().timeIntervalSince(start) < timeout {
+        if (try? js("document.readyState", t)) == "complete" {
+            return String(format: "loaded after %.1fs", Date().timeIntervalSince(start))
+        }
+        await pause(200)
+    }
+    return String(format: "still loading after %.1fs", timeout)
+}
+
+@MainActor
+func browserTool(_ args: [String: Any]) async throws -> [Content] {
+    let op = try args.require("op", args.string)
+    let win = args.int("window") ?? 1
+    var target = TabTarget(window: win, tab: args.int("tab"))
+    let lock = op == "tabs" && !args.bool("capture") ? nil : try await lockInput()
+    defer {
+        if let lock {
+            close(lock)
+        }
+    }
+    var log: [String] = []
+    switch op {
+    case "tabs":
+        log.append(try tabs())
+    case "navigate":
+        let url = try args.require("url", args.string)
+        if args.bool("new_tab") {
+            _ = try osa("tell application \"Orion\" to tell window \(win) to set current tab to (make new tab with properties {URL:\(quote(url))})")
+            target = TabTarget(window: win, tab: nil)
+        } else {
+            _ = try osa("tell application \"Orion\" to set URL of \(target.spec) to \(quote(url))")
+        }
+        log.append(try await waitLoaded(target, timeout: Double(try args.ms("timeout_ms") ?? 15000) / 1000))
+        log.append(try js("document.title + ' | ' + location.href", target))
+    case "js":
+        log.append(try js(try args.require("code", args.string), target))
+    case "select":
+        let tab = try args.require("tab", args.int)
+        _ = try osa("tell application \"Orion\" to set current tab of window \(win) to tab \(tab) of window \(win)")
+        log.append("selected tab \(tab)")
+        target = TabTarget(window: win, tab: nil)
+    case "close":
+        let tab = try args.require("tab", args.int)
+        _ = try osa("tell application \"Orion\" to close tab \(tab) of window \(win)")
+        log.append("closed tab \(tab)")
+        target = TabTarget(window: win, tab: nil)
+    default:
+        throw ToolError("unknown op \(op.debugDescription)")
+    }
+    let present: @MainActor (String) async throws -> Bool = { needle in
+        (try? js("String(!!document.body && document.body.innerText.includes(\(jsString(needle))))", target)) == "true"
+    }
+    if let note = try await waitFor(args, acted: false, present) {
+        log.append(note)
+    }
+    var out: [Content] = []
+    if args.bool("capture") {
+        if let tab = target.tab {
+            _ = try osa("tell application \"Orion\" to set current tab of window \(win) to tab \(tab) of window \(win)")
+            log.append("selected tab \(tab) to capture it")
+            await pause(300)
+        }
+        let id = try orionWindow(win)
+        let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
+        log.append(shot.note)
+        out.append(.png(shot.png))
+    }
+    return [.text(log.joined(separator: "\n"))] + out
+}