Add byre-hands: screen capture and app control tools for agents

2026-10-17 22:43:38 -07:00
parent 658f78934c
commit bb908e493f
14 changed files with 1101 additions and 3 deletions
@@ -0,0 +1,99 @@
+import AppKit
+
+struct ToolError: Error {
+    let message: String
+
+    init(_ message: String) {
+        self.message = message
+    }
+}
+
+enum Content {
+    case text(String)
+    case png(Data)
+
+    var json: [String: Any] {
+        switch self {
+        case .text(let s):
+            return ["type": "text", "text": s]
+        case .png(let d):
+            return ["type": "image", "data": d.base64EncodedString(), "mimeType": "image/png"]
+        }
+    }
+}
+
+struct Tool {
+    let name: String
+    let description: String
+    let schema: String
+    let run: @MainActor ([String: Any]) async throws -> [Content]
+}
+
+@main
+struct Server {
+    static func main() {
+        _ = NSApplication.shared
+        Thread.detachNewThread {
+            while let line = readLine() {
+                let done = DispatchSemaphore(value: 0)
+                Task { @MainActor in
+                    await handle(line)
+                    done.signal()
+                }
+                done.wait()
+            }
+            exit(0)
+        }
+        RunLoop.main.run()
+    }
+
+    @MainActor
+    static func handle(_ line: String) async {
+        guard let data = line.data(using: .utf8),
+              let req = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
+              let id = req["id"]
+        else {
+            return
+        }
+        var resp: [String: Any] = ["jsonrpc": "2.0", "id": id]
+        let params = req["params"] as? [String: Any] ?? [:]
+        switch req["method"] as? String ?? "" {
+        case "initialize":
+            resp["result"] = [
+                "protocolVersion": params["protocolVersion"] as? String ?? "2025-06-18",
+                "capabilities": ["tools": [:]],
+                "serverInfo": ["name": "hands", "version": "0.0.0"],
+            ]
+        case "ping":
+            resp["result"] = [:] as [String: Any]
+        case "tools/list":
+            resp["result"] = ["tools": tools.map { t in
+                ["name": t.name, "description": t.description, "inputSchema": try! JSONSerialization.jsonObject(with: Data(t.schema.utf8))]
+            }]
+        case "tools/call":
+            resp["result"] = await call(params)
+        default:
+            resp["error"] = ["code": -32601, "message": "method not found: \(req["method"] ?? "")"]
+        }
+        let out = try! JSONSerialization.data(withJSONObject: resp, options: .withoutEscapingSlashes)
+        FileHandle.standardOutput.write(out)
+        FileHandle.standardOutput.write(Data("\n".utf8))
+    }
+
+    @MainActor
+    static func call(_ params: [String: Any]) async -> [String: Any] {
+        let name = params["name"] as? String ?? ""
+        let args = params["arguments"] as? [String: Any] ?? [:]
+        guard let tool = tools.first(where: { $0.name == name }) else {
+            return ["content": [Content.text("unknown tool: \(name)").json], "isError": true]
+        }
+        do {
+            let content = try await tool.run(args)
+            return ["content": content.map { $0.json }]
+        } catch let e as ToolError {
+            return ["content": [Content.text(e.message).json], "isError": true]
+        } catch {
+            return ["content": [Content.text("\(error)").json], "isError": true]
+        }
+    }
+}