byre/hands/Sources/byre-hands/Act.swift

import AppKit
import ApplicationServices

func point(_ v: Any?, _ info: WindowInfo, _ table: RefTable) throws -> CGPoint {
    if let ref = v as? String {
        guard let f = frame(try element(ref, in: info, table)) else {
            throw ToolError("\(ref) has no position on screen")
        }
        return CGPoint(x: f.midX, y: f.midY)
    }
    if let xy = v as? [NSNumber], xy.count == 2 {
        return CGPoint(x: info.frame.minX + CGFloat(xy[0].doubleValue), y: info.frame.minY + CGFloat(xy[1].doubleValue))
    }
    throw ToolError("a point is a ref (\"e5\") or window coordinates ([x, y])")
}

func check(_ err: AXError, _ what: String) throws {
    if err != .success {
        throw ToolError("\(what) failed: AXError \(err.rawValue)")
    }
}

let inputKeys = ["click", "double_click", "right_click", "drag", "scroll", "type", "key"]

func isInput(_ step: [String: Any]) -> Bool {
    return step.keys.contains(where: inputKeys.contains)
}

func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> String {
    var info = try window(id)
    if isInput(step) {
        info = try await focusWindow(id, w)
    }
    if let ref = step.string("press") {
        try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXPressAction as CFString), "press \(ref)")
        return "pressed \(ref)"
    }
    if let ref = step.string("confirm") {
        try check(AXUIElementPerformAction(try element(ref, in: info, table), kAXConfirmAction as CFString), "confirm \(ref)")
        return "confirmed \(ref)"
    }
    if let ref = step.string("focus") {
        try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXFocusedAttribute as CFString, kCFBooleanTrue), "focus \(ref)")
        return "focused \(ref)"
    }
    if let ref = step.string("set") {
        let value = try step.require("value", step.string)
        try check(AXUIElementSetAttributeValue(try element(ref, in: info, table), kAXValueAttribute as CFString, value as CFString), "set \(ref)")
        return "set \(ref)"
    }
    if let name = step.string("action") {
        let ref = try step.require("ref", step.string)
        try check(AXUIElementPerformAction(try element(ref, in: info, table), name as CFString), "\(name) \(ref)")
        return "\(name) \(ref)"
    }
    for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
        if step[k] != nil {
            let p = try point(step[k], info, table)
            await click(p, right: right, count: count)
            return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
        }
    }
    if step["drag"] != nil {
        let from = try point(step["drag"], info, table)
        let to = try point(step["to"], info, table)
        await drag(from, to)
        return "dragged"
    }
    if step["scroll"] != nil {
        let p = try point(step["scroll"], info, table)
        let dx = step.int("dx") ?? 0
        let dy = step.int("dy") ?? 0
        guard (-1000...1000).contains(dx), (-1000...1000).contains(dy) else {
            throw ToolError("dx and dy must be between -1000 and 1000 lines")
        }
        scroll(p, dx: Int32(dx), dy: Int32(dy))
        return "scrolled"
    }
    if let s = step.string("type") {
        await type(s)
        return "typed \(s.count) characters"
    }
    if let k = step.string("key") {
        try key(k)
        return "key \(k)"
    }
    if let ms = try step.ms("wait") {
        await pause(ms)
        return "waited \(ms)ms"
    }
    throw ToolError("unknown step \(step.keys.sorted())")
}

func waitFor(_ args: [String: Any], acted: Bool, _ present: (String) async throws -> Bool) async throws -> String? {
    let wait = try args.ms("wait_ms")
    let timeout = Double(try args.ms("timeout_ms") ?? 10000) / 1000
    if let wait {
        await pause(wait)
    } else if acted {
        await pause(300)
    }
    guard let needle = args.string("wait_for") else {
        return nil
    }
    let start = Date()
    while true {
        if try await present(needle) {
            return String(format: "found %@ after %.1fs", needle.debugDescription, Date().timeIntervalSince(start))
        }
        if Date().timeIntervalSince(start) > timeout {
            return String(format: "gave up waiting for %@ after %.1fs", needle.debugDescription, timeout)
        }
        await pause(200)
    }
}

func runSteps(_ steps: [[String: Any]], _ id: Int, _ w: AXUIElement, _ table: RefTable) async throws -> [String] {
    var log: [String] = []
    for (i, step) in steps.enumerated() {
        do {
            log.append("\(i + 1). " + (try await runStep(step, id, w, table)))
            await pause(50)
        } catch let e as ToolError {
            log.append("\(i + 1). " + e.message + "; stopped here")
            break
        }
    }
    return log
}

func actTool(_ args: [String: Any]) async throws -> [Content] {
    let table = currentRefs()
    let id = try args.require("window", args.int)
    let (_, w) = try axWindow(id)
    let steps = args["steps"] as? [[String: Any]] ?? []
    let lock = steps.isEmpty ? nil : try await lockInput()
    defer {
        if let lock {
            close(lock)
        }
    }
    var before = 0
    if !steps.isEmpty, let needle = args.string("wait_for") {
        before = try windowMatches(w, needle)
    }
    var log = try await runSteps(steps, id, w, table)
    if let note = try await waitFor(args, acted: !steps.isEmpty, { try windowMatches(w, $0) > before }) {
        log.append(note)
    }
    var out: [Content] = []
    if args.bool("tree") {
        let current = try window(id)
        log.append("\n" + (try axTree(w, RefSource(pid: current.pid, window: id), treeOptions(args, origin: current.frame.origin))))
    }
    if args.bool("capture") {
        let shot = try await capture(window: id, display: nil, crop: args.dict("crop"), maxEdge: args.double("max_edge") ?? 1568)
        log.append(shot.note)
        out.append(.png(shot.png))
    }
    return [.text(log.joined(separator: "\n"))] + out
}