Paste images in iPhone message boxes; let hands drive the simulator

hands/Sources/byre-hands/Act.swift

@@ -56,8 +56,9 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTab
     for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
         if step[k] != nil {
             let p = try point(step[k], info, table)
-            try await click(p, right: right, count: count)
-            return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
+            let hold = try step.ms("hold") ?? 0
+            try await click(p, right: right, count: count, hold: hold)
+            return "\(k) at screen \(Int(p.x)),\(Int(p.y))" + (hold > 0 ? ", held \(hold)ms" : "")
         }
     }
     if step["drag"] != nil {

hands/Sources/byre-hands/Input.swift

@@ -105,7 +105,7 @@ func post(_ e: CGEvent?) {
 }
 
 func isFront(_ info: WindowInfo) throws -> Bool {
-    guard NSWorkspace.shared.frontmostApplication?.processIdentifier == info.pid,
+    guard onScreenWindows().first?.id == info.id,
           let focused = try attr(appElement(info.pid), kAXFocusedWindowAttribute)
     else {
         return false
@@ -129,17 +129,26 @@ func focusWindow(_ id: Int, _ w: AXUIElement) async throws -> WindowInfo {
     return now
 }
 
-func click(_ p: CGPoint, right: Bool, count: Int) async throws {
+func click(_ p: CGPoint, right: Bool, count: Int, hold: Int = 0) async throws {
     let (down, up): (CGEventType, CGEventType) = right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
     let button: CGMouseButton = right ? .right : .left
     post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: button))
     try await pause(30)
     for i in 1...max(1, count) {
-        for t in [down, up] {
+        let event = { (t: CGEventType) in
             let e = CGEvent(mouseEventSource: nil, mouseType: t, mouseCursorPosition: p, mouseButton: button)
             e?.setIntegerValueField(.mouseEventClickState, value: Int64(i))
             post(e)
         }
+        event(down)
+        do {
+            defer {
+                event(up)
+            }
+            if hold > 0 {
+                try await pause(hold)
+            }
+        }
         try await pause(30)
     }
 }

hands/Sources/byre-hands/Tools.swift

@@ -59,7 +59,7 @@ let tools: [Tool] = [
             + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. "
             + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. "
             + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): "
-            + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
+            + "{\"click\": \"e5\"} or {\"click\": [x, y]} (add \"hold\": 1200 to hold the button down that many ms, a long press in the iOS simulator), {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
             + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). "
             + "And {\"wait\": 500} (ms). [x, y] are window points, from the top left of the window. A failing step stops the rest and says which. "
             + "Other agents share this Mac's apps, mouse and keyboard: a call with steps (and every browser call but tabs) holds them for the whole call, waiting and capture included, so keep such calls short, and waits up to 30s for another agent's call to finish.",