Paste images in iPhone message boxes; let hands drive the simulator

docs/iphone.md

@@ -14,7 +14,7 @@ iOS finds the app's App Intents (the Run workflow shortcut the Action button can
 
 `mac/phone.sh --simulator` builds an ad-hoc-signed app for the iOS simulator instead, at `mac/.build/phone/iphonesimulator/Byre.app`; install it with `xcrun simctl install <device> <app>` and start it with `xcrun simctl launch <device> run.fc.byre.ios`. The simulator ignores entitlements given to `codesign` and refuses to launch an ad-hoc app signed with restricted ones, so the script links `mac/phone/byre.entitlements` into the binary's `__TEXT,__entitlements` section, as Xcode does; without the iCloud entitlement CloudKit raises at launch. Its node signs in to Tailscale like any other: `xcrun simctl launch --console-pty` prints libtailscale's sign-in URL, which changes on every launch.
 
-The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse. Hands' mouse steps (`click`, `drag`, `scroll`) refuse that window, because Device Hub reports no process id; a gesture such as dragging the keyboard away needs mouse events posted with `CGEvent` instead. `xcrun simctl io booted screenshot --display LCD <file>` captures the app at full resolution.
+The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse, though the frames it gives those elements are wrong; take tap points from a `capture` of the window. Hands' mouse steps work there too: a tap is a `click`, and a long press (the edit menu with Paste) is a `click` with `hold`. macOS reports Device Hub as the frontmost app with process id -1, not its window's owner, which is why hands decides a window is in front by the on-screen window order. `xcrun simctl pbcopy` leaves the simulator's clipboard empty; set this Mac's clipboard and run `xcrun simctl pbsync host booted` instead, which also carries images. `xcrun simctl io booted screenshot --display LCD <file>` captures the app at full resolution.
 
 ## Platform facts
 

hands/Sources/byre-hands/Act.swift

@@ -56,8 +56,9 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTab
     for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] {
         if step[k] != nil {
             let p = try point(step[k], info, table)
-            try await click(p, right: right, count: count)
-            return "\(k) at screen \(Int(p.x)),\(Int(p.y))"
+            let hold = try step.ms("hold") ?? 0
+            try await click(p, right: right, count: count, hold: hold)
+            return "\(k) at screen \(Int(p.x)),\(Int(p.y))" + (hold > 0 ? ", held \(hold)ms" : "")
         }
     }
     if step["drag"] != nil {

hands/Sources/byre-hands/Input.swift

@@ -105,7 +105,7 @@ func post(_ e: CGEvent?) {
 }
 
 func isFront(_ info: WindowInfo) throws -> Bool {
-    guard NSWorkspace.shared.frontmostApplication?.processIdentifier == info.pid,
+    guard onScreenWindows().first?.id == info.id,
           let focused = try attr(appElement(info.pid), kAXFocusedWindowAttribute)
     else {
         return false
@@ -129,17 +129,26 @@ func focusWindow(_ id: Int, _ w: AXUIElement) async throws -> WindowInfo {
     return now
 }
 
-func click(_ p: CGPoint, right: Bool, count: Int) async throws {
+func click(_ p: CGPoint, right: Bool, count: Int, hold: Int = 0) async throws {
     let (down, up): (CGEventType, CGEventType) = right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp)
     let button: CGMouseButton = right ? .right : .left
     post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: button))
     try await pause(30)
     for i in 1...max(1, count) {
-        for t in [down, up] {
+        let event = { (t: CGEventType) in
             let e = CGEvent(mouseEventSource: nil, mouseType: t, mouseCursorPosition: p, mouseButton: button)
             e?.setIntegerValueField(.mouseEventClickState, value: Int64(i))
             post(e)
         }
+        event(down)
+        do {
+            defer {
+                event(up)
+            }
+            if hold > 0 {
+                try await pause(hold)
+            }
+        }
         try await pause(30)
     }
 }

hands/Sources/byre-hands/Tools.swift

@@ -59,7 +59,7 @@ let tools: [Tool] = [
             + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. "
             + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. "
             + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): "
-            + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
+            + "{\"click\": \"e5\"} or {\"click\": [x, y]} (add \"hold\": 1200 to hold the button down that many ms, a long press in the iOS simulator), {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), "
             + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). "
             + "And {\"wait\": 500} (ms). [x, y] are window points, from the top left of the window. A failing step stops the rest and says which. "
             + "Other agents share this Mac's apps, mouse and keyboard: a call with steps (and every browser call but tabs) holds them for the whole call, waiting and capture included, so keep such calls short, and waits up to 30s for another agent's call to finish.",

mac/Sources/Byre/Composer.swift

@@ -183,6 +183,16 @@ final class ImageDrafts {
     }
 }
 
+#if os(iOS)
+private struct PastedImage: Transferable {
+    let data: Data
+
+    static var transferRepresentation: some TransferRepresentation {
+        DataRepresentation(importedContentType: .image) { PastedImage(data: $0) }
+    }
+}
+#endif
+
 #if os(macOS)
 struct ComposerTarget {
     let paste: @MainActor (NSPasteboard) -> Bool
@@ -242,6 +252,12 @@ private struct ImageInput: ViewModifier {
         content
             #if os(macOS)
             .focusedValue(\.composer, ComposerTarget(paste: { pb in store.images.add(key, pasteboard: pb, overlays) }))
+            #else
+            .pasteDestination(for: PastedImage.self) { pasted in
+                for p in pasted {
+                    store.images.add(key, p.data, name: nil, overlays)
+                }
+            }
             #endif
             .overlay {
                 if targeted {