From c6da60f7281391b87b187ce06c48bc101597bfc0 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Fri, 23 Oct 2026 06:50:55 -0700 Subject: [PATCH] Paste images in iPhone message boxes; let hands drive the simulator --- docs/iphone.md | 2 +- hands/Sources/byre-hands/Act.swift | 5 +++-- hands/Sources/byre-hands/Input.swift | 15 ++++++++++++--- hands/Sources/byre-hands/Tools.swift | 2 +- mac/Sources/Byre/Composer.swift | 16 ++++++++++++++++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/iphone.md b/docs/iphone.md index ef17042..30f8ca1 100644 --- a/docs/iphone.md +++ b/docs/iphone.md @@ -14,7 +14,7 @@ iOS finds the app's App Intents (the Run workflow shortcut the Action button can `mac/phone.sh --simulator` builds an ad-hoc-signed app for the iOS simulator instead, at `mac/.build/phone/iphonesimulator/Byre.app`; install it with `xcrun simctl install ` and start it with `xcrun simctl launch run.fc.byre.ios`. The simulator ignores entitlements given to `codesign` and refuses to launch an ad-hoc app signed with restricted ones, so the script links `mac/phone/byre.entitlements` into the binary's `__TEXT,__entitlements` section, as Xcode does; without the iCloud entitlement CloudKit raises at launch. Its node signs in to Tailscale like any other: `xcrun simctl launch --console-pty` prints libtailscale's sign-in URL, which changes on every launch. -The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse. Hands' mouse steps (`click`, `drag`, `scroll`) refuse that window, because Device Hub reports no process id; a gesture such as dragging the keyboard away needs mouse events posted with `CGEvent` instead. `xcrun simctl io booted screenshot --display LCD ` captures the app at full resolution. +The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse, though the frames it gives those elements are wrong; take tap points from a `capture` of the window. Hands' mouse steps work there too: a tap is a `click`, and a long press (the edit menu with Paste) is a `click` with `hold`. macOS reports Device Hub as the frontmost app with process id -1, not its window's owner, which is why hands decides a window is in front by the on-screen window order. `xcrun simctl pbcopy` leaves the simulator's clipboard empty; set this Mac's clipboard and run `xcrun simctl pbsync host booted` instead, which also carries images. `xcrun simctl io booted screenshot --display LCD ` captures the app at full resolution. ## Platform facts diff --git a/hands/Sources/byre-hands/Act.swift b/hands/Sources/byre-hands/Act.swift index 7468d11..6c2d4a2 100644 --- a/hands/Sources/byre-hands/Act.swift +++ b/hands/Sources/byre-hands/Act.swift @@ -56,8 +56,9 @@ func runStep(_ step: [String: Any], _ id: Int, _ w: AXUIElement, _ table: RefTab for (k, right, count) in [("click", false, 1), ("double_click", false, 2), ("right_click", true, 1)] { if step[k] != nil { let p = try point(step[k], info, table) - try await click(p, right: right, count: count) - return "\(k) at screen \(Int(p.x)),\(Int(p.y))" + let hold = try step.ms("hold") ?? 0 + try await click(p, right: right, count: count, hold: hold) + return "\(k) at screen \(Int(p.x)),\(Int(p.y))" + (hold > 0 ? ", held \(hold)ms" : "") } } if step["drag"] != nil { diff --git a/hands/Sources/byre-hands/Input.swift b/hands/Sources/byre-hands/Input.swift index 4c20e20..540e1df 100644 --- a/hands/Sources/byre-hands/Input.swift +++ b/hands/Sources/byre-hands/Input.swift @@ -105,7 +105,7 @@ func post(_ e: CGEvent?) { } func isFront(_ info: WindowInfo) throws -> Bool { - guard NSWorkspace.shared.frontmostApplication?.processIdentifier == info.pid, + guard onScreenWindows().first?.id == info.id, let focused = try attr(appElement(info.pid), kAXFocusedWindowAttribute) else { return false @@ -129,17 +129,26 @@ func focusWindow(_ id: Int, _ w: AXUIElement) async throws -> WindowInfo { return now } -func click(_ p: CGPoint, right: Bool, count: Int) async throws { +func click(_ p: CGPoint, right: Bool, count: Int, hold: Int = 0) async throws { let (down, up): (CGEventType, CGEventType) = right ? (.rightMouseDown, .rightMouseUp) : (.leftMouseDown, .leftMouseUp) let button: CGMouseButton = right ? .right : .left post(CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: p, mouseButton: button)) try await pause(30) for i in 1...max(1, count) { - for t in [down, up] { + let event = { (t: CGEventType) in let e = CGEvent(mouseEventSource: nil, mouseType: t, mouseCursorPosition: p, mouseButton: button) e?.setIntegerValueField(.mouseEventClickState, value: Int64(i)) post(e) } + event(down) + do { + defer { + event(up) + } + if hold > 0 { + try await pause(hold) + } + } try await pause(30) } } diff --git a/hands/Sources/byre-hands/Tools.swift b/hands/Sources/byre-hands/Tools.swift index edf114e..b1bcd0c 100644 --- a/hands/Sources/byre-hands/Tools.swift +++ b/hands/Sources/byre-hands/Tools.swift @@ -59,7 +59,7 @@ let tools: [Tool] = [ + "{\"press\": \"e5\"}, {\"set\": \"e7\", \"value\": \"text\"} (replaces a field's value), {\"focus\": \"e7\"}, {\"confirm\": \"e7\"}, {\"action\": \"AXShowMenu\", \"ref\": \"e3\"}. " + "Some apps list confirm but ignore it (Orion's address bar does); then use {\"key\": \"return\"}, and for URLs use browser navigate. " + "Through real mouse and keyboard events, which bring the window to the front and move the cursor of whoever sits at this Mac (use when accessibility doesn't work): " - + "{\"click\": \"e5\"} or {\"click\": [x, y]}, {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), " + + "{\"click\": \"e5\"} or {\"click\": [x, y]} (add \"hold\": 1200 to hold the button down that many ms, a long press in the iOS simulator), {\"double_click\": …}, {\"right_click\": …}, {\"drag\": [x, y], \"to\": [x, y]}, {\"scroll\": [x, y], \"dy\": -5} (lines, negative scrolls down), " + "{\"type\": \"text\"} (into the focused element), {\"key\": \"cmd+shift+t\"} (modifiers cmd shift alt ctrl fn; keys a-z 0-9, return tab space delete escape left right up down home end pageup pagedown f1-f12 and punctuation). " + "And {\"wait\": 500} (ms). [x, y] are window points, from the top left of the window. A failing step stops the rest and says which. " + "Other agents share this Mac's apps, mouse and keyboard: a call with steps (and every browser call but tabs) holds them for the whole call, waiting and capture included, so keep such calls short, and waits up to 30s for another agent's call to finish.", diff --git a/mac/Sources/Byre/Composer.swift b/mac/Sources/Byre/Composer.swift index c1f118b..c946c26 100644 --- a/mac/Sources/Byre/Composer.swift +++ b/mac/Sources/Byre/Composer.swift @@ -183,6 +183,16 @@ final class ImageDrafts { } } +#if os(iOS) +private struct PastedImage: Transferable { + let data: Data + + static var transferRepresentation: some TransferRepresentation { + DataRepresentation(importedContentType: .image) { PastedImage(data: $0) } + } +} +#endif + #if os(macOS) struct ComposerTarget { let paste: @MainActor (NSPasteboard) -> Bool @@ -242,6 +252,12 @@ private struct ImageInput: ViewModifier { content #if os(macOS) .focusedValue(\.composer, ComposerTarget(paste: { pb in store.images.add(key, pasteboard: pb, overlays) })) + #else + .pasteDestination(for: PastedImage.self) { pasted in + for p in pasted { + store.images.add(key, p.data, name: nil, overlays) + } + } #endif .overlay { if targeted {