Paste screenshots into iPhone message boxes; add ⋯ menu to the picker

2026-10-23 07:30:53 -07:00
parent c28b100fd0
commit 48f2ef3ade
4 changed files with 127 additions and 40 deletions
@@ -14,10 +14,11 @@ iOS finds the app's App Intents (the Run workflow shortcut the Action button can

 `mac/phone.sh --simulator` builds an ad-hoc-signed app for the iOS simulator instead, at `mac/.build/phone/iphonesimulator/Byre.app`; install it with `xcrun simctl install <device> <app>` and start it with `xcrun simctl launch <device> run.fc.byre.ios`. The simulator ignores entitlements given to `codesign` and refuses to launch an ad-hoc app signed with restricted ones, so the script links `mac/phone/byre.entitlements` into the binary's `__TEXT,__entitlements` section, as Xcode does; without the iCloud entitlement CloudKit raises at launch. Its node signs in to Tailscale like any other: `xcrun simctl launch --console-pty` prints libtailscale's sign-in URL, which changes on every launch.

-The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse, though the frames it gives those elements are wrong; take tap points from a `capture` of the window. Hands' mouse steps work there too: a tap is a `click`, and a long press (the edit menu with Paste) is a `click` with `hold`. macOS reports Device Hub as the frontmost app with process id -1, not its window's owner, which is why hands decides a window is in front by the on-screen window order. `xcrun simctl pbcopy` leaves the simulator's clipboard empty; set this Mac's clipboard and run `xcrun simctl pbsync host booted` instead, which also carries images. `xcrun simctl io booted screenshot --display LCD <file>` captures the app at full resolution.
+The simulator's window (Xcode's Device Hub) lists the app's own accessibility elements, so hands' `ax_tree` and `act` `press` drive the app without a mouse, though the frames it gives those elements are wrong; take tap points from a `capture` of the window. Hands' mouse steps work there too: a tap is a `click`, and a long press (the edit menu with Paste) is a `click` with `hold`. macOS reports Device Hub as the frontmost app with process id -1, not its window's owner, which is why hands decides a window is in front by the on-screen window order. `xcrun simctl pbcopy` leaves the simulator's clipboard empty; set this Mac's clipboard and run `xcrun simctl pbsync host booted` instead, which also carries images. `xcrun simctl io booted screenshot --display LCD <file>` captures the app at full resolution. To put clipboard types the Mac can't produce (such as `com.apple.uikit.image` alone), compile a small UIKit command-line program with `xcrun --sdk iphonesimulator swiftc -target arm64-apple-ios27.0-simulator` and run it with `xcrun simctl spawn booted <binary>`; it can set `UIPasteboard.general` and print its `types`. The simulator has no Photos app, and its AssistiveTouch menu draws no icons, so an iOS screenshot's Copy and Delete can't be tried there.

 ## Platform facts

 - CloudKit on iOS requires the `remote-notification` background mode in `Info.plist`, even though the app sets up no push of its own.
 - The phone's name is the one typed in Settings, kept in the phone's own `UserDefaults` (not synced through iCloud, since each phone has its own), and `iPhone` until one is typed. It goes in `Byre-Host` and the tailnet node's name, both fixed when the app starts. `UIDevice.current.name` returns just `iPhone` without Apple's `com.apple.developer.device-information.user-assigned-device-name` entitlement, which Apple refused because byre shows the name to other people and sends it to Tailscale and to the agents' model vendors.
+- The message boxes are a `UITextView` subclass (`ImageTextView`), not SwiftUI's `TextEditor`. `TextEditor`'s text view offers Paste only for text, and a `.pasteDestination` on it didn't show Paste for an iPhone screenshot's Copy and Delete on a real phone, although it did for PNG and HEIC in the simulator. The subclass offers Paste whenever `UIPasteboard.general.hasImages` and reads `UIPasteboard.general.images`, the same read as ⋯ → Paste image, which works with screenshots. The edit menu's Paste reads the clipboard without iOS's "Allow Paste" prompt; the ⋯ menu's item gets the prompt.
 - SwiftTerm's iOS `TerminalViewDelegate` requires `requestOpenLink`, which its macOS one provides a default for.
@@ -690,11 +690,7 @@ private struct LineCommentBox: View {
        VStack(alignment: .trailing, spacing: 4) {
            Thumbnails(key: key)
                .frame(maxWidth: .infinity, alignment: .leading)
-            TextEditor(text: $text)
-                .font(Fonts.font(12))
-                .scrollContentBackground(.hidden)
-                .writingToolsBehavior(.disabled)
-                .focused($focused)
+            MessageTextEditor(text: $text, size: 12, key: key, focus: $focused)
                .padding(.horizontal, 5)
                .padding(.vertical, 4)
                .frame(height: 64)
@@ -154,6 +154,19 @@ final class ImageDrafts {
        }
        return false
    }
+    #else
+    func addPasteboard(_ key: String, _ overlays: Overlays) {
+        let images = UIPasteboard.general.images ?? []
+        if images.isEmpty {
+            overlays.showAlert("The clipboard has no image.")
+            return
+        }
+        for image in images {
+            if let data = image.jpegData(compressionQuality: 0.85) {
+                add(key, data, name: nil, overlays)
+            }
+        }
+    }
    #endif

    func add(_ key: String, providers: [NSItemProvider], _ overlays: Overlays) {
@@ -184,15 +197,112 @@ final class ImageDrafts {
 }

 #if os(iOS)
-private struct PastedImage: Transferable {
-    let data: Data
+private final class ImageTextView: UITextView {
+    var pasteImages: () -> Void = {}

-    static var transferRepresentation: some TransferRepresentation {
-        DataRepresentation(importedContentType: .image) { PastedImage(data: $0) }
+    override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool {
+        if action == #selector(paste(_:)) && UIPasteboard.general.hasImages {
+            return true
+        }
+        return super.canPerformAction(action, withSender: sender)
+    }
+
+    override func paste(_ sender: Any?) {
+        if UIPasteboard.general.hasImages {
+            pasteImages()
+            return
+        }
+        super.paste(sender)
+    }
+}
+
+private struct ImageTextEditor: UIViewRepresentable {
+    @Binding var text: String
+    let size: CGFloat
+    var focus: FocusState<Bool>.Binding
+    let paste: () -> Void
+
+    func makeCoordinator() -> Coordinator {
+        Coordinator(self)
+    }
+
+    func makeUIView(context: Context) -> ImageTextView {
+        let v = ImageTextView()
+        v.delegate = context.coordinator
+        v.backgroundColor = .clear
+        v.textColor = .label
+        v.textContainerInset = .zero
+        v.writingToolsBehavior = .none
+        return v
+    }
+
+    func updateUIView(_ v: ImageTextView, context: Context) {
+        context.coordinator.parent = self
+        v.pasteImages = paste
+        let font = Fonts.native(size)
+        if v.font != font {
+            v.font = font
+        }
+        if v.text != text {
+            v.text = text
+        }
+        if focus.wrappedValue != v.isFirstResponder {
+            DispatchQueue.main.async {
+                if focus.wrappedValue && !v.isFirstResponder {
+                    v.becomeFirstResponder()
+                } else if !focus.wrappedValue && v.isFirstResponder {
+                    v.resignFirstResponder()
+                }
+            }
+        }
+    }
+
+    final class Coordinator: NSObject, UITextViewDelegate {
+        var parent: ImageTextEditor
+
+        init(_ parent: ImageTextEditor) {
+            self.parent = parent
+        }
+
+        func textViewDidChange(_ v: UITextView) {
+            parent.text = v.text
+        }
+
+        func textViewDidBeginEditing(_ v: UITextView) {
+            parent.focus.wrappedValue = true
+        }
+
+        func textViewDidEndEditing(_ v: UITextView) {
+            parent.focus.wrappedValue = false
+        }
    }
 }
 #endif

+struct MessageTextEditor: View {
+    @Binding var text: String
+    let size: CGFloat
+    let key: String
+    var focus: FocusState<Bool>.Binding
+    @Environment(Store.self) private var store
+    @Environment(Overlays.self) private var overlays
+
+    var body: some View {
+        #if os(iOS)
+        ImageTextEditor(text: $text, size: size, focus: focus) {
+            store.images.addPasteboard(key, overlays)
+        }
+        .focused(focus)
+        #else
+        TextEditor(text: $text)
+            .font(Fonts.font(size))
+            .scrollContentBackground(.hidden)
+            .writingToolsBehavior(.disabled)
+            .focused(focus)
+        #endif
+    }
+}
+
 #if os(macOS)
 struct ComposerTarget {
    let paste: @MainActor (NSPasteboard) -> Bool
@@ -252,12 +362,6 @@ private struct ImageInput: ViewModifier {
        content
            #if os(macOS)
            .focusedValue(\.composer, ComposerTarget(paste: { pb in store.images.add(key, pasteboard: pb, overlays) }))
-            #else
-            .pasteDestination(for: PastedImage.self) { pasted in
-                for p in pasted {
-                    store.images.add(key, p.data, name: nil, overlays)
-                }
-            }
            #endif
            .overlay {
                if targeted {
@@ -303,7 +407,7 @@ struct ComposerMenu: View {
                    Label("Attach photos", systemImage: "photo")
                }
                Button {
-                    paste()
+                    store.images.addPasteboard(key, overlays)
                } label: {
                    Label("Paste image", systemImage: "doc.on.clipboard")
                }
@@ -337,19 +441,6 @@ struct ComposerMenu: View {
            }
        }
    }
-
-    private func paste() {
-        let images = UIPasteboard.general.images ?? []
-        if images.isEmpty {
-            overlays.showAlert("The clipboard has no image.")
-            return
-        }
-        for image in images {
-            if let data = image.jpegData(compressionQuality: 0.85) {
-                store.images.add(key, data, name: nil, overlays)
-            }
-        }
-    }
 }
 #endif

@@ -419,11 +510,7 @@ struct AgentEditor: View {
    let send: () -> Void

    var body: some View {
-        TextEditor(text: $text)
-            .font(Fonts.font(13))
-            .scrollContentBackground(.hidden)
-            .writingToolsBehavior(.disabled)
-            .focused(focus)
+        MessageTextEditor(text: $text, size: 13, key: key, focus: focus)
            #if os(macOS)
            .onKeyPress(.return, phases: .down) { press in
                let marked = (NSApp.keyWindow?.firstResponder as? NSTextView)?.hasMarkedText() ?? false
@@ -399,7 +399,14 @@ private struct SwitcherBox: View {
                .frame(maxHeight: 110)
                .fixedSize(horizontal: false, vertical: true)
            }
+            #if os(iOS)
+            HStack(alignment: .top, spacing: 8) {
+                editor
+                ComposerMenu(key: model.imageKey, photos: true)
+            }
+            #else
            editor
+            #endif
            ScrollViewReader { proxy in
                ScrollView {
                    VStack(alignment: .leading, spacing: 0) {
@@ -480,11 +487,7 @@ private struct SwitcherBox: View {
            .padding(.vertical, 6)
            .hidden()
            .overlay {
-                TextEditor(text: Binding(get: { model.text }, set: { model.setText($0) }))
-                    .font(Fonts.font(13))
-                    .scrollContentBackground(.hidden)
-                    .writingToolsBehavior(.disabled)
-                    .focused($focused)
+                MessageTextEditor(text: Binding(get: { model.text }, set: { model.setText($0) }), size: 13, key: model.imageKey, focus: $focused)
                    .padding(.horizontal, 5)
                    .padding(.vertical, 6)
            }