Add the macOS client spike and record its SwiftUI gap results

2026-10-16 22:23:31 -07:00
parent 7e0e6d9585
commit c2d0ff77d7
16 changed files with 1281 additions and 20 deletions
@@ -0,0 +1,381 @@
+import AppKit
+import SwiftUI
+import UniformTypeIdentifiers
+
+let maxImageBytes = 5 << 20
+
+struct PendingImage: Identifiable {
+    let id = UUID()
+    let data: Data
+    let image: NSImage
+}
+
+func imageType(_ data: Data) -> String? {
+    let b = [UInt8](data.prefix(12))
+    if b.starts(with: [0x89, 0x50, 0x4E, 0x47]) {
+        return "png"
+    }
+    if b.starts(with: [0xFF, 0xD8, 0xFF]) {
+        return "jpeg"
+    }
+    if b.starts(with: Array("GIF8".utf8)) {
+        return "gif"
+    }
+    if b.count == 12, b.starts(with: Array("RIFF".utf8)), Array(b[8..<12]) == Array("WEBP".utf8) {
+        return "webp"
+    }
+    return nil
+}
+
+func pngFromTIFF(_ data: Data) -> Data? {
+    NSBitmapImageRep(data: data)?.representation(using: .png, properties: [:])
+}
+
+@MainActor
+@Observable
+final class Attachments {
+    var images: [PendingImage] = []
+    let server: Server
+    let name: String
+
+    init(server: Server, name: String) {
+        self.server = server
+        self.name = name
+    }
+
+    func add(_ data: Data, from source: String) {
+        guard let kind = imageType(data), let image = NSImage(data: data) else {
+            server.log("\(name): \(source) gave \(data.count) bytes that aren't PNG, JPEG, GIF or WebP")
+            return
+        }
+        if data.count > maxImageBytes {
+            server.log("\(name): \(source) image is over \(maxImageBytes >> 20) MiB")
+            return
+        }
+        server.log("\(name): \(source) attached a \(kind), \(data.count) bytes")
+        images.append(PendingImage(data: data, image: image))
+    }
+
+    func add(pasteboard pb: NSPasteboard, from source: String) -> Bool {
+        if let urls = pb.readObjects(forClasses: [NSURL.self], options: [.urlReadingFileURLsOnly: true]) as? [URL], !urls.isEmpty {
+            var any = false
+            for url in urls {
+                if let data = try? Data(contentsOf: url), imageType(data) != nil || pngFromTIFF(data) != nil && NSImage(data: data) != nil {
+                    add(imageType(data) != nil ? data : pngFromTIFF(data)!, from: "\(source) file \(url.lastPathComponent)")
+                    any = true
+                }
+            }
+            return any
+        }
+        for t in [NSPasteboard.PasteboardType.png, NSPasteboard.PasteboardType(UTType.jpeg.identifier), NSPasteboard.PasteboardType(UTType.gif.identifier), NSPasteboard.PasteboardType(UTType.webP.identifier)] {
+            if let data = pb.data(forType: t) {
+                add(data, from: "\(source) \(t.rawValue)")
+                return true
+            }
+        }
+        if let tiff = pb.data(forType: .tiff), let png = pngFromTIFF(tiff) {
+            add(png, from: "\(source) public.tiff converted to PNG")
+            return true
+        }
+        return false
+    }
+
+    func add(providers: [NSItemProvider], from source: String) {
+        server.log("\(name): \(source) with types \(providers.map { $0.registeredTypeIdentifiers })")
+        for p in providers {
+            let types = [UTType.png, .jpeg, .gif, .webP, .tiff, .fileURL].filter { p.hasItemConformingToTypeIdentifier($0.identifier) }
+            guard let t = types.first else {
+                continue
+            }
+            _ = p.loadDataRepresentation(for: t) { data, error in
+                Task { @MainActor in
+                    guard let data else {
+                        self.server.log("\(self.name): \(source) load failed: \(String(describing: error))")
+                        return
+                    }
+                    if t == .fileURL, let url = URL(dataRepresentation: data, relativeTo: nil), let file = try? Data(contentsOf: url) {
+                        self.add(imageType(file) != nil ? file : (pngFromTIFF(file) ?? file), from: "\(source) file \(url.lastPathComponent)")
+                    } else if t == .tiff, let png = pngFromTIFF(data) {
+                        self.add(png, from: "\(source) tiff converted to PNG")
+                    } else {
+                        self.add(data, from: "\(source) \(t.identifier)")
+                    }
+                }
+            }
+        }
+    }
+
+    func send(_ text: String) {
+        let images = self.images
+        self.images = []
+        server.log("\(name): send \(text.debugDescription) with \(images.count) images")
+        Task {
+            for img in images {
+                do {
+                    server.log("\(name): uploaded blob \(try await server.upload(img.data))")
+                } catch {
+                    server.log("\(name): upload failed: \(error)")
+                }
+            }
+        }
+    }
+}
+
+struct Thumbnails: View {
+    @Bindable var attachments: Attachments
+
+    var body: some View {
+        HStack {
+            ForEach(attachments.images) { img in
+                ZStack(alignment: .topTrailing) {
+                    Image(nsImage: img.image)
+                        .resizable()
+                        .scaledToFit()
+                        .frame(width: 64, height: 64)
+                    Button("×") {
+                        attachments.images.removeAll { $0.id == img.id }
+                    }
+                }
+            }
+        }
+        .frame(height: attachments.images.isEmpty ? 0 : 64)
+    }
+}
+
+struct ComposerSpike: View {
+    @Environment(Server.self) private var server
+
+    var body: some View {
+        HStack(alignment: .top, spacing: 16) {
+            SwiftUIComposer(attachments: Attachments(server: server, name: "TextEditor"))
+            AppKitComposerPanel(attachments: Attachments(server: server, name: "NSTextView"))
+            RoutedComposer(attachments: Attachments(server: server, name: "Routed"))
+        }
+        .padding(8)
+    }
+}
+
+struct ComposerFocusKey: FocusedValueKey {
+    typealias Value = Attachments
+}
+
+extension FocusedValues {
+    var composer: Attachments? {
+        get {
+            self[ComposerFocusKey.self]
+        }
+        set {
+            self[ComposerFocusKey.self] = newValue
+        }
+    }
+}
+
+struct PasteboardCommands: Commands {
+    @FocusedValue(\.composer) private var composer
+
+    var body: some Commands {
+        CommandGroup(replacing: .pasteboard) {
+            Button("Cut") {
+                NSApp.sendAction(#selector(NSText.cut(_:)), to: nil, from: nil)
+            }
+            .keyboardShortcut("x")
+            Button("Copy") {
+                NSApp.sendAction(#selector(NSText.copy(_:)), to: nil, from: nil)
+            }
+            .keyboardShortcut("c")
+            Button("Paste") {
+                if let composer, composer.add(pasteboard: .general, from: "Edit ▸ Paste") {
+                    return
+                }
+                NSApp.sendAction(#selector(NSText.paste(_:)), to: nil, from: nil)
+            }
+            .keyboardShortcut("v")
+            Button("Select All") {
+                NSApp.sendAction(#selector(NSText.selectAll(_:)), to: nil, from: nil)
+            }
+            .keyboardShortcut("a")
+        }
+    }
+}
+
+struct RoutedComposer: View {
+    @ViewState var attachments: Attachments
+    @ViewState private var text = ""
+    @ViewState private var overlayHits = false
+    @ViewState private var targeted = false
+
+    var body: some View {
+        VStack(alignment: .leading) {
+            Text("SwiftUI TextEditor, paste through Edit ▸ Paste, drop overlay")
+            Toggle("Drop overlay takes hits (off: clicks pass through to the editor)", isOn: $overlayHits)
+            Thumbnails(attachments: attachments)
+            TextEditor(text: $text)
+                .frame(minHeight: 160)
+                .border(targeted ? Color.accentColor : Color.secondary.opacity(0.4))
+                .focusedValue(\.composer, attachments)
+                .onKeyPress(.return, phases: .down) { press in
+                    let marked = (NSApp.keyWindow?.firstResponder as? NSTextView)?.hasMarkedText() ?? false
+                    attachments.server.log("Routed: onKeyPress return, modifiers \(press.modifiers.rawValue), hasMarkedText \(marked)")
+                    if press.modifiers.contains(.shift) || marked {
+                        return .ignored
+                    }
+                    attachments.send(text)
+                    text = ""
+                    return .handled
+                }
+                .overlay {
+                    Color.clear
+                        .contentShape(Rectangle())
+                        .allowsHitTesting(overlayHits)
+                        .onDrop(of: [.image, .fileURL], isTargeted: $targeted) { providers in
+                            attachments.add(providers: providers, from: "overlay onDrop")
+                            return true
+                        }
+                }
+                .onChange(of: targeted) {
+                    attachments.server.log("Routed: overlay drop targeted \(targeted)")
+                }
+        }
+    }
+}
+
+struct SwiftUIComposer: View {
+    @ViewState var attachments: Attachments
+    @ViewState private var text = ""
+    @ViewState private var checkMarked = false
+
+    var body: some View {
+        VStack(alignment: .leading) {
+            Text("SwiftUI TextEditor + onKeyPress, onPasteCommand, onDrop")
+            Toggle("In onKeyPress, also ignore Return while the first responder hasMarkedText()", isOn: $checkMarked)
+            Thumbnails(attachments: attachments)
+            TextEditor(text: $text)
+                .frame(minHeight: 160)
+                .border(Color.secondary.opacity(0.4))
+                .onKeyPress(.return, phases: .down) { press in
+                    let marked = (NSApp.keyWindow?.firstResponder as? NSTextView)?.hasMarkedText() ?? false
+                    attachments.server.log("TextEditor: onKeyPress return, modifiers \(press.modifiers.rawValue), hasMarkedText \(marked)")
+                    if press.modifiers.contains(.shift) {
+                        return .ignored
+                    }
+                    if checkMarked && marked {
+                        return .ignored
+                    }
+                    attachments.send(text)
+                    text = ""
+                    return .handled
+                }
+                .onPasteCommand(of: [.png, .jpeg, .gif, .webP, .tiff, .fileURL]) { providers in
+                    attachments.add(providers: providers, from: "onPasteCommand")
+                }
+                .onDrop(of: [.image, .fileURL], isTargeted: nil) { providers in
+                    attachments.add(providers: providers, from: "onDrop")
+                    return true
+                }
+        }
+    }
+}
+
+struct AppKitComposerPanel: View {
+    @ViewState var attachments: Attachments
+
+    var body: some View {
+        VStack(alignment: .leading) {
+            Text("NSTextView in NSViewRepresentable")
+            Text("doCommand(insertNewline:), paste(_:), performDragOperation")
+            Thumbnails(attachments: attachments)
+            AppKitComposer(attachments: attachments)
+                .frame(minHeight: 160)
+                .border(Color.secondary.opacity(0.4))
+        }
+    }
+}
+
+final class ComposerTextView: NSTextView {
+    var attachments: Attachments!
+
+    override func keyDown(with event: NSEvent) {
+        if event.keyCode == 36 || event.keyCode == 76 {
+            attachments.server.log("NSTextView: keyDown return, modifiers \(event.modifierFlags.intersection(.deviceIndependentFlagsMask).rawValue), hasMarkedText \(hasMarkedText())")
+        }
+        super.keyDown(with: event)
+    }
+
+    override func doCommand(by selector: Selector) {
+        let shift = NSApp.currentEvent?.modifierFlags.contains(.shift) ?? false
+        if selector == #selector(insertNewline(_:)) && !shift {
+            attachments.send(string)
+            string = ""
+            return
+        }
+        if selector == #selector(insertNewline(_:)) {
+            insertNewlineIgnoringFieldEditor(nil)
+            return
+        }
+        super.doCommand(by: selector)
+    }
+
+    override func paste(_ sender: Any?) {
+        if attachments.add(pasteboard: .general, from: "paste(_:)") {
+            return
+        }
+        super.paste(sender)
+    }
+
+    override var readablePasteboardTypes: [NSPasteboard.PasteboardType] {
+        super.readablePasteboardTypes + [.png, .tiff, .fileURL]
+    }
+
+    override func draggingEntered(_ sender: any NSDraggingInfo) -> NSDragOperation {
+        if hasImage(sender.draggingPasteboard) {
+            return .copy
+        }
+        return super.draggingEntered(sender)
+    }
+
+    override func draggingUpdated(_ sender: any NSDraggingInfo) -> NSDragOperation {
+        if hasImage(sender.draggingPasteboard) {
+            return .copy
+        }
+        return super.draggingUpdated(sender)
+    }
+
+    override func performDragOperation(_ sender: any NSDraggingInfo) -> Bool {
+        if hasImage(sender.draggingPasteboard) {
+            return attachments.add(pasteboard: sender.draggingPasteboard, from: "drop")
+        }
+        return super.performDragOperation(sender)
+    }
+
+    private func hasImage(_ pb: NSPasteboard) -> Bool {
+        if let urls = pb.readObjects(forClasses: [NSURL.self], options: [.urlReadingFileURLsOnly: true]) as? [URL], !urls.isEmpty {
+            return urls.contains { (try? Data(contentsOf: $0)).map { imageType($0) != nil || NSImage(data: $0) != nil } ?? false }
+        }
+        return pb.availableType(from: [.png, .tiff, NSPasteboard.PasteboardType(UTType.jpeg.identifier), NSPasteboard.PasteboardType(UTType.gif.identifier), NSPasteboard.PasteboardType(UTType.webP.identifier)]) != nil
+    }
+}
+
+struct AppKitComposer: NSViewRepresentable {
+    let attachments: Attachments
+
+    func makeNSView(context: Context) -> NSScrollView {
+        let scroll = NSScrollView()
+        scroll.hasVerticalScroller = true
+        let tv = ComposerTextView(frame: .zero)
+        tv.attachments = attachments
+        tv.isRichText = false
+        tv.allowsUndo = true
+        tv.font = Fonts.ct(13) as NSFont
+        tv.minSize = .zero
+        tv.maxSize = NSSize(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude)
+        tv.isVerticallyResizable = true
+        tv.isHorizontallyResizable = false
+        tv.autoresizingMask = [.width]
+        tv.textContainer?.widthTracksTextView = true
+        tv.registerForDraggedTypes([.png, .tiff, .fileURL])
+        scroll.documentView = tv
+        return scroll
+    }
+
+    func updateNSView(_ nsView: NSScrollView, context: Context) {}
+}