Open websites in the Page view and let agents drive them

2026-10-21 21:34:26 -07:00
parent d6e1895591
commit 57d660c57f
8 changed files with 786 additions and 390 deletions
@@ -8,10 +8,11 @@ import WebKit
 enum Page: Hashable {
    case file(taskId: String, path: String)
    case preview(taskId: String, port: Int)
+    case web(taskId: String, id: Int)

    var taskId: String {
        switch self {
-        case .file(let taskId, _), .preview(let taskId, _): taskId
+        case .file(let taskId, _), .preview(let taskId, _), .web(let taskId, _): taskId
        }
    }

@@ -27,6 +28,7 @@ enum Page: Hashable {
        switch self {
        case .file(_, let path): (path as NSString).lastPathComponent
        case .preview(let taskId, let port): store.forwards.name(taskId, port) ?? "port \(port)"
+        case .web: store.pages.title(self)
        }
    }

@@ -35,6 +37,7 @@ enum Page: Hashable {
        switch self {
        case .file(let taskId, let path): store.pages.url(taskId, path)
        case .preview(let taskId, let port): store.forwards.url(taskId, port)
+        case .web: store.pages.existing(self)?.url
        }
    }
 }
@@ -61,10 +64,18 @@ extension Store {
        sheet = Sheet(owner: .task(page.taskId), view: .tab(.page))
    }

+    func openWeb(_ taskId: String, _ url: URL) -> Int {
+        route("#/task/\(taskId)")
+        let page = pages.web(taskId, url)
+        openPage(page)
+        return pages.tab(page)
+    }
+
    func closePage(_ page: Page) {
        guard var shown = shownPages[page.taskId], let i = shown.pages.firstIndex(of: page) else {
            return
        }
+        pages.drop(page)
        shown.pages.remove(at: i)
        if shown.pages.isEmpty {
            shownPages[page.taskId] = nil
@@ -82,16 +93,67 @@ extension Store {
    func currentPage() -> Page? {
        selectedTaskId.flatMap { shownPages[$0]?.current }
    }
+
+    func pageView(_ page: Page) -> WKWebView? {
+        if case .web = page {
+            return pages.existing(page)
+        }
+        return page.url(self).map { pages.view(page, $0) }
+    }
+
+    func pageTabs(_ taskId: String) -> String {
+        guard let shown = shownPages[taskId] else {
+            return "This task's page view has no tabs; open one with show_page."
+        }
+        return shown.pages.map { page in
+            var line = "tab \(pages.tab(page)): \(page.title(self))"
+            if let url = pages.existing(page)?.url ?? page.url(self) {
+                line += " — \(url.absoluteString)"
+            }
+            if page == shown.current {
+                line += " (showing)"
+            }
+            return line
+        }.joined(separator: "\n")
+    }
+
+    func drivenPage(_ taskId: String, _ tab: Int?) throws -> WKWebView {
+        guard let shown = shownPages[taskId] else {
+            throw ApiError(status: 0, message: "this task's page view has no tabs; open one with show_page")
+        }
+        var page = shown.current
+        if let tab {
+            guard let p = shown.pages.first(where: { pages.tab($0) == tab }) else {
+                throw ApiError(status: 0, message: "this task's page view has no tab \(tab); drive_page tabs lists them")
+            }
+            page = p
+        }
+        guard let view = pageView(page) else {
+            throw ApiError(status: 0, message: "tab \(pages.tab(page)) isn't ready yet")
+        }
+        return view
+    }
 }

@MainActor
@Observable
 final class Pages {
    private(set) var port: Int?
+    private(set) var titles: [Page: String] = [:]
    @ObservationIgnored var served: [String: [String: Data]] = [:]
+    @ObservationIgnored var popup: ((Page) -> Void)?
    @ObservationIgnored private var listener: PageListener?
+    @ObservationIgnored private var views: [Page: WKWebView] = [:]
+    @ObservationIgnored private var sources: [Page: URL] = [:]
+    @ObservationIgnored private var titleWatches: [Page: NSKeyValueObservation] = [:]
+    @ObservationIgnored private var tabs: [Page: Int] = [:]
+    @ObservationIgnored private var lastTab = 0
+    @ObservationIgnored private let delegate = PageDelegate()

    init(api: API) {
+        delegate.popup = { [weak self] opener, config in
+            self?.adopt(opener, config)
+        }
        listener = PageListener(api: api, ready: { [weak self] port in
            Task { @MainActor in
                self?.port = port
@@ -115,6 +177,108 @@ final class Pages {
        return c.url
    }

+    func existing(_ page: Page) -> WKWebView? {
+        views[page]
+    }
+
+    func view(_ page: Page, _ url: URL) -> WKWebView {
+        let view = views[page] ?? make(page, WKWebViewConfiguration())
+        if sources[page] != url {
+            sources[page] = url
+            view.load(URLRequest(url: url))
+        }
+        return view
+    }
+
+    func web(_ taskId: String, _ url: URL) -> Page {
+        lastTab += 1
+        let page = Page.web(taskId: taskId, id: lastTab)
+        make(page, WKWebViewConfiguration()).load(URLRequest(url: url))
+        return page
+    }
+
+    private func adopt(_ opener: WKWebView, _ config: WKWebViewConfiguration) -> WKWebView? {
+        guard let from = views.first(where: { $0.value === opener })?.key else {
+            return nil
+        }
+        lastTab += 1
+        let page = Page.web(taskId: from.taskId, id: lastTab)
+        let view = make(page, config)
+        popup?(page)
+        return view
+    }
+
+    private func make(_ page: Page, _ config: WKWebViewConfiguration) -> WKWebView {
+        config.websiteDataStore = .default()
+        let view = WKWebView(frame: .zero, configuration: config)
+        view.uiDelegate = delegate
+        view.navigationDelegate = delegate
+        views[page] = view
+        if case .web(_, let id) = page {
+            tabs[page] = id
+            titleWatches[page] = view.observe(\.title) { [weak self] view, _ in
+                Task { @MainActor in
+                    self?.titles[page] = view.title
+                }
+            }
+        }
+        return view
+    }
+
+    func drop(_ page: Page) {
+        views[page] = nil
+        sources[page] = nil
+        titleWatches[page] = nil
+        titles[page] = nil
+        tabs[page] = nil
+    }
+
+    func tab(_ page: Page) -> Int {
+        if let tab = tabs[page] {
+            return tab
+        }
+        lastTab += 1
+        tabs[page] = lastTab
+        return lastTab
+    }
+
+    func title(_ page: Page) -> String {
+        if let title = titles[page], !title.isEmpty {
+            return title
+        }
+        return views[page]?.url?.host() ?? "page"
+    }
+
+    func navigate(_ view: WKWebView, _ url: URL) async throws -> String {
+        try await delegate.wait(view.load(URLRequest(url: url)))
+        return "Loaded \(view.url?.absoluteString ?? url.absoluteString): \(view.title ?? "")"
+    }
+
+    func js(_ view: WKWebView, _ code: String) async throws -> String {
+        let value = try await view.callAsyncJavaScript(code, contentWorld: .page)
+        guard let value, !(value is NSNull) else {
+            return value == nil ? "undefined" : "null"
+        }
+        if let s = value as? String {
+            return s
+        }
+        if let data = try? JSONSerialization.data(withJSONObject: value, options: [.fragmentsAllowed, .sortedKeys]) {
+            return String(decoding: data, as: UTF8.self)
+        }
+        return String(describing: value)
+    }
+
+    func capture(_ view: WKWebView) async throws -> Data {
+        guard unsafe view.window != nil, !view.isHiddenOrHasHiddenAncestor else {
+            throw ApiError(status: 0, message: "the tab isn't on screen: the human has the page view closed or another tab showing")
+        }
+        let image = try await view.takeSnapshot(configuration: nil)
+        guard let tiff = image.tiffRepresentation, let png = NSBitmapImageRep(data: tiff)?.representation(using: .png, properties: [:]) else {
+            throw ApiError(status: 0, message: "couldn't encode the capture as PNG")
+        }
+        return png
+    }
+
    func close() {
        listener?.close()
        listener = nil
@@ -299,8 +463,8 @@ struct PageTabs: View {
            }
            ZStack {
                ForEach(shown.pages, id: \.self) { page in
-                    if let url = page.url(store) {
-                        WebPage(url: url, reloads: page.isFile ? watch.reloads : 0, shown: page == shown.current)
+                    if let view = store.pageView(page) {
+                        WebPage(view: view, reloads: page.isFile ? watch.reloads : 0, shown: page == shown.current, refresh: page.isFile)
                    } else if page == shown.current {
                        EmptyText(text: "Loading…")
                    }
@@ -352,21 +516,87 @@ private struct PageTabButton: View {
    }
 }

-private struct WebPage: NSViewRepresentable {
-    let url: URL
-    let reloads: Int
-    let shown: Bool
+@MainActor
+private final class PageDelegate: NSObject, WKUIDelegate, WKNavigationDelegate {
+    var popup: ((WKWebView, WKWebViewConfiguration) -> WKWebView?)?
+    private var waits: [ObjectIdentifier: CheckedContinuation<Void, any Error>] = [:]

-    final class Coordinator: NSObject, WKUIDelegate {
-        var url: URL?
-        var reloads = 0
+    func wait(_ navigation: WKNavigation?) async throws {
+        guard let navigation else {
+            return
+        }
+        try await withCheckedThrowingContinuation { c in
+            waits[ObjectIdentifier(navigation)] = c
+        }
+    }

-        func webView(_ webView: WKWebView, createWebViewWith configuration: WKWebViewConfiguration, for action: WKNavigationAction, windowFeatures: WKWindowFeatures) -> WKWebView? {
-            if let url = action.request.url {
-                NSWorkspace.shared.open(url)
-            }
+    private func finish(_ navigation: WKNavigation?, _ error: (any Error)?) {
+        guard let navigation, let c = waits.removeValue(forKey: ObjectIdentifier(navigation)) else {
+            return
+        }
+        if let error {
+            c.resume(throwing: error)
+        } else {
+            c.resume()
+        }
+    }
+
+    func webView(_ webView: WKWebView, createWebViewWith configuration: WKWebViewConfiguration, for action: WKNavigationAction, windowFeatures: WKWindowFeatures) -> WKWebView? {
+        popup?(webView, configuration)
+    }
+
+    func webView(_ webView: WKWebView, runJavaScriptAlertPanelWithMessage message: String, initiatedByFrame frame: WKFrameInfo) async {
+        _ = await ask(webView, frame, message, buttons: ["OK"], field: nil)
+    }
+
+    func webView(_ webView: WKWebView, runJavaScriptConfirmPanelWithMessage message: String, initiatedByFrame frame: WKFrameInfo) async -> Bool {
+        await ask(webView, frame, message, buttons: ["OK", "Cancel"], field: nil) == .alertFirstButtonReturn
+    }
+
+    func webView(_ webView: WKWebView, runJavaScriptTextInputPanelWithPrompt prompt: String, defaultText: String?, initiatedByFrame frame: WKFrameInfo) async -> String? {
+        let field = NSTextField(frame: NSRect(x: 0, y: 0, width: 300, height: 24))
+        field.stringValue = defaultText ?? ""
+        guard await ask(webView, frame, prompt, buttons: ["OK", "Cancel"], field: field) == .alertFirstButtonReturn else {
            return nil
        }
+        return field.stringValue
+    }
+
+    private func ask(_ webView: WKWebView, _ frame: WKFrameInfo, _ message: String, buttons: [String], field: NSTextField?) async -> NSApplication.ModalResponse {
+        guard let window = unsafe webView.window, !webView.isHiddenOrHasHiddenAncestor else {
+            return .cancel
+        }
+        let alert = NSAlert()
+        alert.messageText = frame.securityOrigin.host.isEmpty ? "This page says" : "\(frame.securityOrigin.host) says"
+        alert.informativeText = message
+        for title in buttons {
+            alert.addButton(withTitle: title)
+        }
+        alert.accessoryView = field
+        return await alert.beginSheetModal(for: window)
+    }
+
+    func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
+        finish(navigation, nil)
+    }
+
+    func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: any Error) {
+        finish(navigation, error)
+    }
+
+    func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, withError error: any Error) {
+        finish(navigation, error)
+    }
+}
+
+private struct WebPage: NSViewRepresentable {
+    let view: WKWebView
+    let reloads: Int
+    let shown: Bool
+    let refresh: Bool
+
+    final class Coordinator {
+        var reloads = 0
    }

    func makeCoordinator() -> Coordinator {
@@ -374,22 +604,16 @@ private struct WebPage: NSViewRepresentable {
    }

    func makeNSView(context: Context) -> WKWebView {
-        let config = WKWebViewConfiguration()
-        config.websiteDataStore = .nonPersistent()
-        let view = WKWebView(frame: .zero, configuration: config)
-        view.uiDelegate = context.coordinator
+        context.coordinator.reloads = reloads
+        if refresh, !view.isLoading {
+            view.reload()
+        }
        return view
    }

    func updateNSView(_ view: WKWebView, context: Context) {
        view.isHidden = !shown
        let c = context.coordinator
-        if c.url != url {
-            c.url = url
-            c.reloads = reloads
-            view.load(URLRequest(url: url))
-            return
-        }
        if c.reloads != reloads {
            c.reloads = reloads
            view.reload()
@@ -190,11 +190,31 @@ final class Remote {
            let (code, status, signature) = try await sign(Data(req.data?.data ?? []), as: req.ident ?? "", timeoutMs: req.timeoutMs ?? 120_000)
            return ClientReply(id: req.id, exitCode: code, output: status, data: Base64EncodedData(signature))
        case .openPage:
+            if let address = req.url {
+                guard let url = URL(string: address) else {
+                    throw ApiError(status: 0, message: "\(address) isn't a URL")
+                }
+                return ClientReply(id: req.id, output: "tab \(store!.openWeb(req.task ?? "", url))")
+            }
            store!.showPage(req.task ?? "", req.path ?? "")
            return ClientReply(id: req.id)
        case .openFile:
            store!.showInFiles(req.task ?? "", req.path ?? "")
            return ClientReply(id: req.id)
+        case .pageTabs:
+            return ClientReply(id: req.id, output: store!.pageTabs(req.task ?? ""))
+        case .pageNavigate:
+            let view = try store!.drivenPage(req.task ?? "", req.tab)
+            guard let url = URL(string: req.url ?? "") else {
+                throw ApiError(status: 0, message: "\(req.url ?? "") isn't a URL")
+            }
+            return ClientReply(id: req.id, output: try await store!.pages.navigate(view, url))
+        case .pageJs:
+            let view = try store!.drivenPage(req.task ?? "", req.tab)
+            return ClientReply(id: req.id, output: try await store!.pages.js(view, req.code ?? ""))
+        case .pageCapture:
+            let view = try store!.drivenPage(req.task ?? "", req.tab)
+            return ClientReply(id: req.id, data: Base64EncodedData(try await store!.pages.capture(view)))
        }
    }

@@ -114,6 +114,9 @@ final class Store {
        paneFocus = PaneFocus(api: api)
        remote = Remote(api: api)
        remote.store = self
+        pages.popup = { [weak self] page in
+            self?.openPage(page)
+        }
    }

    func sends(_ key: String) -> LocalSends {