From caf0a830a98ba589fddca29d1fca7063a430a7ac Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Wed, 21 Oct 2026 22:31:48 -0700 Subject: [PATCH] Let agents show and drive Page view tabs on the iPhone --- docs/iphone.md | 4 +++- internal/agents/clients.go | 5 +++-- internal/agents/kb/client-exec.md | 2 +- internal/agents/sign.go | 2 +- internal/clients/clients.go | 16 ++++++++++++++-- internal/server/clients.go | 4 ++-- mac/Sources/Byre/API.swift | 1 + mac/Sources/Byre/Pages.swift | 11 +++++++++++ mac/Sources/Byre/Remote.swift | 14 ++++++++++++-- mac/Sources/Byre/Startup.swift | 6 ------ mac/Sources/Byre/Store.swift | 6 ------ 11 files changed, 48 insertions(+), 23 deletions(-) diff --git a/docs/iphone.md b/docs/iphone.md index b36f277..2a78588 100644 --- a/docs/iphone.md +++ b/docs/iphone.md @@ -2,7 +2,9 @@ The iPhone app is the same `Byre` target as the Mac app, built for iOS: platform differences are `#if os(macOS)` / `#if os(iOS)` blocks, with the small shared aliases (`PlatformImage`, `PlatformColor`, `PlatformFont`, `PlatformView`, `touchOnly`, `deviceKind`) in `mac/Sources/Byre/Platform.swift`. Only the layout is the phone's own (`Phone.swift`): a tab bar with Lead, Tasks and More. Tasks lists every project's active tasks with the Mac's own task rows; a task opens pushed, with the Mac's task bar and tab strip on top and the tab's view below in place of the Mac's attached sheet. Zoomed images and diagrams cover the screen. More holds Notes, Workflows, Status and Settings. Every action that focuses a task or project (a row, a link, a notification, the switcher) bumps `Store.reveals`, which brings the Tasks tab forward with the task open; the server's own focus changes don't. -Left out on the phone: the Server tab and running byre locally, the client connection (`Remote`: the phone is never a client for agents' client tools, so `list_clients` doesn't show it), the Sparkle updater, and keyboard shortcuts. Notifications come from the app's own event stream, so they only show while the app is open; push notifications would need APNs on the server. +The phone holds the client connection (`Remote`) like a Mac, so agents can show it pages and files and drive its Page view (`show_page`, `show_file`, `drive_page`); it answers every other client request with an error. It sends `Byre-Device: iPhone` (Macs send `Mac`), and the server signs commits and runs `byre client-exec` without `--client` only on Macs. + +Left out on the phone: the Server tab and running byre locally, the shell, file, hands and signing client requests, the Sparkle updater, and keyboard shortcuts. Notifications come from the app's own event stream, so they only show while the app is open; push notifications would need APNs on the server. ## Building diff --git a/internal/agents/clients.go b/internal/agents/clients.go index f1f83f8..b8d080f 100644 --- a/internal/agents/clients.go +++ b/internal/agents/clients.go @@ -36,7 +36,8 @@ func (m *Manager) clientTools(l *live) []mcp.Tool { Description: "List the Macs with byre's app open: the human's own machines, not the one byre (and your worktree) runs on. " + "Each has its name, which the other client tools take as client, the id of that running app (pass it as client instead when two apps are open on one Mac), the human using it, and when the human last did something in byre from it. " + "A message from the human says which Mac it was sent from (\"Sent from (client )\"); that is the Mac the human sits at, so show things there, naming it by that id. " + - "The most recently active one is listed first.", + "The most recently active one is listed first. " + + "The human's iPhone is listed too, when byre's app is open on it; it takes only show_page, show_file and drive_page, and the other client tools fail there.", InputSchema: json.RawMessage(`{"type": "object", "properties": {}}`), Handler: func(ctx context.Context, raw json.RawMessage) (string, error) { return m.listClients(), nil @@ -289,7 +290,7 @@ func (m *Manager) listClients() string { now := time.Now() b := &strings.Builder{} for i, r := range rows { - fmt.Fprintf(b, "- %s (%s, id %s): ", r.Host, r.Login, r.ID) + fmt.Fprintf(b, "- %s (%s, %s, id %s): ", r.Host, r.Device, r.Login, r.ID) if r.LastActionAt == nil { b.WriteString("no action since byre started") } else { diff --git a/internal/agents/kb/client-exec.md b/internal/agents/kb/client-exec.md index 5c4735b..f5d4b36 100644 --- a/internal/agents/kb/client-exec.md +++ b/internal/agents/kb/client-exec.md @@ -11,7 +11,7 @@ byre client-exec [--client NAME] [--timeout SECS] -- CMD... `byre` is `/Applications/Byre.app/Contents/Helpers/byre` on every Mac with the app. - It runs CMD as `client_bash` does: zsh as a login shell in the home directory, in the app's session, as the human. That session is what GUI-only commands need. -- Without `--client` it picks the most recently active client, the one `list_clients` lists first. +- Without `--client` it picks the most recently active Mac, skipping the iPhone, which runs no commands. - Each argument reaches the client as one word, so shell syntax needs `-- zsh -c '...'`. - The command's stdout and stderr come back separately and byte for byte, once it ends, written to client-exec's own stdout and stderr. - Exit status: the command's own; 75 when no client is connected, it didn't acknowledge the request within 2s, or it disconnected; 124 when `--timeout` (default 30s) killed it. A one-line `byre client-exec: ...` on stderr says why. diff --git a/internal/agents/sign.go b/internal/agents/sign.go index 2120c56..b46a8ab 100644 --- a/internal/agents/sign.go +++ b/internal/agents/sign.go @@ -15,7 +15,7 @@ const signTimeout = 2 * time.Minute func (m *Manager) signer(ctx context.Context, user api.User) func(payload []byte) ([]byte, error) { return func(payload []byte) ([]byte, error) { mine := []clients.Client{} - for _, c := range m.clients.Recent() { + for _, c := range m.clients.RecentMacs() { if c.Login == user.Login { mine = append(mine, c) } diff --git a/internal/clients/clients.go b/internal/clients/clients.go index a989c52..d056fc1 100644 --- a/internal/clients/clients.go +++ b/internal/clients/clients.go @@ -33,6 +33,7 @@ type Client struct { Login string Host string Commit string + Device string ConnectedAt time.Time LastActionAt *time.Time Sync *api.ClientSync @@ -109,7 +110,7 @@ func (r *Registry) Synced(id, login, host, commit string, sync api.ClientSync) { r.entry(id, login, host, commit, time.Now()).Sync = &sync } -func (r *Registry) Serve(ctx context.Context, ws *websocket.Conn, id, login, host, commit string) { +func (r *Registry) Serve(ctx context.Context, ws *websocket.Conn, id, login, host, commit, device string) { ws.SetReadLimit(readLimit) c := &conn{ws: ws, done: make(chan struct{}), pending: map[string]chan api.ClientReply{}} @@ -118,12 +119,13 @@ func (r *Registry) Serve(ctx context.Context, ws *websocket.Conn, id, login, hos e := r.entry(id, login, host, commit, now) old := e.conn e.conn = c + e.Device = device e.ConnectedAt = now r.mu.Unlock() if old != nil { old.ws.Close(websocket.StatusGoingAway, "replaced by a newer connection from this client") } - slog.Info("client connected", "id", id, "host", host, "login", login, "commit", commit) + slog.Info("client connected", "id", id, "host", host, "device", device, "login", login, "commit", commit) r.changed() defer func() { @@ -219,6 +221,16 @@ func (r *Registry) Recent() []Client { return rows } +func (r *Registry) RecentMacs() []Client { + out := []Client{} + for _, c := range r.Recent() { + if c.Device == "Mac" { + out = append(out, c) + } + } + return out +} + func (r *Registry) Call(ctx context.Context, name string, req api.ClientRequest) (api.ClientReply, error) { r.mu.Lock() matches := []entry{} diff --git a/internal/server/clients.go b/internal/server/clients.go index 1bb6a28..3f62b96 100644 --- a/internal/server/clients.go +++ b/internal/server/clients.go @@ -35,7 +35,7 @@ func (s *Server) ConnectClient(w http.ResponseWriter, r *http.Request) { } defer ws.CloseNow() - s.clients.Serve(r.Context(), ws, id, actor(r), host, r.Header.Get("Byre-Commit")) + s.clients.Serve(r.Context(), ws, id, actor(r), host, r.Header.Get("Byre-Commit"), r.Header.Get("Byre-Device")) } func (s *Server) SetClientSync(w http.ResponseWriter, r *http.Request) { @@ -84,7 +84,7 @@ func (s *Server) ExecClient(w http.ResponseWriter, r *http.Request) { name = *req.Client } if name == "" { - recent := s.clients.Recent() + recent := s.clients.RecentMacs() if len(recent) == 0 { writeError(w, http.StatusServiceUnavailable, errors.New("no client Macs are connected")) return diff --git a/mac/Sources/Byre/API.swift b/mac/Sources/Byre/API.swift index 2428971..1947bb4 100644 --- a/mac/Sources/Byre/API.swift +++ b/mac/Sources/Byre/API.swift @@ -118,6 +118,7 @@ final class API: Sendable { config.httpAdditionalHeaders = [ "Byre-Client": clientID, "Byre-Commit": Bundle.main.object(forInfoDictionaryKey: "ByreCommit") as! String, + "Byre-Device": deviceKind, "Byre-Host": deviceName, ] config.proxyConfigurations = [tailnetNode.proxy()] diff --git a/mac/Sources/Byre/Pages.swift b/mac/Sources/Byre/Pages.swift index 0cbed0e..6be2bc1 100644 --- a/mac/Sources/Byre/Pages.swift +++ b/mac/Sources/Byre/Pages.swift @@ -278,6 +278,17 @@ final class Pages { } return png } + #else + func capture(_ view: WKWebView) async throws -> Data { + guard view.window != nil, !view.isHidden else { + throw ApiError(status: 0, message: "the tab isn't on screen: the human has the page view closed or another tab showing") + } + let image = try await view.takeSnapshot(configuration: nil) + guard let png = image.pngData() else { + throw ApiError(status: 0, message: "couldn't encode the capture as PNG") + } + return png + } #endif func close() { diff --git a/mac/Sources/Byre/Remote.swift b/mac/Sources/Byre/Remote.swift index d833fb9..f734a93 100644 --- a/mac/Sources/Byre/Remote.swift +++ b/mac/Sources/Byre/Remote.swift @@ -7,12 +7,13 @@ typealias ClientRequest = Components.Schemas.ClientRequest typealias ClientReply = Components.Schemas.ClientReply typealias ClientSync = Components.Schemas.ClientSync -#if os(macOS) private let retryAfter = Duration.seconds(2) private let checkEvery = Duration.seconds(10) private let staleAfter: TimeInterval = 25 -private let maxOutput = 64 << 10 private let maxMessage = 256 << 20 + +#if os(macOS) +private let maxOutput = 64 << 10 private let maxRead = 64 << 20 private let copyTimeoutMs = 10 * 60 * 1000 private let hands = Hands() @@ -27,6 +28,7 @@ private let signScript = """ esac exec "$program" --status-fd=2 -bsau "$(git config user.signingkey || echo "$1")" >"$2" """ +#endif @MainActor final class Remote { @@ -157,6 +159,10 @@ final class Remote { private func perform(_ req: ClientRequest) async throws -> ClientReply { switch req.op { + #if os(iOS) + case .bash, .exec, .upload, .download, .readFile, .writeFile, .hands, .sign: + throw ApiError(status: 0, message: "an iPhone only shows and drives pages and files in byre's app (show_page, show_file, drive_page)") + #else case .bash: let home = FileManager.default.homeDirectoryForCurrentUser.path let (code, output) = try await spawn(["/bin/zsh", "-l", "-c", req.command ?? ""], in: home, timeoutMs: req.timeoutMs ?? 120_000) @@ -190,6 +196,7 @@ final class Remote { case .sign: let (code, status, signature) = try await sign(Data(req.data?.data ?? []), as: req.ident ?? "", timeoutMs: req.timeoutMs ?? 120_000) return ClientReply(id: req.id, exitCode: code, output: status, data: Base64EncodedData(signature)) + #endif case .openPage: if let address = req.url { guard let url = URL(string: address) else { @@ -219,6 +226,7 @@ final class Remote { } } + #if os(macOS) private func sign(_ payload: Data, as ident: String, timeoutMs: Int) async throws -> (Int, String, Data) { let fm = FileManager.default let dir = fm.temporaryDirectory.appending(path: "byre-sign-\(UUID().uuidString)", directoryHint: .isDirectory) @@ -293,8 +301,10 @@ final class Remote { } try fm.moveItem(at: entries[0], to: target) } + #endif } +#if os(macOS) private func expand(_ path: String) -> URL { URL(filePath: (path as NSString).expandingTildeInPath) } diff --git a/mac/Sources/Byre/Startup.swift b/mac/Sources/Byre/Startup.swift index 739c6df..f3d4582 100644 --- a/mac/Sources/Byre/Startup.swift +++ b/mac/Sources/Byre/Startup.swift @@ -254,13 +254,7 @@ final class Connection { store.startGlobalStream(after: status.lastEventSeq) { [weak self] in self?.reconnected(status.startedAt) } - #if os(macOS) store.remote.start() - #else - if let sync = Servers.shared.sync { - store.reportSync(sync) - } - #endif forwards.sync(store.api, Set(store.activeTasks().map(\.task.id))) Notifier.shared.deliver() return true diff --git a/mac/Sources/Byre/Store.swift b/mac/Sources/Byre/Store.swift index ed12c22..5e1546d 100644 --- a/mac/Sources/Byre/Store.swift +++ b/mac/Sources/Byre/Store.swift @@ -89,9 +89,7 @@ final class Store { let imageCache = ImageCache() let events = Listeners() let paneFocus: PaneFocus - #if os(macOS) let remote: Remote - #endif let forwards: Forwards let pages: Pages @@ -115,10 +113,8 @@ final class Store { self.pages = pages leadFeed = Feed(shown: { true }) paneFocus = PaneFocus(api: api) - #if os(macOS) remote = Remote(api: api) remote.store = self - #endif pages.popup = { [weak self] page in self?.openPage(page) } @@ -144,9 +140,7 @@ final class Store { stopped = true globalStream?.stop() leadStream?.stop() - #if os(macOS) remote.stop() - #endif drafts?.flush() await paneFocus.leave() }