Add a Captures tab showing screen captures from the task's agent

internal/agents/codex.go

@@ -169,8 +169,11 @@ func (m *Manager) codexItemCompleted(ctx context.Context, l *live, s *codex.Sess
 		return m.emit(ctx, l, agentActor, "agent.vendor", payload)
 	}
 
-	content, isError := codexToolResult(item)
+	content, isError, err := codexToolResult(item)
-	err := m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content})
+	if err != nil {
+		return err
+	}
+	err = m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content})
 	if err != nil {
 		return err
 	}
@@ -210,24 +213,51 @@ func imageAlt(prompt string) string {
 	return alt
 }
 
-func codexToolResult(item codex.ThreadItem) (string, bool) {
+func codexToolResult(item codex.ThreadItem) (any, bool, error) {
 	status := codexStatus(item)
 	switch item.Type {
 	case codex.ThreadItemTypeCommandExecution:
-		return item.AggregatedOutput, status != "completed" || item.ExitCode != 0
+		return item.AggregatedOutput, status != "completed" || item.ExitCode != 0, nil
 	case codex.ThreadItemTypeDynamicToolCall:
-		texts := []string{}
+		blocks := []map[string]any{}
 		for _, c := range item.ContentItems {
-			texts = append(texts, c.Text)
+			if c.Type != codex.DynamicToolCallOutputContentItemTypeInputImage {
+				blocks = append(blocks, map[string]any{"type": "text", "text": c.Text})
+				continue
+			}
+			mime, data, ok := strings.Cut(strings.TrimPrefix(c.ImageURL, "data:"), ";base64,")
+			if !ok {
+				return nil, false, fmt.Errorf("dynamic tool image is not a base64 data url")
+			}
+			blocks = append(blocks, imageBlock(mime, data))
 		}
-		return strings.Join(texts, "\n"), !item.Success
+		return blocks, !item.Success, nil
 	case codex.ThreadItemTypeMcpToolCall:
 		if item.Error != nil {
-			return item.Error.Message, true
+			return item.Error.Message, true, nil
 		}
-		return string(item.Result), false
+		res := struct {
+			Content []map[string]any `json:"content"`
+			IsError bool             `json:"isError"`
+		}{}
+		err := json.Unmarshal(item.Result, &res)
+		if err != nil {
+			return nil, false, fmt.Errorf("decode mcp tool result: %w", err)
+		}
+		for i, b := range res.Content {
+			if b["type"] == "image" {
+				mime, _ := b["mimeType"].(string)
+				data, _ := b["data"].(string)
+				res.Content[i] = imageBlock(mime, data)
+			}
+		}
+		return res.Content, res.IsError, nil
 	}
-	return status, status != "completed"
+	return status, status != "completed", nil
+}
+
+func imageBlock(mime, data string) map[string]any {
+	return map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": mime, "data": data}}
 }
 
 func (m *Manager) readCodexRateLimit() {

mac/Sources/Byre/Captures.swift

@@ -0,0 +1,52 @@
+import SwiftUI
+
+struct CapturesView: View {
+    let captures: [Capture]
+
+    var body: some View {
+        ScrollView {
+            LazyVStack(alignment: .leading, spacing: 20) {
+                ForEach(captures.reversed()) { c in
+                    CaptureItem(capture: c)
+                }
+            }
+            .padding(16)
+        }
+        .background(Palette.bg)
+    }
+}
+
+private struct CaptureItem: View {
+    let capture: Capture
+    @ViewState private var image: PlatformImage?
+
+    var body: some View {
+        VStack(alignment: .leading, spacing: 6) {
+            HStack(alignment: .firstTextBaseline, spacing: 8) {
+                Text(capture.tool)
+                    .font(Fonts.font(13, weight: 700))
+                Text(clockText(capture.when))
+                    .foregroundStyle(Palette.muted)
+                Spacer(minLength: 0)
+            }
+            if !capture.note.isEmpty {
+                Text(verbatim: capture.note)
+                    .font(Fonts.font(12, mono: true))
+                    .foregroundStyle(Palette.muted)
+                    .lineLimit(4)
+                    .textSelection(.enabled)
+            }
+            if let image {
+                Image(platformImage: image)
+                    .resizable()
+                    .scaledToFit()
+                    .frame(maxWidth: image.size.width, alignment: .leading)
+                    .clipShape(RoundedRectangle(cornerRadius: 6))
+                    .overlay(RoundedRectangle(cornerRadius: 6).stroke(Palette.border))
+            }
+        }
+        .task(id: capture.id) {
+            image = Data(base64Encoded: capture.base64).flatMap { PlatformImage(data: $0) }
+        }
+    }
+}

mac/Sources/Byre/Feed.swift

@@ -20,10 +20,19 @@ final class Listeners<T> {
     }
 }
 
+struct Capture: Identifiable {
+    let id: String
+    let tool: String
+    let note: String
+    let when: Date
+    let base64: String
+}
+
 @MainActor
 @Observable
 final class Feed {
     private(set) var events: [ByreEvent] = []
+    private(set) var captures: [Capture] = []
     private(set) var ready = false
     private(set) var inherited = 0
     let appended = Listeners<[ByreEvent]>()
@@ -31,6 +40,7 @@ final class Feed {
     @ObservationIgnored private var last: Int64 = 0
     @ObservationIgnored private var buffer: [ByreEvent] = []
     @ObservationIgnored private var scheduled = false
+    @ObservationIgnored private var toolNames: [String: String] = [:]
     @ObservationIgnored private let shown: () -> Bool
 
     init(shown: @escaping () -> Bool) {
@@ -66,6 +76,30 @@ final class Feed {
 
     func prepend(_ older: [ByreEvent]) {
         events.insert(contentsOf: older, at: 0)
+        captures.insert(contentsOf: findCaptures(older), at: 0)
+    }
+
+    private func findCaptures(_ batch: [ByreEvent]) -> [Capture] {
+        var found: [Capture] = []
+        for ev in batch {
+            let p = ev.payload.value
+            if ev.kind == "agent.tool_call" {
+                toolNames[jsString(p["id"])] = toolLabel(jsString(p["name"]))
+                continue
+            }
+            guard ev.kind == "agent.tool_result", let list = p["content"] as? [(any Sendable)?] else {
+                continue
+            }
+            let blocks = list.compactMap { $0 as? JSONDict }
+            let note = blocks.compactMap { str($0, "type") == "text" ? str($0, "text") : nil }.first ?? ""
+            for (i, b) in blocks.enumerated() {
+                guard str(b, "type") == "image", let source = dict(b, "source"), let data = str(source, "data") else {
+                    continue
+                }
+                found.append(Capture(id: "\(ev.seq)/\(i)", tool: toolNames[jsString(p["tool_use_id"])] ?? "", note: note, when: ev.createdAt, base64: data))
+            }
+        }
+        return found
     }
 
     private func flush() {
@@ -97,6 +131,10 @@ final class Feed {
             return
         }
         events.append(contentsOf: fresh)
+        let found = findCaptures(fresh)
+        if !found.isEmpty {
+            captures.append(contentsOf: found)
+        }
         if shown() {
             appended.send(fresh)
         }

mac/Sources/Byre/Panes.swift

@@ -7,6 +7,7 @@ let tabLabels: [TaskTab: String] = [
     .changes: "Changes",
     .files: "Files",
     .page: "Page",
+    .captures: "Captures",
     .terminal: "Terminal",
     .workflow: "Workflow",
 ]
@@ -251,6 +252,8 @@ struct TaskTabBody: View {
             if let shown = store.selectedTaskId.flatMap({ store.shownPages[$0] }) {
                 PageTabs(shown: shown)
             }
+        case .captures:
+            CapturesView(captures: store.selectedTaskId.map { store.captures($0) } ?? [])
         case .terminal:
             TerminalTab()
         case .workflow:
@@ -419,6 +422,7 @@ struct TabStrip: View {
             switch t {
             case .board: return team
             case .page: return store.shownPages[taskId] != nil
+            case .captures: return !store.captures(taskId).isEmpty
             case .workflow: return false
             default: return true
             }

mac/Sources/Byre/Store.swift

@@ -3,10 +3,10 @@ import Observation
 import SwiftData
 
 enum TaskTab: String {
-    case transcript, board, changes, files, page, terminal, workflow
+    case transcript, board, changes, files, page, captures, terminal, workflow
 }
 
-let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .terminal, .workflow]
+let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .captures, .terminal, .workflow]
 
 enum JumpTarget: Equatable {
     case question(String)
@@ -215,6 +215,10 @@ final class Store {
         selectedTaskId.flatMap { taskFeeds[$0] }
     }
 
+    func captures(_ taskId: String) -> [Capture] {
+        taskFeeds[taskId]?.captures ?? []
+    }
+
     func activeTasks() -> [(project: Project, task: ByreTask)] {
         projects.flatMap { project in
             (tasks[project.id] ?? []).filter { $0.status == .active }.map { (project, $0) }