diff --git a/internal/agents/codex.go b/internal/agents/codex.go index 447c3c6..4b4c180 100644 --- a/internal/agents/codex.go +++ b/internal/agents/codex.go @@ -169,8 +169,11 @@ func (m *Manager) codexItemCompleted(ctx context.Context, l *live, s *codex.Sess return m.emit(ctx, l, agentActor, "agent.vendor", payload) } - content, isError := codexToolResult(item) - err := m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content}) + content, isError, err := codexToolResult(item) + if err != nil { + return err + } + err = m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content}) if err != nil { return err } @@ -210,24 +213,51 @@ func imageAlt(prompt string) string { return alt } -func codexToolResult(item codex.ThreadItem) (string, bool) { +func codexToolResult(item codex.ThreadItem) (any, bool, error) { status := codexStatus(item) switch item.Type { case codex.ThreadItemTypeCommandExecution: - return item.AggregatedOutput, status != "completed" || item.ExitCode != 0 + return item.AggregatedOutput, status != "completed" || item.ExitCode != 0, nil case codex.ThreadItemTypeDynamicToolCall: - texts := []string{} + blocks := []map[string]any{} for _, c := range item.ContentItems { - texts = append(texts, c.Text) + if c.Type != codex.DynamicToolCallOutputContentItemTypeInputImage { + blocks = append(blocks, map[string]any{"type": "text", "text": c.Text}) + continue + } + mime, data, ok := strings.Cut(strings.TrimPrefix(c.ImageURL, "data:"), ";base64,") + if !ok { + return nil, false, fmt.Errorf("dynamic tool image is not a base64 data url") + } + blocks = append(blocks, imageBlock(mime, data)) } - return strings.Join(texts, "\n"), !item.Success + return blocks, !item.Success, nil case codex.ThreadItemTypeMcpToolCall: if item.Error != nil { - return item.Error.Message, true + return item.Error.Message, true, nil } - return string(item.Result), false + res := struct { + Content []map[string]any `json:"content"` + IsError bool `json:"isError"` + }{} + err := json.Unmarshal(item.Result, &res) + if err != nil { + return nil, false, fmt.Errorf("decode mcp tool result: %w", err) + } + for i, b := range res.Content { + if b["type"] == "image" { + mime, _ := b["mimeType"].(string) + data, _ := b["data"].(string) + res.Content[i] = imageBlock(mime, data) + } + } + return res.Content, res.IsError, nil } - return status, status != "completed" + return status, status != "completed", nil +} + +func imageBlock(mime, data string) map[string]any { + return map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": mime, "data": data}} } func (m *Manager) readCodexRateLimit() { diff --git a/mac/Sources/Byre/Captures.swift b/mac/Sources/Byre/Captures.swift new file mode 100644 index 0000000..fe7b824 --- /dev/null +++ b/mac/Sources/Byre/Captures.swift @@ -0,0 +1,52 @@ +import SwiftUI + +struct CapturesView: View { + let captures: [Capture] + + var body: some View { + ScrollView { + LazyVStack(alignment: .leading, spacing: 20) { + ForEach(captures.reversed()) { c in + CaptureItem(capture: c) + } + } + .padding(16) + } + .background(Palette.bg) + } +} + +private struct CaptureItem: View { + let capture: Capture + @ViewState private var image: PlatformImage? + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .firstTextBaseline, spacing: 8) { + Text(capture.tool) + .font(Fonts.font(13, weight: 700)) + Text(clockText(capture.when)) + .foregroundStyle(Palette.muted) + Spacer(minLength: 0) + } + if !capture.note.isEmpty { + Text(verbatim: capture.note) + .font(Fonts.font(12, mono: true)) + .foregroundStyle(Palette.muted) + .lineLimit(4) + .textSelection(.enabled) + } + if let image { + Image(platformImage: image) + .resizable() + .scaledToFit() + .frame(maxWidth: image.size.width, alignment: .leading) + .clipShape(RoundedRectangle(cornerRadius: 6)) + .overlay(RoundedRectangle(cornerRadius: 6).stroke(Palette.border)) + } + } + .task(id: capture.id) { + image = Data(base64Encoded: capture.base64).flatMap { PlatformImage(data: $0) } + } + } +} diff --git a/mac/Sources/Byre/Feed.swift b/mac/Sources/Byre/Feed.swift index 8f02c28..b6a8127 100644 --- a/mac/Sources/Byre/Feed.swift +++ b/mac/Sources/Byre/Feed.swift @@ -20,10 +20,19 @@ final class Listeners { } } +struct Capture: Identifiable { + let id: String + let tool: String + let note: String + let when: Date + let base64: String +} + @MainActor @Observable final class Feed { private(set) var events: [ByreEvent] = [] + private(set) var captures: [Capture] = [] private(set) var ready = false private(set) var inherited = 0 let appended = Listeners<[ByreEvent]>() @@ -31,6 +40,7 @@ final class Feed { @ObservationIgnored private var last: Int64 = 0 @ObservationIgnored private var buffer: [ByreEvent] = [] @ObservationIgnored private var scheduled = false + @ObservationIgnored private var toolNames: [String: String] = [:] @ObservationIgnored private let shown: () -> Bool init(shown: @escaping () -> Bool) { @@ -66,6 +76,30 @@ final class Feed { func prepend(_ older: [ByreEvent]) { events.insert(contentsOf: older, at: 0) + captures.insert(contentsOf: findCaptures(older), at: 0) + } + + private func findCaptures(_ batch: [ByreEvent]) -> [Capture] { + var found: [Capture] = [] + for ev in batch { + let p = ev.payload.value + if ev.kind == "agent.tool_call" { + toolNames[jsString(p["id"])] = toolLabel(jsString(p["name"])) + continue + } + guard ev.kind == "agent.tool_result", let list = p["content"] as? [(any Sendable)?] else { + continue + } + let blocks = list.compactMap { $0 as? JSONDict } + let note = blocks.compactMap { str($0, "type") == "text" ? str($0, "text") : nil }.first ?? "" + for (i, b) in blocks.enumerated() { + guard str(b, "type") == "image", let source = dict(b, "source"), let data = str(source, "data") else { + continue + } + found.append(Capture(id: "\(ev.seq)/\(i)", tool: toolNames[jsString(p["tool_use_id"])] ?? "", note: note, when: ev.createdAt, base64: data)) + } + } + return found } private func flush() { @@ -97,6 +131,10 @@ final class Feed { return } events.append(contentsOf: fresh) + let found = findCaptures(fresh) + if !found.isEmpty { + captures.append(contentsOf: found) + } if shown() { appended.send(fresh) } diff --git a/mac/Sources/Byre/Panes.swift b/mac/Sources/Byre/Panes.swift index 836bf16..c3369e0 100644 --- a/mac/Sources/Byre/Panes.swift +++ b/mac/Sources/Byre/Panes.swift @@ -7,6 +7,7 @@ let tabLabels: [TaskTab: String] = [ .changes: "Changes", .files: "Files", .page: "Page", + .captures: "Captures", .terminal: "Terminal", .workflow: "Workflow", ] @@ -251,6 +252,8 @@ struct TaskTabBody: View { if let shown = store.selectedTaskId.flatMap({ store.shownPages[$0] }) { PageTabs(shown: shown) } + case .captures: + CapturesView(captures: store.selectedTaskId.map { store.captures($0) } ?? []) case .terminal: TerminalTab() case .workflow: @@ -419,6 +422,7 @@ struct TabStrip: View { switch t { case .board: return team case .page: return store.shownPages[taskId] != nil + case .captures: return !store.captures(taskId).isEmpty case .workflow: return false default: return true } diff --git a/mac/Sources/Byre/Store.swift b/mac/Sources/Byre/Store.swift index 57a4870..59266dc 100644 --- a/mac/Sources/Byre/Store.swift +++ b/mac/Sources/Byre/Store.swift @@ -3,10 +3,10 @@ import Observation import SwiftData enum TaskTab: String { - case transcript, board, changes, files, page, terminal, workflow + case transcript, board, changes, files, page, captures, terminal, workflow } -let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .terminal, .workflow] +let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .captures, .terminal, .workflow] enum JumpTarget: Equatable { case question(String) @@ -215,6 +215,10 @@ final class Store { selectedTaskId.flatMap { taskFeeds[$0] } } + func captures(_ taskId: String) -> [Capture] { + taskFeeds[taskId]?.captures ?? [] + } + func activeTasks() -> [(project: Project, task: ByreTask)] { projects.flatMap { project in (tasks[project.id] ?? []).filter { $0.status == .active }.map { (project, $0) }