ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 14 Wiki Activity

Add a Captures tab showing screen captures from the task's agent

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-22 20:53:28 -07:00
parent de9c9a684c
commit 5fc8a88a82
5 changed files with 140 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/agents/codex.go
+40 -10
View File
@@ -169,8 +169,11 @@ func (m *Manager) codexItemCompleted(ctx context.Context, l *live, s *codex.Sess
return m.emit(ctx, l, agentActor, "agent.vendor", payload)
}
content, isError := codexToolResult(item)
err := m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content})
content, isError, err := codexToolResult(item)
if err != nil {
return err
}
err = m.emit(ctx, l, agentActor, "agent.tool_result", map[string]any{"tool_use_id": item.ID, "is_error": isError, "content": content})
if err != nil {
return err
}
@@ -210,24 +213,51 @@ func imageAlt(prompt string) string {
return alt
}
func codexToolResult(item codex.ThreadItem) (string, bool) {
func codexToolResult(item codex.ThreadItem) (any, bool, error) {
status := codexStatus(item)
switch item.Type {
case codex.ThreadItemTypeCommandExecution:
return item.AggregatedOutput, status != "completed" || item.ExitCode != 0
return item.AggregatedOutput, status != "completed" || item.ExitCode != 0, nil
case codex.ThreadItemTypeDynamicToolCall:
texts := []string{}
blocks := []map[string]any{}
for _, c := range item.ContentItems {
texts = append(texts, c.Text)
if c.Type != codex.DynamicToolCallOutputContentItemTypeInputImage {
blocks = append(blocks, map[string]any{"type": "text", "text": c.Text})
continue
}
return strings.Join(texts, "\n"), !item.Success
mime, data, ok := strings.Cut(strings.TrimPrefix(c.ImageURL, "data:"), ";base64,")
if !ok {
return nil, false, fmt.Errorf("dynamic tool image is not a base64 data url")
}
blocks = append(blocks, imageBlock(mime, data))
}
return blocks, !item.Success, nil
case codex.ThreadItemTypeMcpToolCall:
if item.Error != nil {
return item.Error.Message, true
return item.Error.Message, true, nil
}
return string(item.Result), false
res := struct {
Content []map[string]any `json:"content"`
IsError bool `json:"isError"`
}{}
err := json.Unmarshal(item.Result, &res)
if err != nil {
return nil, false, fmt.Errorf("decode mcp tool result: %w", err)
}
return status, status != "completed"
for i, b := range res.Content {
if b["type"] == "image" {
mime, _ := b["mimeType"].(string)
data, _ := b["data"].(string)
res.Content[i] = imageBlock(mime, data)
}
}
return res.Content, res.IsError, nil
}
return status, status != "completed", nil
}
func imageBlock(mime, data string) map[string]any {
return map[string]any{"type": "image", "source": map[string]any{"type": "base64", "media_type": mime, "data": data}}
}
func (m *Manager) readCodexRateLimit() {
mac/Sources/Byre/Captures.swift
+52
View File
@@ -0,0 +1,52 @@
import SwiftUI
struct CapturesView: View {
let captures: [Capture]
var body: some View {
ScrollView {
LazyVStack(alignment: .leading, spacing: 20) {
ForEach(captures.reversed()) { c in
CaptureItem(capture: c)
}
}
.padding(16)
}
.background(Palette.bg)
}
}
private struct CaptureItem: View {
let capture: Capture
@ViewState private var image: PlatformImage?
var body: some View {
VStack(alignment: .leading, spacing: 6) {
HStack(alignment: .firstTextBaseline, spacing: 8) {
Text(capture.tool)
.font(Fonts.font(13, weight: 700))
Text(clockText(capture.when))
.foregroundStyle(Palette.muted)
Spacer(minLength: 0)
}
if !capture.note.isEmpty {
Text(verbatim: capture.note)
.font(Fonts.font(12, mono: true))
.foregroundStyle(Palette.muted)
.lineLimit(4)
.textSelection(.enabled)
}
if let image {
Image(platformImage: image)
.resizable()
.scaledToFit()
.frame(maxWidth: image.size.width, alignment: .leading)
.clipShape(RoundedRectangle(cornerRadius: 6))
.overlay(RoundedRectangle(cornerRadius: 6).stroke(Palette.border))
}
}
.task(id: capture.id) {
image = Data(base64Encoded: capture.base64).flatMap { PlatformImage(data: $0) }
}
}
}
mac/Sources/Byre/Feed.swift
+38
View File
@@ -20,10 +20,19 @@ final class Listeners<T> {
}
}
struct Capture: Identifiable {
let id: String
let tool: String
let note: String
let when: Date
let base64: String
}
@MainActor
@Observable
final class Feed {
private(set) var events: [ByreEvent] = []
private(set) var captures: [Capture] = []
private(set) var ready = false
private(set) var inherited = 0
let appended = Listeners<[ByreEvent]>()
@@ -31,6 +40,7 @@ final class Feed {
@ObservationIgnored private var last: Int64 = 0
@ObservationIgnored private var buffer: [ByreEvent] = []
@ObservationIgnored private var scheduled = false
@ObservationIgnored private var toolNames: [String: String] = [:]
@ObservationIgnored private let shown: () -> Bool
init(shown: @escaping () -> Bool) {
@@ -66,6 +76,30 @@ final class Feed {
func prepend(_ older: [ByreEvent]) {
events.insert(contentsOf: older, at: 0)
captures.insert(contentsOf: findCaptures(older), at: 0)
}
private func findCaptures(_ batch: [ByreEvent]) -> [Capture] {
var found: [Capture] = []
for ev in batch {
let p = ev.payload.value
if ev.kind == "agent.tool_call" {
toolNames[jsString(p["id"])] = toolLabel(jsString(p["name"]))
continue
}
guard ev.kind == "agent.tool_result", let list = p["content"] as? [(any Sendable)?] else {
continue
}
let blocks = list.compactMap { $0 as? JSONDict }
let note = blocks.compactMap { str($0, "type") == "text" ? str($0, "text") : nil }.first ?? ""
for (i, b) in blocks.enumerated() {
guard str(b, "type") == "image", let source = dict(b, "source"), let data = str(source, "data") else {
continue
}
found.append(Capture(id: "\(ev.seq)/\(i)", tool: toolNames[jsString(p["tool_use_id"])] ?? "", note: note, when: ev.createdAt, base64: data))
}
}
return found
}
private func flush() {
@@ -97,6 +131,10 @@ final class Feed {
return
}
events.append(contentsOf: fresh)
let found = findCaptures(fresh)
if !found.isEmpty {
captures.append(contentsOf: found)
}
if shown() {
appended.send(fresh)
}
mac/Sources/Byre/Panes.swift
+4
View File
@@ -7,6 +7,7 @@ let tabLabels: [TaskTab: String] = [
.changes: "Changes",
.files: "Files",
.page: "Page",
.captures: "Captures",
.terminal: "Terminal",
.workflow: "Workflow",
]
@@ -251,6 +252,8 @@ struct TaskTabBody: View {
if let shown = store.selectedTaskId.flatMap({ store.shownPages[$0] }) {
PageTabs(shown: shown)
}
case .captures:
CapturesView(captures: store.selectedTaskId.map { store.captures($0) } ?? [])
case .terminal:
TerminalTab()
case .workflow:
@@ -419,6 +422,7 @@ struct TabStrip: View {
switch t {
case .board: return team
case .page: return store.shownPages[taskId] != nil
case .captures: return !store.captures(taskId).isEmpty
case .workflow: return false
default: return true
}
mac/Sources/Byre/Store.swift
+6 -2
View File
@@ -3,10 +3,10 @@ import Observation
import SwiftData
enum TaskTab: String {
case transcript, board, changes, files, page, terminal, workflow
case transcript, board, changes, files, page, captures, terminal, workflow
}
let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .terminal, .workflow]
let taskTabs: [TaskTab] = [.transcript, .board, .changes, .files, .page, .captures, .terminal, .workflow]
enum JumpTarget: Equatable {
case question(String)
@@ -215,6 +215,10 @@ final class Store {
selectedTaskId.flatMap { taskFeeds[$0] }
}
func captures(_ taskId: String) -> [Capture] {
taskFeeds[taskId]?.captures ?? []
}
func activeTasks() -> [(project: Project, task: ByreTask)] {
projects.flatMap { project in
(tasks[project.id] ?? []).filter { $0.status == .active }.map { (project, $0) }