Add a Captures tab showing screen captures from the task's agent

This commit is contained in:
Ian Gulliver
2026-10-22 20:53:28 -07:00
parent de9c9a684c
commit 5fc8a88a82
5 changed files with 140 additions and 12 deletions
+38
View File
@@ -20,10 +20,19 @@ final class Listeners<T> {
}
}
struct Capture: Identifiable {
let id: String
let tool: String
let note: String
let when: Date
let base64: String
}
@MainActor
@Observable
final class Feed {
private(set) var events: [ByreEvent] = []
private(set) var captures: [Capture] = []
private(set) var ready = false
private(set) var inherited = 0
let appended = Listeners<[ByreEvent]>()
@@ -31,6 +40,7 @@ final class Feed {
@ObservationIgnored private var last: Int64 = 0
@ObservationIgnored private var buffer: [ByreEvent] = []
@ObservationIgnored private var scheduled = false
@ObservationIgnored private var toolNames: [String: String] = [:]
@ObservationIgnored private let shown: () -> Bool
init(shown: @escaping () -> Bool) {
@@ -66,6 +76,30 @@ final class Feed {
func prepend(_ older: [ByreEvent]) {
events.insert(contentsOf: older, at: 0)
captures.insert(contentsOf: findCaptures(older), at: 0)
}
private func findCaptures(_ batch: [ByreEvent]) -> [Capture] {
var found: [Capture] = []
for ev in batch {
let p = ev.payload.value
if ev.kind == "agent.tool_call" {
toolNames[jsString(p["id"])] = toolLabel(jsString(p["name"]))
continue
}
guard ev.kind == "agent.tool_result", let list = p["content"] as? [(any Sendable)?] else {
continue
}
let blocks = list.compactMap { $0 as? JSONDict }
let note = blocks.compactMap { str($0, "type") == "text" ? str($0, "text") : nil }.first ?? ""
for (i, b) in blocks.enumerated() {
guard str(b, "type") == "image", let source = dict(b, "source"), let data = str(source, "data") else {
continue
}
found.append(Capture(id: "\(ev.seq)/\(i)", tool: toolNames[jsString(p["tool_use_id"])] ?? "", note: note, when: ev.createdAt, base64: data))
}
}
return found
}
private func flush() {
@@ -97,6 +131,10 @@ final class Feed {
return
}
events.append(contentsOf: fresh)
let found = findCaptures(fresh)
if !found.isEmpty {
captures.append(contentsOf: found)
}
if shown() {
appended.send(fresh)
}