Fix diagram layout and clicks; tell agents to yield to image messages

internal/agents/inbox.go

@@ -387,19 +387,27 @@ type midTurn struct {
 	taskInputs []store.TaskInput
 	leadInputs []store.LeadInput
 	responses  []string
+	hinted     int64
 }
 
 func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 	if l.owner != "" {
-		inputs := m.takeLeadQueuedMidTurn(ctx, l)
+		inputs, held := m.takeLeadQueuedMidTurn(ctx, l)
-		if len(inputs) == 0 {
+		parts := []string{}
-			return midTurn{}
+		if len(inputs) > 0 {
-		}
 			msgs := []queued{}
 			for _, in := range inputs {
 				msgs = append(msgs, queued{actor: in.Actor, text: in.Text, sendID: in.SendID, client: in.Client})
 			}
-		return midTurn{text: m.recordQueued(ctx, l, msgs), leadInputs: inputs}
+			parts = append(parts, m.recordQueued(ctx, l, msgs))
+		}
+		mt := midTurn{leadInputs: inputs}
+		if hint := m.imagesHint(l, held); hint != "" {
+			parts = append(parts, hint)
+			mt.hinted = held.id
+		}
+		mt.text = strings.Join(parts, "\n\n")
+		return mt
 	}
 
 	unlock := m.lockDelivery(l.id)
@@ -419,7 +427,8 @@ func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 
 	mt := midTurn{}
 	parts := []string{}
-	mt.taskInputs = m.takeQueuedMidTurn(ctx, l)
+	inputs, held := m.takeQueuedMidTurn(ctx, l)
+	mt.taskInputs = inputs
 	if len(mt.taskInputs) > 0 {
 		msgs := []queued{}
 		for _, in := range mt.taskInputs {
@@ -427,6 +436,10 @@ func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 		}
 		parts = append(parts, m.recordQueued(ctx, l, msgs))
 	}
+	if hint := m.imagesHint(l, held); hint != "" {
+		parts = append(parts, hint)
+		mt.hinted = held.id
+	}
 	responses, ids := m.responsesMidTurn(ctx, l)
 	if responses != "" {
 		parts = append(parts, responses)
@@ -472,6 +485,13 @@ func (m *Manager) restoreMidTurn(ctx context.Context, l *live, mt midTurn) {
 			slog.Error("mark responses undelivered", "session", l.id, "err", err, "responses", mt.responses)
 		}
 	}
+	if mt.hinted != 0 {
+		m.mu.Lock()
+		if l.hintedImages == mt.hinted {
+			l.hintedImages = 0
+		}
+		m.mu.Unlock()
+	}
 	slog.Info("mid-turn delivery restored", "session", l.id, "inputs", len(mt.taskInputs)+len(mt.leadInputs), "responses", mt.responses)
 }
 

internal/agents/manager.go

@@ -122,6 +122,8 @@ type live struct {
 	lastText    string
 	subagents   map[string]bool
 
+	hintedImages int64
+
 	contextModel  string
 	contextUsed   int
 	contextWindow int

internal/agents/queue.go

@@ -176,12 +176,17 @@ type queued struct {
 	client string
 }
 
-func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) []store.TaskInput {
+type heldImages struct {
+	id    int64
+	actor string
+}
+
+func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) ([]store.TaskInput, heldImages) {
 	m.mu.Lock()
 	compacting := l.compacting
 	m.mu.Unlock()
 	if compacting {
-		return nil
+		return nil, heldImages{}
 	}
 
 	unlock := m.lockDelivery(queueLockKey(l.taskID))
@@ -190,34 +195,36 @@ func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) []store.TaskIn
 	inputs, err := m.store.TaskInputs(ctx, l.taskID, l.member)
 	if err != nil {
 		slog.Error("list queued messages", "session", l.id, "err", err)
-		return nil
+		return nil, heldImages{}
 	}
 	out := []store.TaskInput{}
 	ids := []int64{}
+	held := heldImages{}
 	for _, in := range inputs {
 		if len(in.Images) > 0 {
+			held = heldImages{in.ID, in.Actor}
 			break
 		}
 		out = append(out, in)
 		ids = append(ids, in.ID)
 	}
 	if len(ids) == 0 {
-		return nil
+		return nil, held
 	}
 	err = m.store.DeleteTaskInputs(ctx, ids)
 	if err != nil {
 		slog.Error("delete queued messages", "session", l.id, "err", err)
-		return nil
+		return nil, held
 	}
-	return out
+	return out, held
 }
 
-func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) []store.LeadInput {
+func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) ([]store.LeadInput, heldImages) {
 	m.mu.Lock()
 	compacting := l.compacting
 	m.mu.Unlock()
 	if compacting {
-		return nil
+		return nil, heldImages{}
 	}
 
 	m.leadMu.Lock()
@@ -226,29 +233,44 @@ func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) []store.Le
 	inputs, err := m.store.LeadInputs(ctx, l.owner)
 	if err != nil {
 		slog.Error("list held lead messages", "session", l.id, "err", err)
-		return nil
+		return nil, heldImages{}
 	}
 	out := []store.LeadInput{}
 	ids := []int64{}
+	held := heldImages{}
 	for _, in := range inputs {
 		if in.Kind != store.LeadMessage {
 			continue
 		}
 		if len(in.Images) > 0 {
+			held = heldImages{in.ID, in.Actor}
 			break
 		}
 		out = append(out, in)
 		ids = append(ids, in.ID)
 	}
 	if len(ids) == 0 {
-		return nil
+		return nil, held
 	}
 	err = m.store.DeleteLeadInputs(ctx, ids)
 	if err != nil {
 		slog.Error("delete held lead messages", "session", l.id, "err", err)
-		return nil
+		return nil, held
 	}
-	return out
+	return out, held
+}
+
+func (m *Manager) imagesHint(l *live, held heldImages) string {
+	if held.id == 0 {
+		return ""
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if l.hintedImages == held.id {
+		return ""
+	}
+	l.hintedImages = held.id
+	return fmt.Sprintf("%s sent you a message with images, which byre can only hand you as a new turn. Finish the step you're on, end this turn with a short summary, and don't start more work: they're waiting for you to read it.", held.actor)
 }
 
 func (m *Manager) recordQueued(ctx context.Context, l *live, msgs []queued) string {

mac/Sources/Byre/BoxDiagram.swift

@@ -380,6 +380,7 @@ struct BoxLayout {
 struct BoxFocus {
     let boxes: Set<String>
     let edges: Set<Int>
+    let outlined: Set<String>
 
     static func box(_ id: String, _ layout: BoxLayout) -> BoxFocus {
         var boxes: Set<String> = [id]
@@ -389,12 +390,12 @@ struct BoxFocus {
             boxes.insert(r.edge.from)
             boxes.insert(r.edge.to)
         }
-        return BoxFocus(boxes: boxes, edges: edges)
+        return BoxFocus(boxes: boxes, edges: edges, outlined: [id])
     }
 
     static func preset(_ p: BoxPreset, _ layout: BoxLayout) -> BoxFocus {
         let edges = layout.edges.indices.filter { p.ids.contains(layout.edges[$0].edge.from) && p.ids.contains(layout.edges[$0].edge.to) }
-        return BoxFocus(boxes: p.ids, edges: Set(edges))
+        return BoxFocus(boxes: p.ids, edges: Set(edges), outlined: p.ids)
     }
 
     func dims(_ b: PlacedBox, _ layout: BoxLayout) -> Bool {
@@ -412,6 +413,7 @@ private let headGap: CGFloat = 24
 private let childGap: CGFloat = 16
 private let boxGap: CGFloat = 24
 private let rowGap: CGFloat = 52
+private let laneGap: CGFloat = 24
 private let columnGap: CGFloat = 80
 private let margin: CGFloat = 20
 private let labelRoom: CGFloat = 28
@@ -675,11 +677,26 @@ private func placeRows(_ rows: [BoxRow], _ edges: [BoxEdge]) -> (boxes: [PlacedB
     let here = Set(rows.flatMap { $0.main + $0.aside }.flatMap(ids))
     let crossing = edges.filter { here.contains($0.from) && here.contains($0.to) && asideIDs.contains($0.from) != asideIDs.contains($0.to) }.compactMap(\.label)
     let asideX = margin + mainWidth + crossing.map { ceil(textSize($0, edgeLabelFont).width) + labelRoom }.reduce(columnGap, max)
+    var rowOf: [String: Int] = [:]
+    for (i, r) in rows.enumerated() {
+        for id in r.main.flatMap(ids) {
+            rowOf[id] = i
+        }
+    }
+    func gap(below i: Int) -> CGFloat {
+        let n = edges.filter { e in
+            guard let a = rowOf[e.from], let b = rowOf[e.to] else {
+                return false
+            }
+            return min(a, b) <= i && max(a, b) > i
+        }.count
+        return max(rowGap, CGFloat(n + 1) * laneGap)
+    }
     var boxes: [PlacedBox] = []
     var y = margin
     var asideBottom = margin - rowGap
     var bottom = margin
-    for r in rows {
+    for (i, r) in rows.enumerated() {
         let height = r.main.map { sizer.naturalSize($0).height }.max() ?? 0
         sizer.lay(r.main, from: margin, width: mainWidth, gaps: sizer.gaps(r.main, boxGap)) { i, x0, x1 in
             sizer.place(r.main[i], CGRect(x: x0, y: y, width: x1 - x0, height: height), 0, nil, &boxes)
@@ -695,7 +712,7 @@ private func placeRows(_ rows: [BoxRow], _ edges: [BoxEdge]) -> (boxes: [PlacedB
             asideBottom = ay - rowGap
         }
         bottom = max(bottom, y + height, asideBottom)
-        y += height + (r.main.isEmpty ? 0 : rowGap)
+        y += height + (r.main.isEmpty ? 0 : gap(below: i))
     }
     let width = (asideWidth > 0 ? asideX + asideWidth : margin + mainWidth) + margin
     return (boxes, CGSize(width: width, height: bottom + margin))
@@ -731,8 +748,13 @@ private func routeLevel(_ level: String, _ boxes: [PlacedBox], _ size: CGSize, _
         }
         routed.append(RoutedEdge(edge: e, points: pts, labelRect: labelRect))
     }
-    let fit = CGSize(width: ceil(labels.map(\.maxX).reduce(size.width - margin, max) + margin), height: ceil(labels.map(\.maxY).reduce(size.height - margin, max) + margin))
+    let dx = max(0, ceil(margin - labels.map(\.minX).reduce(margin, min)))
-    return BoxLayout(level: level, boxes: boxes, index: index, edges: routed, presets: presets, size: fit)
+    let dy = max(0, ceil(margin - labels.map(\.minY).reduce(margin, min)))
+    let fit = CGSize(width: ceil(labels.map(\.maxX).reduce(size.width - margin, max) + margin) + dx, height: ceil(labels.map(\.maxY).reduce(size.height - margin, max) + margin) + dy)
+    let shifted = routed.map { r in
+        RoutedEdge(edge: r.edge, points: r.points.map { CGPoint(x: $0.x + dx, y: $0.y + dy) }, labelRect: r.labelRect?.offsetBy(dx: dx, dy: dy))
+    }
+    return BoxLayout(level: level, boxes: boxes.map { $0.moved(dx, dy) }, index: index, edges: shifted, presets: presets, size: fit)
 }
 
 private struct Anchor {
@@ -868,17 +890,24 @@ private func labelSpot(_ size: CGSize, _ pts: [CGPoint], headers: [CGRect], outl
         return area(box, c)
     }
     func overlap(_ c: CGRect) -> CGFloat {
-        let inside = c.intersection(bounds)
+        var covered = headers.reduce(0) { $0 + area($1, c) }
-        let outside = c.width * c.height - (inside.isNull ? 0 : inside.width * inside.height)
-        var covered = headers.reduce(outside) { $0 + area($1, c) }
         covered = outlines.reduce(covered) { $0 + straddles($1, c) }
         covered = labels.reduce(covered) { $0 + area($1, c, 12) }
         return covered * 100 + lines.reduce(0) { $0 + area($1, c) }
     }
+    func outside(_ c: CGRect) -> CGFloat {
+        let inside = c.intersection(bounds)
+        return c.width * c.height - (inside.isNull ? 0 : inside.width * inside.height)
+    }
     return candidates.min { a, b in
         let oa = overlap(a.rect)
         let ob = overlap(b.rect)
-        return oa != ob ? oa < ob : a.order < b.order
+        if oa != ob {
+            return oa < ob
+        }
+        let xa = outside(a.rect)
+        let xb = outside(b.rect)
+        return xa != xb ? xa < xb : a.order < b.order
     }!.rect
 }
 
@@ -967,6 +996,15 @@ private struct Router {
                 mark(CGRect(x: r.maxX - cell, y: r.minY, width: cell * 3, height: r.height), &level)
             }
         }
+        let toAncestors = Set(lineage(e.to))
+        if let outer = lineage(e.from).first(where: toAncestors.contains) {
+            let c = cells(boxes[index[outer]!].rect)
+            for y in 0..<rows {
+                for x in 0..<cols where x <= c.x0 || x >= c.x1 || y <= c.y0 || y >= c.y1 {
+                    grid[y * cols + x] = true
+                }
+            }
+        }
         return (grid, upright, level)
     }
 
@@ -1314,7 +1352,7 @@ struct BoxCanvas: View {
                 ctx.opacity = 0.3
             }
             let path = Path(roundedRect: b.rect, cornerRadius: corner(b))
-            let lit = focus?.boxes.contains(b.node.id) == true
+            let lit = focus?.outlined.contains(b.node.id) == true
             let stroke = lit ? Palette.accent : b.kind == .ghost ? Palette.muted : Palette.border
             if b.kind == .box {
                 ctx.fill(path, with: .color(fill(b)))
@@ -1515,25 +1553,23 @@ struct DiagramViewer: View {
     }
 }
 
-private struct TappableCanvas: View {
+private struct SizedCanvas: View {
     let layout: BoxLayout
     let focus: BoxFocus?
-    let tap: (String?) -> Void
-    let open: (String) -> Void
 
     var body: some View {
         BoxCanvas(layout: layout, focus: focus)
             .frame(width: layout.size.width, height: layout.size.height)
-            .contentShape(Rectangle())
+            .diagramMenu(layout)
-            .onTapGesture(coordinateSpace: .local) { p in
+    }
+}
+
+private func click(_ layout: BoxLayout, _ p: CGPoint, tap: (String?) -> Void, open: (String) -> Void) {
     if let id = layout.opener(at: p) {
         open(id)
     } else {
         tap(layout.hit(p))
     }
-            }
-            .diagramMenu(layout)
-    }
 }
 
 extension View {
@@ -1584,6 +1620,26 @@ private struct ZoomingCanvas: NSViewRepresentable {
     let tap: (String?) -> Void
     let open: (String) -> Void
 
+    @MainActor
+    final class Coordinator: NSObject {
+        var canvas: ZoomingCanvas
+
+        init(_ canvas: ZoomingCanvas) {
+            self.canvas = canvas
+        }
+
+        @objc func clicked(_ g: NSClickGestureRecognizer) {
+            guard let view = g.view else {
+                return
+            }
+            click(canvas.layout, g.location(in: view), tap: canvas.tap, open: canvas.open)
+        }
+    }
+
+    func makeCoordinator() -> Coordinator {
+        Coordinator(self)
+    }
+
     func makeNSView(context: Context) -> FittingScrollView {
         let scroll = FittingScrollView()
         scroll.contentView = CenteringClipView()
@@ -1594,17 +1650,34 @@ private struct ZoomingCanvas: NSViewRepresentable {
         scroll.allowsMagnification = true
         scroll.minMagnification = 0.25
         scroll.maxMagnification = 6
-        let host = NSHostingView(rootView: TappableCanvas(layout: layout, focus: focus, tap: tap, open: open))
+        let host = NSHostingView(rootView: SizedCanvas(layout: layout, focus: focus))
         host.frame = CGRect(origin: .zero, size: layout.size)
+        host.addGestureRecognizer(NSClickGestureRecognizer(target: context.coordinator, action: #selector(Coordinator.clicked)))
         scroll.documentView = host
         return scroll
     }
 
     func updateNSView(_ scroll: FittingScrollView, context: Context) {
-        (scroll.documentView as? NSHostingView<TappableCanvas>)?.rootView = TappableCanvas(layout: layout, focus: focus, tap: tap, open: open)
+        context.coordinator.canvas = self
+        (scroll.documentView as? NSHostingView<SizedCanvas>)?.rootView = SizedCanvas(layout: layout, focus: focus)
     }
 }
 #else
+private struct TappableCanvas: View {
+    let layout: BoxLayout
+    let focus: BoxFocus?
+    let tap: (String?) -> Void
+    let open: (String) -> Void
+
+    var body: some View {
+        SizedCanvas(layout: layout, focus: focus)
+            .contentShape(Rectangle())
+            .onTapGesture(coordinateSpace: .local) { p in
+                click(layout, p, tap: tap, open: open)
+            }
+    }
+}
+
 private final class FittingScrollView: UIScrollView, UIScrollViewDelegate {
     let host: UIHostingController<TappableCanvas>
     private var fitted = false

mac/Sources/Byre/DiagramHTML.swift

@@ -149,7 +149,8 @@ func diagramHTML(_ diagram: DiagramLayout) -> String {
     html,body{margin:0;height:100%;background:var(--bg);color:var(--text);font-family:Atkinson,-apple-system,sans-serif}
     body{display:flex;flex-direction:column}
     nav{display:flex;flex-wrap:wrap;align-items:center;gap:6px;padding:12px 12px 0;font:700 13px Atkinson,-apple-system,sans-serif}
-    nav:empty{display:none}
+    #trail:empty,#bar[hidden]{display:none}
+    #bar{min-height:27px}
     nav .sep{color:var(--muted)}
     nav .grow{flex:1}
     nav a{color:var(--accent);cursor:pointer}
@@ -197,16 +198,17 @@ func diagramHTML(_ diagram: DiagramLayout) -> String {
         t.append(l===level?el("span","",D.levels[l].title):el("a","",D.levels[l].title,()=>go(l,ls[i+1])))})}
       render();fit()}
     function render(){
-      let boxes=null,edges=new Set();
+      let boxes=null,edges=new Set(),outlined=new Set();
-      if(picked!=null){boxes=new Set([picked]);L.edges.forEach((e,i)=>{if(e.from==picked||e.to==picked){edges.add(i);boxes.add(e.from);boxes.add(e.to)}})}
+      if(picked!=null){boxes=new Set([picked]);outlined=new Set([picked]);L.edges.forEach((e,i)=>{if(e.from==picked||e.to==picked){edges.add(i);boxes.add(e.from);boxes.add(e.to)}})}
-      else if(preset!=null){const ids=L.presets[preset].ids;boxes=new Set(ids);L.edges.forEach((e,i)=>{if(boxes.has(e.from)&&boxes.has(e.to))edges.add(i)})}
+      else if(preset!=null){const ids=L.presets[preset].ids;boxes=new Set(ids);outlined=boxes;L.edges.forEach((e,i)=>{if(boxes.has(e.from)&&boxes.has(e.to))edges.add(i)})}
       const g=shown();
       g.querySelectorAll(".box").forEach(g=>{const id=g.dataset.id;
-        g.classList.toggle("lit",!!boxes&&boxes.has(id));
+        g.classList.toggle("lit",outlined.has(id));
         const dim=!!boxes&&!boxes.has(id)&&!anc(id).some(a=>boxes.has(a))&&![...boxes].some(b=>anc(b).includes(id));
         g.classList.toggle("dim",dim)});
       g.querySelectorAll(".edge").forEach(g=>{const i=+g.dataset.i;g.classList.toggle("lit",edges.has(i));g.classList.toggle("faint",!!boxes&&!edges.has(i))});
       const bar=document.getElementById("bar");bar.replaceChildren();
+      bar.hidden=!L.presets.length&&!L.boxes.some(b=>b.kind==="box"&&b.opens);
       L.presets.forEach((p,i)=>{const b=el("button",i===preset?"on":"",p.name,()=>{picked=null;preset=preset===i?null:i;render()});bar.append(b)});
       const opening=picked!=null&&L.boxes.find(b=>b.id===picked&&b.kind==="box"&&b.opens);
       if(opening){bar.append(el("span","grow",""));bar.append(el("button","on","Open "+D.levels[picked].title+" ›",()=>go(picked,null)))}