Fix diagram layout and clicks; tell agents to yield to image messages

internal/agents/inbox.go

@@ -387,19 +387,27 @@ type midTurn struct {
 	taskInputs []store.TaskInput
 	leadInputs []store.LeadInput
 	responses  []string
+	hinted     int64
 }
 
 func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 	if l.owner != "" {
-		inputs := m.takeLeadQueuedMidTurn(ctx, l)
-		if len(inputs) == 0 {
-			return midTurn{}
+		inputs, held := m.takeLeadQueuedMidTurn(ctx, l)
+		parts := []string{}
+		if len(inputs) > 0 {
+			msgs := []queued{}
+			for _, in := range inputs {
+				msgs = append(msgs, queued{actor: in.Actor, text: in.Text, sendID: in.SendID, client: in.Client})
+			}
+			parts = append(parts, m.recordQueued(ctx, l, msgs))
 		}
-		msgs := []queued{}
-		for _, in := range inputs {
-			msgs = append(msgs, queued{actor: in.Actor, text: in.Text, sendID: in.SendID, client: in.Client})
+		mt := midTurn{leadInputs: inputs}
+		if hint := m.imagesHint(l, held); hint != "" {
+			parts = append(parts, hint)
+			mt.hinted = held.id
 		}
-		return midTurn{text: m.recordQueued(ctx, l, msgs), leadInputs: inputs}
+		mt.text = strings.Join(parts, "\n\n")
+		return mt
 	}
 
 	unlock := m.lockDelivery(l.id)
@@ -419,7 +427,8 @@ func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 
 	mt := midTurn{}
 	parts := []string{}
-	mt.taskInputs = m.takeQueuedMidTurn(ctx, l)
+	inputs, held := m.takeQueuedMidTurn(ctx, l)
+	mt.taskInputs = inputs
 	if len(mt.taskInputs) > 0 {
 		msgs := []queued{}
 		for _, in := range mt.taskInputs {
@@ -427,6 +436,10 @@ func (m *Manager) deliverMidTurn(ctx context.Context, l *live) midTurn {
 		}
 		parts = append(parts, m.recordQueued(ctx, l, msgs))
 	}
+	if hint := m.imagesHint(l, held); hint != "" {
+		parts = append(parts, hint)
+		mt.hinted = held.id
+	}
 	responses, ids := m.responsesMidTurn(ctx, l)
 	if responses != "" {
 		parts = append(parts, responses)
@@ -472,6 +485,13 @@ func (m *Manager) restoreMidTurn(ctx context.Context, l *live, mt midTurn) {
 			slog.Error("mark responses undelivered", "session", l.id, "err", err, "responses", mt.responses)
 		}
 	}
+	if mt.hinted != 0 {
+		m.mu.Lock()
+		if l.hintedImages == mt.hinted {
+			l.hintedImages = 0
+		}
+		m.mu.Unlock()
+	}
 	slog.Info("mid-turn delivery restored", "session", l.id, "inputs", len(mt.taskInputs)+len(mt.leadInputs), "responses", mt.responses)
 }
 

internal/agents/manager.go

@@ -122,6 +122,8 @@ type live struct {
 	lastText    string
 	subagents   map[string]bool
 
+	hintedImages int64
+
 	contextModel  string
 	contextUsed   int
 	contextWindow int

internal/agents/queue.go

@@ -176,12 +176,17 @@ type queued struct {
 	client string
 }
 
-func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) []store.TaskInput {
+type heldImages struct {
+	id    int64
+	actor string
+}
+
+func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) ([]store.TaskInput, heldImages) {
 	m.mu.Lock()
 	compacting := l.compacting
 	m.mu.Unlock()
 	if compacting {
-		return nil
+		return nil, heldImages{}
 	}
 
 	unlock := m.lockDelivery(queueLockKey(l.taskID))
@@ -190,34 +195,36 @@ func (m *Manager) takeQueuedMidTurn(ctx context.Context, l *live) []store.TaskIn
 	inputs, err := m.store.TaskInputs(ctx, l.taskID, l.member)
 	if err != nil {
 		slog.Error("list queued messages", "session", l.id, "err", err)
-		return nil
+		return nil, heldImages{}
 	}
 	out := []store.TaskInput{}
 	ids := []int64{}
+	held := heldImages{}
 	for _, in := range inputs {
 		if len(in.Images) > 0 {
+			held = heldImages{in.ID, in.Actor}
 			break
 		}
 		out = append(out, in)
 		ids = append(ids, in.ID)
 	}
 	if len(ids) == 0 {
-		return nil
+		return nil, held
 	}
 	err = m.store.DeleteTaskInputs(ctx, ids)
 	if err != nil {
 		slog.Error("delete queued messages", "session", l.id, "err", err)
-		return nil
+		return nil, held
 	}
-	return out
+	return out, held
 }
 
-func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) []store.LeadInput {
+func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) ([]store.LeadInput, heldImages) {
 	m.mu.Lock()
 	compacting := l.compacting
 	m.mu.Unlock()
 	if compacting {
-		return nil
+		return nil, heldImages{}
 	}
 
 	m.leadMu.Lock()
@@ -226,29 +233,44 @@ func (m *Manager) takeLeadQueuedMidTurn(ctx context.Context, l *live) []store.Le
 	inputs, err := m.store.LeadInputs(ctx, l.owner)
 	if err != nil {
 		slog.Error("list held lead messages", "session", l.id, "err", err)
-		return nil
+		return nil, heldImages{}
 	}
 	out := []store.LeadInput{}
 	ids := []int64{}
+	held := heldImages{}
 	for _, in := range inputs {
 		if in.Kind != store.LeadMessage {
 			continue
 		}
 		if len(in.Images) > 0 {
+			held = heldImages{in.ID, in.Actor}
 			break
 		}
 		out = append(out, in)
 		ids = append(ids, in.ID)
 	}
 	if len(ids) == 0 {
-		return nil
+		return nil, held
 	}
 	err = m.store.DeleteLeadInputs(ctx, ids)
 	if err != nil {
 		slog.Error("delete held lead messages", "session", l.id, "err", err)
-		return nil
+		return nil, held
 	}
-	return out
+	return out, held
+}
+
+func (m *Manager) imagesHint(l *live, held heldImages) string {
+	if held.id == 0 {
+		return ""
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if l.hintedImages == held.id {
+		return ""
+	}
+	l.hintedImages = held.id
+	return fmt.Sprintf("%s sent you a message with images, which byre can only hand you as a new turn. Finish the step you're on, end this turn with a short summary, and don't start more work: they're waiting for you to read it.", held.actor)
 }
 
 func (m *Manager) recordQueued(ctx context.Context, l *live, msgs []queued) string {