From fae286a7d8a242fe9222cc02d594f095ed45d757 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Mon, 19 Oct 2026 19:01:24 -0700 Subject: [PATCH] errors at top of status are distracting --- docs/operations.md | 2 +- internal/agents/health.go | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 1613f10..b482363 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -264,7 +264,7 @@ Every change broadcasts `admin.health`. A Claude `rejected` status still drives Agent health is `internal/agents/health.go`: - **Processes.** `agent_processes` lists every process byre is tracking (idle sessions keep theirs), with session, task, vendor, pid, the session's state, `last_event_at` (when the process last wrote any line to stdout, or when it started; `claude.Session.LastLine`) and `health`. `health` is `exited` when the stream has closed or `kill(pid, 0)` says the pid is gone, `stuck` when the session is `running` and nothing has arrived for `StuckAfter`, otherwise `ok`. The stream does mark long tool calls: while a tool runs, Claude Code sends a `tool_progress` line with `"heartbeat": true` every 30 seconds, so a long `go test` keeps `last_event_at` fresh and a silent stream means nothing is happening. -- **Zombies.** `running_without_process`: a session marked `running` whose process byre isn't tracking or has `exited`. `process_without_session`: a `claude` child of the byre process (from `ps -A -o pid=,ppid=,comm=`) whose pid belongs to no tracked process, or a tracked process still alive for a `completed` session. A live process for an `idle` session is normal and not a zombie. +- **Zombies.** `running_without_process`: a session marked `running` whose process byre isn't tracking or has `exited`. `process_without_session`: a `claude` child of the byre process (from `ps -A -o pid=,ppid=,comm=`) whose pid belongs to no tracked process, or a tracked process still alive for a `completed` session. A live process for an `idle` session is normal and not a zombie. While a session's process is starting (in `Manager.starting`, between its turn marking it `running` and its process being tracked), neither kind is reported: that session isn't a `running_without_process`, and no untracked `claude` child is a `process_without_session`, since the starting one can't be told apart from an orphan until it is tracked. A real orphan shows at the next check with nothing starting. `Manager.Health` reads the `running` sessions and the `claude` children first and the tracked and starting processes after them, so a start seen in either is already registered by then. - **Watching.** `Manager.Watch` recomputes health every `healthInterval`. When the set of problems (non-`ok` processes and zombies) changes, it logs each new one as `agent health problem` and broadcasts an unstored `admin.health` event (`seq` 0). Nothing is killed or repaired automatically. - **Protocol errors.** `internal/claude` reports a stdout line that isn't a JSON object, doesn't parse, or has a `type` outside `knownTypes` (the line is still passed on), and the manager adds lines whose content it fails to decode. `protocol_errors.counts` counts them per vendor since byre started; `recent` holds the last `keptBadLines`, newest first, each with session, time, reason, the raw line cut to `badLineBytes` and its full `size`. They live in memory only, and each one also broadcasts `admin.health`. diff --git a/internal/agents/health.go b/internal/agents/health.go index b0d3e53..d101cf2 100644 --- a/internal/agents/health.go +++ b/internal/agents/health.go @@ -107,11 +107,26 @@ type procInfo struct { } func (m *Manager) Health(ctx context.Context) (Health, error) { + running, err := m.store.SessionsInState(ctx, api.AgentSessionStateRunning) + if err != nil { + return Health{}, err + } + + children, err := claudeChildren(ctx, os.Getpid()) + if err != nil { + return Health{}, err + } + + // Snapshot live and starting after running and children, so a start that shows in either is already registered. m.mu.Lock() all := []*live{} for _, l := range m.live { all = append(all, l) } + starting := map[string]bool{} + for id := range m.starting { + starting[id] = true + } m.mu.Unlock() procs := []procInfo{} @@ -132,20 +147,10 @@ func (m *Manager) Health(ctx context.Context) (Health, error) { }) } - running, err := m.store.SessionsInState(ctx, api.AgentSessionStateRunning) - if err != nil { - return Health{}, err - } - - children, err := claudeChildren(ctx, os.Getpid()) - if err != nil { - return Health{}, err - } - - return assess(time.Now(), procs, running, children), nil + return assess(time.Now(), procs, running, children, starting), nil } -func assess(now time.Time, procs []procInfo, running []api.AgentSession, children []int) Health { +func assess(now time.Time, procs []procInfo, running []api.AgentSession, children []int, starting map[string]bool) Health { slices.SortFunc(procs, func(a, b procInfo) int { return strings.Compare(a.sessionID, b.sessionID) }) h := Health{Processes: []api.AgentProcess{}, Zombies: []api.Zombie{}} @@ -187,7 +192,7 @@ func assess(now time.Time, procs []procInfo, running []api.AgentSession, childre for _, sess := range running { p, ok := bySession[sess.Id] - if ok && !p.exited { + if (ok && !p.exited) || starting[sess.Id] { continue } detail := "no agent process" @@ -203,7 +208,7 @@ func assess(now time.Time, procs []procInfo, running []api.AgentSession, childre } for _, pid := range children { - if tracked[pid] { + if tracked[pid] || len(starting) > 0 { continue } h.Zombies = append(h.Zombies, api.Zombie{