Show agent liveness, stuck/zombie detection and parse errors in admin

web/src/admin.ts

@@ -1,8 +1,8 @@
-import { get, type ByreEvent, type Status, type TurnRef } from "./api.js";
+import { get, type AgentProcess, type ByreEvent, type Status, type TurnRef, type Zombie } from "./api.js";
 import { ago, errorText, h, replace } from "./dom.js";
 import { findTask, on, onEach, state } from "./state.js";
 
-const refreshKinds = new Set(["session.created", "session.state", "task.created", "task.merged", "task.discarded", "project.created"]);
+const refreshKinds = new Set(["session.created", "session.state", "task.created", "task.merged", "task.discarded", "project.created", "admin.health"]);
 
 export function mountAdmin(root: HTMLElement): () => void {
   let status: Status | undefined;
@@ -16,17 +16,21 @@ export function mountAdmin(root: HTMLElement): () => void {
       replace(root, h("div", { class: error ? "empty error-text" : "empty" }, error || "Loading…"));
       return;
     }
+    const scrollTop = root.querySelector(".admin-view")?.scrollTop ?? 0;
+    const open = new Set([...root.querySelectorAll<HTMLDetailsElement>(".problems details[open]")].map((d) => d.dataset["vendor"] ?? ""));
     replace(
       root,
       h(
         "div",
         { class: "admin-view" },
         error && h("p", { class: "error-text" }, error),
+        problemsSection(status, open),
         turnsSection(status),
         rateLimitsSection(status.rate_limits),
         serverSection(status),
       ),
     );
+    root.querySelector(".admin-view")!.scrollTop = scrollTop;
   };
 
   const refresh = async () => {
@@ -59,8 +63,20 @@ export function mountAdmin(root: HTMLElement): () => void {
       void refresh();
       return;
     }
-    if (status && ev.seq > status.last_event_seq) {
+    if (!status) {
+      return;
+    }
+    let changed = false;
+    const proc = status.agent_processes.find((p) => p.session_id === ev.agent_session_id);
+    if (proc && Date.parse(ev.created_at) - Date.parse(proc.last_event_at) >= 1000) {
+      proc.last_event_at = ev.created_at;
+      changed = true;
+    }
+    if (ev.seq > status.last_event_seq) {
       status.last_event_seq = ev.seq;
+      changed = true;
+    }
+    if (changed) {
       render();
     }
   };
@@ -102,9 +118,9 @@ function turnsSection(s: Status): HTMLElement {
       h("div", { class: "cap-tile" }, h("div", { class: "cap-label" }, "Queued"), h("div", { class: s.queued_turns.length > 0 ? "cap-value warn" : "cap-value" }, s.queued_turns.length)),
     ),
     h("h3", {}, "Running"),
-    turnList(s.running_turns, "running", "No turns running."),
+    turnList(s.running_turns, "running", "No turns running.", s.agent_processes),
     h("h3", {}, "Queued"),
-    turnList(s.queued_turns, "queued", "Nothing queued."),
+    turnList(s.queued_turns, "queued", "Nothing queued.", []),
   );
 }
 
@@ -118,7 +134,7 @@ function capTile(label: string, running: number, cap: number | undefined): HTMLE
   );
 }
 
-function turnList(turns: TurnRef[], kind: "running" | "queued", empty: string): HTMLElement {
+function turnList(turns: TurnRef[], kind: "running" | "queued", empty: string, procs: AgentProcess[]): HTMLElement {
   return h(
     "ul",
     { class: "task-list" },
@@ -133,6 +149,7 @@ function turnList(turns: TurnRef[], kind: "running" | "queued", empty: string):
           { href: `#/task/${t.task_id}` },
           kind === "queued" ? h("span", { class: "queue-pos" }, `${i + 1}.`) : h("span", { class: "dot running", title: "running" }),
           h("span", { class: "task-title" }, project && h("span", { class: "project-name" }, `${project.name} ▸ `), task?.title ?? t.task_id),
+          liveness(procs.find((p) => p.session_id === t.session_id)),
           h("span", { class: "task-state" }, t.vendor),
         ),
       );
@@ -141,6 +158,75 @@ function turnList(turns: TurnRef[], kind: "running" | "queued", empty: string):
   );
 }
 
+function taskLink(taskId: string | undefined): HTMLElement | string {
+  if (!taskId) {
+    return "—";
+  }
+  const task = findTask(taskId);
+  const project = task ? state.projects.find((p) => p.id === task.project_id) : undefined;
+  return h("a", { href: `#/task/${taskId}` }, project ? `${project.name} ▸ ` : "", task?.title ?? taskId);
+}
+
+function since(iso: string): string {
+  const seconds = Math.max(0, Math.round((Date.now() - Date.parse(iso)) / 1000));
+  if (seconds < 60) {
+    return `${seconds}s ago`;
+  }
+  if (seconds < 3600) {
+    return `${Math.floor(seconds / 60)}m ${seconds % 60}s ago`;
+  }
+  return ago(iso);
+}
+
+function liveness(p: AgentProcess | undefined): HTMLElement | false {
+  if (!p) {
+    return false;
+  }
+  if (p.health === "ok") {
+    return h("span", { class: "task-state" }, since(p.last_event_at));
+  }
+  return h("span", { class: "task-state danger" }, `${p.health} · ${since(p.last_event_at)}`);
+}
+
+const zombieLabel: Record<Zombie["kind"], string> = {
+  running_without_process: "Running with no process",
+  process_without_session: "Process with no session",
+};
+
+function problemsSection(s: Status, openVendors: Set<string>): HTMLElement | false {
+  const pe = s.protocol_errors;
+  const vendors = Object.keys(pe.counts).sort();
+  if (s.zombies.length === 0 && vendors.length === 0) {
+    return false;
+  }
+  return h(
+    "section",
+    { class: "admin-section" },
+    h("h2", {}, "Problems"),
+    h(
+      "ul",
+      { class: "problems" },
+      ...s.zombies.map((z) => h("li", {}, h("span", { class: "danger" }, zombieLabel[z.kind]), " ", z.task_id ? taskLink(z.task_id) : "", " ", h("span", { class: "hint" }, z.detail))),
+      ...vendors.map((v) => {
+        const latest = pe.recent.find((e) => e.vendor === v);
+        const n = pe.counts[v] ?? 0;
+        return h(
+          "li",
+          {},
+          h("span", { class: "danger" }, `${v}: ${n} bad stream line${n === 1 ? "" : "s"}`),
+          latest &&
+            h(
+              "details",
+              { data: { vendor: v }, open: openVendors.has(v) },
+              h("summary", {}, `latest ${since(latest.at)}: ${latest.reason}`),
+              h("pre", { class: "raw-line" }, latest.raw, new TextEncoder().encode(latest.raw).length < latest.size ? `… (${bytes(latest.size)})` : ""),
+            ),
+        );
+      }),
+    ),
+  );
+}
+
 function rateLimitsSection(limits: Status["rate_limits"]): HTMLElement {
   const vendors = Object.keys(limits).sort();
   return h(

web/src/api.gen.ts

@@ -732,6 +732,13 @@ export interface components {
         Status: {
             /** @description Agent processes currently running. */
             live_agents: number;
+            /** @description Every agent process byre is tracking, idle sessions included. */
+            agent_processes: components["schemas"]["AgentProcess"][];
+            /** @description Sessions marked running with no live process, and agent processes byre has no session for. */
+            zombies: components["schemas"]["Zombie"][];
+            /** @description How long a running turn may go without a stream event before it counts as stuck. */
+            stuck_after_seconds: number;
+            protocol_errors: components["schemas"]["ProtocolErrors"];
             /** @description Turns running now, oldest session first. */
             running_turns: components["schemas"]["TurnRef"][];
             /** @description Turns waiting for room under the caps, in the order they will start. */
@@ -764,6 +771,51 @@ export interface components {
             task_id: string;
             vendor: string;
         };
+        AgentProcess: {
+            session_id: string;
+            task_id: string;
+            vendor: string;
+            pid: number;
+            /** @description The session's state */
+            state: string;
+            /**
+             * Format: date-time
+             * @description When the process last wrote a line to its stream
+             */
+            last_event_at: string;
+            /**
+             * @description stuck means a turn is running and no stream event has arrived for stuck_after_seconds; exited means the process is gone.
+             * @enum {string}
+             */
+            health: "ok" | "stuck" | "exited";
+        };
+        Zombie: {
+            /** @enum {string} */
+            kind: "running_without_process" | "process_without_session";
+            detail: string;
+            session_id?: string;
+            task_id?: string;
+            pid?: number;
+        };
+        ProtocolErrors: {
+            /** @description Stream lines the adapter couldn't parse or didn't recognise since byre started, keyed by vendor. */
+            counts: {
+                [key: string]: number;
+            };
+            /** @description The most recent ones, newest first. */
+            recent: components["schemas"]["ProtocolError"][];
+        };
+        ProtocolError: {
+            vendor: string;
+            session_id: string;
+            /** Format: date-time */
+            at: string;
+            reason: string;
+            /** @description The line as received */
+            raw: string;
+            /** @description The line's full length in bytes. */
+            size: number;
+        };
     };
     responses: {
         /** @description Error. */

web/src/api.ts

@@ -14,6 +14,9 @@ export type ByreEvent = Schemas["Event"];
 export type Status = Schemas["Status"];
 export type NotificationPrefs = Schemas["NotificationPrefs"];
 export type TurnRef = Schemas["TurnRef"];
+export type AgentProcess = Schemas["AgentProcess"];
+export type Zombie = Schemas["Zombie"];
+export type ProtocolErrors = Schemas["ProtocolErrors"];
 
 type Method = "get" | "post" | "put" | "delete";
 

web/src/state.ts

@@ -131,6 +131,7 @@ export function handleEvent(ev: ByreEvent): void {
     if (ev.task_id && ev.task_id === state.selectedTaskId && !liveDuringLoad) {
       notify("stream", ev);
     }
+    notify("event", ev);
     return;
   }