Render Mermaid diagrams and worktree images in agent Markdown

web/dist/inbox.js

@@ -1,7 +1,8 @@
 import { post } from "./api.js";
 import { mountChanges } from "./changes.js";
 import { ago, errorText, h, replace, showAlert } from "./dom.js";
-import { inlineMarkdown, markdown } from "./markdown.js";
+import { image, inlineMarkdown, markdown } from "./markdown.js";
+import { diagram } from "./mermaid.js";
 import { findTask, onEach, state } from "./state.js";
 const checked = new Map();
 const texts = new Map();
@@ -277,10 +278,10 @@ function card(item) {
     };
     switch (item.kind) {
         case "question":
-            replace(body, markdown(String(p["context"]), "md context", choice), markdown(String(p["question"]), "md question"), h("p", { class: "why" }, h("strong", {}, "Blocked on: "), inlineMarkdown(String(p["why_blocked"]))), attachments(p["attachments"] ?? []));
+            replace(body, markdown(String(p["context"]), "md context", { choice, taskId: item.task_id }), markdown(String(p["question"]), "md question", { taskId: item.task_id }), h("p", { class: "why" }, h("strong", {}, "Blocked on: "), inlineMarkdown(String(p["why_blocked"]), { taskId: item.task_id })), attachments(p["attachments"] ?? [], item.task_id));
             break;
         case "report":
-            replace(body, markdown(String(p["text"] ?? ""), "md summary"));
+            replace(body, markdown(String(p["text"] ?? ""), "md summary", { taskId: item.task_id }));
             break;
         case "session_error":
             replace(body, h("p", { class: "error-text" }, String(p["error"])));
@@ -301,7 +302,7 @@ function thread(item) {
     }
     return h("ul", { class: "thread" }, ...item.responses.map((r) => h("li", {}, h("strong", {}, r.user), ` ${r.type.replace("_", " ")}`, r.options ? `: ${r.options.join("; ")}` : "", r.text ? ` — ${r.text}` : "")));
 }
-function attachments(list) {
+function attachments(list, taskId) {
     if (list.length === 0) {
         return null;
     }
@@ -315,9 +316,9 @@ function attachments(list) {
             case "diff":
                 return h("li", {}, "diff ", h("code", {}, a.ref ?? ""), caption);
             case "image":
-                return h("li", {}, a.image?.startsWith("data:image/") ? h("img", { src: a.image, alt: a.caption ?? "" }) : null, caption);
+                return h("li", {}, a.image || a.path ? image(a.image || a.path, a.caption ?? "", taskId) : null, caption);
             case "diagram":
-                return h("li", {}, h("pre", { class: "mermaid-source" }, a.mermaid ?? ""), caption);
+                return h("li", {}, diagram(a.mermaid ?? ""), caption);
             case "task":
                 return h("li", {}, h("a", { href: `#/task/${a.task_id}` }, "task ", a.task_id ?? ""), caption);
             case "snapshot":

web/dist/markdown.js

@@ -1,22 +1,23 @@
 import { h } from "./dom.js";
+import { diagram } from "./mermaid.js";
 const choiceMarker = /^(\([ *]\)|\[[ xX]\])\s+(\S.*?)\s*$/;
 const detailsOpen = /^\s*<details>\s*(?:<summary>(.*?)<\/summary>)?\s*$/i;
 const summaryLine = /^\s*<summary>(.*?)<\/summary>\s*$/i;
 const detailsClose = /^\s*<\/details>\s*$/i;
-const fence = /^\s*(```|~~~)/;
+const fence = /^\s*(```|~~~)\s*(\S*)/;
 const heading = /^(#{1,6})\s+(.*)$/;
 const bullet = /^(\s*)[-*+]\s+(.*)$/;
 const ordered = /^(\s*)(\d+)[.)]\s+(.*)$/;
 const quote = /^\s*>\s?(.*)$/;
 const rule = /^\s*([-*_])(\s*\1){2,}\s*$/;
 const tableSep = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$/;
-export function markdown(src, className = "md", choice) {
-    return h("div", { class: className }, ...blocks(src.replace(/\r\n?/g, "\n").split("\n"), choice));
+export function markdown(src, className = "md", opts = {}) {
+    return h("div", { class: className }, ...blocks(src.replace(/\r\n?/g, "\n").split("\n"), opts));
 }
-export function inlineMarkdown(src) {
-    return h("span", { class: "md-inline" }, ...inline(src));
+export function inlineMarkdown(src, opts = {}) {
+    return h("span", { class: "md-inline" }, ...inline(src, opts));
 }
-function blocks(lines, choice) {
+function blocks(lines, opts) {
     const out = [];
     let i = 0;
     while (i < lines.length) {
@@ -35,7 +36,7 @@ function blocks(lines, choice) {
                 i++;
             }
             i++;
-            out.push(h("pre", { class: "md-code" }, h("code", {}, body.join("\n"))));
+            out.push(f[2].toLowerCase() === "mermaid" ? diagram(body.join("\n")) : h("pre", { class: "md-code" }, h("code", {}, body.join("\n"))));
             continue;
         }
         if (detailsClose.test(line)) {
@@ -63,13 +64,13 @@ function blocks(lines, choice) {
                 }
                 body.push(l);
             }
-            out.push(h("details", { class: "md-details" }, h("summary", {}, ...inline(title?.trim() || "Details")), h("div", { class: "md-details-body" }, ...blocks(body, choice))));
+            out.push(h("details", { class: "md-details" }, h("summary", {}, ...inline(title?.trim() || "Details", opts)), h("div", { class: "md-details-body" }, ...blocks(body, opts))));
             continue;
         }
         const hd = heading.exec(line);
         if (hd) {
             const level = Math.min(hd[1].length + 2, 6);
-            out.push(h(`h${level}`, {}, ...inline(hd[2])));
+            out.push(h(`h${level}`, {}, ...inline(hd[2], opts)));
             i++;
             continue;
         }
@@ -84,11 +85,11 @@ function blocks(lines, choice) {
                 body.push(quote.exec(lines[i])[1]);
                 i++;
             }
-            out.push(h("blockquote", {}, ...blocks(body)));
+            out.push(h("blockquote", {}, ...blocks(body, { taskId: opts.taskId })));
             continue;
         }
         if (bullet.test(line) || ordered.test(line)) {
-            const [list, next] = parseList(lines, i, choice);
+            const [list, next] = parseList(lines, i, opts);
             out.push(list);
             i = next;
             continue;
@@ -100,7 +101,7 @@ function blocks(lines, choice) {
                 rows.push(cells(lines[i]));
                 i++;
             }
-            out.push(table(rows));
+            out.push(table(rows, opts));
             continue;
         }
         const para = [];
@@ -108,7 +109,7 @@ function blocks(lines, choice) {
             para.push(lines[i].trim());
             i++;
         }
-        out.push(h("p", {}, ...joinLines(para)));
+        out.push(h("p", {}, ...joinLines(para, opts)));
     }
     return out;
 }
@@ -124,20 +125,20 @@ function startsBlock(lines, i) {
         ordered.test(line) ||
         (line.includes("|") && i + 1 < lines.length && tableSep.test(lines[i + 1])));
 }
-function joinLines(lines) {
+function joinLines(lines, opts) {
     const out = [];
     lines.forEach((l, n) => {
         if (n > 0) {
             out.push(h("br"));
         }
-        out.push(...inline(l));
+        out.push(...inline(l, opts));
     });
     return out;
 }
 function indentOf(line) {
     return line.length - line.trimStart().length;
 }
-function parseList(lines, start, choice) {
+function parseList(lines, start, opts) {
     const first = lines[start];
     const isOrdered = !bullet.test(first);
     const base = indentOf(first);
@@ -175,17 +176,17 @@ function parseList(lines, start, choice) {
             body.push(indentOf(next) > base ? next.slice(Math.min(indentOf(next), base + 2)) : next.trim());
             i++;
         }
-        list.append(item(body, isOrdered ? undefined : choice));
+        list.append(item(body, isOrdered ? { taskId: opts.taskId } : opts));
         while (i < lines.length && lines[i].trim() === "" && lines[i + 1] !== undefined && indentOf(lines[i + 1]) === base && (isOrdered ? ordered : bullet).test(lines[i + 1])) {
             i++;
         }
     }
     return [list, i];
 }
-function item(body, choice) {
+function item(body, opts) {
     const li = h("li");
-    const m = choice && choiceMarker.exec(body[0]);
-    const picker = m ? choice(m[2]) : null;
+    const m = opts.choice && choiceMarker.exec(body[0]);
+    const picker = m ? opts.choice(m[2]) : null;
     if (picker) {
         li.className = "choice";
         li.append(picker);
@@ -193,7 +194,7 @@ function item(body, choice) {
     const first = picker ? [] : [body[0]];
     const rest = body.slice(1);
     if (rest.every((l) => !startsBlockLine(l)) && !rest.includes("")) {
-        li.append(...joinLines([...first, ...rest].map((l) => l.trim())));
+        li.append(...joinLines([...first, ...rest].map((l) => l.trim()), opts));
         return li;
     }
     const head = first;
@@ -202,7 +203,7 @@ function item(body, choice) {
         head.push(body[k].trim());
         k++;
     }
-    li.append(...joinLines(head), ...blocks(body.slice(k), choice));
+    li.append(...joinLines(head, opts), ...blocks(body.slice(k), opts));
     return li;
 }
 function startsBlockLine(line) {
@@ -218,12 +219,12 @@ function cells(line) {
     }
     return s.split("|").map((c) => c.trim());
 }
-function table(rows) {
+function table(rows, opts) {
     const [head, ...body] = rows;
-    return h("div", { class: "md-table" }, h("table", {}, h("thead", {}, h("tr", {}, ...head.map((c) => h("th", {}, ...inline(c))))), h("tbody", {}, ...body.map((r) => h("tr", {}, ...r.map((c) => h("td", {}, ...inline(c))))))));
+    return h("div", { class: "md-table" }, h("table", {}, h("thead", {}, h("tr", {}, ...head.map((c) => h("th", {}, ...inline(c, opts))))), h("tbody", {}, ...body.map((r) => h("tr", {}, ...r.map((c) => h("td", {}, ...inline(c, opts))))))));
 }
-const inlineToken = /`([^`]+)`|\*\*(.+?)\*\*|__(.+?)__|\*([^*\s][^*]*?)\*|(?<![\w])_([^_\s][^_]*?)_(?![\w])|\[([^\]]+)\]\(([^)\s]+)\)|(https?:\/\/[^\s<>()]+[^\s<>().,;:!?'"])/;
-function inline(src) {
+const inlineToken = /`([^`]+)`|\*\*(.+?)\*\*|__(.+?)__|\*([^*\s][^*]*?)\*|(?<![\w])_([^_\s][^_]*?)_(?![\w])|!\[([^\]]*)\]\((?:<([^>]+)>|([^)]+))\)|\[([^\]]+)\]\(([^)\s]+)\)|(https?:\/\/[^\s<>()]+[^\s<>().,;:!?'"])/;
+function inline(src, opts) {
     const out = [];
     let rest = src;
     while (rest.length > 0) {
@@ -235,18 +236,21 @@ function inline(src) {
         if (m.index > 0) {
             out.push(document.createTextNode(rest.slice(0, m.index)));
         }
-        const [whole, code, strong1, strong2, em1, em2, linkText, linkHref, bare] = m;
+        const [whole, code, strong1, strong2, em1, em2, imgAlt, imgAngled, imgSrc, linkText, linkHref, bare] = m;
         if (code !== undefined) {
             out.push(h("code", {}, code));
         }
         else if (strong1 !== undefined || strong2 !== undefined) {
-            out.push(h("strong", {}, ...inline(strong1 ?? strong2)));
+            out.push(h("strong", {}, ...inline(strong1 ?? strong2, opts)));
         }
         else if (em1 !== undefined || em2 !== undefined) {
-            out.push(h("em", {}, ...inline(em1 ?? em2)));
+            out.push(h("em", {}, ...inline(em1 ?? em2, opts)));
+        }
+        else if (imgAlt !== undefined) {
+            out.push(image((imgAngled ?? imgSrc).trim(), imgAlt, opts.taskId));
         }
         else if (linkText !== undefined) {
-            out.push(link(linkHref, inline(linkText)));
+            out.push(link(linkHref, inline(linkText, opts)));
         }
         else if (bare !== undefined) {
             out.push(link(bare, [document.createTextNode(bare)]));
@@ -264,4 +268,30 @@ function link(href, children) {
     }
     return h("span", {}, ...children);
 }
+function worktreeImageURL(path, taskId) {
+    if (/^data:image\/(png|jpeg|gif|webp);/i.test(path)) {
+        return path;
+    }
+    let file = path;
+    if (/^file:\/\//i.test(file)) {
+        file = decodeURI(file.slice("file://".length));
+    }
+    else if (/^[a-z][a-z0-9+.-]*:/i.test(file)) {
+        return null;
+    }
+    if (!taskId) {
+        return null;
+    }
+    return `/api/tasks/${encodeURIComponent(taskId)}/image?path=${encodeURIComponent(file)}`;
+}
+export function image(src, alt, taskId) {
+    const url = worktreeImageURL(src, taskId);
+    if (!url) {
+        return link(src, [document.createTextNode(alt || src)]);
+    }
+    const img = h("img", { src: url, alt, loading: "lazy" });
+    const el = h("a", { class: "md-img", href: url, target: "_blank", rel: "noopener" }, img);
+    img.onerror = () => el.replaceWith(h("span", { class: "md-img-missing" }, `image not available: ${src}`));
+    return el;
+}
 //# sourceMappingURL=markdown.js.map

web/dist/mermaid.js

@@ -0,0 +1,55 @@
+import { errorText, h } from "./dom.js";
+let loading;
+let next = 0;
+function load() {
+    loading ??= new Promise((resolve, reject) => {
+        const script = h("script", { src: "/vendor/mermaid.min.js" });
+        script.onload = () => {
+            const m = window.mermaid;
+            const css = getComputedStyle(document.documentElement);
+            const v = (name) => css.getPropertyValue(name).trim();
+            m.initialize({
+                startOnLoad: false,
+                securityLevel: "strict",
+                suppressErrorRendering: true,
+                theme: "base",
+                fontFamily: v("--font"),
+                themeVariables: {
+                    fontFamily: v("--font"),
+                    fontSize: "13px",
+                    background: v("--bg"),
+                    primaryColor: v("--surface"),
+                    primaryTextColor: v("--text"),
+                    primaryBorderColor: v("--border"),
+                    secondaryColor: v("--accent-tint"),
+                    tertiaryColor: v("--raised"),
+                    lineColor: v("--muted"),
+                    textColor: v("--text"),
+                    noteBkgColor: v("--warn-tint"),
+                    noteTextColor: v("--text"),
+                    noteBorderColor: v("--border"),
+                },
+            });
+            resolve(m);
+        };
+        script.onerror = () => reject(new Error("couldn't load /vendor/mermaid.min.js"));
+        document.head.append(script);
+    });
+    return loading;
+}
+export function diagram(src) {
+    const el = h("div", { class: "mermaid-diagram" }, h("pre", { class: "mermaid-source" }, src));
+    void render(el, src);
+    return el;
+}
+async function render(el, src) {
+    try {
+        const m = await load();
+        const { svg } = await m.render(`mermaid-${next++}`, src);
+        el.innerHTML = svg;
+    }
+    catch (err) {
+        el.append(h("div", { class: "mermaid-error" }, `diagram not rendered: ${errorText(err)}`));
+    }
+}
+//# sourceMappingURL=mermaid.js.map

web/dist/mermaid.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mermaid.js","sourceRoot":"","sources":["../src/mermaid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,UAAU,CAAC;AAaxC,IAAI,OAAqC,CAAC;AAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;AAEb,SAAS,IAAI;IACX,OAAO,KAAK,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,MAAM,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE;YACnB,MAAM,CAAC,GAAG,MAAM,CAAC,OAAQ,CAAC;YAC1B,MAAM,GAAG,GAAG,gBAAgB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;YACvD,MAAM,CAAC,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,CAAC,CAAC,UAAU,CAAC;gBACX,WAAW,EAAE,KAAK;gBAClB,aAAa,EAAE,QAAQ;gBACvB,sBAAsB,EAAE,IAAI;gBAC5B,KAAK,EAAE,MAAM;gBACb,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC;gBACvB,cAAc,EAAE;oBACd,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC;oBACvB,QAAQ,EAAE,MAAM;oBAChB,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC;oBACrB,YAAY,EAAE,CAAC,CAAC,WAAW,CAAC;oBAC5B,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC;oBAC7B,kBAAkB,EAAE,CAAC,CAAC,UAAU,CAAC;oBACjC,cAAc,EAAE,CAAC,CAAC,eAAe,CAAC;oBAClC,aAAa,EAAE,CAAC,CAAC,UAAU,CAAC;oBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC;oBACvB,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC;oBACtB,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC;oBAC9B,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC;oBAC1B,eAAe,EAAE,CAAC,CAAC,UAAU,CAAC;iBAC/B;aACF,CAAC,CAAC;YACH,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,CAAC,CAAC;QACF,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC,CAAC;QACjF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAW;IACjC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9F,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,EAAe,EAAE,GAAW;IAChD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,MAAM,IAAI,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACzD,EAAE,CAAC,SAAS,GAAG,GAAG,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,yBAAyB,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC;AACH,CAAC"}

web/dist/task.js

@@ -290,7 +290,7 @@ export function mountTranscript(root) {
             }
             stream.timer = 0;
             const pinned = transcript.scrollHeight - transcript.scrollTop - transcript.clientHeight < 80;
-            replace(stream.el, markdown(stream.text, "text md"));
+            replace(stream.el, markdown(stream.text, "text md", { taskId: state.selectedTaskId ?? undefined }));
             if (pinned) {
                 transcript.scrollTop = transcript.scrollHeight;
             }
@@ -687,6 +687,7 @@ function runSummary(calls) {
 }
 function renderEvent(ev, ctx) {
     const p = ev.payload;
+    const md = { taskId: ev.task_id };
     const when = h("time", { dateTime: ev.created_at, title: ev.created_at }, ago(ev.created_at));
     switch (ev.kind) {
         case "user.message":
@@ -695,14 +696,14 @@ function renderEvent(ev, ctx) {
                 const text = String(p["text"]);
                 const first = text.split("\n")[0] ?? "";
                 const brief = first.length > 120 ? `${first.slice(0, 120)}…` : first;
-                return h("details", { class: "tool byre-prompt" }, h("summary", {}, h("span", { class: "tool-name" }, "byre"), " ", h("span", { class: "tool-brief" }, brief), " ", when), markdown(text, "text md"));
+                return h("details", { class: "tool byre-prompt" }, h("summary", {}, h("span", { class: "tool-name" }, "byre"), " ", h("span", { class: "tool-brief" }, brief), " ", when), markdown(text, "text md", md));
             }
-            return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), messageImages(p["images"]), String(p["text"] ?? "").trim() ? markdown(String(p["text"]), "text md") : null);
+            return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), messageImages(p["images"]), String(p["text"] ?? "").trim() ? markdown(String(p["text"]), "text md", md) : null);
         case "agent.message":
-            ctx.lastMessage = h("div", { class: "msg agent" }, markdown(String(p["text"]), "text md"));
+            ctx.lastMessage = h("div", { class: "msg agent" }, markdown(String(p["text"]), "text md", md));
             return ctx.lastMessage;
         case "agent.thinking":
-            return h("details", { class: "thinking" }, h("summary", {}, "thinking"), markdown(String(p["text"]), "text md"));
+            return h("details", { class: "thinking" }, h("summary", {}, "thinking"), markdown(String(p["text"]), "text md", md));
         case "agent.tool_result": {
             const target = ctx.tools.get(String(p["tool_use_id"]));
             const body = h("pre", {}, resultText(p["content"]));