Image input

web/src/api.gen.ts

@@ -299,6 +299,41 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/blobs": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /** @description Stores an image for use in a message. The body is the raw image; its type is detected from the bytes and must be PNG, JPEG, GIF or WebP. At most 5 MiB. */
+        post: operations["uploadBlob"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/blobs/{blobId}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                blobId: string;
+            };
+            cookie?: never;
+        };
+        get: operations["getBlob"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/inbox": {
         parameters: {
             query?: never;
@@ -491,12 +526,24 @@ export interface components {
         SessionCreate: {
             prompt: string;
             model?: string;
+            /** @description Blob ids from uploadBlob, sent to the agent as images alongside the prompt. */
+            images?: string[];
         };
         MessageCreate: {
+            /** @description May be empty when images are given. */
             text: string;
+            /** @description Blob ids from uploadBlob, sent to the agent as images alongside the text. */
+            images?: string[];
             /** @description Model for this turn and later ones. When it differs from the session's model, the session switches before the message is sent. */
             model?: string;
         };
+        Blob: {
+            id: string;
+            content_type: string;
+            created_by: string;
+            /** Format: date-time */
+            created_at: string;
+        };
         Attachment: {
             /** @enum {string} */
             kind: "file" | "diff" | "image" | "diagram" | "task" | "snapshot" | "url";
@@ -1120,6 +1167,61 @@ export interface operations {
             409: components["responses"]["Error"];
         };
     };
+    uploadBlob: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "image/png": string;
+                "image/jpeg": string;
+                "image/gif": string;
+                "image/webp": string;
+            };
+        };
+        responses: {
+            /** @description The stored blob. */
+            201: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["Blob"];
+                };
+            };
+            400: components["responses"]["Error"];
+            413: components["responses"]["Error"];
+        };
+    };
+    getBlob: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                blobId: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description The blob's bytes, with its content type. */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "image/png": string;
+                    "image/jpeg": string;
+                    "image/gif": string;
+                    "image/webp": string;
+                };
+            };
+            404: components["responses"]["Error"];
+        };
+    };
     listInbox: {
         parameters: {
             query?: {

web/src/api.ts

@@ -23,7 +23,11 @@ type Op<P extends keyof paths, M extends Method> = NonNullable<paths[P][M]>;
 type Params<O> = O extends { parameters: infer X } ? X : never;
 type PathParams<O> = Params<O> extends { path: infer X } ? X : never;
 type QueryParams<O> = Params<O> extends { query?: infer X } ? X : never;
-type Body<O> = O extends { requestBody: { content: { "application/json": infer B } } } ? B : never;
+type Body<O> = O extends { requestBody: { content: { "application/json": infer B } } }
+  ? B
+  : O extends { requestBody: { content: Record<`image/${string}`, unknown> } }
+    ? Blob
+    : never;
 type Success<O> = O extends { responses: infer R } ? R[Extract<keyof R, 200 | 201 | 202>] : never;
 type Result<O> = Success<O> extends { content: { "application/json": infer J } } ? J : never;
 
@@ -64,7 +68,10 @@ export async function request<M extends Method, P extends PathsWith<M>>(
   }
 
   const init: RequestInit = { method: method.toUpperCase() };
-  if (opts.body !== undefined) {
+  if (opts.body instanceof Blob) {
+    init.headers = { "Content-Type": opts.body.type || "application/octet-stream" };
+    init.body = opts.body;
+  } else if (opts.body !== undefined) {
     init.headers = { "Content-Type": "application/json" };
     init.body = JSON.stringify(opts.body);
   }

web/src/task.ts

@@ -325,21 +325,141 @@ function keepDraft(input: HTMLTextAreaElement, key: string): () => void {
   return () => composerDrafts.delete(key);
 }
 
+type PendingImage = { file: Blob; url: string };
+
+const imageDrafts = new Map<string, PendingImage[]>();
+
+const imageTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
+const maxImageBytes = 5 << 20;
+
+function imageTray(form: HTMLFormElement, input: HTMLTextAreaElement, key: string): { el: HTMLElement; take: () => PendingImage[]; restore: (imgs: PendingImage[]) => void } {
+  const el = h("div", { class: "compose-images" });
+  const render = () => {
+    const imgs = imageDrafts.get(key) ?? [];
+    el.hidden = imgs.length === 0;
+    replace(
+      el,
+      ...imgs.map((img) => {
+        const remove = h("button", { type: "button", class: "remove", title: "Remove image" }, "×");
+        remove.onclick = () => {
+          URL.revokeObjectURL(img.url);
+          const rest = (imageDrafts.get(key) ?? []).filter((i) => i !== img);
+          if (rest.length > 0) {
+            imageDrafts.set(key, rest);
+          } else {
+            imageDrafts.delete(key);
+          }
+          render();
+        };
+        return h("div", { class: "thumb" }, h("img", { src: img.url, alt: "attached image" }), remove);
+      }),
+    );
+  };
+  const add = (files: Iterable<File>) => {
+    const added: PendingImage[] = [];
+    for (const f of files) {
+      if (!imageTypes.includes(f.type)) {
+        alert(`${f.name || "That file"} isn't a PNG, JPEG, GIF or WebP image.`);
+        continue;
+      }
+      if (f.size > maxImageBytes) {
+        alert(`${f.name || "That image"} is over ${maxImageBytes >> 20} MiB.`);
+        continue;
+      }
+      added.push({ file: f, url: URL.createObjectURL(f) });
+    }
+    if (added.length === 0) {
+      return false;
+    }
+    imageDrafts.set(key, [...(imageDrafts.get(key) ?? []), ...added]);
+    render();
+    return true;
+  };
+
+  input.addEventListener("paste", (e) => {
+    const files = [...(e.clipboardData?.files ?? [])].filter((f) => f.type.startsWith("image/"));
+    if (files.length > 0) {
+      e.preventDefault();
+      add(files);
+    }
+  });
+  form.addEventListener("dragover", (e) => {
+    if (e.dataTransfer?.types.includes("Files")) {
+      e.preventDefault();
+      form.classList.add("dropping");
+    }
+  });
+  form.addEventListener("dragleave", (e) => {
+    if (!form.contains(e.relatedTarget as Node | null)) {
+      form.classList.remove("dropping");
+    }
+  });
+  form.addEventListener("drop", (e) => {
+    form.classList.remove("dropping");
+    const files = [...(e.dataTransfer?.files ?? [])];
+    if (files.length > 0) {
+      e.preventDefault();
+      add(files);
+    }
+  });
+
+  render();
+  return {
+    el,
+    take: () => {
+      const imgs = imageDrafts.get(key) ?? [];
+      imageDrafts.delete(key);
+      render();
+      return imgs;
+    },
+    restore: (imgs) => {
+      if (imgs.length > 0) {
+        imageDrafts.set(key, [...imgs, ...(imageDrafts.get(key) ?? [])]);
+      }
+      render();
+    },
+  };
+}
+
+async function uploadImages(imgs: PendingImage[]): Promise<string[] | undefined> {
+  if (imgs.length === 0) {
+    return undefined;
+  }
+  const ids: string[] = [];
+  for (const img of imgs) {
+    const blob = await post("/api/blobs", { body: img.file });
+    ids.push(blob.id);
+  }
+  return ids;
+}
+
+function releaseImages(imgs: PendingImage[]): void {
+  for (const img of imgs) {
+    URL.revokeObjectURL(img.url);
+  }
+}
+
 function startForm(taskId: string): HTMLElement {
-  const prompt = h("textarea", { placeholder: "What should the agent do?", rows: 4, required: true });
+  const prompt = h("textarea", { placeholder: "What should the agent do? Paste or drop images to attach them.", rows: 4, required: true });
   const clearDraft = keepDraft(prompt, `start:${taskId}`);
   const model = modelSelect(`start:${taskId}`, models[0]!);
-  const form = h("form", { class: "compose" }, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
+  const form = h("form", { class: "compose" });
+  const tray = imageTray(form, prompt, `start:${taskId}`);
+  form.append(tray.el, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
   form.onsubmit = async (e) => {
     e.preventDefault();
     const value = prompt.value;
+    const imgs = tray.take();
     prompt.value = "";
     clearDraft();
     try {
-      await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value } });
+      const images = await uploadImages(imgs);
+      await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value, images } });
+      releaseImages(imgs);
     } catch (err) {
       composerDrafts.set(`start:${taskId}`, value);
       prompt.value = value;
+      tray.restore(imgs);
       alert(errorText(err));
     }
   };
@@ -360,7 +480,10 @@ function modelSelect(key: string, current: string): HTMLSelectElement {
 }
 
 function messageForm(taskId: string, sessionId: string, sessionState: string, currentModel: string): HTMLElement {
-  const text = h("textarea", { placeholder: sessionState === "idle" ? "Message the agent" : "Wait for the turn to end, or interrupt it", rows: 3 });
+  const text = h("textarea", {
+    placeholder: sessionState === "idle" ? "Message the agent. Paste or drop images to attach them." : "Wait for the turn to end, or interrupt it",
+    rows: 3,
+  });
   const clearDraft = keepDraft(text, `message:${taskId}`);
   const send = h("button", { type: "submit", disabled: sessionState !== "idle" }, "Send");
   const interrupt = h(
@@ -377,24 +500,30 @@ function messageForm(taskId: string, sessionId: string, sessionState: string, cu
   };
 
   const model = modelSelect(`message:${sessionId}`, currentModel);
-  const form = h("form", { class: "compose" }, text, h("div", { class: "compose-actions" }, model, interrupt, send));
+  const form = h("form", { class: "compose" });
+  const tray = imageTray(form, text, `message:${taskId}`);
+  form.append(tray.el, text, h("div", { class: "compose-actions" }, model, interrupt, send));
   form.onsubmit = async (e) => {
     e.preventDefault();
     const value = text.value;
-    if (!value.trim()) {
+    if (!value.trim() && !imageDrafts.has(`message:${taskId}`)) {
       return;
     }
+    const imgs = tray.take();
     text.value = "";
     clearDraft();
     try {
+      const images = await uploadImages(imgs);
       await post("/api/sessions/{sessionId}/messages", {
         path: { sessionId },
-        body: { text: value, model: model.value !== currentModel ? model.value : undefined },
+        body: { text: value, images, model: model.value !== currentModel ? model.value : undefined },
       });
       modelChoices.delete(`message:${sessionId}`);
+      releaseImages(imgs);
     } catch (err) {
       composerDrafts.set(`message:${taskId}`, value);
       text.value = value;
+      tray.restore(imgs);
       alert(errorText(err));
     }
   };
@@ -492,7 +621,13 @@ function renderEvent(ev: ByreEvent, ctx: Ctx): HTMLElement | null {
         }
         return h("details", { class: "note" }, h("summary", {}, first, " ", when), markdown(rest.join("\n"), "text md"));
       }
-      return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), markdown(String(p["text"]), "text md"));
+      return h(
+        "div",
+        { class: "msg user" },
+        h("div", { class: "who" }, ev.actor, " ", when),
+        messageImages(p["images"]),
+        String(p["text"] ?? "").trim() ? markdown(String(p["text"]), "text md") : null,
+      );
 
     case "agent.message":
       ctx.lastMessage = h("div", { class: "msg agent" }, markdown(String(p["text"]), "text md"));
@@ -557,6 +692,20 @@ function renderEvent(ev: ByreEvent, ctx: Ctx): HTMLElement | null {
   return null;
 }
 
+function messageImages(ids: unknown): HTMLElement | null {
+  if (!Array.isArray(ids) || ids.length === 0) {
+    return null;
+  }
+  return h(
+    "div",
+    { class: "msg-images" },
+    ...ids.map((id) => {
+      const src = `/api/blobs/${encodeURIComponent(String(id))}`;
+      return h("a", { href: src, target: "_blank", rel: "noopener" }, h("img", { src, alt: "attached image", loading: "lazy" }));
+    }),
+  );
+}
+
 async function closeTask(task: Task, prompt: (u: Unmerged) => string, path: "/api/tasks/{taskId}/merge" | "/api/tasks/{taskId}/discard") {
   try {
     const u = await get("/api/tasks/{taskId}/unmerged", { path: { taskId: task.id } });