ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Image input

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-14 20:07:26 -07:00
parent bd8702ee87
commit 9e741ede32
27 changed files with 1044 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/task.ts
+157 -8
View File
@@ -325,21 +325,141 @@ function keepDraft(input: HTMLTextAreaElement, key: string): () => void {
return () => composerDrafts.delete(key);
}
type PendingImage = { file: Blob; url: string };
const imageDrafts = new Map<string, PendingImage[]>();
const imageTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
const maxImageBytes = 5 << 20;
function imageTray(form: HTMLFormElement, input: HTMLTextAreaElement, key: string): { el: HTMLElement; take: () => PendingImage[]; restore: (imgs: PendingImage[]) => void } {
const el = h("div", { class: "compose-images" });
const render = () => {
const imgs = imageDrafts.get(key) ?? [];
el.hidden = imgs.length === 0;
replace(
el,
...imgs.map((img) => {
const remove = h("button", { type: "button", class: "remove", title: "Remove image" }, "×");
remove.onclick = () => {
URL.revokeObjectURL(img.url);
const rest = (imageDrafts.get(key) ?? []).filter((i) => i !== img);
if (rest.length > 0) {
imageDrafts.set(key, rest);
} else {
imageDrafts.delete(key);
}
render();
};
return h("div", { class: "thumb" }, h("img", { src: img.url, alt: "attached image" }), remove);
}),
);
};
const add = (files: Iterable<File>) => {
const added: PendingImage[] = [];
for (const f of files) {
if (!imageTypes.includes(f.type)) {
alert(`${f.name || "That file"} isn't a PNG, JPEG, GIF or WebP image.`);
continue;
}
if (f.size > maxImageBytes) {
alert(`${f.name || "That image"} is over ${maxImageBytes >> 20} MiB.`);
continue;
}
added.push({ file: f, url: URL.createObjectURL(f) });
}
if (added.length === 0) {
return false;
}
imageDrafts.set(key, [...(imageDrafts.get(key) ?? []), ...added]);
render();
return true;
};
input.addEventListener("paste", (e) => {
const files = [...(e.clipboardData?.files ?? [])].filter((f) => f.type.startsWith("image/"));
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
form.addEventListener("dragover", (e) => {
if (e.dataTransfer?.types.includes("Files")) {
e.preventDefault();
form.classList.add("dropping");
}
});
form.addEventListener("dragleave", (e) => {
if (!form.contains(e.relatedTarget as Node | null)) {
form.classList.remove("dropping");
}
});
form.addEventListener("drop", (e) => {
form.classList.remove("dropping");
const files = [...(e.dataTransfer?.files ?? [])];
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
render();
return {
el,
take: () => {
const imgs = imageDrafts.get(key) ?? [];
imageDrafts.delete(key);
render();
return imgs;
},
restore: (imgs) => {
if (imgs.length > 0) {
imageDrafts.set(key, [...imgs, ...(imageDrafts.get(key) ?? [])]);
}
render();
},
};
}
async function uploadImages(imgs: PendingImage[]): Promise<string[] | undefined> {
if (imgs.length === 0) {
return undefined;
}
const ids: string[] = [];
for (const img of imgs) {
const blob = await post("/api/blobs", { body: img.file });
ids.push(blob.id);
}
return ids;
}
function releaseImages(imgs: PendingImage[]): void {
for (const img of imgs) {
URL.revokeObjectURL(img.url);
}
}
function startForm(taskId: string): HTMLElement {
const prompt = h("textarea", { placeholder: "What should the agent do?", rows: 4, required: true });
const prompt = h("textarea", { placeholder: "What should the agent do? Paste or drop images to attach them.", rows: 4, required: true });
const clearDraft = keepDraft(prompt, `start:${taskId}`);
const model = modelSelect(`start:${taskId}`, models[0]!);
const form = h("form", { class: "compose" }, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
const form = h("form", { class: "compose" });
const tray = imageTray(form, prompt, `start:${taskId}`);
form.append(tray.el, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
form.onsubmit = async (e) => {
e.preventDefault();
const value = prompt.value;
const imgs = tray.take();
prompt.value = "";
clearDraft();
try {
await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value } });
const images = await uploadImages(imgs);
await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value, images } });
releaseImages(imgs);
} catch (err) {
composerDrafts.set(`start:${taskId}`, value);
prompt.value = value;
tray.restore(imgs);
alert(errorText(err));
}
};
@@ -360,7 +480,10 @@ function modelSelect(key: string, current: string): HTMLSelectElement {
}
function messageForm(taskId: string, sessionId: string, sessionState: string, currentModel: string): HTMLElement {
const text = h("textarea", { placeholder: sessionState === "idle" ? "Message the agent" : "Wait for the turn to end, or interrupt it", rows: 3 });
const text = h("textarea", {
placeholder: sessionState === "idle" ? "Message the agent. Paste or drop images to attach them." : "Wait for the turn to end, or interrupt it",
rows: 3,
});
const clearDraft = keepDraft(text, `message:${taskId}`);
const send = h("button", { type: "submit", disabled: sessionState !== "idle" }, "Send");
const interrupt = h(
@@ -377,24 +500,30 @@ function messageForm(taskId: string, sessionId: string, sessionState: string, cu
};
const model = modelSelect(`message:${sessionId}`, currentModel);
const form = h("form", { class: "compose" }, text, h("div", { class: "compose-actions" }, model, interrupt, send));
const form = h("form", { class: "compose" });
const tray = imageTray(form, text, `message:${taskId}`);
form.append(tray.el, text, h("div", { class: "compose-actions" }, model, interrupt, send));
form.onsubmit = async (e) => {
e.preventDefault();
const value = text.value;
if (!value.trim()) {
if (!value.trim() && !imageDrafts.has(`message:${taskId}`)) {
return;
}
const imgs = tray.take();
text.value = "";
clearDraft();
try {
const images = await uploadImages(imgs);
await post("/api/sessions/{sessionId}/messages", {
path: { sessionId },
body: { text: value, model: model.value !== currentModel ? model.value : undefined },
body: { text: value, images, model: model.value !== currentModel ? model.value : undefined },
});
modelChoices.delete(`message:${sessionId}`);
releaseImages(imgs);
} catch (err) {
composerDrafts.set(`message:${taskId}`, value);
text.value = value;
tray.restore(imgs);
alert(errorText(err));
}
};
@@ -492,7 +621,13 @@ function renderEvent(ev: ByreEvent, ctx: Ctx): HTMLElement | null {
}
return h("details", { class: "note" }, h("summary", {}, first, " ", when), markdown(rest.join("\n"), "text md"));
}
return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), markdown(String(p["text"]), "text md"));
return h(
"div",
{ class: "msg user" },
h("div", { class: "who" }, ev.actor, " ", when),
messageImages(p["images"]),
String(p["text"] ?? "").trim() ? markdown(String(p["text"]), "text md") : null,
);
case "agent.message":
ctx.lastMessage = h("div", { class: "msg agent" }, markdown(String(p["text"]), "text md"));
@@ -557,6 +692,20 @@ function renderEvent(ev: ByreEvent, ctx: Ctx): HTMLElement | null {
return null;
}
function messageImages(ids: unknown): HTMLElement | null {
if (!Array.isArray(ids) || ids.length === 0) {
return null;
}
return h(
"div",
{ class: "msg-images" },
...ids.map((id) => {
const src = `/api/blobs/${encodeURIComponent(String(id))}`;
return h("a", { href: src, target: "_blank", rel: "noopener" }, h("img", { src, alt: "attached image", loading: "lazy" }));
}),
);
}
async function closeTask(task: Task, prompt: (u: Unmerged) => string, path: "/api/tasks/{taskId}/merge" | "/api/tasks/{taskId}/discard") {
try {
const u = await get("/api/tasks/{taskId}/unmerged", { path: { taskId: task.id } });