ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Image input

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-14 20:07:26 -07:00
parent bd8702ee87
commit 9e741ede32
27 changed files with 1044 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/dist/api.js
Vendored
+5 -1
View File
@@ -24,7 +24,11 @@ export async function request(method, path, opts = {}) {
url += `?${query}`;
}
const init = { method: method.toUpperCase() };
if (opts.body !== undefined) {
if (opts.body instanceof Blob) {
init.headers = { "Content-Type": opts.body.type || "application/octet-stream" };
init.body = opts.body;
}
else if (opts.body !== undefined) {
init.headers = { "Content-Type": "application/json" };
init.body = JSON.stringify(opts.body);
}
web/dist/api.js.map
Vendored
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAmCA,MAAM,OAAO,QAAS,SAAQ,KAAK;IACZ;IAArB,YAAqB,MAAc,EAAE,OAAe;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;QADI,WAAM,GAAN,MAAM,CAAQ;IAEnC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAS,EACT,IAAO,EACP,OAA0B,EAAE;IAE5B,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IAChE,IAAI,GAAG,GAAW,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE;QAC9D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC,EAAE,CAAC;QACzF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;IAC3D,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAG,IAAyB,CAAC,OAAO,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,IAAwB,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,CAA6B,IAAO,EAAE,IAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACrH,MAAM,CAAC,MAAM,IAAI,GAAG,CAA8B,IAAO,EAAE,IAA6B,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACzH,MAAM,CAAC,MAAM,GAAG,GAAG,CAA6B,IAAO,EAAE,IAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAErH,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,OAAgC,EAAE,OAAqC;IAC9G,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;IAC7D,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,CAAC,SAAS,GAAG,CAAC,GAAyB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAc,CAAC,CAAC;IAC7F,OAAO,MAAM,CAAC;AAChB,CAAC"}
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAuCA,MAAM,OAAO,QAAS,SAAQ,KAAK;IACZ;IAArB,YAAqB,MAAc,EAAE,OAAe;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;QADI,WAAM,GAAN,MAAM,CAAQ;IAEnC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAS,EACT,IAAO,EACP,OAA0B,EAAE;IAE5B,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IAChE,IAAI,GAAG,GAAW,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE;QAC9D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC,EAAE,CAAC;QACzF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;IAC3D,IAAI,IAAI,CAAC,IAAI,YAAY,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,0BAA0B,EAAE,CAAC;QAChF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACxB,CAAC;SAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAG,IAAyB,CAAC,OAAO,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,IAAwB,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,CAA6B,IAAO,EAAE,IAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACrH,MAAM,CAAC,MAAM,IAAI,GAAG,CAA8B,IAAO,EAAE,IAA6B,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACzH,MAAM,CAAC,MAAM,GAAG,GAAG,CAA6B,IAAO,EAAE,IAA4B,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAErH,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,OAAgC,EAAE,OAAqC;IAC9G,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;IAC7D,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,CAAC,SAAS,GAAG,CAAC,GAAyB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAc,CAAC,CAAC;IAC7F,OAAO,MAAM,CAAC;AAChB,CAAC"}
web/dist/task.js
Vendored
+135 -8
View File
@@ -285,22 +285,131 @@ function keepDraft(input, key) {
});
return () => composerDrafts.delete(key);
}
const imageDrafts = new Map();
const imageTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
const maxImageBytes = 5 << 20;
function imageTray(form, input, key) {
const el = h("div", { class: "compose-images" });
const render = () => {
const imgs = imageDrafts.get(key) ?? [];
el.hidden = imgs.length === 0;
replace(el, ...imgs.map((img) => {
const remove = h("button", { type: "button", class: "remove", title: "Remove image" }, "×");
remove.onclick = () => {
URL.revokeObjectURL(img.url);
const rest = (imageDrafts.get(key) ?? []).filter((i) => i !== img);
if (rest.length > 0) {
imageDrafts.set(key, rest);
}
else {
imageDrafts.delete(key);
}
render();
};
return h("div", { class: "thumb" }, h("img", { src: img.url, alt: "attached image" }), remove);
}));
};
const add = (files) => {
const added = [];
for (const f of files) {
if (!imageTypes.includes(f.type)) {
alert(`${f.name || "That file"} isn't a PNG, JPEG, GIF or WebP image.`);
continue;
}
if (f.size > maxImageBytes) {
alert(`${f.name || "That image"} is over ${maxImageBytes >> 20} MiB.`);
continue;
}
added.push({ file: f, url: URL.createObjectURL(f) });
}
if (added.length === 0) {
return false;
}
imageDrafts.set(key, [...(imageDrafts.get(key) ?? []), ...added]);
render();
return true;
};
input.addEventListener("paste", (e) => {
const files = [...(e.clipboardData?.files ?? [])].filter((f) => f.type.startsWith("image/"));
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
form.addEventListener("dragover", (e) => {
if (e.dataTransfer?.types.includes("Files")) {
e.preventDefault();
form.classList.add("dropping");
}
});
form.addEventListener("dragleave", (e) => {
if (!form.contains(e.relatedTarget)) {
form.classList.remove("dropping");
}
});
form.addEventListener("drop", (e) => {
form.classList.remove("dropping");
const files = [...(e.dataTransfer?.files ?? [])];
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
render();
return {
el,
take: () => {
const imgs = imageDrafts.get(key) ?? [];
imageDrafts.delete(key);
render();
return imgs;
},
restore: (imgs) => {
if (imgs.length > 0) {
imageDrafts.set(key, [...imgs, ...(imageDrafts.get(key) ?? [])]);
}
render();
},
};
}
async function uploadImages(imgs) {
if (imgs.length === 0) {
return undefined;
}
const ids = [];
for (const img of imgs) {
const blob = await post("/api/blobs", { body: img.file });
ids.push(blob.id);
}
return ids;
}
function releaseImages(imgs) {
for (const img of imgs) {
URL.revokeObjectURL(img.url);
}
}
function startForm(taskId) {
const prompt = h("textarea", { placeholder: "What should the agent do?", rows: 4, required: true });
const prompt = h("textarea", { placeholder: "What should the agent do? Paste or drop images to attach them.", rows: 4, required: true });
const clearDraft = keepDraft(prompt, `start:${taskId}`);
const model = modelSelect(`start:${taskId}`, models[0]);
const form = h("form", { class: "compose" }, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
const form = h("form", { class: "compose" });
const tray = imageTray(form, prompt, `start:${taskId}`);
form.append(tray.el, prompt, h("div", { class: "compose-actions" }, model, h("button", { type: "submit" }, "Start agent")));
form.onsubmit = async (e) => {
e.preventDefault();
const value = prompt.value;
const imgs = tray.take();
prompt.value = "";
clearDraft();
try {
await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value } });
const images = await uploadImages(imgs);
await post("/api/tasks/{taskId}/sessions", { path: { taskId }, body: { prompt: value, model: model.value, images } });
releaseImages(imgs);
}
catch (err) {
composerDrafts.set(`start:${taskId}`, value);
prompt.value = value;
tray.restore(imgs);
alert(errorText(err));
}
};
@@ -318,7 +427,10 @@ function modelSelect(key, current) {
return select;
}
function messageForm(taskId, sessionId, sessionState, currentModel) {
const text = h("textarea", { placeholder: sessionState === "idle" ? "Message the agent" : "Wait for the turn to end, or interrupt it", rows: 3 });
const text = h("textarea", {
placeholder: sessionState === "idle" ? "Message the agent. Paste or drop images to attach them." : "Wait for the turn to end, or interrupt it",
rows: 3,
});
const clearDraft = keepDraft(text, `message:${taskId}`);
const send = h("button", { type: "submit", disabled: sessionState !== "idle" }, "Send");
const interrupt = h("button", { type: "button", class: "danger", disabled: sessionState !== "running" && sessionState !== "queued" }, sessionState === "queued" ? "Cancel queued turn" : "Interrupt");
@@ -331,25 +443,31 @@ function messageForm(taskId, sessionId, sessionState, currentModel) {
}
};
const model = modelSelect(`message:${sessionId}`, currentModel);
const form = h("form", { class: "compose" }, text, h("div", { class: "compose-actions" }, model, interrupt, send));
const form = h("form", { class: "compose" });
const tray = imageTray(form, text, `message:${taskId}`);
form.append(tray.el, text, h("div", { class: "compose-actions" }, model, interrupt, send));
form.onsubmit = async (e) => {
e.preventDefault();
const value = text.value;
if (!value.trim()) {
if (!value.trim() && !imageDrafts.has(`message:${taskId}`)) {
return;
}
const imgs = tray.take();
text.value = "";
clearDraft();
try {
const images = await uploadImages(imgs);
await post("/api/sessions/{sessionId}/messages", {
path: { sessionId },
body: { text: value, model: model.value !== currentModel ? model.value : undefined },
body: { text: value, images, model: model.value !== currentModel ? model.value : undefined },
});
modelChoices.delete(`message:${sessionId}`);
releaseImages(imgs);
}
catch (err) {
composerDrafts.set(`message:${taskId}`, value);
text.value = value;
tray.restore(imgs);
alert(errorText(err));
}
};
@@ -430,7 +548,7 @@ function renderEvent(ev, ctx) {
}
return h("details", { class: "note" }, h("summary", {}, first, " ", when), markdown(rest.join("\n"), "text md"));
}
return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), markdown(String(p["text"]), "text md"));
return h("div", { class: "msg user" }, h("div", { class: "who" }, ev.actor, " ", when), messageImages(p["images"]), String(p["text"] ?? "").trim() ? markdown(String(p["text"]), "text md") : null);
case "agent.message":
ctx.lastMessage = h("div", { class: "msg agent" }, markdown(String(p["text"]), "text md"));
return ctx.lastMessage;
@@ -483,6 +601,15 @@ function renderEvent(ev, ctx) {
}
return null;
}
function messageImages(ids) {
if (!Array.isArray(ids) || ids.length === 0) {
return null;
}
return h("div", { class: "msg-images" }, ...ids.map((id) => {
const src = `/api/blobs/${encodeURIComponent(String(id))}`;
return h("a", { href: src, target: "_blank", rel: "noopener" }, h("img", { src, alt: "attached image", loading: "lazy" }));
}));
}
async function closeTask(task, prompt, path) {
try {
const u = await get("/api/tasks/{taskId}/unmerged", { path: { taskId: task.id } });
web/dist/task.js.map
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long