ai/byre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Accept pasted images in inbox Other and Reply answers

Browse Source
This commit is contained in:
Ian Gulliver
2026-10-16 22:49:06 -07:00
parent a9bec7a7ea
commit ea90178fce
19 changed files with 579 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/api.gen.ts
+5 -1
View File
@@ -1380,6 +1380,8 @@ export interface components {
type: components["schemas"]["InboxResponseType"];
options?: string[];
text?: string;
/** @description Blob ids sent to the agent as images alongside the response. */
images: string[];
attachments: components["schemas"]["Attachment"][];
/** Format: date-time */
created_at: string;
@@ -1391,8 +1393,10 @@ export interface components {
type: components["schemas"]["InboxResponseType"];
/** @description option: the chosen options; exactly one for a pick-one question, one or more for a pick-any question. */
options?: string[];
/** @description other: the text sent to the agent. option on a pick-any question: optional text sent along with the checked options. */
/** @description other: the text sent to the agent, which may be empty when images are given. option on a pick-any question: optional text sent along with the checked options. */
text?: string;
/** @description other, or option on a pick-any question: blob ids from uploadBlob, sent to the agent as images alongside the response. */
images?: string[];
attachments?: components["schemas"]["Attachment"][];
};
InboxRespond: {
web/src/images.ts
+122
View File
@@ -0,0 +1,122 @@
import { post } from "./api.js";
import { h, replace, showAlert } from "./dom.js";
export type PendingImage = { file: Blob; url: string };
export const imageDrafts = new Map<string, PendingImage[]>();
const imageTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
const maxImageBytes = 5 << 20;
export function imageTray(
drop: HTMLElement,
input: HTMLTextAreaElement,
key: string,
onCount: (n: number) => void,
): { el: HTMLElement; take: () => PendingImage[]; restore: (imgs: PendingImage[]) => void } {
const el = h("div", { class: "compose-images" });
const render = () => {
const imgs = imageDrafts.get(key) ?? [];
onCount(imgs.length);
el.hidden = imgs.length === 0;
replace(
el,
...imgs.map((img) => {
const remove = h("button", { type: "button", class: "remove", title: "Remove image" }, "×");
remove.onclick = () => {
URL.revokeObjectURL(img.url);
const rest = (imageDrafts.get(key) ?? []).filter((i) => i !== img);
if (rest.length > 0) {
imageDrafts.set(key, rest);
} else {
imageDrafts.delete(key);
}
render();
};
return h("div", { class: "thumb" }, h("img", { src: img.url, alt: "attached image" }), remove);
}),
);
};
const add = (files: Iterable<File>) => {
const added: PendingImage[] = [];
for (const f of files) {
if (!imageTypes.includes(f.type)) {
void showAlert(`${f.name || "That file"} isn't a PNG, JPEG, GIF or WebP image.`);
continue;
}
if (f.size > maxImageBytes) {
void showAlert(`${f.name || "That image"} is over ${maxImageBytes >> 20} MiB.`);
continue;
}
added.push({ file: f, url: URL.createObjectURL(f) });
}
if (added.length === 0) {
return false;
}
imageDrafts.set(key, [...(imageDrafts.get(key) ?? []), ...added]);
render();
return true;
};
input.addEventListener("paste", (e) => {
const files = [...(e.clipboardData?.files ?? [])].filter((f) => f.type.startsWith("image/"));
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
drop.addEventListener("dragover", (e) => {
if (e.dataTransfer?.types.includes("Files")) {
e.preventDefault();
drop.classList.add("dropping");
}
});
drop.addEventListener("dragleave", (e) => {
if (!drop.contains(e.relatedTarget as Node | null)) {
drop.classList.remove("dropping");
}
});
drop.addEventListener("drop", (e) => {
drop.classList.remove("dropping");
const files = [...(e.dataTransfer?.files ?? [])];
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
render();
return {
el,
take: () => {
const imgs = imageDrafts.get(key) ?? [];
imageDrafts.delete(key);
render();
return imgs;
},
restore: (imgs) => {
if (imgs.length > 0) {
imageDrafts.set(key, [...imgs, ...(imageDrafts.get(key) ?? [])]);
}
render();
},
};
}
export async function uploadImages(imgs: PendingImage[]): Promise<string[] | undefined> {
if (imgs.length === 0) {
return undefined;
}
const ids: string[] = [];
for (const img of imgs) {
const blob = await post("/api/blobs", { body: img.file });
ids.push(blob.id);
}
return ids;
}
export function releaseImages(imgs: PendingImage[]): void {
for (const img of imgs) {
URL.revokeObjectURL(img.url);
}
}
web/src/inbox.ts
+36 -22
View File
@@ -3,11 +3,14 @@ import { mountChanges } from "./changes.js";
import { ago, errorText, h, replace, showAlert } from "./dom.js";
import { image, inlineMarkdown, markdown } from "./markdown.js";
import { checkedAnswer, clearAnswer, dropTextAnswer, flushDrafts, hasTextAnswer, setCheckedAnswer, setTextAnswer, textAnswer } from "./drafts.js";
import { imageDrafts, imageTray, releaseImages, uploadImages } from "./images.js";
import { diagram } from "./mermaid.js";
import { focusTab, onEach, state } from "./state.js";
import { taskStyle } from "./taskcolor.js";
const delegateText = "Use your judgment for this decision and continue within the existing task instructions.";
type Tray = ReturnType<typeof imageTray>;
const delegateText ="Use your judgment for this decision and continue within the existing task instructions.";
export function mountInbox(root: HTMLElement): () => void {
const list = h("div", { class: "inbox-list" });
@@ -112,19 +115,24 @@ function card(item: InboxItem): HTMLElement {
c.disabled = on;
}
};
const submit = async (response: InboxResponseCreate) => {
const imageKey = `inbox:${item.id}`;
const submit = async (response: InboxResponseCreate, tray?: Tray) => {
if (busy()) {
return;
}
lock(true);
const imgs = tray?.take() ?? [];
try {
await post("/api/inbox/responses", { body: { responses: [response] } });
const images = await uploadImages(imgs);
await post("/api/inbox/responses", { body: { responses: [{ ...response, images }] } });
releaseImages(imgs);
clearAnswer(item.id);
const others = state.inbox.some((i) => i.task_id === item.task_id && i.id !== item.id);
if (!others && state.focus.tab === "inbox" && state.selectedTaskId === item.task_id) {
focusTab("transcript");
}
} catch (err) {
tray?.restore(imgs);
void showAlert(errorText(err));
lock(false);
renderActions();
@@ -164,11 +172,17 @@ function card(item: InboxItem): HTMLElement {
return h("label", { class: rec ? "option choice-box rec" : "option choice-box" }, h("div", { class: "choice-head" }, input, name, tag));
};
const textEntry = (label: string, placeholder: string, shared?: { onInput: () => void; onSubmit: () => void }) => {
const open = hasTextAnswer(item.id);
const textEntry = (label: string, placeholder: string, shared?: { onInput: () => void; onSubmit: (tray: Tray) => void }) => {
const open = hasTextAnswer(item.id) || imageDrafts.has(imageKey);
const text = h("textarea", { rows: 3, placeholder, value: textAnswer(item.id) ?? "" });
const send = h("button", { type: "button", class: "primary", disabled: !text.value.trim() }, "Submit");
const editor = h("div", { class: "text-editor", hidden: !open }, text, !shared && h("div", { class: "submit-row" }, send));
const send = h("button", { type: "button", class: "primary" }, "Submit");
const editor = h("div", { class: "text-editor", hidden: !open });
const empty = () => !text.value.trim() && !imageDrafts.has(imageKey);
const tray = imageTray(editor, text, imageKey, () => {
send.disabled = empty();
shared?.onInput();
});
editor.append(tray.el, text, shared ? "" : h("div", { class: "submit-row" }, send));
const block = clickable(h("div", { class: "pick other", role: "button", tabIndex: 0, hidden: open }, label), () => {
setTextAnswer(item.id, "");
block.hidden = true;
@@ -177,21 +191,21 @@ function card(item: InboxItem): HTMLElement {
});
const go = () => {
if (shared) {
shared.onSubmit();
shared.onSubmit(tray);
return;
}
if (text.value.trim()) {
void submit({ item_id: item.id, type: "other", text: text.value });
if (!empty()) {
void submit({ item_id: item.id, type: "other", text: text.value }, tray);
}
};
send.onclick = go;
text.oninput = () => {
setTextAnswer(item.id, text.value);
send.disabled = !text.value.trim();
send.disabled = empty();
shared?.onInput();
};
text.onblur = () => {
if (!text.value.trim() && !busy()) {
if (empty() && !busy()) {
dropTextAnswer(item.id);
editor.hidden = true;
block.hidden = false;
@@ -205,7 +219,7 @@ function card(item: InboxItem): HTMLElement {
go();
}
};
return h("div", { class: "text-entry" }, block, editor);
return { el: h("div", { class: "text-entry" }, block, editor), tray };
};
const delegate = () => {
@@ -221,28 +235,28 @@ function card(item: InboxItem): HTMLElement {
const send = h("button", { type: "button", class: "primary" }, "Submit");
const typed = () => (textAnswer(item.id) ?? "").trim();
const refresh = () => {
send.disabled = !typed() && (!multiple || checkedAnswer(item.id).length === 0);
send.disabled = !typed() && !imageDrafts.has(imageKey) && (!multiple || checkedAnswer(item.id).length === 0);
};
const go = () => {
const go = (tray: Tray) => {
const options = multiple ? checkedAnswer(item.id) : [];
if (options.length > 0) {
void submit({ item_id: item.id, type: "option", options, text: typed() ? textAnswer(item.id) : undefined });
void submit({ item_id: item.id, type: "option", options, text: typed() ? textAnswer(item.id) : undefined }, tray);
return;
}
if (typed()) {
void submit({ item_id: item.id, type: "other", text: textAnswer(item.id)! });
if (typed() || imageDrafts.has(imageKey)) {
void submit({ item_id: item.id, type: "other", text: textAnswer(item.id) ?? "" }, tray);
}
};
const other = textEntry("Other…", placeholder, { onInput: refresh, onSubmit: go });
send.onclick = go;
send.onclick = () => go(other.tray);
refresh();
replace(actions, other, h("div", { class: "submit-row" }, delegate(), send));
replace(actions, other.el, h("div", { class: "submit-row" }, delegate(), send));
return;
}
default: {
const dismiss = h("button", { type: "button" }, "Dismiss");
dismiss.onclick = () => void submit({ item_id: item.id, type: "dismiss" });
replace(actions, h("div", { class: "reply-row" }, textEntry("Reply…", "Reply to the agent"), dismiss));
replace(actions, h("div", { class: "reply-row" }, textEntry("Reply…", "Reply to the agent").el, dismiss));
}
}
};
@@ -293,7 +307,7 @@ function thread(item: InboxItem): HTMLElement | null {
"ul",
{ class: "thread" },
...item.responses.map((r) =>
h("li", {}, h("strong", {}, r.user), ` ${r.type.replace("_", " ")}`, r.options ? `: ${r.options.join("; ")}` : "", r.text ? ` — ${r.text}` : ""),
h("li", {}, h("strong", {}, r.user), ` ${r.type.replace("_", " ")}`, r.options ? `: ${r.options.join("; ")}` : "", r.text ? ` — ${r.text}` : "", r.images.length > 0 ? ` (${r.images.length === 1 ? "1 image" : `${r.images.length} images`})` : ""),
),
);
}
web/src/task.ts
+2 -116
View File
@@ -4,6 +4,7 @@ import { markdown } from "./markdown.js";
import { previewURL } from "./preview.js";
import { findingCount, reviewCounts, severityCounts, unignoredFindings } from "./reviews.js";
import { composerText, endSend, flushDrafts, forgetLostImages, lostImageCount, setComposerImages, setComposerText, startSend } from "./drafts.js";
import { imageDrafts, imageTray, releaseImages, uploadImages, type PendingImage } from "./images.js";
import { currentSession, findTask, focusTab, focusTask, markSummaryRead, on, onEach, pickReview, pinSnapshot, setFocus, showReview, state, transcriptLoading, type Topic } from "./state.js";
import { taskStyle } from "./taskcolor.js";
import { fillTodoLinks } from "./todolink.js";
@@ -771,121 +772,6 @@ function lostImagesNote(key: string): HTMLElement | null {
return note;
}
type PendingImage = { file: Blob; url: string };
const imageDrafts = new Map<string, PendingImage[]>();
const imageTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
const maxImageBytes = 5 << 20;
function imageTray(form: HTMLFormElement, input: HTMLTextAreaElement, key: string): { el: HTMLElement; take: () => PendingImage[]; restore: (imgs: PendingImage[]) => void } {
const el = h("div", { class: "compose-images" });
const render = () => {
const imgs = imageDrafts.get(key) ?? [];
setComposerImages(key, imgs.length);
el.hidden = imgs.length === 0;
replace(
el,
...imgs.map((img) => {
const remove = h("button", { type: "button", class: "remove", title: "Remove image" }, "×");
remove.onclick = () => {
URL.revokeObjectURL(img.url);
const rest = (imageDrafts.get(key) ?? []).filter((i) => i !== img);
if (rest.length > 0) {
imageDrafts.set(key, rest);
} else {
imageDrafts.delete(key);
}
render();
};
return h("div", { class: "thumb" }, h("img", { src: img.url, alt: "attached image" }), remove);
}),
);
};
const add = (files: Iterable<File>) => {
const added: PendingImage[] = [];
for (const f of files) {
if (!imageTypes.includes(f.type)) {
void showAlert(`${f.name || "That file"} isn't a PNG, JPEG, GIF or WebP image.`);
continue;
}
if (f.size > maxImageBytes) {
void showAlert(`${f.name || "That image"} is over ${maxImageBytes >> 20} MiB.`);
continue;
}
added.push({ file: f, url: URL.createObjectURL(f) });
}
if (added.length === 0) {
return false;
}
imageDrafts.set(key, [...(imageDrafts.get(key) ?? []), ...added]);
render();
return true;
};
input.addEventListener("paste", (e) => {
const files = [...(e.clipboardData?.files ?? [])].filter((f) => f.type.startsWith("image/"));
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
form.addEventListener("dragover", (e) => {
if (e.dataTransfer?.types.includes("Files")) {
e.preventDefault();
form.classList.add("dropping");
}
});
form.addEventListener("dragleave", (e) => {
if (!form.contains(e.relatedTarget as Node | null)) {
form.classList.remove("dropping");
}
});
form.addEventListener("drop", (e) => {
form.classList.remove("dropping");
const files = [...(e.dataTransfer?.files ?? [])];
if (files.length > 0) {
e.preventDefault();
add(files);
}
});
render();
return {
el,
take: () => {
const imgs = imageDrafts.get(key) ?? [];
imageDrafts.delete(key);
render();
return imgs;
},
restore: (imgs) => {
if (imgs.length > 0) {
imageDrafts.set(key, [...imgs, ...(imageDrafts.get(key) ?? [])]);
}
render();
},
};
}
async function uploadImages(imgs: PendingImage[]): Promise<string[] | undefined> {
if (imgs.length === 0) {
return undefined;
}
const ids: string[] = [];
for (const img of imgs) {
const blob = await post("/api/blobs", { body: img.file });
ids.push(blob.id);
}
return ids;
}
function releaseImages(imgs: PendingImage[]): void {
for (const img of imgs) {
URL.revokeObjectURL(img.url);
}
}
const modelChoices = new Map<string, string>();
function vendorOf(model: string): string {
@@ -931,7 +817,7 @@ function composeForm(
const modelKey = `message:${sessionId ?? key}`;
const model = modelSelect(modelKey, currentModel, vendor);
const form = h("form", { class: "compose" });
const tray = imageTray(form, text, draftKey);
const tray = imageTray(form, text, draftKey, (n) => setComposerImages(draftKey, n));
form.append(lostImagesNote(draftKey) ?? "", tray.el, text, h("div", { class: "compose-actions" }, model, interrupt, send));
form.onsubmit = async (e) => {
e.preventDefault();