From 2aba7455311ea355e8b613271c8fac39a8c71235 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Thu, 22 Oct 2026 19:16:47 -0700 Subject: [PATCH] Add generate_image tool and show Codex images inline --- internal/agents/codex.go | 22 ++++++ internal/agents/imagegen.go | 125 +++++++++++++++++++++++++++++++++++ internal/agents/kb/images.md | 15 +++++ internal/agents/manager.go | 1 + 4 files changed, 163 insertions(+) create mode 100644 internal/agents/imagegen.go create mode 100644 internal/agents/kb/images.md diff --git a/internal/agents/codex.go b/internal/agents/codex.go index d278931..447c3c6 100644 --- a/internal/agents/codex.go +++ b/internal/agents/codex.go @@ -3,6 +3,7 @@ package agents import ( "context" "encoding/json" + "fmt" "log/slog" "os" "strings" @@ -143,6 +144,16 @@ func (m *Manager) codexItemCompleted(ctx context.Context, l *live, s *codex.Sess return nil } return m.emit(ctx, l, agentActor, "agent.thinking", map[string]any{"text": text}) + + case codex.ThreadItemTypeImageGeneration: + if item.SavedPath == "" || l.reviewID != "" || l.owner != "" { + return nil + } + path, err := copyToMedia(MediaDir(l.dir), string(item.SavedPath)) + if err != nil { + return err + } + return m.emit(ctx, l, agentActor, "agent.message", map[string]any{"text": fmt.Sprintf("![%s](<%s>)", imageAlt(item.RevisedPrompt), path)}) } if !codexToolItems[item.Type] { @@ -188,6 +199,17 @@ func (m *Manager) codexItemCompleted(ctx context.Context, l *live, s *codex.Sess return nil } +func imageAlt(prompt string) string { + alt := strings.NewReplacer("[", "", "]", "", "\n", " ").Replace(strings.TrimSpace(prompt)) + if r := []rune(alt); len(r) > 120 { + alt = string(r[:120]) + "…" + } + if alt == "" { + return "generated image" + } + return alt +} + func codexToolResult(item codex.ThreadItem) (string, bool) { status := codexStatus(item) switch item.Type { diff --git a/internal/agents/imagegen.go b/internal/agents/imagegen.go new file mode 100644 index 0000000..e0cff5e --- /dev/null +++ b/internal/agents/imagegen.go @@ -0,0 +1,125 @@ +package agents + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "g.fc.run/ai/byre/internal/codex" + "g.fc.run/ai/byre/internal/mcp" +) + +const imageGenGuidance = "You make images for another agent. Call your image_gen tool exactly once with the prompt you are given, unchanged, then reply with only the word done." + +func (m *Manager) generateImageTool(l *live) mcp.Tool { + return mcp.Tool{ + Name: "generate_image", + Description: "Generate an image (a photo, illustration, texture, sprite, mockup) with OpenAI's image model through byre's Codex sign-in, saved as a PNG in the task's media folder. " + + "The reply gives its path: show it inline by embedding it in your final message as a Markdown image, ![what it shows](), or copy it into the worktree when the project uses it. " + + "Use it whenever the human asks for a picture, rather than drawing one in SVG or HTML. It takes a minute or two.", + InputSchema: json.RawMessage(`{ + "type": "object", + "properties": { + "prompt": {"type": "string", "description": "What to draw: subject, style, composition, lighting, and any exact text."} + }, + "required": ["prompt"] + }`), + Handler: func(ctx context.Context, raw json.RawMessage) (string, error) { + return m.generateImage(ctx, l, raw) + }, + } +} + +func (m *Manager) generateImage(ctx context.Context, l *live, raw json.RawMessage) (string, error) { + args := struct { + Prompt string `json:"prompt"` + }{} + err := json.Unmarshal(raw, &args) + if err != nil { + return "", fmt.Errorf("malformed generate_image call: %w", err) + } + if strings.TrimSpace(args.Prompt) == "" { + return "", fmt.Errorf("generate_image needs a prompt") + } + + media := MediaDir(l.dir) + err = os.MkdirAll(m.codexHome, 0o700) + if err != nil { + return "", err + } + s, err := codex.Start(ctx, codex.Options{ + Cwd: media, + Env: []string{"CODEX_HOME=" + m.codexHome}, + Guidance: imageGenGuidance, + }) + if err != nil { + return "", fmt.Errorf("start codex: %w", err) + } + defer s.Close() + + _, err = s.Send(ctx, args.Prompt) + if err != nil { + return "", fmt.Errorf("start codex turn: %w", err) + } + + saved := "" + for { + select { + case <-ctx.Done(): + s.Kill() + return "", fmt.Errorf("no image after %s", mcp.Timeout(ctx)) + case ev, ok := <-s.Events(): + if !ok { + return "", fmt.Errorf("codex exited before the image was made") + } + switch ev.Method { + case codex.MethodItemCompleted: + n := codex.ItemCompletedNotification{} + err := json.Unmarshal(ev.Params, &n) + if err != nil { + return "", err + } + if n.Item.Type != codex.ThreadItemTypeImageGeneration { + continue + } + if n.Item.Failure != nil { + return "", fmt.Errorf("image generation failed: %s (limit %s)", n.Item.Failure.Type, n.Item.Failure.LimitID) + } + saved = string(n.Item.SavedPath) + case codex.MethodTurnCompleted: + n := codex.TurnCompletedNotification{} + err := json.Unmarshal(ev.Params, &n) + if err != nil { + return "", err + } + if n.Turn.Error != nil { + return "", fmt.Errorf("codex: %s", n.Turn.Error.Message) + } + if saved == "" { + return "", fmt.Errorf("codex finished without making an image") + } + path, err := copyToMedia(media, saved) + if err != nil { + return "", err + } + return fmt.Sprintf("Saved %s", path), nil + } + } + } +} + +func copyToMedia(media, src string) (string, error) { + data, err := os.ReadFile(src) + if err != nil { + return "", err + } + err = os.MkdirAll(media, 0o755) + if err != nil { + return "", err + } + dest := filepath.Join(media, filepath.Base(src)) + return dest, os.WriteFile(dest, data, 0o644) +} diff --git a/internal/agents/kb/images.md b/internal/agents/kb/images.md new file mode 100644 index 0000000..3e74cbc --- /dev/null +++ b/internal/agents/kb/images.md @@ -0,0 +1,15 @@ +How agents make real images (photos, illustrations) with generate_image, and how they show inline in the transcript. + +# Generating images + +`generate_image` makes a raster image from a prompt and saves it as a PNG in the task's media folder (`-media/`, outside the worktree, deleted when the task closes). Embed the path it returns in your final message as `![what it shows]()` and the human sees it inline in the transcript; copy it into the worktree when the project itself uses it. + +It runs one Codex turn on the Mac byre runs on, under byre's Codex home, and calls Codex's built-in image tool (OpenAI's image model), so: + +- It needs Codex signed in there (the `setup` article covers it); until then it fails with `401 Unauthorized`. +- Each image counts against the ChatGPT plan's Codex usage, whichever vendor the calling agent runs on. +- One call takes a minute or two; give it a timeout of at least 3 minutes. + +Codex agents (gpt-* models) also have the image tool built in. An image one makes is copied into the media folder and shown inline in the transcript by byre, with no Markdown needed. + +Write the prompt as a short spec: subject, style or medium, composition, lighting, and any exact text in quotes. For a change, generate again with the one change stated; it can't edit an earlier image in place. diff --git a/internal/agents/manager.go b/internal/agents/manager.go index 932c893..e79ead2 100644 --- a/internal/agents/manager.go +++ b/internal/agents/manager.go @@ -885,6 +885,7 @@ func (m *Manager) startLive(ctx context.Context, sess api.AgentSession, task api return nil, err } g += screenshotGuidance + tools = append(tools, m.generateImageTool(l)) hands, err := m.handsMCP(sess.Id, media) if err != nil { return nil, err