Image input

internal/server/blobs.go

@@ -0,0 +1,83 @@
+package server
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	"g.fc.run/misc/byre/internal/store"
+)
+
+const maxBlobBytes = 5 << 20
+
+var imageTypes = map[string]bool{
+	"image/png":  true,
+	"image/jpeg": true,
+	"image/gif":  true,
+	"image/webp": true,
+}
+
+func (s *Server) UploadBlob(w http.ResponseWriter, r *http.Request) {
+	data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBlobBytes))
+	maxErr := &http.MaxBytesError{}
+	if errors.As(err, &maxErr) {
+		writeError(w, http.StatusRequestEntityTooLarge, fmt.Errorf("images are limited to %d MiB", maxBlobBytes>>20))
+		return
+	}
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+
+	contentType := http.DetectContentType(data)
+	if !imageTypes[contentType] {
+		writeError(w, http.StatusBadRequest, fmt.Errorf("unsupported image type %s; use PNG, JPEG, GIF or WebP", contentType))
+		return
+	}
+
+	b, err := s.store.CreateBlob(r.Context(), contentType, data, actor(r))
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusCreated, b)
+}
+
+func (s *Server) GetBlob(w http.ResponseWriter, r *http.Request, blobID string) {
+	b, path, err := s.store.GetBlob(r.Context(), blobID)
+	if err != nil {
+		writeStoreError(w, err)
+		return
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	defer f.Close()
+
+	w.Header().Set("Content-Type", b.ContentType)
+	w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
+	http.ServeContent(w, r, "", b.CreatedAt, f)
+}
+
+func (s *Server) images(w http.ResponseWriter, r *http.Request, ids *[]string) ([]string, bool) {
+	if ids == nil {
+		return nil, true
+	}
+	for _, id := range *ids {
+		_, _, err := s.store.GetBlob(r.Context(), id)
+		if errors.Is(err, store.ErrNotFound) {
+			writeError(w, http.StatusBadRequest, err)
+			return nil, false
+		}
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, err)
+			return nil, false
+		}
+	}
+	return *ids, true
+}

internal/server/server.go

@@ -495,7 +495,12 @@ func (s *Server) CreateSession(w http.ResponseWriter, r *http.Request, taskID st
 		model = *req.Model
 	}
 
-	sess, err := s.agents.Create(r.Context(), t, u, req.Prompt, model)
+	images, ok := s.images(w, r, req.Images)
+	if !ok {
+		return
+	}
+
+	sess, err := s.agents.Create(r.Context(), t, u, req.Prompt, model, images...)
 	if err != nil {
 		writeAgentError(w, err)
 		return
@@ -518,8 +523,13 @@ func (s *Server) SendMessage(w http.ResponseWriter, r *http.Request, sessionID s
 		return
 	}
 
-	if strings.TrimSpace(req.Text) == "" {
-		writeError(w, http.StatusBadRequest, errors.New("text is required"))
+	images, ok := s.images(w, r, req.Images)
+	if !ok {
+		return
+	}
+
+	if strings.TrimSpace(req.Text) == "" && len(images) == 0 {
+		writeError(w, http.StatusBadRequest, errors.New("text or images are required"))
 		return
 	}
 
@@ -528,7 +538,7 @@ func (s *Server) SendMessage(w http.ResponseWriter, r *http.Request, sessionID s
 		model = *req.Model
 	}
 
-	sess, err := s.agents.Send(r.Context(), sessionID, actor(r), req.Text, model)
+	sess, err := s.agents.Send(r.Context(), sessionID, actor(r), req.Text, model, images...)
 	if err != nil {
 		writeAgentError(w, err)
 		return