helios/heliosian
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serve media from drive via fully in-memory blob store

Browse Source
This commit is contained in:
Ian Gulliver
2026-08-15 21:18:33 -07:00
parent 00239d027c
commit 6c0b1d34c5
8 changed files with 377 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2 -1
View File
@@ -31,10 +31,11 @@ require (
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/image v0.45.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/text v0.41.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect
google.golang.org/grpc v1.83.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
go.sum
+4
View File
@@ -67,6 +67,8 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
@@ -78,6 +80,8 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
internal/auth/auth.go
+1 -1
View File
@@ -52,7 +52,7 @@ func (a *Auth) Wrap(next http.Handler) http.Handler {
}
email := a.sessionEmail(r)
if email == "" {
if strings.Contains(r.URL.Path, "/api/") {
if strings.Contains(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/blob/") {
http.Error(w, "unauthenticated", http.StatusUnauthorized)
return
}
internal/blob/blob.go
+260
View File
@@ -0,0 +1,260 @@
// Package blob serves directory media from drive, held fully in memory with startup-generated thumbnails.
package blob
import (
"bytes"
"context"
"fmt"
"image"
"image/jpeg"
"io"
"log"
"net/http"
"path"
"strings"
"sync"
"time"
_ "image/gif"
_ "image/png"
"golang.org/x/image/draw"
_ "golang.org/x/image/webp"
"heliosian/internal/data"
"google.golang.org/api/drive/v3"
"google.golang.org/api/option"
)
const (
folderMime = "application/vnd.google-apps.folder"
refreshInterval = 5 * time.Minute
thumbWidth = 480
)
type entry struct {
id string
mimeType string
data []byte
thumb []byte
}
type listed struct {
id string
mimeType string
}
type Store struct {
service *drive.Service
root string
mu sync.RWMutex
entries map[string]*entry
}
func New() (*Store, error) {
service, err := drive.NewService(context.Background(),
option.WithCredentialsFile(data.KeyFile),
option.WithScopes(drive.DriveReadonlyScope))
if err != nil {
return nil, err
}
drives, err := service.Drives.List().Do()
if err != nil {
return nil, fmt.Errorf("list shared drives: %w", err)
}
if len(drives.Drives) != 1 {
return nil, fmt.Errorf("expected one shared drive visible to the service account, found %d", len(drives.Drives))
}
s := &Store{service: service, root: drives.Drives[0].Id, entries: map[string]*entry{}}
if err := s.refresh(); err != nil {
return nil, err
}
go s.refreshLoop()
return s, nil
}
func Register(mux *http.ServeMux, s *Store) {
mux.HandleFunc("GET /blob/{folder}/{name}", s.serve)
}
func (s *Store) refreshLoop() {
for range time.Tick(refreshInterval) {
if err := s.refresh(); err != nil {
log.Printf("[ERROR] blob refresh: %v", err)
}
}
}
func (s *Store) refresh() error {
start := time.Now()
listing := map[string]listed{}
for _, folderName := range []string{"people", "families"} {
folderID, err := s.subfolder(folderName)
if err != nil {
return err
}
token := ""
for {
call := s.service.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
return fmt.Errorf("list %s: %w", folderName, err)
}
for _, f := range list.Files {
base := strings.TrimSuffix(f.Name, path.Ext(f.Name))
if strings.HasSuffix(base, "-thumb") {
continue
}
listing[folderName+"/"+base] = listed{id: f.Id, mimeType: f.MimeType}
}
if list.NextPageToken == "" {
break
}
token = list.NextPageToken
}
}
s.mu.RLock()
missing := []string{}
for key, l := range listing {
if cached, ok := s.entries[key]; !ok || cached.id != l.id {
missing = append(missing, key)
}
}
s.mu.RUnlock()
fetched := map[string]*entry{}
var fetchedMu sync.Mutex
work := make(chan string)
errs := make(chan error, 1)
var wg sync.WaitGroup
for range 12 {
wg.Add(1)
go func() {
defer wg.Done()
for key := range work {
l := listing[key]
body, err := s.download(l.id)
if err == nil && strings.HasPrefix(l.mimeType, "image/") {
var thumb []byte
thumb, err = thumbnail(body)
if err == nil {
fetchedMu.Lock()
fetched[key] = &entry{id: l.id, mimeType: l.mimeType, data: body, thumb: thumb}
fetchedMu.Unlock()
continue
}
} else if err == nil {
fetchedMu.Lock()
fetched[key] = &entry{id: l.id, mimeType: l.mimeType, data: body}
fetchedMu.Unlock()
continue
}
select {
case errs <- fmt.Errorf("load %s: %w", key, err):
default:
}
return
}
}()
}
for _, key := range missing {
work <- key
}
close(work)
wg.Wait()
select {
case err := <-errs:
return err
default:
}
next := make(map[string]*entry, len(listing))
var totalBytes int64
s.mu.Lock()
for key, l := range listing {
if e, ok := fetched[key]; ok {
next[key] = e
} else if cached, ok := s.entries[key]; ok && cached.id == l.id {
next[key] = cached
}
}
s.entries = next
for _, e := range next {
totalBytes += int64(len(e.data) + len(e.thumb))
}
s.mu.Unlock()
log.Printf("blob store: %d files, %d fetched, %.1f MB in memory in %s",
len(next), len(fetched), float64(totalBytes)/1e6, time.Since(start).Round(time.Millisecond))
return nil
}
func (s *Store) subfolder(name string) (string, error) {
list, err := s.service.Files.List().
Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, s.root, folderMime)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id)").Do()
if err != nil {
return "", fmt.Errorf("find folder %s: %w", name, err)
}
if len(list.Files) != 1 {
return "", fmt.Errorf("expected one %s folder, found %d", name, len(list.Files))
}
return list.Files[0].Id, nil
}
func (s *Store) download(id string) ([]byte, error) {
resp, err := s.service.Files.Get(id).SupportsAllDrives(true).Download()
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
func (s *Store) serve(w http.ResponseWriter, r *http.Request) {
key := r.PathValue("folder") + "/" + r.PathValue("name")
s.mu.RLock()
e, ok := s.entries[key]
s.mu.RUnlock()
if !ok {
http.NotFound(w, r)
return
}
if r.URL.Query().Get("thumb") == "1" {
if e.thumb == nil {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "image/jpeg")
w.Write(e.thumb)
return
}
w.Header().Set("Content-Type", e.mimeType)
w.Write(e.data)
}
func thumbnail(src []byte) ([]byte, error) {
img, _, err := image.Decode(bytes.NewReader(src))
if err != nil {
return nil, err
}
bounds := img.Bounds()
if bounds.Dx() > thumbWidth {
height := bounds.Dy() * thumbWidth / bounds.Dx()
scaled := image.NewRGBA(image.Rect(0, 0, thumbWidth, height))
draw.CatmullRom.Scale(scaled, scaled.Bounds(), img, bounds, draw.Over, nil)
img = scaled
}
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
internal/directory/load.go
+12 -4
View File
@@ -52,8 +52,8 @@ func LoadModel(source data.Source) (*Model, error) {
IsStudent: row["Is Student?"] == "TRUE",
Pronouns: row["Pronouns"],
Facts: row["Facts"],
PronunciationURL: row["Pronunciation"],
PhotoURL: row["Primary Photo"],
PronunciationURL: blobURL("people", email, "pronunciation", row["Pronunciation"]),
PhotoURL: blobURL("people", email, "photo", row["Primary Photo"]),
Grade: row["Grade"],
Classroom: row["Class"],
Section: row["Section"],
@@ -88,13 +88,13 @@ func LoadModel(source data.Source) (*Model, error) {
}
}
if acc.family.PhotoURL == "" {
acc.family.PhotoURL = row["Family Photo"]
acc.family.PhotoURL = blobURL("families", p.FamilyKey, "photo", row["Family Photo"])
}
if acc.family.PhotoCaption == "" {
acc.family.PhotoCaption = row["Family Photo Description"]
}
if acc.family.PronunciationURL == "" {
acc.family.PronunciationURL = row["Family Pronunciation"]
acc.family.PronunciationURL = blobURL("families", p.FamilyKey, "pronunciation", row["Family Pronunciation"])
}
if p.IsParent {
acc.hasParent = true
@@ -188,6 +188,14 @@ func LoadModel(source data.Source) (*Model, error) {
return model, nil
}
func blobURL(folder, email, kind, source string) string {
if source == "" {
return ""
}
local, _, _ := strings.Cut(email, "@")
return "/blob/" + folder + "/" + local + "-" + kind
}
func surname(fullName string) string {
fields := strings.Fields(fullName)
if len(fields) == 0 {
main.go
+8
View File
@@ -9,6 +9,7 @@ import (
"os"
"heliosian/internal/auth"
"heliosian/internal/blob"
"heliosian/internal/data"
"heliosian/internal/directory"
)
@@ -72,6 +73,13 @@ func main() {
mux := http.NewServeMux()
authn.Register(mux)
directory.Register(mux, cache)
if os.Getenv("DIRECTORY_SHEET") != "" {
store, err := blob.New()
if err != nil {
log.Fatalf("[ERROR] blob store: %v", err)
}
blob.Register(mux, store)
}
mux.Handle("GET /{$}", http.RedirectHandler("/people", http.StatusFound))
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static"))))
port := os.Getenv("PORT")
tools/importblobs/main.go
+83 -23
View File
@@ -45,7 +45,21 @@ func localPart(email string) string {
return name
}
func findSharedFolder(svc *drive.Service) string {
func findRoot(svc *drive.Service) string {
drives, err := svc.Drives.List().Do()
if err != nil {
log.Fatalf("[ERROR] list shared drives: %v", err)
}
if len(drives.Drives) == 1 {
log.Printf("using shared drive %q (%s)", drives.Drives[0].Name, drives.Drives[0].Id)
return drives.Drives[0].Id
}
if len(drives.Drives) > 1 {
for _, d := range drives.Drives {
log.Printf("candidate shared drive: %s (%s)", d.Name, d.Id)
}
log.Fatalf("[ERROR] service account can see %d shared drives; pass -folder", len(drives.Drives))
}
list, err := svc.Files.List().
Q("mimeType = '" + folderMime + "' and sharedWithMe = true and trashed = false").
Fields("files(id, name)").Do()
@@ -56,15 +70,16 @@ func findSharedFolder(svc *drive.Service) string {
for _, f := range list.Files {
log.Printf("candidate folder: %s (%s)", f.Name, f.Id)
}
log.Fatalf("[ERROR] expected exactly one folder shared with the service account, found %d; pass -folder", len(list.Files))
log.Fatalf("[ERROR] expected one shared drive or one shared folder, found %d folders; pass -folder", len(list.Files))
}
log.Printf("using folder %q (%s)", list.Files[0].Name, list.Files[0].Id)
log.Printf("using folder %q (%s); note: uploads into personal drives fail on service account quota", list.Files[0].Name, list.Files[0].Id)
return list.Files[0].Id
}
func ensureFolder(svc *drive.Service, parent, name string) string {
list, err := svc.Files.List().
Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id)").Do()
if err != nil {
log.Fatalf("[ERROR] find folder %s: %v", name, err)
@@ -72,7 +87,8 @@ func ensureFolder(svc *drive.Service, parent, name string) string {
if len(list.Files) > 0 {
return list.Files[0].Id
}
created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}).Fields("id").Do()
created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}).
SupportsAllDrives(true).Fields("id").Do()
if err != nil {
log.Fatalf("[ERROR] create folder %s: %v", name, err)
}
@@ -81,11 +97,13 @@ func ensureFolder(svc *drive.Service, parent, name string) string {
func listBases(svc *drive.Service, folderID string) map[string]bool {
bases := map[string]bool{}
deleted := 0
token := ""
for {
call := svc.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
Fields("nextPageToken, files(name)").PageSize(1000)
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(id, name)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
@@ -94,15 +112,53 @@ func listBases(svc *drive.Service, folderID string) map[string]bool {
log.Fatalf("[ERROR] list folder contents: %v", err)
}
for _, f := range list.Files {
bases[strings.TrimSuffix(f.Name, path.Ext(f.Name))] = true
base := strings.TrimSuffix(f.Name, path.Ext(f.Name))
if strings.HasSuffix(base, "-thumb") {
_, err := svc.Files.Update(f.Id, &drive.File{Trashed: true}).SupportsAllDrives(true).Do()
if err != nil {
log.Fatalf("[ERROR] trash stale thumb %s: %v", f.Name, err)
}
deleted++
continue
}
bases[base] = true
}
if list.NextPageToken == "" {
if deleted > 0 {
log.Printf("deleted %d stale thumbs", deleted)
}
return bases
}
token = list.NextPageToken
}
}
func folderStats(svc *drive.Service, folderID string) (int64, int64) {
var files, bytes int64
token := ""
for {
call := svc.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(size)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
log.Fatalf("[ERROR] list folder for stats: %v", err)
}
for _, f := range list.Files {
files++
bytes += f.Size
}
if list.NextPageToken == "" {
return files, bytes
}
token = list.NextPageToken
}
}
func fetch(client *http.Client, url string) ([]byte, string, error) {
resp, err := client.Get(url)
if err != nil {
@@ -156,7 +212,7 @@ func main() {
root := *folderID
if root == "" {
root = findSharedFolder(svc)
root = findRoot(svc)
}
folders := map[string]string{
"people": ensureFolder(svc, root, "people"),
@@ -200,7 +256,7 @@ func main() {
client := &http.Client{Timeout: 60 * time.Second}
var mu sync.Mutex
uploaded, failed := 0, 0
uploaded := 0
work := make(chan task)
var wg sync.WaitGroup
for range 6 {
@@ -210,24 +266,19 @@ func main() {
for t := range work {
body, contentType, err := fetch(client, t.url)
if err != nil {
mu.Lock()
failed++
log.Printf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err)
mu.Unlock()
continue
log.Fatalf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err)
}
name := t.base + extension(contentType, t.url)
_, err = svc.Files.Create(&drive.File{Name: name, Parents: []string{folders[t.folderName]}}).
Media(bytes.NewReader(body), googleapi.ContentType(contentType)).Fields("id").Do()
mu.Lock()
Media(bytes.NewReader(body), googleapi.ContentType(contentType)).
SupportsAllDrives(true).Fields("id").Do()
if err != nil {
failed++
log.Printf("[ERROR] upload %s/%s: %v", t.folderName, name, err)
} else {
uploaded++
if uploaded%50 == 0 {
log.Printf("uploaded %d/%d", uploaded, len(pending))
}
log.Fatalf("[ERROR] upload %s/%s: %v", t.folderName, name, err)
}
mu.Lock()
uploaded++
if uploaded%50 == 0 {
log.Printf("uploaded %d/%d", uploaded, len(pending))
}
mu.Unlock()
}
@@ -238,5 +289,14 @@ func main() {
}
close(work)
wg.Wait()
log.Printf("done: %d uploaded, %d failed, %d skipped", uploaded, failed, skipped)
log.Printf("done: %d uploaded, %d skipped", uploaded, skipped)
var totalFiles, totalBytes int64
for _, folderName := range []string{"people", "families"} {
files, bytes := folderStats(svc, folders[folderName])
totalFiles += files
totalBytes += bytes
log.Printf("%s: %d files, %.1f MB", folderName, files, float64(bytes)/1e6)
}
log.Printf("total: %d files, %d bytes (%.1f MB)", totalFiles, totalBytes, float64(totalBytes)/1e6)
}
web/static/directory/app.js
+7 -3
View File
@@ -101,10 +101,14 @@ function firstName(fullName) {
return fullName.trim().split(/\s+/)[0];
}
function thumbUrl(url) {
return url ? url + '?thumb=1' : url;
}
function photoOrInitials(url, name, className) {
if (url) {
const img = el('img', className);
img.src = url;
img.src = thumbUrl(url);
img.loading = 'lazy';
img.alt = '';
return img;
@@ -176,7 +180,7 @@ function renderStudents(grid) {
card.href = personLink(p);
if (p.photoUrl) {
const img = el('img', 'student-photo');
img.src = p.photoUrl;
img.src = thumbUrl(p.photoUrl);
img.loading = 'lazy';
img.alt = '';
card.append(img);
@@ -391,7 +395,7 @@ function memberRow(p, label, sub) {
row.href = personLink(p);
if (p.photoUrl) {
const img = el('img', 'member-thumb');
img.src = p.photoUrl;
img.src = thumbUrl(p.photoUrl);
img.loading = 'lazy';
img.alt = '';
row.append(img);