diff --git a/go.mod b/go.mod index 4b5b0a2..28854b7 100644 --- a/go.mod +++ b/go.mod @@ -31,10 +31,11 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect golang.org/x/crypto v0.54.0 // indirect + golang.org/x/image v0.45.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260807164820-c8921c73eeea // indirect google.golang.org/grpc v1.83.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/go.sum b/go.sum index e90f985..84bd9bd 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/ go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -78,6 +80,8 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/internal/auth/auth.go b/internal/auth/auth.go index ca4a4bc..7124532 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -52,7 +52,7 @@ func (a *Auth) Wrap(next http.Handler) http.Handler { } email := a.sessionEmail(r) if email == "" { - if strings.Contains(r.URL.Path, "/api/") { + if strings.Contains(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/blob/") { http.Error(w, "unauthenticated", http.StatusUnauthorized) return } diff --git a/internal/blob/blob.go b/internal/blob/blob.go new file mode 100644 index 0000000..423014c --- /dev/null +++ b/internal/blob/blob.go @@ -0,0 +1,260 @@ +// Package blob serves directory media from drive, held fully in memory with startup-generated thumbnails. +package blob + +import ( + "bytes" + "context" + "fmt" + "image" + "image/jpeg" + "io" + "log" + "net/http" + "path" + "strings" + "sync" + "time" + + _ "image/gif" + _ "image/png" + + "golang.org/x/image/draw" + _ "golang.org/x/image/webp" + + "heliosian/internal/data" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +const ( + folderMime = "application/vnd.google-apps.folder" + refreshInterval = 5 * time.Minute + thumbWidth = 480 +) + +type entry struct { + id string + mimeType string + data []byte + thumb []byte +} + +type listed struct { + id string + mimeType string +} + +type Store struct { + service *drive.Service + root string + mu sync.RWMutex + entries map[string]*entry +} + +func New() (*Store, error) { + service, err := drive.NewService(context.Background(), + option.WithCredentialsFile(data.KeyFile), + option.WithScopes(drive.DriveReadonlyScope)) + if err != nil { + return nil, err + } + drives, err := service.Drives.List().Do() + if err != nil { + return nil, fmt.Errorf("list shared drives: %w", err) + } + if len(drives.Drives) != 1 { + return nil, fmt.Errorf("expected one shared drive visible to the service account, found %d", len(drives.Drives)) + } + s := &Store{service: service, root: drives.Drives[0].Id, entries: map[string]*entry{}} + if err := s.refresh(); err != nil { + return nil, err + } + go s.refreshLoop() + return s, nil +} + +func Register(mux *http.ServeMux, s *Store) { + mux.HandleFunc("GET /blob/{folder}/{name}", s.serve) +} + +func (s *Store) refreshLoop() { + for range time.Tick(refreshInterval) { + if err := s.refresh(); err != nil { + log.Printf("[ERROR] blob refresh: %v", err) + } + } +} + +func (s *Store) refresh() error { + start := time.Now() + listing := map[string]listed{} + for _, folderName := range []string{"people", "families"} { + folderID, err := s.subfolder(folderName) + if err != nil { + return err + } + token := "" + for { + call := s.service.Files.List(). + Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000) + if token != "" { + call = call.PageToken(token) + } + list, err := call.Do() + if err != nil { + return fmt.Errorf("list %s: %w", folderName, err) + } + for _, f := range list.Files { + base := strings.TrimSuffix(f.Name, path.Ext(f.Name)) + if strings.HasSuffix(base, "-thumb") { + continue + } + listing[folderName+"/"+base] = listed{id: f.Id, mimeType: f.MimeType} + } + if list.NextPageToken == "" { + break + } + token = list.NextPageToken + } + } + + s.mu.RLock() + missing := []string{} + for key, l := range listing { + if cached, ok := s.entries[key]; !ok || cached.id != l.id { + missing = append(missing, key) + } + } + s.mu.RUnlock() + + fetched := map[string]*entry{} + var fetchedMu sync.Mutex + work := make(chan string) + errs := make(chan error, 1) + var wg sync.WaitGroup + for range 12 { + wg.Add(1) + go func() { + defer wg.Done() + for key := range work { + l := listing[key] + body, err := s.download(l.id) + if err == nil && strings.HasPrefix(l.mimeType, "image/") { + var thumb []byte + thumb, err = thumbnail(body) + if err == nil { + fetchedMu.Lock() + fetched[key] = &entry{id: l.id, mimeType: l.mimeType, data: body, thumb: thumb} + fetchedMu.Unlock() + continue + } + } else if err == nil { + fetchedMu.Lock() + fetched[key] = &entry{id: l.id, mimeType: l.mimeType, data: body} + fetchedMu.Unlock() + continue + } + select { + case errs <- fmt.Errorf("load %s: %w", key, err): + default: + } + return + } + }() + } + for _, key := range missing { + work <- key + } + close(work) + wg.Wait() + select { + case err := <-errs: + return err + default: + } + + next := make(map[string]*entry, len(listing)) + var totalBytes int64 + s.mu.Lock() + for key, l := range listing { + if e, ok := fetched[key]; ok { + next[key] = e + } else if cached, ok := s.entries[key]; ok && cached.id == l.id { + next[key] = cached + } + } + s.entries = next + for _, e := range next { + totalBytes += int64(len(e.data) + len(e.thumb)) + } + s.mu.Unlock() + log.Printf("blob store: %d files, %d fetched, %.1f MB in memory in %s", + len(next), len(fetched), float64(totalBytes)/1e6, time.Since(start).Round(time.Millisecond)) + return nil +} + +func (s *Store) subfolder(name string) (string, error) { + list, err := s.service.Files.List(). + Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, s.root, folderMime)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("files(id)").Do() + if err != nil { + return "", fmt.Errorf("find folder %s: %w", name, err) + } + if len(list.Files) != 1 { + return "", fmt.Errorf("expected one %s folder, found %d", name, len(list.Files)) + } + return list.Files[0].Id, nil +} + +func (s *Store) download(id string) ([]byte, error) { + resp, err := s.service.Files.Get(id).SupportsAllDrives(true).Download() + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) +} + +func (s *Store) serve(w http.ResponseWriter, r *http.Request) { + key := r.PathValue("folder") + "/" + r.PathValue("name") + s.mu.RLock() + e, ok := s.entries[key] + s.mu.RUnlock() + if !ok { + http.NotFound(w, r) + return + } + if r.URL.Query().Get("thumb") == "1" { + if e.thumb == nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "image/jpeg") + w.Write(e.thumb) + return + } + w.Header().Set("Content-Type", e.mimeType) + w.Write(e.data) +} + +func thumbnail(src []byte) ([]byte, error) { + img, _, err := image.Decode(bytes.NewReader(src)) + if err != nil { + return nil, err + } + bounds := img.Bounds() + if bounds.Dx() > thumbWidth { + height := bounds.Dy() * thumbWidth / bounds.Dx() + scaled := image.NewRGBA(image.Rect(0, 0, thumbWidth, height)) + draw.CatmullRom.Scale(scaled, scaled.Bounds(), img, bounds, draw.Over, nil) + img = scaled + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/directory/load.go b/internal/directory/load.go index 5c1324f..e1f3d25 100644 --- a/internal/directory/load.go +++ b/internal/directory/load.go @@ -52,8 +52,8 @@ func LoadModel(source data.Source) (*Model, error) { IsStudent: row["Is Student?"] == "TRUE", Pronouns: row["Pronouns"], Facts: row["Facts"], - PronunciationURL: row["Pronunciation"], - PhotoURL: row["Primary Photo"], + PronunciationURL: blobURL("people", email, "pronunciation", row["Pronunciation"]), + PhotoURL: blobURL("people", email, "photo", row["Primary Photo"]), Grade: row["Grade"], Classroom: row["Class"], Section: row["Section"], @@ -88,13 +88,13 @@ func LoadModel(source data.Source) (*Model, error) { } } if acc.family.PhotoURL == "" { - acc.family.PhotoURL = row["Family Photo"] + acc.family.PhotoURL = blobURL("families", p.FamilyKey, "photo", row["Family Photo"]) } if acc.family.PhotoCaption == "" { acc.family.PhotoCaption = row["Family Photo Description"] } if acc.family.PronunciationURL == "" { - acc.family.PronunciationURL = row["Family Pronunciation"] + acc.family.PronunciationURL = blobURL("families", p.FamilyKey, "pronunciation", row["Family Pronunciation"]) } if p.IsParent { acc.hasParent = true @@ -188,6 +188,14 @@ func LoadModel(source data.Source) (*Model, error) { return model, nil } +func blobURL(folder, email, kind, source string) string { + if source == "" { + return "" + } + local, _, _ := strings.Cut(email, "@") + return "/blob/" + folder + "/" + local + "-" + kind +} + func surname(fullName string) string { fields := strings.Fields(fullName) if len(fields) == 0 { diff --git a/main.go b/main.go index a75f7c0..8e81cf5 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "os" "heliosian/internal/auth" + "heliosian/internal/blob" "heliosian/internal/data" "heliosian/internal/directory" ) @@ -72,6 +73,13 @@ func main() { mux := http.NewServeMux() authn.Register(mux) directory.Register(mux, cache) + if os.Getenv("DIRECTORY_SHEET") != "" { + store, err := blob.New() + if err != nil { + log.Fatalf("[ERROR] blob store: %v", err) + } + blob.Register(mux, store) + } mux.Handle("GET /{$}", http.RedirectHandler("/people", http.StatusFound)) mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static")))) port := os.Getenv("PORT") diff --git a/tools/importblobs/main.go b/tools/importblobs/main.go index 9f0dafd..b3a9c84 100644 --- a/tools/importblobs/main.go +++ b/tools/importblobs/main.go @@ -45,7 +45,21 @@ func localPart(email string) string { return name } -func findSharedFolder(svc *drive.Service) string { +func findRoot(svc *drive.Service) string { + drives, err := svc.Drives.List().Do() + if err != nil { + log.Fatalf("[ERROR] list shared drives: %v", err) + } + if len(drives.Drives) == 1 { + log.Printf("using shared drive %q (%s)", drives.Drives[0].Name, drives.Drives[0].Id) + return drives.Drives[0].Id + } + if len(drives.Drives) > 1 { + for _, d := range drives.Drives { + log.Printf("candidate shared drive: %s (%s)", d.Name, d.Id) + } + log.Fatalf("[ERROR] service account can see %d shared drives; pass -folder", len(drives.Drives)) + } list, err := svc.Files.List(). Q("mimeType = '" + folderMime + "' and sharedWithMe = true and trashed = false"). Fields("files(id, name)").Do() @@ -56,15 +70,16 @@ func findSharedFolder(svc *drive.Service) string { for _, f := range list.Files { log.Printf("candidate folder: %s (%s)", f.Name, f.Id) } - log.Fatalf("[ERROR] expected exactly one folder shared with the service account, found %d; pass -folder", len(list.Files)) + log.Fatalf("[ERROR] expected one shared drive or one shared folder, found %d folders; pass -folder", len(list.Files)) } - log.Printf("using folder %q (%s)", list.Files[0].Name, list.Files[0].Id) + log.Printf("using folder %q (%s); note: uploads into personal drives fail on service account quota", list.Files[0].Name, list.Files[0].Id) return list.Files[0].Id } func ensureFolder(svc *drive.Service, parent, name string) string { list, err := svc.Files.List(). Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). Fields("files(id)").Do() if err != nil { log.Fatalf("[ERROR] find folder %s: %v", name, err) @@ -72,7 +87,8 @@ func ensureFolder(svc *drive.Service, parent, name string) string { if len(list.Files) > 0 { return list.Files[0].Id } - created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}).Fields("id").Do() + created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}). + SupportsAllDrives(true).Fields("id").Do() if err != nil { log.Fatalf("[ERROR] create folder %s: %v", name, err) } @@ -81,11 +97,13 @@ func ensureFolder(svc *drive.Service, parent, name string) string { func listBases(svc *drive.Service, folderID string) map[string]bool { bases := map[string]bool{} + deleted := 0 token := "" for { call := svc.Files.List(). Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). - Fields("nextPageToken, files(name)").PageSize(1000) + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("nextPageToken, files(id, name)").PageSize(1000) if token != "" { call = call.PageToken(token) } @@ -94,15 +112,53 @@ func listBases(svc *drive.Service, folderID string) map[string]bool { log.Fatalf("[ERROR] list folder contents: %v", err) } for _, f := range list.Files { - bases[strings.TrimSuffix(f.Name, path.Ext(f.Name))] = true + base := strings.TrimSuffix(f.Name, path.Ext(f.Name)) + if strings.HasSuffix(base, "-thumb") { + _, err := svc.Files.Update(f.Id, &drive.File{Trashed: true}).SupportsAllDrives(true).Do() + if err != nil { + log.Fatalf("[ERROR] trash stale thumb %s: %v", f.Name, err) + } + deleted++ + continue + } + bases[base] = true } if list.NextPageToken == "" { + if deleted > 0 { + log.Printf("deleted %d stale thumbs", deleted) + } return bases } token = list.NextPageToken } } +func folderStats(svc *drive.Service, folderID string) (int64, int64) { + var files, bytes int64 + token := "" + for { + call := svc.Files.List(). + Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("nextPageToken, files(size)").PageSize(1000) + if token != "" { + call = call.PageToken(token) + } + list, err := call.Do() + if err != nil { + log.Fatalf("[ERROR] list folder for stats: %v", err) + } + for _, f := range list.Files { + files++ + bytes += f.Size + } + if list.NextPageToken == "" { + return files, bytes + } + token = list.NextPageToken + } +} + func fetch(client *http.Client, url string) ([]byte, string, error) { resp, err := client.Get(url) if err != nil { @@ -156,7 +212,7 @@ func main() { root := *folderID if root == "" { - root = findSharedFolder(svc) + root = findRoot(svc) } folders := map[string]string{ "people": ensureFolder(svc, root, "people"), @@ -200,7 +256,7 @@ func main() { client := &http.Client{Timeout: 60 * time.Second} var mu sync.Mutex - uploaded, failed := 0, 0 + uploaded := 0 work := make(chan task) var wg sync.WaitGroup for range 6 { @@ -210,24 +266,19 @@ func main() { for t := range work { body, contentType, err := fetch(client, t.url) if err != nil { - mu.Lock() - failed++ - log.Printf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err) - mu.Unlock() - continue + log.Fatalf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err) } name := t.base + extension(contentType, t.url) _, err = svc.Files.Create(&drive.File{Name: name, Parents: []string{folders[t.folderName]}}). - Media(bytes.NewReader(body), googleapi.ContentType(contentType)).Fields("id").Do() - mu.Lock() + Media(bytes.NewReader(body), googleapi.ContentType(contentType)). + SupportsAllDrives(true).Fields("id").Do() if err != nil { - failed++ - log.Printf("[ERROR] upload %s/%s: %v", t.folderName, name, err) - } else { - uploaded++ - if uploaded%50 == 0 { - log.Printf("uploaded %d/%d", uploaded, len(pending)) - } + log.Fatalf("[ERROR] upload %s/%s: %v", t.folderName, name, err) + } + mu.Lock() + uploaded++ + if uploaded%50 == 0 { + log.Printf("uploaded %d/%d", uploaded, len(pending)) } mu.Unlock() } @@ -238,5 +289,14 @@ func main() { } close(work) wg.Wait() - log.Printf("done: %d uploaded, %d failed, %d skipped", uploaded, failed, skipped) + log.Printf("done: %d uploaded, %d skipped", uploaded, skipped) + + var totalFiles, totalBytes int64 + for _, folderName := range []string{"people", "families"} { + files, bytes := folderStats(svc, folders[folderName]) + totalFiles += files + totalBytes += bytes + log.Printf("%s: %d files, %.1f MB", folderName, files, float64(bytes)/1e6) + } + log.Printf("total: %d files, %d bytes (%.1f MB)", totalFiles, totalBytes, float64(totalBytes)/1e6) } diff --git a/web/static/directory/app.js b/web/static/directory/app.js index a6e593c..7a8e572 100644 --- a/web/static/directory/app.js +++ b/web/static/directory/app.js @@ -101,10 +101,14 @@ function firstName(fullName) { return fullName.trim().split(/\s+/)[0]; } +function thumbUrl(url) { + return url ? url + '?thumb=1' : url; +} + function photoOrInitials(url, name, className) { if (url) { const img = el('img', className); - img.src = url; + img.src = thumbUrl(url); img.loading = 'lazy'; img.alt = ''; return img; @@ -176,7 +180,7 @@ function renderStudents(grid) { card.href = personLink(p); if (p.photoUrl) { const img = el('img', 'student-photo'); - img.src = p.photoUrl; + img.src = thumbUrl(p.photoUrl); img.loading = 'lazy'; img.alt = ''; card.append(img); @@ -391,7 +395,7 @@ function memberRow(p, label, sub) { row.href = personLink(p); if (p.photoUrl) { const img = el('img', 'member-thumb'); - img.src = p.photoUrl; + img.src = thumbUrl(p.photoUrl); img.loading = 'lazy'; img.alt = ''; row.append(img);