From 366e3bb8a2429ea896266e3006d1cb9930f5bd08 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Sun, 16 Aug 2026 11:22:59 -0700 Subject: [PATCH] Add server wrapper, blob fetch logging, sheet cell tool; drop importblobs --- internal/blob/blob.go | 6 + tools/importblobs/main.go | 302 -------------------------------- tools/probeblob/main.go | 41 +++++ tools/renamefamilyblobs/main.go | 126 +++++++++++++ tools/setcell/main.go | 30 ++++ tools/startserver/main.go | 77 ++++++++ 6 files changed, 280 insertions(+), 302 deletions(-) delete mode 100644 tools/importblobs/main.go create mode 100644 tools/probeblob/main.go create mode 100644 tools/renamefamilyblobs/main.go create mode 100644 tools/setcell/main.go create mode 100644 tools/startserver/main.go diff --git a/internal/blob/blob.go b/internal/blob/blob.go index 75cc11f..39a7b82 100644 --- a/internal/blob/blob.go +++ b/internal/blob/blob.go @@ -143,7 +143,13 @@ func (s *Store) refresh() error { defer wg.Done() for key := range work { l := listing[key] + fetchStart := time.Now() body, err := s.download(l.id) + if err == nil { + log.Printf("blob fetch: %s %d bytes in %s", key, len(body), time.Since(fetchStart).Round(time.Millisecond)) + } else { + log.Printf("[ERROR] blob fetch: %s: %v", key, err) + } if err == nil && strings.HasPrefix(l.mimeType, "image/") { var thumb []byte thumb, err = thumbnail(body) diff --git a/tools/importblobs/main.go b/tools/importblobs/main.go deleted file mode 100644 index 85c4d3b..0000000 --- a/tools/importblobs/main.go +++ /dev/null @@ -1,302 +0,0 @@ -// Command importblobs copies directory media into the drive folder under sane names. -package main - -import ( - "bytes" - "context" - "flag" - "fmt" - "io" - "log" - "net/http" - "path" - "strings" - "sync" - "time" - - "heliosian/internal/data" - "heliosian/internal/directory" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" - "google.golang.org/api/option" -) - -const folderMime = "application/vnd.google-apps.folder" - -var extensions = map[string]string{ - "image/jpeg": ".jpg", - "image/png": ".png", - "image/webp": ".webp", - "image/gif": ".gif", - "audio/mpeg": ".mp3", - "audio/mp3": ".mp3", - "audio/mp4": ".m4a", - "audio/wav": ".wav", -} - -type task struct { - folderName string - base string - url string -} - -func localPart(email string) string { - name, _, _ := strings.Cut(email, "@") - return name -} - -func findRoot(svc *drive.Service) string { - drives, err := svc.Drives.List().Do() - if err != nil { - log.Fatalf("[ERROR] list shared drives: %v", err) - } - if len(drives.Drives) == 1 { - log.Printf("using shared drive %q (%s)", drives.Drives[0].Name, drives.Drives[0].Id) - return drives.Drives[0].Id - } - if len(drives.Drives) > 1 { - for _, d := range drives.Drives { - log.Printf("candidate shared drive: %s (%s)", d.Name, d.Id) - } - log.Fatalf("[ERROR] service account can see %d shared drives; pass -folder", len(drives.Drives)) - } - list, err := svc.Files.List(). - Q("mimeType = '" + folderMime + "' and sharedWithMe = true and trashed = false"). - Fields("files(id, name)").Do() - if err != nil { - log.Fatalf("[ERROR] list shared folders: %v", err) - } - if len(list.Files) != 1 { - for _, f := range list.Files { - log.Printf("candidate folder: %s (%s)", f.Name, f.Id) - } - log.Fatalf("[ERROR] expected one shared drive or one shared folder, found %d folders; pass -folder", len(list.Files)) - } - log.Printf("using folder %q (%s); note: uploads into personal drives fail on service account quota", list.Files[0].Name, list.Files[0].Id) - return list.Files[0].Id -} - -func ensureFolder(svc *drive.Service, parent, name string) string { - list, err := svc.Files.List(). - Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("files(id)").Do() - if err != nil { - log.Fatalf("[ERROR] find folder %s: %v", name, err) - } - if len(list.Files) > 0 { - return list.Files[0].Id - } - created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}). - SupportsAllDrives(true).Fields("id").Do() - if err != nil { - log.Fatalf("[ERROR] create folder %s: %v", name, err) - } - return created.Id -} - -func listBases(svc *drive.Service, folderID string) map[string]bool { - bases := map[string]bool{} - deleted := 0 - token := "" - for { - call := svc.Files.List(). - Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("nextPageToken, files(id, name)").PageSize(1000) - if token != "" { - call = call.PageToken(token) - } - list, err := call.Do() - if err != nil { - log.Fatalf("[ERROR] list folder contents: %v", err) - } - for _, f := range list.Files { - base := strings.TrimSuffix(f.Name, path.Ext(f.Name)) - if strings.HasSuffix(base, "-thumb") { - _, err := svc.Files.Update(f.Id, &drive.File{Trashed: true}).SupportsAllDrives(true).Do() - if err != nil { - log.Fatalf("[ERROR] trash stale thumb %s: %v", f.Name, err) - } - deleted++ - continue - } - bases[base] = true - } - if list.NextPageToken == "" { - if deleted > 0 { - log.Printf("deleted %d stale thumbs", deleted) - } - return bases - } - token = list.NextPageToken - } -} - -func folderStats(svc *drive.Service, folderID string) (int64, int64) { - var files, bytes int64 - token := "" - for { - call := svc.Files.List(). - Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("nextPageToken, files(size)").PageSize(1000) - if token != "" { - call = call.PageToken(token) - } - list, err := call.Do() - if err != nil { - log.Fatalf("[ERROR] list folder for stats: %v", err) - } - for _, f := range list.Files { - files++ - bytes += f.Size - } - if list.NextPageToken == "" { - return files, bytes - } - token = list.NextPageToken - } -} - -func fetch(client *http.Client, url string) ([]byte, string, error) { - resp, err := client.Get(url) - if err != nil { - return nil, "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, "", fmt.Errorf("status %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, "", err - } - contentType, _, _ := strings.Cut(resp.Header.Get("Content-Type"), ";") - return body, strings.TrimSpace(contentType), nil -} - -func extension(contentType, url string) string { - if ext, ok := extensions[contentType]; ok { - return ext - } - if ext := path.Ext(strings.SplitN(path.Base(url), "?", 2)[0]); ext != "" && len(ext) <= 5 { - return ext - } - return ".bin" -} - -func main() { - sheetID := flag.String("sheet", "", "directory spreadsheet id") - folderID := flag.String("folder", "", "drive folder id (default: the folder shared with the service account)") - flag.Parse() - if *sheetID == "" { - log.Fatal("[ERROR] -sheet is required") - } - - source, err := data.NewSheet(map[string]string{"directory": *sheetID}) - if err != nil { - log.Fatalf("[ERROR] sheet source: %v", err) - } - model, err := directory.LoadModel(source, nil) - if err != nil { - log.Fatalf("[ERROR] load model: %v", err) - } - - svc, err := drive.NewService(context.Background(), - option.WithCredentialsFile(data.KeyFile), - option.WithScopes(drive.DriveScope)) - if err != nil { - log.Fatalf("[ERROR] drive client: %v", err) - } - - root := *folderID - if root == "" { - root = findRoot(svc) - } - folders := map[string]string{ - "people": ensureFolder(svc, root, "people"), - "families": ensureFolder(svc, root, "families"), - } - - tasks := []task{} - for _, p := range model.People { - if p.PhotoURL != "" { - tasks = append(tasks, task{"people", localPart(p.Email) + "-photo", p.PhotoURL}) - } - if p.PronunciationURL != "" { - tasks = append(tasks, task{"people", localPart(p.Email) + "-pronunciation", p.PronunciationURL}) - } - } - for key, f := range model.Families { - if f.PhotoURL != "" { - tasks = append(tasks, task{"families", localPart(key) + "-photo", f.PhotoURL}) - } - if f.PronunciationURL != "" { - tasks = append(tasks, task{"families", localPart(key) + "-pronunciation", f.PronunciationURL}) - } - } - - pending := []task{} - skipped := 0 - for folderName, id := range folders { - bases := listBases(svc, id) - for _, t := range tasks { - if t.folderName != folderName { - continue - } - if bases[t.base] { - skipped++ - } else { - pending = append(pending, t) - } - } - } - log.Printf("%d media files in model, %d already imported, %d to fetch", len(tasks), skipped, len(pending)) - - client := &http.Client{Timeout: 60 * time.Second} - var mu sync.Mutex - uploaded := 0 - work := make(chan task) - var wg sync.WaitGroup - for range 6 { - wg.Add(1) - go func() { - defer wg.Done() - for t := range work { - body, contentType, err := fetch(client, t.url) - if err != nil { - log.Fatalf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err) - } - name := t.base + extension(contentType, t.url) - _, err = svc.Files.Create(&drive.File{Name: name, Parents: []string{folders[t.folderName]}}). - Media(bytes.NewReader(body), googleapi.ContentType(contentType)). - SupportsAllDrives(true).Fields("id").Do() - if err != nil { - log.Fatalf("[ERROR] upload %s/%s: %v", t.folderName, name, err) - } - mu.Lock() - uploaded++ - if uploaded%50 == 0 { - log.Printf("uploaded %d/%d", uploaded, len(pending)) - } - mu.Unlock() - } - }() - } - for _, t := range pending { - work <- t - } - close(work) - wg.Wait() - log.Printf("done: %d uploaded, %d skipped", uploaded, skipped) - - var totalFiles, totalBytes int64 - for _, folderName := range []string{"people", "families"} { - files, bytes := folderStats(svc, folders[folderName]) - totalFiles += files - totalBytes += bytes - log.Printf("%s: %d files, %.1f MB", folderName, files, float64(bytes)/1e6) - } - log.Printf("total: %d files, %d bytes (%.1f MB)", totalFiles, totalBytes, float64(totalBytes)/1e6) -} diff --git a/tools/probeblob/main.go b/tools/probeblob/main.go new file mode 100644 index 0000000..3f2480a --- /dev/null +++ b/tools/probeblob/main.go @@ -0,0 +1,41 @@ +// Command probeblob times the download of a few drive media files. +package main + +import ( + "context" + "fmt" + "io" + "log" + "time" + + "heliosian/internal/data" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +func main() { + svc, err := drive.NewService(context.Background(), + option.WithCredentialsFile(data.KeyFile), + option.WithScopes(drive.DriveScope)) + if err != nil { + log.Fatalf("[ERROR] drive client: %v", err) + } + list, err := svc.Files.List(). + Q("mimeType != 'application/vnd.google-apps.folder' and trashed = false"). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("files(id, name, size)").PageSize(5).Do() + if err != nil { + log.Fatalf("[ERROR] list: %v", err) + } + for _, f := range list.Files { + start := time.Now() + resp, err := svc.Files.Get(f.Id).SupportsAllDrives(true).Download() + if err != nil { + fmt.Printf("%s (%d bytes): request error after %s: %v\n", f.Name, f.Size, time.Since(start).Round(time.Millisecond), err) + continue + } + n, err := io.Copy(io.Discard, resp.Body) + resp.Body.Close() + fmt.Printf("%s: %d bytes in %s (err %v)\n", f.Name, n, time.Since(start).Round(time.Millisecond), err) + } +} diff --git a/tools/renamefamilyblobs/main.go b/tools/renamefamilyblobs/main.go new file mode 100644 index 0000000..9c189f8 --- /dev/null +++ b/tools/renamefamilyblobs/main.go @@ -0,0 +1,126 @@ +// Command renamefamilyblobs renames family media in the drive from parent-email names to family key hashes. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "path" + "strings" + + "heliosian/internal/data" + "heliosian/internal/directory" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +const folderMime = "application/vnd.google-apps.folder" + +func main() { + sheet := flag.String("sheet", "", "spreadsheet id") + flag.Parse() + if *sheet == "" { + log.Fatal("[ERROR] -sheet is required") + } + source, err := data.NewSheet(map[string]string{"directory": *sheet}) + if err != nil { + log.Fatalf("[ERROR] sheet source: %v", err) + } + model, err := directory.LoadModel(source, nil) + if err != nil { + log.Fatalf("[ERROR] load model: %v", err) + } + localToHash := map[string]string{} + for _, family := range model.Families { + for _, adult := range family.AdultEmails { + local, _, _ := strings.Cut(adult, "@") + if existing, ok := localToHash[local]; ok && existing != family.Key { + log.Fatalf("[ERROR] adult local part %s maps to two families", local) + } + localToHash[local] = family.Key + } + } + + svc, err := drive.NewService(context.Background(), + option.WithCredentialsFile(data.KeyFile), + option.WithScopes(drive.DriveScope)) + if err != nil { + log.Fatalf("[ERROR] drive client: %v", err) + } + drives, err := svc.Drives.List().Do() + if err != nil { + log.Fatalf("[ERROR] list shared drives: %v", err) + } + if len(drives.Drives) != 1 { + log.Fatalf("[ERROR] expected one shared drive, found %d", len(drives.Drives)) + } + folderList, err := svc.Files.List(). + Q(fmt.Sprintf("name = 'families' and '%s' in parents and mimeType = '%s' and trashed = false", drives.Drives[0].Id, folderMime)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("files(id)").Do() + if err != nil { + log.Fatalf("[ERROR] find families folder: %v", err) + } + if len(folderList.Files) != 1 { + log.Fatalf("[ERROR] expected one families folder, found %d", len(folderList.Files)) + } + folderID := folderList.Files[0].Id + + renamed, kept, unknown := 0, 0, 0 + token := "" + for { + call := svc.Files.List(). + Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). + SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). + Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000) + if token != "" { + call = call.PageToken(token) + } + list, err := call.Do() + if err != nil { + log.Fatalf("[ERROR] list families folder: %v", err) + } + for _, f := range list.Files { + if f.MimeType == folderMime { + continue + } + ext := path.Ext(f.Name) + base := strings.TrimSuffix(f.Name, ext) + kind := "" + for _, k := range []string{"-photo", "-pronunciation"} { + if strings.HasSuffix(base, k) { + kind = k + } + } + if kind == "" { + log.Printf("[ERROR] unrecognized file name %q, leaving it", f.Name) + unknown++ + continue + } + local := strings.TrimSuffix(base, kind) + hash, ok := localToHash[local] + if !ok { + if _, isCurrent := model.Families[local]; isCurrent { + kept++ + continue + } + log.Printf("[ERROR] file %q matches no family adult, leaving it", f.Name) + unknown++ + continue + } + newName := hash + kind + ext + _, err := svc.Files.Update(f.Id, &drive.File{Name: newName}).SupportsAllDrives(true).Do() + if err != nil { + log.Fatalf("[ERROR] rename %q to %q: %v", f.Name, newName, err) + } + log.Printf("renamed %q -> %q", f.Name, newName) + renamed++ + } + if list.NextPageToken == "" { + break + } + token = list.NextPageToken + } + log.Printf("done: %d renamed, %d already keyed by hash, %d left untouched", renamed, kept, unknown) +} diff --git a/tools/setcell/main.go b/tools/setcell/main.go new file mode 100644 index 0000000..bcfa3cb --- /dev/null +++ b/tools/setcell/main.go @@ -0,0 +1,30 @@ +// Command setcell sets one cell in a sheet tab by key column, appending the row if missing. +package main + +import ( + "flag" + "log" + + "heliosian/internal/data" +) + +func main() { + sheet := flag.String("sheet", "", "spreadsheet id") + tab := flag.String("tab", "", "tab title") + keyCol := flag.String("keycol", "Email", "key column name") + key := flag.String("key", "", "key value") + col := flag.String("col", "", "column to set") + value := flag.String("value", "", "value to write") + flag.Parse() + if *sheet == "" || *tab == "" || *key == "" || *col == "" { + log.Fatal("[ERROR] -sheet, -tab, -key, and -col are required") + } + source, err := data.NewSheet(map[string]string{"directory": *sheet}) + if err != nil { + log.Fatalf("[ERROR] sheet source: %v", err) + } + if err := source.Upsert("directory", *tab, *keyCol, *key, *col, *value); err != nil { + log.Fatalf("[ERROR] set %s[%s=%s].%s: %v", *tab, *keyCol, *key, *col, err) + } + log.Printf("set %s[%s=%s].%s = %q", *tab, *keyCol, *key, *col, *value) +} diff --git a/tools/startserver/main.go b/tools/startserver/main.go new file mode 100644 index 0000000..bf781fa --- /dev/null +++ b/tools/startserver/main.go @@ -0,0 +1,77 @@ +// Command startserver launches the app, waits for it to listen, prints the pid and an auth header, and leaves it running. +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "flag" + "fmt" + "log" + "os" + "os/exec" + "strings" + "time" +) + +func main() { + email := flag.String("email", "dev@heliosschool.org", "session email for the minted cookie") + flag.Parse() + key := os.Getenv("SESSION_KEY") + if key == "" { + log.Fatal("[ERROR] SESSION_KEY is required (the server and the minted cookie must share it)") + } + if os.Getenv("DIRECTORY_SHEET") == "" { + log.Fatal("[ERROR] DIRECTORY_SHEET is required") + } + + logFile, err := os.Create("/tmp/heliosian-server.log") + if err != nil { + log.Fatalf("[ERROR] create server log: %v", err) + } + cmd := exec.Command("go", "run", ".") + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Start(); err != nil { + log.Fatalf("[ERROR] start server: %v", err) + } + + deadline := time.Now().Add(10 * time.Minute) + for { + if time.Now().After(deadline) { + cmd.Process.Kill() + log.Fatalf("[ERROR] server did not start within 10 minutes; log: /tmp/heliosian-server.log") + } + content, err := os.ReadFile("/tmp/heliosian-server.log") + if err != nil { + log.Fatalf("[ERROR] read server log: %v", err) + } + if strings.Contains(string(content), "listening on ") { + break + } + if cmd.ProcessState != nil || !processAlive(cmd.Process.Pid) { + fmt.Print(string(content)) + log.Fatal("[ERROR] server exited before listening") + } + time.Sleep(time.Second) + } + + payload := fmt.Sprintf("%s|%d", *email, time.Now().Add(24*time.Hour).Unix()) + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(payload)) + cookie := base64.RawURLEncoding.EncodeToString([]byte(payload)) + "." + + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + + content, _ := os.ReadFile("/tmp/heliosian-server.log") + for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { + fmt.Println(line) + } + fmt.Printf("pid: %d\n", cmd.Process.Pid) + fmt.Println("log: /tmp/heliosian-server.log") + fmt.Printf("header: Cookie: session=%s\n", cookie) + cmd.Process.Release() +} + +func processAlive(pid int) bool { + return exec.Command("kill", "-0", fmt.Sprint(pid)).Run() == nil +}