Add server wrapper, blob fetch logging, sheet cell tool; drop importblobs

This commit is contained in:
Ian Gulliver
2026-08-16 11:22:59 -07:00
parent bbff936f39
commit 366e3bb8a2
6 changed files with 280 additions and 302 deletions
+6
View File
@@ -143,7 +143,13 @@ func (s *Store) refresh() error {
defer wg.Done()
for key := range work {
l := listing[key]
fetchStart := time.Now()
body, err := s.download(l.id)
if err == nil {
log.Printf("blob fetch: %s %d bytes in %s", key, len(body), time.Since(fetchStart).Round(time.Millisecond))
} else {
log.Printf("[ERROR] blob fetch: %s: %v", key, err)
}
if err == nil && strings.HasPrefix(l.mimeType, "image/") {
var thumb []byte
thumb, err = thumbnail(body)
-302
View File
@@ -1,302 +0,0 @@
// Command importblobs copies directory media into the drive folder under sane names.
package main
import (
"bytes"
"context"
"flag"
"fmt"
"io"
"log"
"net/http"
"path"
"strings"
"sync"
"time"
"heliosian/internal/data"
"heliosian/internal/directory"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
)
const folderMime = "application/vnd.google-apps.folder"
var extensions = map[string]string{
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"audio/mpeg": ".mp3",
"audio/mp3": ".mp3",
"audio/mp4": ".m4a",
"audio/wav": ".wav",
}
type task struct {
folderName string
base string
url string
}
func localPart(email string) string {
name, _, _ := strings.Cut(email, "@")
return name
}
func findRoot(svc *drive.Service) string {
drives, err := svc.Drives.List().Do()
if err != nil {
log.Fatalf("[ERROR] list shared drives: %v", err)
}
if len(drives.Drives) == 1 {
log.Printf("using shared drive %q (%s)", drives.Drives[0].Name, drives.Drives[0].Id)
return drives.Drives[0].Id
}
if len(drives.Drives) > 1 {
for _, d := range drives.Drives {
log.Printf("candidate shared drive: %s (%s)", d.Name, d.Id)
}
log.Fatalf("[ERROR] service account can see %d shared drives; pass -folder", len(drives.Drives))
}
list, err := svc.Files.List().
Q("mimeType = '" + folderMime + "' and sharedWithMe = true and trashed = false").
Fields("files(id, name)").Do()
if err != nil {
log.Fatalf("[ERROR] list shared folders: %v", err)
}
if len(list.Files) != 1 {
for _, f := range list.Files {
log.Printf("candidate folder: %s (%s)", f.Name, f.Id)
}
log.Fatalf("[ERROR] expected one shared drive or one shared folder, found %d folders; pass -folder", len(list.Files))
}
log.Printf("using folder %q (%s); note: uploads into personal drives fail on service account quota", list.Files[0].Name, list.Files[0].Id)
return list.Files[0].Id
}
func ensureFolder(svc *drive.Service, parent, name string) string {
list, err := svc.Files.List().
Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id)").Do()
if err != nil {
log.Fatalf("[ERROR] find folder %s: %v", name, err)
}
if len(list.Files) > 0 {
return list.Files[0].Id
}
created, err := svc.Files.Create(&drive.File{Name: name, MimeType: folderMime, Parents: []string{parent}}).
SupportsAllDrives(true).Fields("id").Do()
if err != nil {
log.Fatalf("[ERROR] create folder %s: %v", name, err)
}
return created.Id
}
func listBases(svc *drive.Service, folderID string) map[string]bool {
bases := map[string]bool{}
deleted := 0
token := ""
for {
call := svc.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(id, name)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
log.Fatalf("[ERROR] list folder contents: %v", err)
}
for _, f := range list.Files {
base := strings.TrimSuffix(f.Name, path.Ext(f.Name))
if strings.HasSuffix(base, "-thumb") {
_, err := svc.Files.Update(f.Id, &drive.File{Trashed: true}).SupportsAllDrives(true).Do()
if err != nil {
log.Fatalf("[ERROR] trash stale thumb %s: %v", f.Name, err)
}
deleted++
continue
}
bases[base] = true
}
if list.NextPageToken == "" {
if deleted > 0 {
log.Printf("deleted %d stale thumbs", deleted)
}
return bases
}
token = list.NextPageToken
}
}
func folderStats(svc *drive.Service, folderID string) (int64, int64) {
var files, bytes int64
token := ""
for {
call := svc.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(size)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
log.Fatalf("[ERROR] list folder for stats: %v", err)
}
for _, f := range list.Files {
files++
bytes += f.Size
}
if list.NextPageToken == "" {
return files, bytes
}
token = list.NextPageToken
}
}
func fetch(client *http.Client, url string) ([]byte, string, error) {
resp, err := client.Get(url)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("status %s", resp.Status)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
contentType, _, _ := strings.Cut(resp.Header.Get("Content-Type"), ";")
return body, strings.TrimSpace(contentType), nil
}
func extension(contentType, url string) string {
if ext, ok := extensions[contentType]; ok {
return ext
}
if ext := path.Ext(strings.SplitN(path.Base(url), "?", 2)[0]); ext != "" && len(ext) <= 5 {
return ext
}
return ".bin"
}
func main() {
sheetID := flag.String("sheet", "", "directory spreadsheet id")
folderID := flag.String("folder", "", "drive folder id (default: the folder shared with the service account)")
flag.Parse()
if *sheetID == "" {
log.Fatal("[ERROR] -sheet is required")
}
source, err := data.NewSheet(map[string]string{"directory": *sheetID})
if err != nil {
log.Fatalf("[ERROR] sheet source: %v", err)
}
model, err := directory.LoadModel(source, nil)
if err != nil {
log.Fatalf("[ERROR] load model: %v", err)
}
svc, err := drive.NewService(context.Background(),
option.WithCredentialsFile(data.KeyFile),
option.WithScopes(drive.DriveScope))
if err != nil {
log.Fatalf("[ERROR] drive client: %v", err)
}
root := *folderID
if root == "" {
root = findRoot(svc)
}
folders := map[string]string{
"people": ensureFolder(svc, root, "people"),
"families": ensureFolder(svc, root, "families"),
}
tasks := []task{}
for _, p := range model.People {
if p.PhotoURL != "" {
tasks = append(tasks, task{"people", localPart(p.Email) + "-photo", p.PhotoURL})
}
if p.PronunciationURL != "" {
tasks = append(tasks, task{"people", localPart(p.Email) + "-pronunciation", p.PronunciationURL})
}
}
for key, f := range model.Families {
if f.PhotoURL != "" {
tasks = append(tasks, task{"families", localPart(key) + "-photo", f.PhotoURL})
}
if f.PronunciationURL != "" {
tasks = append(tasks, task{"families", localPart(key) + "-pronunciation", f.PronunciationURL})
}
}
pending := []task{}
skipped := 0
for folderName, id := range folders {
bases := listBases(svc, id)
for _, t := range tasks {
if t.folderName != folderName {
continue
}
if bases[t.base] {
skipped++
} else {
pending = append(pending, t)
}
}
}
log.Printf("%d media files in model, %d already imported, %d to fetch", len(tasks), skipped, len(pending))
client := &http.Client{Timeout: 60 * time.Second}
var mu sync.Mutex
uploaded := 0
work := make(chan task)
var wg sync.WaitGroup
for range 6 {
wg.Add(1)
go func() {
defer wg.Done()
for t := range work {
body, contentType, err := fetch(client, t.url)
if err != nil {
log.Fatalf("[ERROR] fetch %s/%s: %v", t.folderName, t.base, err)
}
name := t.base + extension(contentType, t.url)
_, err = svc.Files.Create(&drive.File{Name: name, Parents: []string{folders[t.folderName]}}).
Media(bytes.NewReader(body), googleapi.ContentType(contentType)).
SupportsAllDrives(true).Fields("id").Do()
if err != nil {
log.Fatalf("[ERROR] upload %s/%s: %v", t.folderName, name, err)
}
mu.Lock()
uploaded++
if uploaded%50 == 0 {
log.Printf("uploaded %d/%d", uploaded, len(pending))
}
mu.Unlock()
}
}()
}
for _, t := range pending {
work <- t
}
close(work)
wg.Wait()
log.Printf("done: %d uploaded, %d skipped", uploaded, skipped)
var totalFiles, totalBytes int64
for _, folderName := range []string{"people", "families"} {
files, bytes := folderStats(svc, folders[folderName])
totalFiles += files
totalBytes += bytes
log.Printf("%s: %d files, %.1f MB", folderName, files, float64(bytes)/1e6)
}
log.Printf("total: %d files, %d bytes (%.1f MB)", totalFiles, totalBytes, float64(totalBytes)/1e6)
}
+41
View File
@@ -0,0 +1,41 @@
// Command probeblob times the download of a few drive media files.
package main
import (
"context"
"fmt"
"io"
"log"
"time"
"heliosian/internal/data"
"google.golang.org/api/drive/v3"
"google.golang.org/api/option"
)
func main() {
svc, err := drive.NewService(context.Background(),
option.WithCredentialsFile(data.KeyFile),
option.WithScopes(drive.DriveScope))
if err != nil {
log.Fatalf("[ERROR] drive client: %v", err)
}
list, err := svc.Files.List().
Q("mimeType != 'application/vnd.google-apps.folder' and trashed = false").
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id, name, size)").PageSize(5).Do()
if err != nil {
log.Fatalf("[ERROR] list: %v", err)
}
for _, f := range list.Files {
start := time.Now()
resp, err := svc.Files.Get(f.Id).SupportsAllDrives(true).Download()
if err != nil {
fmt.Printf("%s (%d bytes): request error after %s: %v\n", f.Name, f.Size, time.Since(start).Round(time.Millisecond), err)
continue
}
n, err := io.Copy(io.Discard, resp.Body)
resp.Body.Close()
fmt.Printf("%s: %d bytes in %s (err %v)\n", f.Name, n, time.Since(start).Round(time.Millisecond), err)
}
}
+126
View File
@@ -0,0 +1,126 @@
// Command renamefamilyblobs renames family media in the drive from parent-email names to family key hashes.
package main
import (
"context"
"flag"
"fmt"
"log"
"path"
"strings"
"heliosian/internal/data"
"heliosian/internal/directory"
"google.golang.org/api/drive/v3"
"google.golang.org/api/option"
)
const folderMime = "application/vnd.google-apps.folder"
func main() {
sheet := flag.String("sheet", "", "spreadsheet id")
flag.Parse()
if *sheet == "" {
log.Fatal("[ERROR] -sheet <spreadsheet id> is required")
}
source, err := data.NewSheet(map[string]string{"directory": *sheet})
if err != nil {
log.Fatalf("[ERROR] sheet source: %v", err)
}
model, err := directory.LoadModel(source, nil)
if err != nil {
log.Fatalf("[ERROR] load model: %v", err)
}
localToHash := map[string]string{}
for _, family := range model.Families {
for _, adult := range family.AdultEmails {
local, _, _ := strings.Cut(adult, "@")
if existing, ok := localToHash[local]; ok && existing != family.Key {
log.Fatalf("[ERROR] adult local part %s maps to two families", local)
}
localToHash[local] = family.Key
}
}
svc, err := drive.NewService(context.Background(),
option.WithCredentialsFile(data.KeyFile),
option.WithScopes(drive.DriveScope))
if err != nil {
log.Fatalf("[ERROR] drive client: %v", err)
}
drives, err := svc.Drives.List().Do()
if err != nil {
log.Fatalf("[ERROR] list shared drives: %v", err)
}
if len(drives.Drives) != 1 {
log.Fatalf("[ERROR] expected one shared drive, found %d", len(drives.Drives))
}
folderList, err := svc.Files.List().
Q(fmt.Sprintf("name = 'families' and '%s' in parents and mimeType = '%s' and trashed = false", drives.Drives[0].Id, folderMime)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id)").Do()
if err != nil {
log.Fatalf("[ERROR] find families folder: %v", err)
}
if len(folderList.Files) != 1 {
log.Fatalf("[ERROR] expected one families folder, found %d", len(folderList.Files))
}
folderID := folderList.Files[0].Id
renamed, kept, unknown := 0, 0, 0
token := ""
for {
call := svc.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
log.Fatalf("[ERROR] list families folder: %v", err)
}
for _, f := range list.Files {
if f.MimeType == folderMime {
continue
}
ext := path.Ext(f.Name)
base := strings.TrimSuffix(f.Name, ext)
kind := ""
for _, k := range []string{"-photo", "-pronunciation"} {
if strings.HasSuffix(base, k) {
kind = k
}
}
if kind == "" {
log.Printf("[ERROR] unrecognized file name %q, leaving it", f.Name)
unknown++
continue
}
local := strings.TrimSuffix(base, kind)
hash, ok := localToHash[local]
if !ok {
if _, isCurrent := model.Families[local]; isCurrent {
kept++
continue
}
log.Printf("[ERROR] file %q matches no family adult, leaving it", f.Name)
unknown++
continue
}
newName := hash + kind + ext
_, err := svc.Files.Update(f.Id, &drive.File{Name: newName}).SupportsAllDrives(true).Do()
if err != nil {
log.Fatalf("[ERROR] rename %q to %q: %v", f.Name, newName, err)
}
log.Printf("renamed %q -> %q", f.Name, newName)
renamed++
}
if list.NextPageToken == "" {
break
}
token = list.NextPageToken
}
log.Printf("done: %d renamed, %d already keyed by hash, %d left untouched", renamed, kept, unknown)
}
+30
View File
@@ -0,0 +1,30 @@
// Command setcell sets one cell in a sheet tab by key column, appending the row if missing.
package main
import (
"flag"
"log"
"heliosian/internal/data"
)
func main() {
sheet := flag.String("sheet", "", "spreadsheet id")
tab := flag.String("tab", "", "tab title")
keyCol := flag.String("keycol", "Email", "key column name")
key := flag.String("key", "", "key value")
col := flag.String("col", "", "column to set")
value := flag.String("value", "", "value to write")
flag.Parse()
if *sheet == "" || *tab == "" || *key == "" || *col == "" {
log.Fatal("[ERROR] -sheet, -tab, -key, and -col are required")
}
source, err := data.NewSheet(map[string]string{"directory": *sheet})
if err != nil {
log.Fatalf("[ERROR] sheet source: %v", err)
}
if err := source.Upsert("directory", *tab, *keyCol, *key, *col, *value); err != nil {
log.Fatalf("[ERROR] set %s[%s=%s].%s: %v", *tab, *keyCol, *key, *col, err)
}
log.Printf("set %s[%s=%s].%s = %q", *tab, *keyCol, *key, *col, *value)
}
+77
View File
@@ -0,0 +1,77 @@
// Command startserver launches the app, waits for it to listen, prints the pid and an auth header, and leaves it running.
package main
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"flag"
"fmt"
"log"
"os"
"os/exec"
"strings"
"time"
)
func main() {
email := flag.String("email", "dev@heliosschool.org", "session email for the minted cookie")
flag.Parse()
key := os.Getenv("SESSION_KEY")
if key == "" {
log.Fatal("[ERROR] SESSION_KEY is required (the server and the minted cookie must share it)")
}
if os.Getenv("DIRECTORY_SHEET") == "" {
log.Fatal("[ERROR] DIRECTORY_SHEET is required")
}
logFile, err := os.Create("/tmp/heliosian-server.log")
if err != nil {
log.Fatalf("[ERROR] create server log: %v", err)
}
cmd := exec.Command("go", "run", ".")
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
log.Fatalf("[ERROR] start server: %v", err)
}
deadline := time.Now().Add(10 * time.Minute)
for {
if time.Now().After(deadline) {
cmd.Process.Kill()
log.Fatalf("[ERROR] server did not start within 10 minutes; log: /tmp/heliosian-server.log")
}
content, err := os.ReadFile("/tmp/heliosian-server.log")
if err != nil {
log.Fatalf("[ERROR] read server log: %v", err)
}
if strings.Contains(string(content), "listening on ") {
break
}
if cmd.ProcessState != nil || !processAlive(cmd.Process.Pid) {
fmt.Print(string(content))
log.Fatal("[ERROR] server exited before listening")
}
time.Sleep(time.Second)
}
payload := fmt.Sprintf("%s|%d", *email, time.Now().Add(24*time.Hour).Unix())
mac := hmac.New(sha256.New, []byte(key))
mac.Write([]byte(payload))
cookie := base64.RawURLEncoding.EncodeToString([]byte(payload)) + "." +
base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
content, _ := os.ReadFile("/tmp/heliosian-server.log")
for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") {
fmt.Println(line)
}
fmt.Printf("pid: %d\n", cmd.Process.Pid)
fmt.Println("log: /tmp/heliosian-server.log")
fmt.Printf("header: Cookie: session=%s\n", cookie)
cmd.Process.Release()
}
func processAlive(pid int) bool {
return exec.Command("kill", "-0", fmt.Sprint(pid)).Run() == nil
}