From 97af37ba40f41c98cf5016151cfa33a60cd1b830 Mon Sep 17 00:00:00 2001 From: Ian Gulliver Date: Sat, 15 Aug 2026 16:22:25 -0700 Subject: [PATCH] Load sheet into in-memory data model and serve it as the API --- docs/dev.md | 6 + internal/data/sheet.go | 77 ++++++++++++ internal/directory/cache.go | 55 ++++++++ internal/directory/directory.go | 73 ++--------- internal/directory/load.go | 215 ++++++++++++++++++++++++++++++++ internal/directory/model.go | 65 ++++++++++ main.go | 22 +++- tools/sheets/main.go | 10 ++ web/directory/static/app.js | 76 ++++++++--- web/directory/static/style.css | 1 + 10 files changed, 513 insertions(+), 87 deletions(-) create mode 100644 internal/data/sheet.go create mode 100644 internal/directory/cache.go create mode 100644 internal/directory/load.go create mode 100644 internal/directory/model.go diff --git a/docs/dev.md b/docs/dev.md index b73adb4..091b588 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -12,6 +12,12 @@ The server listens on http://localhost:8080 (override with `PORT`). Templates, s The server reads local data from `sampledata/`, mirroring the production Sheets layout: one directory per app, one CSV file per table, first row is the schema. It goes through the same data-source interface production backends implement, so app code never knows which backend it is talking to. +## Real data + + DIRECTORY_SHEET= go run . + +switches the directory app to the Google Sheets source. At startup the directory tables are read from the spreadsheet and normalized into the in-memory data model (see `docs/data.md`); the server refuses to start if that load fails, and the model reloads every five minutes. Requires a service account key in `creds/` (any `*.json`; the directory is gitignored) with the Sheets API enabled and the spreadsheet shared read-only with the service account. Real data never leaves the process: nothing is written to disk. + ## Layout - `main.go` — server entry point and app routing diff --git a/internal/data/sheet.go b/internal/data/sheet.go new file mode 100644 index 0000000..1347569 --- /dev/null +++ b/internal/data/sheet.go @@ -0,0 +1,77 @@ +package data + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" +) + +type Sheet struct { + service *sheets.Service + spreadsheets map[string]string +} + +func KeyFile() (string, error) { + matches, err := filepath.Glob("creds/*.json") + if err != nil { + return "", err + } + if len(matches) == 0 { + return "", fmt.Errorf("no service account key found in creds/") + } + return matches[0], nil +} + +func NewSheet(keyFile string, spreadsheets map[string]string) (*Sheet, error) { + service, err := sheets.NewService(context.Background(), + option.WithCredentialsFile(keyFile), + option.WithScopes(sheets.SpreadsheetsReadonlyScope)) + if err != nil { + return nil, err + } + return &Sheet{service: service, spreadsheets: spreadsheets}, nil +} + +func (s *Sheet) Table(app, name string) ([]map[string]string, error) { + id, ok := s.spreadsheets[app] + if !ok { + return nil, fmt.Errorf("no spreadsheet configured for app %q", app) + } + resp, err := s.service.Spreadsheets.Values.Get(id, "'"+strings.ReplaceAll(name, "'", "''")+"'").Do() + if err != nil { + return nil, err + } + return toRecords(resp.Values), nil +} + +func toRecords(values [][]interface{}) []map[string]string { + if len(values) == 0 { + return nil + } + header := make([]string, len(values[0])) + for i, cell := range values[0] { + header[i] = strings.TrimSpace(fmt.Sprint(cell)) + } + records := []map[string]string{} + for _, row := range values[1:] { + record := map[string]string{} + for i, cell := range row { + if i >= len(header) || header[i] == "" { + continue + } + value := strings.TrimSpace(fmt.Sprint(cell)) + if value == "" { + continue + } + record[header[i]] = value + } + if len(record) > 0 { + records = append(records, record) + } + } + return records +} diff --git a/internal/directory/cache.go b/internal/directory/cache.go new file mode 100644 index 0000000..2a7df8e --- /dev/null +++ b/internal/directory/cache.go @@ -0,0 +1,55 @@ +package directory + +import ( + "log" + "sync" + "time" + + "heliosian/internal/data" +) + +const refreshInterval = 5 * time.Minute + +type Cache struct { + source data.Source + mu sync.RWMutex + model *Model +} + +func NewCache(source data.Source) (*Cache, error) { + c := &Cache{source: source} + if err := c.refresh(); err != nil { + return nil, err + } + go c.refreshLoop() + return c, nil +} + +func (c *Cache) Model() *Model { + c.mu.RLock() + defer c.mu.RUnlock() + return c.model +} + +func (c *Cache) refreshLoop() { + for range time.Tick(refreshInterval) { + if err := c.refresh(); err != nil { + log.Printf("[ERROR] directory model refresh: %v", err) + } + } +} + +func (c *Cache) refresh() error { + start := time.Now() + model, err := LoadModel(c.source) + if err != nil { + return err + } + c.mu.Lock() + c.model = model + c.mu.Unlock() + log.Printf("loaded directory model: %d people, %d families, %d classrooms, %d sections in %s", + len(model.People), len(model.Families), len(model.Classrooms), len(model.Sections), + time.Since(start).Round(time.Millisecond)) + return nil +} diff --git a/internal/directory/directory.go b/internal/directory/directory.go index 58c1391..bc0d831 100644 --- a/internal/directory/directory.go +++ b/internal/directory/directory.go @@ -3,23 +3,19 @@ package directory import ( "encoding/json" - "fmt" "html/template" "log" "net/http" - "sort" - - "heliosian/internal/data" ) type app struct { - source data.Source + cache *Cache } -func Register(mux *http.ServeMux, source data.Source) { - a := app{source: source} +func Register(mux *http.ServeMux, cache *Cache) { + a := app{cache: cache} mux.HandleFunc("GET /directory/{$}", a.index) - mux.HandleFunc("GET /directory/api/people", a.people) + mux.HandleFunc("GET /directory/api/model", a.model) mux.Handle("GET /directory/static/", http.StripPrefix("/directory/static/", http.FileServer(http.Dir("web/directory/static")))) } @@ -34,65 +30,10 @@ func (a app) index(w http.ResponseWriter, r *http.Request) { } } -type person struct { - ID string `json:"id"` - FirstName string `json:"firstName"` - LastName string `json:"lastName"` - Role string `json:"role"` - Grade string `json:"grade"` - Email string `json:"email"` - Phone string `json:"phone"` - Pronunciation string `json:"pronunciation"` - FamilyName string `json:"familyName"` - Address string `json:"address"` - FamilyPhone string `json:"familyPhone"` -} - -func (a app) people(w http.ResponseWriter, r *http.Request) { - families, err := a.source.Table("directory", "families") - if err != nil { - serverError(w, err) - return - } - familiesByID := map[string]map[string]string{} - for _, family := range families { - familiesByID[family["id"]] = family - } - rows, err := a.source.Table("directory", "people") - if err != nil { - serverError(w, err) - return - } - people := []person{} - for _, row := range rows { - family, ok := familiesByID[row["family_id"]] - if !ok { - serverError(w, fmt.Errorf("person %s has unknown family_id %q", row["id"], row["family_id"])) - return - } - people = append(people, person{ - ID: row["id"], - FirstName: row["first_name"], - LastName: row["last_name"], - Role: row["role"], - Grade: row["grade"], - Email: row["email"], - Phone: row["phone"], - Pronunciation: row["pronunciation"], - FamilyName: family["name"], - Address: family["address"], - FamilyPhone: family["phone"], - }) - } - sort.Slice(people, func(i, j int) bool { - if people[i].LastName != people[j].LastName { - return people[i].LastName < people[j].LastName - } - return people[i].FirstName < people[j].FirstName - }) +func (a app) model(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(people); err != nil { - log.Printf("[ERROR] encode people: %v", err) + if err := json.NewEncoder(w).Encode(a.cache.Model()); err != nil { + log.Printf("[ERROR] encode model: %v", err) } } diff --git a/internal/directory/load.go b/internal/directory/load.go new file mode 100644 index 0000000..5c1324f --- /dev/null +++ b/internal/directory/load.go @@ -0,0 +1,215 @@ +package directory + +import ( + "sort" + "strconv" + "strings" + + "heliosian/internal/data" +) + +const appName = "directory" + +func LoadModel(source data.Source) (*Model, error) { + tables := map[string][]map[string]string{} + for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} { + rows, err := source.Table(appName, name) + if err != nil { + return nil, err + } + tables[name] = rows + } + + model := &Model{Families: map[string]Family{}, RoomParents: map[string][]string{}} + + staffByEmail := map[string]map[string]string{} + for _, row := range tables["Staff Details"] { + email := strings.ToLower(row["Email Lower"]) + if email != "" { + staffByEmail[email] = row + } + } + + type familyAcc struct { + family Family + hasParent bool + hasStudent bool + } + families := map[string]*familyAcc{} + byEmail := map[string]Person{} + for _, row := range tables["Basic Directory"] { + email := strings.ToLower(row["Email Lower"]) + if email == "" { + continue + } + p := Person{ + Email: email, + FullName: row["Full Name"], + LegalName: row["Legal Name"], + PreferredName: row["Preferred Name"], + IsStaff: row["Is Staff?"] == "TRUE", + IsParent: row["Is Parent?"] == "TRUE", + IsStudent: row["Is Student?"] == "TRUE", + Pronouns: row["Pronouns"], + Facts: row["Facts"], + PronunciationURL: row["Pronunciation"], + PhotoURL: row["Primary Photo"], + Grade: row["Grade"], + Classroom: row["Class"], + Section: row["Section"], + Phone: row["Phone Number"], + FamilyKey: strings.ToLower(row["Family Key"]), + } + for _, contact := range strings.Split(row["Parent Contact Emails"], ",") { + if contact = strings.ToLower(strings.TrimSpace(contact)); contact != "" { + p.ParentContactEmails = append(p.ParentContactEmails, contact) + } + } + if details, ok := staffByEmail[email]; ok { + p.JobTitle = details["Job Title"] + p.Department = details["Department"] + p.GradeBand = details["Grade Band"] + } + model.People = append(model.People, p) + byEmail[email] = p + + if p.FamilyKey == "" { + continue + } + acc, ok := families[p.FamilyKey] + if !ok { + acc = &familyAcc{family: Family{Key: p.FamilyKey}} + families[p.FamilyKey] = acc + } + if acc.family.Address == "" && row["Address 1"] != "" { + acc.family.Address = row["Address 1"] + if row["Address 2"] != "" { + acc.family.Address += ", " + row["Address 2"] + } + } + if acc.family.PhotoURL == "" { + acc.family.PhotoURL = row["Family Photo"] + } + if acc.family.PhotoCaption == "" { + acc.family.PhotoCaption = row["Family Photo Description"] + } + if acc.family.PronunciationURL == "" { + acc.family.PronunciationURL = row["Family Pronunciation"] + } + if p.IsParent { + acc.hasParent = true + acc.family.AdultEmails = append(acc.family.AdultEmails, email) + } + if p.IsStudent { + acc.hasStudent = true + acc.family.KidEmails = append(acc.family.KidEmails, email) + } + } + for key, acc := range families { + if !acc.hasParent && !acc.hasStudent { + continue + } + acc.family.Name = familyName(acc.family, byEmail) + model.Families[key] = acc.family + } + + sort.Slice(model.People, func(i, j int) bool { + si, sj := surname(model.People[i].FullName), surname(model.People[j].FullName) + if si != sj { + return si < sj + } + return model.People[i].FullName < model.People[j].FullName + }) + + for _, row := range tables["Classrooms"] { + if row["Class"] == "" { + continue + } + model.Classrooms = append(model.Classrooms, Classroom{ + Name: row["Class"], + ImageURL: row["Classroom Image"], + HasSections: row["Has Sections"] == "TRUE", + }) + } + + for _, row := range tables["Schedules"] { + if row["Classroom"] == "" { + continue + } + section := Section{Classroom: row["Classroom"], Name: row["Section"], GradeBand: row["Grade Band"]} + for _, column := range []string{"Teacher 1", "Teacher 2", "Teacher 3"} { + if row[column] != "" { + section.Teachers = append(section.Teachers, row[column]) + } + } + model.Sections = append(model.Sections, section) + } + + for _, row := range tables["Grade Lookup"] { + if row["Current Grade"] == "" { + continue + } + model.Grades = append(model.Grades, Grade{ + Name: row["Current Grade"], + NextName: row["Next Grade"], + Band: row["Current Gradeband"], + NextBand: row["Next Gradeband"], + }) + } + + for _, row := range tables["Room Parents"] { + band, email := row["Gradeband"], strings.ToLower(row["Email Address"]) + if band == "" || email == "" { + continue + } + model.RoomParents[band] = append(model.RoomParents[band], email) + } + + type department struct { + name string + order float64 + } + departments := []department{} + for _, row := range tables["Departments"] { + if row["Department"] == "" { + continue + } + order, err := strconv.ParseFloat(row["Order"], 64) + if err != nil { + order = float64(len(departments)) + } + departments = append(departments, department{name: row["Department"], order: order}) + } + sort.SliceStable(departments, func(i, j int) bool { return departments[i].order < departments[j].order }) + for _, d := range departments { + model.Departments = append(model.Departments, d.name) + } + + return model, nil +} + +func surname(fullName string) string { + fields := strings.Fields(fullName) + if len(fields) == 0 { + return "" + } + return fields[len(fields)-1] +} + +func familyName(f Family, byEmail map[string]Person) string { + members := append(append([]string{}, f.KidEmails...), f.AdultEmails...) + seen := map[string]bool{} + names := []string{} + for _, email := range members { + s := surname(byEmail[email].FullName) + if s == "" || seen[s] { + continue + } + seen[s] = true + names = append(names, s) + } + if len(names) == 0 { + return "" + } + return strings.Join(names, " & ") + " Family" +} diff --git a/internal/directory/model.go b/internal/directory/model.go new file mode 100644 index 0000000..cd0e128 --- /dev/null +++ b/internal/directory/model.go @@ -0,0 +1,65 @@ +package directory + +type Person struct { + Email string `json:"email"` + FullName string `json:"fullName"` + LegalName string `json:"legalName,omitempty"` + PreferredName string `json:"preferredName,omitempty"` + IsStaff bool `json:"isStaff"` + IsParent bool `json:"isParent"` + IsStudent bool `json:"isStudent"` + Pronouns string `json:"pronouns,omitempty"` + Facts string `json:"facts,omitempty"` + PronunciationURL string `json:"pronunciationUrl,omitempty"` + PhotoURL string `json:"photoUrl,omitempty"` + Grade string `json:"grade,omitempty"` + Classroom string `json:"classroom,omitempty"` + Section string `json:"section,omitempty"` + Phone string `json:"phone,omitempty"` + FamilyKey string `json:"familyKey,omitempty"` + ParentContactEmails []string `json:"parentContactEmails,omitempty"` + JobTitle string `json:"jobTitle,omitempty"` + Department string `json:"department,omitempty"` + GradeBand string `json:"gradeBand,omitempty"` +} + +type Family struct { + Key string `json:"key"` + Name string `json:"name,omitempty"` + Address string `json:"address,omitempty"` + PhotoURL string `json:"photoUrl,omitempty"` + PhotoCaption string `json:"photoCaption,omitempty"` + PronunciationURL string `json:"pronunciationUrl,omitempty"` + AdultEmails []string `json:"adultEmails,omitempty"` + KidEmails []string `json:"kidEmails,omitempty"` +} + +type Classroom struct { + Name string `json:"name"` + ImageURL string `json:"imageUrl,omitempty"` + HasSections bool `json:"hasSections"` +} + +type Section struct { + Classroom string `json:"classroom"` + Name string `json:"name,omitempty"` + Teachers []string `json:"teachers,omitempty"` + GradeBand string `json:"gradeBand,omitempty"` +} + +type Grade struct { + Name string `json:"name"` + NextName string `json:"nextName,omitempty"` + Band string `json:"band,omitempty"` + NextBand string `json:"nextBand,omitempty"` +} + +type Model struct { + People []Person `json:"people"` + Families map[string]Family `json:"families"` + Classrooms []Classroom `json:"classrooms"` + Sections []Section `json:"sections"` + Grades []Grade `json:"grades"` + RoomParents map[string][]string `json:"roomParents"` + Departments []string `json:"departments"` +} diff --git a/main.go b/main.go index 5f67ee0..c2ce51c 100644 --- a/main.go +++ b/main.go @@ -10,9 +10,29 @@ import ( "heliosian/internal/directory" ) +func directorySource() data.Source { + sheetID := os.Getenv("DIRECTORY_SHEET") + if sheetID == "" { + return data.Dir{Root: "sampledata"} + } + keyFile, err := data.KeyFile() + if err != nil { + log.Fatalf("[ERROR] %v", err) + } + source, err := data.NewSheet(keyFile, map[string]string{"directory": sheetID}) + if err != nil { + log.Fatalf("[ERROR] load directory sheet: %v", err) + } + return source +} + func main() { + cache, err := directory.NewCache(directorySource()) + if err != nil { + log.Fatalf("[ERROR] load directory data: %v", err) + } mux := http.NewServeMux() - directory.Register(mux, data.Dir{Root: "sampledata"}) + directory.Register(mux, cache) mux.Handle("GET /{$}", http.RedirectHandler("/directory/", http.StatusFound)) port := os.Getenv("PORT") if port == "" { diff --git a/tools/sheets/main.go b/tools/sheets/main.go index 3f71679..0725b3c 100644 --- a/tools/sheets/main.go +++ b/tools/sheets/main.go @@ -26,6 +26,7 @@ func main() { tab := flag.String("tab", "", "print rows of this tab instead of the overview") rows := flag.Int("rows", 0, "with -tab, print only n rows") from := flag.Int("from", 1, "with -tab and -rows, first row to print") + cells := flag.Bool("cells", false, "with -tab, print each non-empty cell with its column index") flag.Parse() if *sheet == "" { log.Fatal("[ERROR] -sheet is required") @@ -46,6 +47,15 @@ func main() { log.Fatalf("[ERROR] read tab %s: %v", *tab, err) } for _, row := range resp.Values { + if *cells { + for i, cell := range row { + if s := fmt.Sprint(cell); s != "" { + fmt.Printf(" %d: %q\n", i, s) + } + } + fmt.Println("---") + continue + } fmt.Println(row) } return diff --git a/web/directory/static/app.js b/web/directory/static/app.js index bb41c0a..c0c2ca5 100644 --- a/web/directory/static/app.js +++ b/web/directory/static/app.js @@ -1,4 +1,4 @@ -const state = {people: []}; +const state = {model: null}; function hue(text) { let h = 0; @@ -19,20 +19,51 @@ function el(tag, className, text) { return node; } -function card(p) { - const root = el('div', 'card'); - const avatar = el('div', 'avatar', (p.firstName[0] || '') + (p.lastName[0] || '')); - avatar.style.background = `hsl(${hue(p.firstName + p.lastName)} 60% 45%)`; - root.append(avatar); - const info = el('div'); - info.append(el('div', 'name', `${p.firstName} ${p.lastName}`)); - if (p.pronunciation) { - info.append(el('div', 'pronunciation', p.pronunciation)); +function initials(name) { + const words = name.trim().split(/\s+/); + if (words.length === 0 || !words[0]) { + return '?'; } - const who = p.role === 'student' ? `Student, grade ${p.grade}` : 'Parent'; - info.append(el('div', 'detail', `${who} · ${p.familyName}`)); - info.append(el('div', 'detail', p.address)); - const contact = [p.phone || p.familyPhone, p.email].filter(Boolean).join(' · '); + const first = words[0][0]; + return words.length > 1 ? first + words[words.length - 1][0] : first; +} + +function roles(p) { + return [p.isStudent && 'Student', p.isParent && 'Parent', p.isStaff && 'Staff'].filter(Boolean); +} + +function card(p, family) { + const root = el('div', 'card'); + if (p.photoUrl) { + const img = el('img', 'avatar'); + img.src = p.photoUrl; + img.loading = 'lazy'; + root.append(img); + } else { + const avatar = el('div', 'avatar', initials(p.fullName)); + avatar.style.background = `hsl(${hue(p.fullName)} 60% 45%)`; + root.append(avatar); + } + const info = el('div'); + info.append(el('div', 'name', p.fullName)); + if (p.pronouns) { + info.append(el('div', 'pronouns', p.pronouns)); + } + const who = [roles(p).join(' / ')]; + if (p.grade) { + who.push([p.grade, p.classroom, p.section].filter(Boolean).join(' ▶ ')); + } + if (p.jobTitle) { + who.push(p.jobTitle); + } + info.append(el('div', 'detail', who.join(' · '))); + if (family && family.name) { + info.append(el('div', 'detail', family.name)); + } + if (family && family.address) { + info.append(el('div', 'detail', family.address)); + } + const contact = [p.phone, p.email].filter(Boolean).join(' · '); if (contact) { info.append(el('div', 'detail', contact)); } @@ -44,23 +75,28 @@ function render() { const q = document.querySelector('#search').value.trim().toLowerCase(); const list = document.querySelector('#people'); list.replaceChildren(); - const matches = state.people.filter(p => - `${p.firstName} ${p.lastName} ${p.familyName}`.toLowerCase().includes(q)); + if (!state.model) { + return; + } + const matches = state.model.people.filter(p => { + const family = state.model.families[p.familyKey]; + return `${p.fullName} ${family ? family.name : ''}`.toLowerCase().includes(q); + }); if (matches.length === 0) { list.append(el('div', 'empty', 'No matches.')); return; } for (const p of matches) { - list.append(card(p)); + list.append(card(p, state.model.families[p.familyKey])); } } async function load() { - const res = await fetch('/directory/api/people'); + const res = await fetch('/directory/api/model'); if (!res.ok) { - throw new Error(`loading people failed: ${res.status}`); + throw new Error(`loading model failed: ${res.status}`); } - state.people = await res.json(); + state.model = await res.json(); render(); } diff --git a/web/directory/static/style.css b/web/directory/static/style.css index 278c6d7..125617b 100644 --- a/web/directory/static/style.css +++ b/web/directory/static/style.css @@ -70,6 +70,7 @@ header h1 { justify-content: center; font-weight: 600; flex-shrink: 0; + object-fit: cover; } .name {