Load sheet into in-memory data model and serve it as the API

docs/dev.md

@@ -12,6 +12,12 @@ The server listens on http://localhost:8080 (override with `PORT`). Templates, s
 
 The server reads local data from `sampledata/`, mirroring the production Sheets layout: one directory per app, one CSV file per table, first row is the schema. It goes through the same data-source interface production backends implement, so app code never knows which backend it is talking to.
 
+## Real data
+
+    DIRECTORY_SHEET=<spreadsheet id> go run .
+
+switches the directory app to the Google Sheets source. At startup the directory tables are read from the spreadsheet and normalized into the in-memory data model (see `docs/data.md`); the server refuses to start if that load fails, and the model reloads every five minutes. Requires a service account key in `creds/` (any `*.json`; the directory is gitignored) with the Sheets API enabled and the spreadsheet shared read-only with the service account. Real data never leaves the process: nothing is written to disk.
+
 ## Layout
 
 - `main.go` — server entry point and app routing

internal/data/sheet.go

@@ -0,0 +1,77 @@
+package data
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"google.golang.org/api/option"
+	"google.golang.org/api/sheets/v4"
+)
+
+type Sheet struct {
+	service      *sheets.Service
+	spreadsheets map[string]string
+}
+
+func KeyFile() (string, error) {
+	matches, err := filepath.Glob("creds/*.json")
+	if err != nil {
+		return "", err
+	}
+	if len(matches) == 0 {
+		return "", fmt.Errorf("no service account key found in creds/")
+	}
+	return matches[0], nil
+}
+
+func NewSheet(keyFile string, spreadsheets map[string]string) (*Sheet, error) {
+	service, err := sheets.NewService(context.Background(),
+		option.WithCredentialsFile(keyFile),
+		option.WithScopes(sheets.SpreadsheetsReadonlyScope))
+	if err != nil {
+		return nil, err
+	}
+	return &Sheet{service: service, spreadsheets: spreadsheets}, nil
+}
+
+func (s *Sheet) Table(app, name string) ([]map[string]string, error) {
+	id, ok := s.spreadsheets[app]
+	if !ok {
+		return nil, fmt.Errorf("no spreadsheet configured for app %q", app)
+	}
+	resp, err := s.service.Spreadsheets.Values.Get(id, "'"+strings.ReplaceAll(name, "'", "''")+"'").Do()
+	if err != nil {
+		return nil, err
+	}
+	return toRecords(resp.Values), nil
+}
+
+func toRecords(values [][]interface{}) []map[string]string {
+	if len(values) == 0 {
+		return nil
+	}
+	header := make([]string, len(values[0]))
+	for i, cell := range values[0] {
+		header[i] = strings.TrimSpace(fmt.Sprint(cell))
+	}
+	records := []map[string]string{}
+	for _, row := range values[1:] {
+		record := map[string]string{}
+		for i, cell := range row {
+			if i >= len(header) || header[i] == "" {
+				continue
+			}
+			value := strings.TrimSpace(fmt.Sprint(cell))
+			if value == "" {
+				continue
+			}
+			record[header[i]] = value
+		}
+		if len(record) > 0 {
+			records = append(records, record)
+		}
+	}
+	return records
+}

internal/directory/cache.go

@@ -0,0 +1,55 @@
+package directory
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	"heliosian/internal/data"
+)
+
+const refreshInterval = 5 * time.Minute
+
+type Cache struct {
+	source data.Source
+	mu     sync.RWMutex
+	model  *Model
+}
+
+func NewCache(source data.Source) (*Cache, error) {
+	c := &Cache{source: source}
+	if err := c.refresh(); err != nil {
+		return nil, err
+	}
+	go c.refreshLoop()
+	return c, nil
+}
+
+func (c *Cache) Model() *Model {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.model
+}
+
+func (c *Cache) refreshLoop() {
+	for range time.Tick(refreshInterval) {
+		if err := c.refresh(); err != nil {
+			log.Printf("[ERROR] directory model refresh: %v", err)
+		}
+	}
+}
+
+func (c *Cache) refresh() error {
+	start := time.Now()
+	model, err := LoadModel(c.source)
+	if err != nil {
+		return err
+	}
+	c.mu.Lock()
+	c.model = model
+	c.mu.Unlock()
+	log.Printf("loaded directory model: %d people, %d families, %d classrooms, %d sections in %s",
+		len(model.People), len(model.Families), len(model.Classrooms), len(model.Sections),
+		time.Since(start).Round(time.Millisecond))
+	return nil
+}

internal/directory/directory.go

@@ -3,23 +3,19 @@ package directory
 
 import (
 	"encoding/json"
-	"fmt"
 	"html/template"
 	"log"
 	"net/http"
-	"sort"
-
-	"heliosian/internal/data"
 )
 
 type app struct {
-	source data.Source
+	cache *Cache
 }
 
-func Register(mux *http.ServeMux, source data.Source) {
-	a := app{source: source}
+func Register(mux *http.ServeMux, cache *Cache) {
+	a := app{cache: cache}
 	mux.HandleFunc("GET /directory/{$}", a.index)
-	mux.HandleFunc("GET /directory/api/people", a.people)
+	mux.HandleFunc("GET /directory/api/model", a.model)
 	mux.Handle("GET /directory/static/", http.StripPrefix("/directory/static/", http.FileServer(http.Dir("web/directory/static"))))
 }
 
@@ -34,65 +30,10 @@ func (a app) index(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-type person struct {
-	ID            string `json:"id"`
-	FirstName     string `json:"firstName"`
-	LastName      string `json:"lastName"`
-	Role          string `json:"role"`
-	Grade         string `json:"grade"`
-	Email         string `json:"email"`
-	Phone         string `json:"phone"`
-	Pronunciation string `json:"pronunciation"`
-	FamilyName    string `json:"familyName"`
-	Address       string `json:"address"`
-	FamilyPhone   string `json:"familyPhone"`
-}
-
-func (a app) people(w http.ResponseWriter, r *http.Request) {
-	families, err := a.source.Table("directory", "families")
-	if err != nil {
-		serverError(w, err)
-		return
-	}
-	familiesByID := map[string]map[string]string{}
-	for _, family := range families {
-		familiesByID[family["id"]] = family
-	}
-	rows, err := a.source.Table("directory", "people")
-	if err != nil {
-		serverError(w, err)
-		return
-	}
-	people := []person{}
-	for _, row := range rows {
-		family, ok := familiesByID[row["family_id"]]
-		if !ok {
-			serverError(w, fmt.Errorf("person %s has unknown family_id %q", row["id"], row["family_id"]))
-			return
-		}
-		people = append(people, person{
-			ID:            row["id"],
-			FirstName:     row["first_name"],
-			LastName:      row["last_name"],
-			Role:          row["role"],
-			Grade:         row["grade"],
-			Email:         row["email"],
-			Phone:         row["phone"],
-			Pronunciation: row["pronunciation"],
-			FamilyName:    family["name"],
-			Address:       family["address"],
-			FamilyPhone:   family["phone"],
-		})
-	}
-	sort.Slice(people, func(i, j int) bool {
-		if people[i].LastName != people[j].LastName {
-			return people[i].LastName < people[j].LastName
-		}
-		return people[i].FirstName < people[j].FirstName
-	})
+func (a app) model(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(people); err != nil {
-		log.Printf("[ERROR] encode people: %v", err)
+	if err := json.NewEncoder(w).Encode(a.cache.Model()); err != nil {
+		log.Printf("[ERROR] encode model: %v", err)
 	}
 }
 

internal/directory/load.go

@@ -0,0 +1,215 @@
+package directory
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+
+	"heliosian/internal/data"
+)
+
+const appName = "directory"
+
+func LoadModel(source data.Source) (*Model, error) {
+	tables := map[string][]map[string]string{}
+	for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} {
+		rows, err := source.Table(appName, name)
+		if err != nil {
+			return nil, err
+		}
+		tables[name] = rows
+	}
+
+	model := &Model{Families: map[string]Family{}, RoomParents: map[string][]string{}}
+
+	staffByEmail := map[string]map[string]string{}
+	for _, row := range tables["Staff Details"] {
+		email := strings.ToLower(row["Email Lower"])
+		if email != "" {
+			staffByEmail[email] = row
+		}
+	}
+
+	type familyAcc struct {
+		family     Family
+		hasParent  bool
+		hasStudent bool
+	}
+	families := map[string]*familyAcc{}
+	byEmail := map[string]Person{}
+	for _, row := range tables["Basic Directory"] {
+		email := strings.ToLower(row["Email Lower"])
+		if email == "" {
+			continue
+		}
+		p := Person{
+			Email:            email,
+			FullName:         row["Full Name"],
+			LegalName:        row["Legal Name"],
+			PreferredName:    row["Preferred Name"],
+			IsStaff:          row["Is Staff?"] == "TRUE",
+			IsParent:         row["Is Parent?"] == "TRUE",
+			IsStudent:        row["Is Student?"] == "TRUE",
+			Pronouns:         row["Pronouns"],
+			Facts:            row["Facts"],
+			PronunciationURL: row["Pronunciation"],
+			PhotoURL:         row["Primary Photo"],
+			Grade:            row["Grade"],
+			Classroom:        row["Class"],
+			Section:          row["Section"],
+			Phone:            row["Phone Number"],
+			FamilyKey:        strings.ToLower(row["Family Key"]),
+		}
+		for _, contact := range strings.Split(row["Parent Contact Emails"], ",") {
+			if contact = strings.ToLower(strings.TrimSpace(contact)); contact != "" {
+				p.ParentContactEmails = append(p.ParentContactEmails, contact)
+			}
+		}
+		if details, ok := staffByEmail[email]; ok {
+			p.JobTitle = details["Job Title"]
+			p.Department = details["Department"]
+			p.GradeBand = details["Grade Band"]
+		}
+		model.People = append(model.People, p)
+		byEmail[email] = p
+
+		if p.FamilyKey == "" {
+			continue
+		}
+		acc, ok := families[p.FamilyKey]
+		if !ok {
+			acc = &familyAcc{family: Family{Key: p.FamilyKey}}
+			families[p.FamilyKey] = acc
+		}
+		if acc.family.Address == "" && row["Address 1"] != "" {
+			acc.family.Address = row["Address 1"]
+			if row["Address 2"] != "" {
+				acc.family.Address += ", " + row["Address 2"]
+			}
+		}
+		if acc.family.PhotoURL == "" {
+			acc.family.PhotoURL = row["Family Photo"]
+		}
+		if acc.family.PhotoCaption == "" {
+			acc.family.PhotoCaption = row["Family Photo Description"]
+		}
+		if acc.family.PronunciationURL == "" {
+			acc.family.PronunciationURL = row["Family Pronunciation"]
+		}
+		if p.IsParent {
+			acc.hasParent = true
+			acc.family.AdultEmails = append(acc.family.AdultEmails, email)
+		}
+		if p.IsStudent {
+			acc.hasStudent = true
+			acc.family.KidEmails = append(acc.family.KidEmails, email)
+		}
+	}
+	for key, acc := range families {
+		if !acc.hasParent && !acc.hasStudent {
+			continue
+		}
+		acc.family.Name = familyName(acc.family, byEmail)
+		model.Families[key] = acc.family
+	}
+
+	sort.Slice(model.People, func(i, j int) bool {
+		si, sj := surname(model.People[i].FullName), surname(model.People[j].FullName)
+		if si != sj {
+			return si < sj
+		}
+		return model.People[i].FullName < model.People[j].FullName
+	})
+
+	for _, row := range tables["Classrooms"] {
+		if row["Class"] == "" {
+			continue
+		}
+		model.Classrooms = append(model.Classrooms, Classroom{
+			Name:        row["Class"],
+			ImageURL:    row["Classroom Image"],
+			HasSections: row["Has Sections"] == "TRUE",
+		})
+	}
+
+	for _, row := range tables["Schedules"] {
+		if row["Classroom"] == "" {
+			continue
+		}
+		section := Section{Classroom: row["Classroom"], Name: row["Section"], GradeBand: row["Grade Band"]}
+		for _, column := range []string{"Teacher 1", "Teacher 2", "Teacher 3"} {
+			if row[column] != "" {
+				section.Teachers = append(section.Teachers, row[column])
+			}
+		}
+		model.Sections = append(model.Sections, section)
+	}
+
+	for _, row := range tables["Grade Lookup"] {
+		if row["Current Grade"] == "" {
+			continue
+		}
+		model.Grades = append(model.Grades, Grade{
+			Name:     row["Current Grade"],
+			NextName: row["Next Grade"],
+			Band:     row["Current Gradeband"],
+			NextBand: row["Next Gradeband"],
+		})
+	}
+
+	for _, row := range tables["Room Parents"] {
+		band, email := row["Gradeband"], strings.ToLower(row["Email Address"])
+		if band == "" || email == "" {
+			continue
+		}
+		model.RoomParents[band] = append(model.RoomParents[band], email)
+	}
+
+	type department struct {
+		name  string
+		order float64
+	}
+	departments := []department{}
+	for _, row := range tables["Departments"] {
+		if row["Department"] == "" {
+			continue
+		}
+		order, err := strconv.ParseFloat(row["Order"], 64)
+		if err != nil {
+			order = float64(len(departments))
+		}
+		departments = append(departments, department{name: row["Department"], order: order})
+	}
+	sort.SliceStable(departments, func(i, j int) bool { return departments[i].order < departments[j].order })
+	for _, d := range departments {
+		model.Departments = append(model.Departments, d.name)
+	}
+
+	return model, nil
+}
+
+func surname(fullName string) string {
+	fields := strings.Fields(fullName)
+	if len(fields) == 0 {
+		return ""
+	}
+	return fields[len(fields)-1]
+}
+
+func familyName(f Family, byEmail map[string]Person) string {
+	members := append(append([]string{}, f.KidEmails...), f.AdultEmails...)
+	seen := map[string]bool{}
+	names := []string{}
+	for _, email := range members {
+		s := surname(byEmail[email].FullName)
+		if s == "" || seen[s] {
+			continue
+		}
+		seen[s] = true
+		names = append(names, s)
+	}
+	if len(names) == 0 {
+		return ""
+	}
+	return strings.Join(names, " & ") + " Family"
+}

internal/directory/model.go

@@ -0,0 +1,65 @@
+package directory
+
+type Person struct {
+	Email               string   `json:"email"`
+	FullName            string   `json:"fullName"`
+	LegalName           string   `json:"legalName,omitempty"`
+	PreferredName       string   `json:"preferredName,omitempty"`
+	IsStaff             bool     `json:"isStaff"`
+	IsParent            bool     `json:"isParent"`
+	IsStudent           bool     `json:"isStudent"`
+	Pronouns            string   `json:"pronouns,omitempty"`
+	Facts               string   `json:"facts,omitempty"`
+	PronunciationURL    string   `json:"pronunciationUrl,omitempty"`
+	PhotoURL            string   `json:"photoUrl,omitempty"`
+	Grade               string   `json:"grade,omitempty"`
+	Classroom           string   `json:"classroom,omitempty"`
+	Section             string   `json:"section,omitempty"`
+	Phone               string   `json:"phone,omitempty"`
+	FamilyKey           string   `json:"familyKey,omitempty"`
+	ParentContactEmails []string `json:"parentContactEmails,omitempty"`
+	JobTitle            string   `json:"jobTitle,omitempty"`
+	Department          string   `json:"department,omitempty"`
+	GradeBand           string   `json:"gradeBand,omitempty"`
+}
+
+type Family struct {
+	Key              string   `json:"key"`
+	Name             string   `json:"name,omitempty"`
+	Address          string   `json:"address,omitempty"`
+	PhotoURL         string   `json:"photoUrl,omitempty"`
+	PhotoCaption     string   `json:"photoCaption,omitempty"`
+	PronunciationURL string   `json:"pronunciationUrl,omitempty"`
+	AdultEmails      []string `json:"adultEmails,omitempty"`
+	KidEmails        []string `json:"kidEmails,omitempty"`
+}
+
+type Classroom struct {
+	Name        string `json:"name"`
+	ImageURL    string `json:"imageUrl,omitempty"`
+	HasSections bool   `json:"hasSections"`
+}
+
+type Section struct {
+	Classroom string   `json:"classroom"`
+	Name      string   `json:"name,omitempty"`
+	Teachers  []string `json:"teachers,omitempty"`
+	GradeBand string   `json:"gradeBand,omitempty"`
+}
+
+type Grade struct {
+	Name     string `json:"name"`
+	NextName string `json:"nextName,omitempty"`
+	Band     string `json:"band,omitempty"`
+	NextBand string `json:"nextBand,omitempty"`
+}
+
+type Model struct {
+	People      []Person            `json:"people"`
+	Families    map[string]Family   `json:"families"`
+	Classrooms  []Classroom         `json:"classrooms"`
+	Sections    []Section           `json:"sections"`
+	Grades      []Grade             `json:"grades"`
+	RoomParents map[string][]string `json:"roomParents"`
+	Departments []string            `json:"departments"`
+}

main.go

@@ -10,9 +10,29 @@ import (
 	"heliosian/internal/directory"
 )
 
+func directorySource() data.Source {
+	sheetID := os.Getenv("DIRECTORY_SHEET")
+	if sheetID == "" {
+		return data.Dir{Root: "sampledata"}
+	}
+	keyFile, err := data.KeyFile()
+	if err != nil {
+		log.Fatalf("[ERROR] %v", err)
+	}
+	source, err := data.NewSheet(keyFile, map[string]string{"directory": sheetID})
+	if err != nil {
+		log.Fatalf("[ERROR] load directory sheet: %v", err)
+	}
+	return source
+}
+
 func main() {
+	cache, err := directory.NewCache(directorySource())
+	if err != nil {
+		log.Fatalf("[ERROR] load directory data: %v", err)
+	}
 	mux := http.NewServeMux()
-	directory.Register(mux, data.Dir{Root: "sampledata"})
+	directory.Register(mux, cache)
 	mux.Handle("GET /{$}", http.RedirectHandler("/directory/", http.StatusFound))
 	port := os.Getenv("PORT")
 	if port == "" {

tools/sheets/main.go

@@ -26,6 +26,7 @@ func main() {
 	tab := flag.String("tab", "", "print rows of this tab instead of the overview")
 	rows := flag.Int("rows", 0, "with -tab, print only n rows")
 	from := flag.Int("from", 1, "with -tab and -rows, first row to print")
+	cells := flag.Bool("cells", false, "with -tab, print each non-empty cell with its column index")
 	flag.Parse()
 	if *sheet == "" {
 		log.Fatal("[ERROR] -sheet <spreadsheet id> is required")
@@ -46,6 +47,15 @@ func main() {
 			log.Fatalf("[ERROR] read tab %s: %v", *tab, err)
 		}
 		for _, row := range resp.Values {
+			if *cells {
+				for i, cell := range row {
+					if s := fmt.Sprint(cell); s != "" {
+						fmt.Printf("  %d: %q\n", i, s)
+					}
+				}
+				fmt.Println("---")
+				continue
+			}
 			fmt.Println(row)
 		}
 		return

web/directory/static/app.js

@@ -1,4 +1,4 @@
-const state = {people: []};
+const state = {model: null};
 
 function hue(text) {
   let h = 0;
@@ -19,20 +19,51 @@ function el(tag, className, text) {
   return node;
 }
 
-function card(p) {
-  const root = el('div', 'card');
-  const avatar = el('div', 'avatar', (p.firstName[0] || '') + (p.lastName[0] || ''));
-  avatar.style.background = `hsl(${hue(p.firstName + p.lastName)} 60% 45%)`;
-  root.append(avatar);
-  const info = el('div');
-  info.append(el('div', 'name', `${p.firstName} ${p.lastName}`));
-  if (p.pronunciation) {
-    info.append(el('div', 'pronunciation', p.pronunciation));
+function initials(name) {
+  const words = name.trim().split(/\s+/);
+  if (words.length === 0 || !words[0]) {
+    return '?';
   }
-  const who = p.role === 'student' ? `Student, grade ${p.grade}` : 'Parent';
-  info.append(el('div', 'detail', `${who} · ${p.familyName}`));
-  info.append(el('div', 'detail', p.address));
-  const contact = [p.phone || p.familyPhone, p.email].filter(Boolean).join(' · ');
+  const first = words[0][0];
+  return words.length > 1 ? first + words[words.length - 1][0] : first;
+}
+
+function roles(p) {
+  return [p.isStudent && 'Student', p.isParent && 'Parent', p.isStaff && 'Staff'].filter(Boolean);
+}
+
+function card(p, family) {
+  const root = el('div', 'card');
+  if (p.photoUrl) {
+    const img = el('img', 'avatar');
+    img.src = p.photoUrl;
+    img.loading = 'lazy';
+    root.append(img);
+  } else {
+    const avatar = el('div', 'avatar', initials(p.fullName));
+    avatar.style.background = `hsl(${hue(p.fullName)} 60% 45%)`;
+    root.append(avatar);
+  }
+  const info = el('div');
+  info.append(el('div', 'name', p.fullName));
+  if (p.pronouns) {
+    info.append(el('div', 'pronouns', p.pronouns));
+  }
+  const who = [roles(p).join(' / ')];
+  if (p.grade) {
+    who.push([p.grade, p.classroom, p.section].filter(Boolean).join(' ▶ '));
+  }
+  if (p.jobTitle) {
+    who.push(p.jobTitle);
+  }
+  info.append(el('div', 'detail', who.join(' · ')));
+  if (family && family.name) {
+    info.append(el('div', 'detail', family.name));
+  }
+  if (family && family.address) {
+    info.append(el('div', 'detail', family.address));
+  }
+  const contact = [p.phone, p.email].filter(Boolean).join(' · ');
   if (contact) {
     info.append(el('div', 'detail', contact));
   }
@@ -44,23 +75,28 @@ function render() {
   const q = document.querySelector('#search').value.trim().toLowerCase();
   const list = document.querySelector('#people');
   list.replaceChildren();
-  const matches = state.people.filter(p =>
-    `${p.firstName} ${p.lastName} ${p.familyName}`.toLowerCase().includes(q));
+  if (!state.model) {
+    return;
+  }
+  const matches = state.model.people.filter(p => {
+    const family = state.model.families[p.familyKey];
+    return `${p.fullName} ${family ? family.name : ''}`.toLowerCase().includes(q);
+  });
   if (matches.length === 0) {
     list.append(el('div', 'empty', 'No matches.'));
     return;
   }
   for (const p of matches) {
-    list.append(card(p));
+    list.append(card(p, state.model.families[p.familyKey]));
   }
 }
 
 async function load() {
-  const res = await fetch('/directory/api/people');
+  const res = await fetch('/directory/api/model');
   if (!res.ok) {
-    throw new Error(`loading people failed: ${res.status}`);
+    throw new Error(`loading model failed: ${res.status}`);
   }
-  state.people = await res.json();
+  state.model = await res.json();
   render();
 }
 

web/directory/static/style.css

@@ -70,6 +70,7 @@ header h1 {
   justify-content: center;
   font-weight: 600;
   flex-shrink: 0;
+  object-fit: cover;
 }
 
 .name {