Load the directory model from the layered Veracross import

internal/blob/blob.go

@@ -231,6 +231,13 @@ func (s *Store) Refresh() error {
 	return s.refresh()
 }
 
+func (s *Store) Has(key string) bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	_, ok := s.entries[key]
+	return ok
+}
+
 func (s *Store) Upload(folder, base, ext, mimeType string, content []byte) (string, error) {
 	folderID, err := s.subfolder(folder)
 	if err != nil {

internal/data/data.go

@@ -3,40 +3,35 @@ package data
 
 import (
 	"encoding/csv"
-	"fmt"
 	"os"
 	"path/filepath"
 )
 
 type Source interface {
-	Table(app, name string) ([]map[string]string, error)
+	Table(app, name string) ([]string, []map[string]string, error)
 }
 
 type Dir struct {
 	Root string
 }
 
-func (d Dir) Table(app, name string) ([]map[string]string, error) {
+func (d Dir) Table(app, name string) ([]string, []map[string]string, error) {
 	f, err := os.Open(filepath.Join(d.Root, app, name+".csv"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defer f.Close()
 	rows, err := csv.NewReader(f).ReadAll()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	if len(rows) == 0 {
+	values := make([][]interface{}, len(rows))
-		return nil, fmt.Errorf("table %s/%s has no header row", app, name)
+	for i, row := range rows {
+		cells := make([]interface{}, len(row))
+		for j, cell := range row {
+			cells[j] = cell
 		}
-	header := rows[0]
+		values[i] = cells
-	records := []map[string]string{}
-	for _, row := range rows[1:] {
-		record := map[string]string{}
-		for i, column := range header {
-			record[column] = row[i]
 	}
-		records = append(records, record)
+	return parseTable(name, values)
-	}
-	return records, nil
 }

internal/data/sheet.go

@@ -26,24 +26,24 @@ func NewSheet(spreadsheets map[string]string) (*Sheet, error) {
 	return &Sheet{service: service, spreadsheets: spreadsheets}, nil
 }
 
-func (s *Sheet) Table(app, name string) ([]map[string]string, error) {
+func (s *Sheet) Table(app, name string) ([]string, []map[string]string, error) {
 	id, ok := s.spreadsheets[app]
 	if !ok {
-		return nil, fmt.Errorf("no spreadsheet configured for app %q", app)
+		return nil, nil, fmt.Errorf("no spreadsheet configured for app %q", app)
 	}
-	resp, err := s.service.Spreadsheets.Values.Get(id, "'"+strings.ReplaceAll(name, "'", "''")+"'").Do()
+	resp, err := s.service.Spreadsheets.Values.Get(id, quoteTab(name)).Do()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return toRecords(resp.Values), nil
+	return parseTable(name, resp.Values)
 }
 
-func (s *Sheet) SetColumn(app, table, keyColumn, keyValue, column, value string) error {
+func (s *Sheet) Upsert(app, table, keyColumn, keyValue, column, value string) error {
 	id, ok := s.spreadsheets[app]
 	if !ok {
 		return fmt.Errorf("no spreadsheet configured for app %q", app)
 	}
-	quoted := "'" + strings.ReplaceAll(table, "'", "''") + "'"
+	quoted := quoteTab(table)
 	resp, err := s.service.Spreadsheets.Values.Get(id, quoted).Do()
 	if err != nil {
 		return err
@@ -74,7 +74,17 @@ func (s *Sheet) SetColumn(app, table, keyColumn, keyValue, column, value string)
 		})
 	}
 	if len(ranges) == 0 {
-		return fmt.Errorf("no row in %s has %s = %q", table, keyColumn, keyValue)
+		width := max(keyIdx, colIdx) + 1
+		row := make([]interface{}, width)
+		for i := range row {
+			row[i] = ""
+		}
+		row[keyIdx] = keyValue
+		row[colIdx] = value
+		_, err := s.service.Spreadsheets.Values.Append(id, quoted, &sheets.ValueRange{
+			Values: [][]interface{}{row},
+		}).ValueInputOption("RAW").InsertDataOption("INSERT_ROWS").Do()
+		return err
 	}
 	_, err = s.service.Spreadsheets.Values.BatchUpdate(id, &sheets.BatchUpdateValuesRequest{
 		ValueInputOption: "RAW",
@@ -99,7 +109,7 @@ func (s *Sheet) Append(app, table string, header, row []string) error {
 			break
 		}
 	}
-	quoted := "'" + strings.ReplaceAll(table, "'", "''") + "'"
+	quoted := quoteTab(table)
 	if !exists {
 		_, err := s.service.Spreadsheets.BatchUpdate(id, &sheets.BatchUpdateSpreadsheetRequest{
 			Requests: []*sheets.Request{{AddSheet: &sheets.AddSheetRequest{
@@ -127,6 +137,10 @@ func (s *Sheet) appendRow(id, quotedTable string, row []string) error {
 	return err
 }
 
+func quoteTab(title string) string {
+	return "'" + strings.ReplaceAll(title, "'", "''") + "'"
+}
+
 func columnName(idx int) string {
 	name := ""
 	for idx >= 0 {
@@ -136,13 +150,19 @@ func columnName(idx int) string {
 	return name
 }
 
-func toRecords(values [][]interface{}) []map[string]string {
+func parseTable(name string, values [][]interface{}) ([]string, []map[string]string, error) {
 	if len(values) == 0 {
-		return nil
+		return nil, nil, fmt.Errorf("table %s has no header row", name)
 	}
 	header := make([]string, len(values[0]))
+	seen := map[string]bool{}
 	for i, cell := range values[0] {
-		header[i] = strings.TrimSpace(fmt.Sprint(cell))
+		h := strings.TrimSpace(fmt.Sprint(cell))
+		if h != "" && seen[h] {
+			return nil, nil, fmt.Errorf("table %s has duplicate header %q", name, h)
+		}
+		seen[h] = true
+		header[i] = h
 	}
 	records := []map[string]string{}
 	for _, row := range values[1:] {
@@ -161,5 +181,5 @@ func toRecords(values [][]interface{}) []map[string]string {
 			records = append(records, record)
 		}
 	}
-	return records
+	return header, records, nil
 }

internal/directory/cache.go

@@ -14,12 +14,13 @@ const refreshInterval = 5 * time.Minute
 type Cache struct {
 	source   data.Source
 	geocoder *geocode.Client
+	blobs    BlobChecker
 	mu       sync.RWMutex
 	model    *Model
 }
 
-func NewCache(source data.Source, geocoder *geocode.Client) (*Cache, error) {
+func NewCache(source data.Source, geocoder *geocode.Client, blobs BlobChecker) (*Cache, error) {
-	c := &Cache{source: source, geocoder: geocoder}
+	c := &Cache{source: source, geocoder: geocoder, blobs: blobs}
 	if err := c.refresh(); err != nil {
 		return nil, err
 	}
@@ -47,7 +48,7 @@ func (c *Cache) refreshLoop() {
 
 func (c *Cache) refresh() error {
 	start := time.Now()
-	model, err := LoadModel(c.source)
+	model, err := LoadModel(c.source, c.blobs)
 	if err != nil {
 		return err
 	}

internal/directory/load.go

@@ -1,8 +1,13 @@
 package directory
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
 	"sort"
-	"strconv"
 	"strings"
 
 	"heliosian/internal/data"
@@ -10,110 +15,454 @@ import (
 
 const appName = "directory"
 
-func LoadModel(source data.Source) (*Model, error) {
+var gradeOrder = []string{
-	tables := map[string][]map[string]string{}
+	"Kindergarten", "Grade 1", "Grade 2", "Grade 3", "Grade 4",
-	for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} {
+	"Grade 5", "Grade 6", "Grade 7", "Grade 8",
-		rows, err := source.Table(appName, name)
+}
+
+var gradeBands = map[string]string{
+	"Kindergarten": "Hummingbirds",
+	"Grade 1":      "Halcons",
+	"Grade 2":      "Halcons",
+	"Grade 3":      "Jayvens",
+	"Grade 4":      "Jayvens",
+	"Grade 5":      "Cospreys",
+	"Grade 6":      "Cospreys",
+	"Grade 7":      "Hegrets",
+	"Grade 8":      "Hegrets",
+}
+
+var departmentOrder = []string{
+	"Admin and Office Staff",
+	"Co-Curriculars and Specialists",
+	"Classroom Teachers",
+	"Facilities Staff",
+}
+
+var importColumns = []string{
+	"entry_sort_name", "student_full_name", "student_classifications", "student_email", "student_phone_mobile",
+}
+
+var overrideColumns = []string{
+	"Email", "Added", "Full Name", "Legal Name", "Preferred Name",
+	"Is Student", "Is Parent", "Is Staff", "New to Helios", "Pronouns", "Facts",
+	"Grade", "Classroom", "Crew", "Phone", "Job Title", "Department", "Grade Band", "Room Parent",
+	"Address", "Family Phone", "Family Photo Caption",
+}
+
+type BlobChecker interface {
+	Has(key string) bool
+}
+
+type parsedName struct {
+	display, legal, preferred string
+}
+
+var nameForm = regexp.MustCompile(`^(.+?) \((.+?)\) (.+)$`)
+
+func parseName(raw string) parsedName {
+	raw = strings.Join(strings.Fields(raw), " ")
+	m := nameForm.FindStringSubmatch(raw)
+	if m == nil {
+		return parsedName{display: raw, legal: raw}
+	}
+	return parsedName{display: m[1] + " " + m[3], legal: m[2] + " " + m[3], preferred: m[1]}
+}
+
+func splitHomeroom(homeroom string) (classroom, crew string) {
+	fields := strings.Fields(homeroom)
+	if len(fields) == 1 {
+		return homeroom, ""
+	}
+	return fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ")
+}
+
+func normName(raw string) string {
+	return strings.ToLower(strings.Join(strings.Fields(raw), " "))
+}
+
+type household struct {
+	adults  []string
+	kids    []string
+	address string
+	phone   string
+}
+
+func requireColumns(table string, header, wanted []string) error {
+	present := map[string]bool{}
+	for _, h := range header {
+		present[h] = true
+	}
+	for _, w := range wanted {
+		if !present[w] {
+			return fmt.Errorf("table %s is missing column %q", table, w)
+		}
+	}
+	return nil
+}
+
+func familyHash(members []string) string {
+	sorted := append([]string{}, members...)
+	sort.Strings(sorted)
+	sum := sha256.Sum256([]byte(strings.Join(sorted, "\n")))
+	return hex.EncodeToString(sum[:])[:16]
+}
+
+func LoadModel(source data.Source, blobs BlobChecker) (*Model, error) {
+	importHeader, importRows, err := source.Table(appName, "Veracross Import")
 	if err != nil {
 		return nil, err
 	}
-		tables[name] = rows
+	if err := requireColumns("Veracross Import", importHeader, importColumns); err != nil {
+		return nil, err
+	}
+	mapHeader, mapRows, err := source.Table(appName, "Name to Email")
+	if err != nil {
+		return nil, err
+	}
+	if err := requireColumns("Name to Email", mapHeader, []string{"Name", "Email"}); err != nil {
+		return nil, err
+	}
+	overrideHeader, overrideRows, err := source.Table(appName, "Overrides")
+	if err != nil {
+		return nil, err
+	}
+	if err := requireColumns("Overrides", overrideHeader, overrideColumns); err != nil {
+		return nil, err
+	}
+
+	nameToEmail := map[string]string{}
+	for _, row := range mapRows {
+		name, email := normName(row["Name"]), strings.ToLower(row["Email"])
+		if name == "" || email == "" {
+			return nil, fmt.Errorf("name to email row %v is incomplete", row)
+		}
+		if _, ok := nameToEmail[name]; ok {
+			return nil, fmt.Errorf("name to email has duplicate name %q", row["Name"])
+		}
+		nameToEmail[name] = email
+	}
+	mappingUses := map[string]int{}
+
+	people := map[string]*Person{}
+	order := []string{}
+	households := map[string]*household{}
+	householdOrder := []string{}
+	personHouseholds := map[string][]string{}
+
+	addAdult := func(rawName, email, phone string) error {
+		n := parseName(rawName)
+		if p, ok := people[email]; ok {
+			if p.FullName != n.display || p.LegalName != n.legal || p.PreferredName != n.preferred {
+				return fmt.Errorf("adult %s has conflicting names %q and %q", email, p.FullName, rawName)
+			}
+			if p.Phone != "" && phone != "" && p.Phone != phone {
+				return fmt.Errorf("adult %s has conflicting phones %q and %q", email, p.Phone, phone)
+			}
+			if p.Phone == "" {
+				p.Phone = phone
+			}
+			p.IsParent = true
+			return nil
+		}
+		people[email] = &Person{
+			Email: email, FullName: n.display, LegalName: n.legal, PreferredName: n.preferred,
+			Phone: phone, IsParent: true,
+		}
+		order = append(order, email)
+		return nil
+	}
+
+	for _, row := range importRows {
+		rawName := row["student_full_name"]
+		if rawName == "" {
+			return nil, fmt.Errorf("import row %v has no student name", row)
+		}
+		var classifications struct {
+			GradeLevel string `json:"grade_level"`
+			Homeroom   string `json:"homeroom"`
+		}
+		if err := json.Unmarshal([]byte(row["student_classifications"]), &classifications); err != nil {
+			return nil, fmt.Errorf("student %s classifications: %w", rawName, err)
+		}
+		if gradeBands[classifications.GradeLevel] == "" {
+			return nil, fmt.Errorf("student %s has unknown grade %q", rawName, classifications.GradeLevel)
+		}
+		if classifications.Homeroom == "" {
+			return nil, fmt.Errorf("student %s has no homeroom", rawName)
+		}
+		classroom, crew := splitHomeroom(classifications.Homeroom)
+
+		email := strings.ToLower(row["student_email"])
+		if mapped, ok := nameToEmail[normName(rawName)]; ok {
+			email = mapped
+			mappingUses[normName(rawName)]++
+		}
+		if email == "" {
+			return nil, fmt.Errorf("student %s has no email and no name to email entry", rawName)
+		}
+		if _, ok := people[email]; ok {
+			return nil, fmt.Errorf("student email %s appears twice", email)
+		}
+		n := parseName(rawName)
+		student := &Person{
+			Email: email, FullName: n.display, LegalName: n.legal, PreferredName: n.preferred,
+			IsStudent: true, Grade: classifications.GradeLevel, Classroom: classroom, Section: crew,
+			Phone: row["student_phone_mobile"],
+		}
+		people[email] = student
+		order = append(order, email)
+
+		for _, hn := range []string{"1", "2"} {
+			adults := []string{}
+			for _, pn := range []string{"1", "2"} {
+				prefix := "household_" + hn + "_person_" + pn + "_"
+				if row[prefix+"full_name"] == "" {
+					continue
+				}
+				adultEmail := strings.ToLower(row[prefix+"email"])
+				if adultEmail == "" {
+					return nil, fmt.Errorf("adult %q of student %s has no email", row[prefix+"full_name"], rawName)
+				}
+				phone := row[prefix+"phone_mobile"]
+				if phone == "" {
+					phone = row[prefix+"phone_business"]
+				}
+				if err := addAdult(row[prefix+"full_name"], adultEmail, phone); err != nil {
+					return nil, err
+				}
+				adults = append(adults, adultEmail)
+			}
+			if len(adults) == 0 {
+				continue
+			}
+			address, phone := row["household_"+hn+"_address"], row["household_"+hn+"_phone"]
+			setKey := strings.Join(func() []string {
+				s := append([]string{}, adults...)
+				sort.Strings(s)
+				return s
+			}(), "\n")
+			hh, ok := households[setKey]
+			if !ok {
+				hh = &household{adults: adults, address: address, phone: phone}
+				households[setKey] = hh
+				householdOrder = append(householdOrder, setKey)
+				for _, a := range adults {
+					if len(personHouseholds[a]) > 0 {
+						return nil, fmt.Errorf("adult %s belongs to more than one household", a)
+					}
+					personHouseholds[a] = append(personHouseholds[a], setKey)
+				}
+			} else if hh.address != address || hh.phone != phone {
+				return nil, fmt.Errorf("household of %v has conflicting address or phone across rows", adults)
+			}
+			hh.kids = append(hh.kids, email)
+			personHouseholds[email] = append(personHouseholds[email], setKey)
+			student.ParentContactEmails = append(student.ParentContactEmails, adults...)
+		}
+	}
+
+	for name := range nameToEmail {
+		switch mappingUses[name] {
+		case 0:
+			return nil, fmt.Errorf("name to email entry %q matches no import row", name)
+		case 1:
+		default:
+			return nil, fmt.Errorf("name to email entry %q matches %d import rows", name, mappingUses[name])
+		}
+	}
+
+	type familyCells struct {
+		address, phone, caption string
+		hasAddress, hasPhone, hasCaption bool
+	}
+	familyOverrides := map[string]familyCells{}
+	bandSet := map[string]bool{}
+	for _, band := range gradeBands {
+		bandSet[band] = true
+	}
+	roomParents := map[string][]string{}
+
+	seenOverride := map[string]bool{}
+	for _, row := range overrideRows {
+		email := strings.ToLower(row["Email"])
+		if email == "" {
+			return nil, fmt.Errorf("overrides row %v has no email", row)
+		}
+		if seenOverride[email] {
+			return nil, fmt.Errorf("overrides has duplicate email %s", email)
+		}
+		seenOverride[email] = true
+		added := row["Added"] == "TRUE"
+		p, exists := people[email]
+		if added && exists {
+			return nil, fmt.Errorf("overrides row %s is flagged added but the import covers this person", email)
+		}
+		if !added && !exists {
+			return nil, fmt.Errorf("overrides row %s matches no imported person", email)
+		}
+		if added {
+			p = &Person{Email: email}
+			people[email] = p
+			order = append(order, email)
+		}
+
+		apply := func(cell string, field *string) {
+			switch cell {
+			case "":
+			case "-":
+				*field = ""
+			default:
+				*field = cell
+			}
+		}
+		applyBool := func(column string, field *bool) error {
+			switch row[column] {
+			case "":
+			case "-", "FALSE":
+				*field = false
+			case "TRUE":
+				*field = true
+			default:
+				return fmt.Errorf("overrides row %s has invalid %s %q", email, column, row[column])
+			}
+			return nil
+		}
+		apply(row["Full Name"], &p.FullName)
+		apply(row["Legal Name"], &p.LegalName)
+		apply(row["Preferred Name"], &p.PreferredName)
+		for column, field := range map[string]*bool{
+			"Is Student": &p.IsStudent, "Is Parent": &p.IsParent, "Is Staff": &p.IsStaff, "New to Helios": &p.IsNew,
+		} {
+			if err := applyBool(column, field); err != nil {
+				return nil, err
+			}
+		}
+		apply(row["Pronouns"], &p.Pronouns)
+		apply(row["Facts"], &p.Facts)
+		if cell := row["Grade"]; cell != "" && cell != "-" && !added && gradeBands[cell] == "" {
+			return nil, fmt.Errorf("overrides row %s has unknown grade %q", email, cell)
+		}
+		apply(row["Grade"], &p.Grade)
+		apply(row["Classroom"], &p.Classroom)
+		apply(row["Crew"], &p.Section)
+		apply(row["Phone"], &p.Phone)
+		apply(row["Job Title"], &p.JobTitle)
+		apply(row["Department"], &p.Department)
+		if cell := row["Grade Band"]; cell != "" && cell != "-" && !bandSet[cell] {
+			return nil, fmt.Errorf("overrides row %s has unknown grade band %q", email, cell)
+		}
+		apply(row["Grade Band"], &p.GradeBand)
+		if cell := row["Room Parent"]; cell != "" && cell != "-" {
+			if !bandSet[cell] {
+				return nil, fmt.Errorf("overrides row %s has unknown room parent band %q", email, cell)
+			}
+			roomParents[cell] = append(roomParents[cell], email)
+		}
+
+		cells := familyCells{}
+		if cell := row["Address"]; cell != "" {
+			cells.hasAddress = true
+			if cell != "-" {
+				cells.address = cell
+			}
+		}
+		if cell := row["Family Phone"]; cell != "" {
+			cells.hasPhone = true
+			if cell != "-" {
+				cells.phone = cell
+			}
+		}
+		if cell := row["Family Photo Caption"]; cell != "" {
+			cells.hasCaption = true
+			if cell != "-" {
+				cells.caption = cell
+			}
+		}
+		if cells.hasAddress || cells.hasPhone || cells.hasCaption {
+			familyOverrides[email] = cells
+		}
+
+		if added {
+			if p.FullName == "" {
+				return nil, fmt.Errorf("added row %s has no full name", email)
+			}
+			if !p.IsStudent && !p.IsParent && !p.IsStaff {
+				return nil, fmt.Errorf("added row %s has no role", email)
+			}
+		}
 	}
 
 	model := &Model{Families: map[string]Family{}, RoomParents: map[string][]string{}}
 
-	staffByEmail := map[string]map[string]string{}
+	familyKeys := map[string]string{}
-	for _, row := range tables["Staff Details"] {
+	for _, setKey := range householdOrder {
-		email := strings.ToLower(row["Email Lower"])
+		hh := households[setKey]
-		if email != "" {
+		members := append(append([]string{}, hh.adults...), hh.kids...)
-			staffByEmail[email] = row
+		key := familyHash(members)
+		familyKeys[setKey] = key
+		family := Family{
+			Key:         key,
+			Address:     hh.address,
+			Phone:       hh.phone,
+			AdultEmails: hh.adults,
+			KidEmails:   hh.kids,
+		}
+		family.Name = familyNameFor(family, people)
+		model.Families[key] = family
+	}
+	for email, sets := range personHouseholds {
+		if p := people[email]; p.FamilyKey == "" {
+			p.FamilyKey = familyKeys[sets[0]]
+		}
+	}
+	for email, cells := range familyOverrides {
+		p := people[email]
+		if !p.IsParent {
+			return nil, fmt.Errorf("overrides row %s has family cells but %s is not a parent", email, email)
+		}
+		sets := personHouseholds[email]
+		if len(sets) != 1 {
+			return nil, fmt.Errorf("overrides row %s has family cells but %s has no household", email, email)
+		}
+		key := familyKeys[sets[0]]
+		family := model.Families[key]
+		if cells.hasAddress {
+			family.Address = cells.address
+		}
+		if cells.hasPhone {
+			family.Phone = cells.phone
+		}
+		if cells.hasCaption {
+			family.PhotoCaption = cells.caption
+		}
+		model.Families[key] = family
+	}
+
+	if blobs != nil {
+		for _, p := range people {
+			local, _, _ := strings.Cut(p.Email, "@")
+			if blobs.Has("people/" + local + "-photo") {
+				p.PhotoURL = "/blob/people/" + local + "-photo"
+			}
+			if blobs.Has("people/" + local + "-pronunciation") {
+				p.PronunciationURL = "/blob/people/" + local + "-pronunciation"
+			}
+		}
+		for key, family := range model.Families {
+			if blobs.Has("families/" + key + "-photo") {
+				family.PhotoURL = "/blob/families/" + key + "-photo"
+			}
+			if blobs.Has("families/" + key + "-pronunciation") {
+				family.PronunciationURL = "/blob/families/" + key + "-pronunciation"
+			}
+			model.Families[key] = family
 		}
 	}
 
-	type familyAcc struct {
+	for _, email := range order {
-		family     Family
+		model.People = append(model.People, *people[email])
-		hasParent  bool
-		hasStudent bool
 	}
-	families := map[string]*familyAcc{}
-	byEmail := map[string]Person{}
-	for _, row := range tables["Basic Directory"] {
-		email := strings.ToLower(row["Email Lower"])
-		if email == "" {
-			continue
-		}
-		p := Person{
-			Email:            email,
-			FullName:         row["Full Name"],
-			LegalName:        row["Legal Name"],
-			PreferredName:    row["Preferred Name"],
-			IsStaff:          row["Is Staff?"] == "TRUE",
-			IsParent:         row["Is Parent?"] == "TRUE",
-			IsStudent:        row["Is Student?"] == "TRUE",
-			IsNew:            row["Is Totally New"] == "TRUE",
-			Pronouns:         row["Pronouns"],
-			Facts:            row["Facts"],
-			PronunciationURL: blobURL("people", email, "pronunciation", row["Pronunciation"]),
-			PhotoURL:         blobURL("people", email, "photo", row["Primary Photo"]),
-			Grade:            row["Grade"],
-			Classroom:        row["Class"],
-			Section:          row["Section"],
-			Phone:            row["Phone Number"],
-			FamilyKey:        strings.ToLower(row["Family Key"]),
-		}
-		for _, contact := range strings.Split(row["Parent Contact Emails"], ",") {
-			if contact = strings.ToLower(strings.TrimSpace(contact)); contact != "" {
-				p.ParentContactEmails = append(p.ParentContactEmails, contact)
-			}
-		}
-		if details, ok := staffByEmail[email]; ok {
-			p.JobTitle = details["Job Title"]
-			p.Department = details["Department"]
-			p.GradeBand = details["Grade Band"]
-		}
-		model.People = append(model.People, p)
-		byEmail[email] = p
-
-		if p.FamilyKey == "" {
-			continue
-		}
-		acc, ok := families[p.FamilyKey]
-		if !ok {
-			acc = &familyAcc{family: Family{Key: p.FamilyKey}}
-			families[p.FamilyKey] = acc
-		}
-		if acc.family.Address == "" && row["Address 1"] != "" {
-			acc.family.Address = row["Address 1"]
-			if row["Address 2"] != "" {
-				acc.family.Address += ", " + row["Address 2"]
-			}
-		}
-		if acc.family.PhotoURL == "" {
-			acc.family.PhotoURL = blobURL("families", p.FamilyKey, "photo", row["Family Photo"])
-		}
-		if acc.family.PhotoCaption == "" {
-			acc.family.PhotoCaption = row["Family Photo Description"]
-		}
-		if acc.family.PronunciationURL == "" {
-			acc.family.PronunciationURL = blobURL("families", p.FamilyKey, "pronunciation", row["Family Pronunciation"])
-		}
-		if p.IsParent {
-			acc.hasParent = true
-			acc.family.AdultEmails = append(acc.family.AdultEmails, email)
-		}
-		if p.IsStudent {
-			acc.hasStudent = true
-			acc.family.KidEmails = append(acc.family.KidEmails, email)
-		}
-	}
-	for key, acc := range families {
-		if !acc.hasParent && !acc.hasStudent {
-			continue
-		}
-		acc.family.Name = familyName(acc.family, byEmail)
-		model.Families[key] = acc.family
-	}
-
 	sort.Slice(model.People, func(i, j int) bool {
 		si, sj := surname(model.People[i].FullName), surname(model.People[j].FullName)
 		if si != sj {
@@ -122,83 +471,136 @@ func LoadModel(source data.Source) (*Model, error) {
 		return model.People[i].FullName < model.People[j].FullName
 	})
 
-	for _, row := range tables["Classrooms"] {
+	type classroomInfo struct {
-		if row["Class"] == "" {
+		crews    map[string]bool
+		minGrade int
+		bands    map[string]bool
+	}
+	classrooms := map[string]*classroomInfo{}
+	for _, p := range model.People {
+		if p.Classroom == "" || !p.IsStudent || gradeBands[p.Grade] == "" {
 			continue
 		}
+		info, ok := classrooms[p.Classroom]
+		if !ok {
+			info = &classroomInfo{crews: map[string]bool{}, minGrade: len(gradeOrder), bands: map[string]bool{}}
+			classrooms[p.Classroom] = info
+		}
+		if p.Section != "" {
+			info.crews[p.Section] = true
+		}
+		for i, g := range gradeOrder {
+			if g == p.Grade && i < info.minGrade {
+				info.minGrade = i
+			}
+		}
+		info.bands[gradeBands[p.Grade]] = true
+	}
+	classroomNames := []string{}
+	for name := range classrooms {
+		classroomNames = append(classroomNames, name)
+	}
+	sort.Slice(classroomNames, func(i, j int) bool {
+		ci, cj := classrooms[classroomNames[i]], classrooms[classroomNames[j]]
+		if ci.minGrade != cj.minGrade {
+			return ci.minGrade < cj.minGrade
+		}
+		return classroomNames[i] < classroomNames[j]
+	})
+	for _, name := range classroomNames {
+		info := classrooms[name]
+		if len(info.bands) != 1 {
+			return nil, fmt.Errorf("classroom %s spans multiple grade bands", name)
+		}
 		imageURL := ""
-		if row["Classroom Image"] != "" {
+		imagePath := "web/static/brand/classrooms/classroom-" + strings.ToLower(name) + ".jpg"
-			imageURL = "/static/brand/classrooms/classroom-" + strings.ToLower(row["Class"]) + ".jpg"
+		if _, err := os.Stat(imagePath); err == nil {
+			imageURL = "/static/brand/classrooms/classroom-" + strings.ToLower(name) + ".jpg"
 		}
 		model.Classrooms = append(model.Classrooms, Classroom{
-			Name:        row["Class"],
+			Name:        name,
 			ImageURL:    imageURL,
-			HasSections: row["Has Sections"] == "TRUE",
+			HasSections: len(info.crews) > 0,
 		})
 	}
 
-	for _, row := range tables["Schedules"] {
+	for _, p := range model.People {
-		if row["Classroom"] == "" {
+		if !p.IsStaff || p.Classroom == "" {
 			continue
 		}
-		section := Section{Classroom: row["Classroom"], Name: row["Section"], GradeBand: row["Grade Band"]}
+		info, ok := classrooms[p.Classroom]
-		for _, column := range []string{"Teacher 1", "Teacher 2", "Teacher 3"} {
+		if !ok {
-			if row[column] != "" {
+			return nil, fmt.Errorf("staff %s is assigned to unknown classroom %q", p.Email, p.Classroom)
-				section.Teachers = append(section.Teachers, row[column])
+		}
+		if p.Section != "" && !info.crews[p.Section] {
+			return nil, fmt.Errorf("staff %s is assigned to unknown crew %q of %s", p.Email, p.Section, p.Classroom)
+		}
+	}
+	for _, name := range classroomNames {
+		info := classrooms[name]
+		band := ""
+		for b := range info.bands {
+			band = b
+		}
+		crews := []string{}
+		for crew := range info.crews {
+			crews = append(crews, crew)
+		}
+		sort.Strings(crews)
+		if len(crews) == 0 {
+			crews = []string{""}
+		}
+		for _, crew := range crews {
+			section := Section{Classroom: name, Name: crew, GradeBand: band}
+			for _, p := range model.People {
+				if p.IsStaff && p.Classroom == name && p.Section == crew {
+					section.Teachers = append(section.Teachers, p.Email)
 				}
 			}
 			model.Sections = append(model.Sections, section)
 		}
-
-	for _, row := range tables["Grade Lookup"] {
-		if row["Current Grade"] == "" {
-			continue
-		}
-		model.Grades = append(model.Grades, Grade{
-			Name:     row["Current Grade"],
-			NextName: row["Next Grade"],
-			Band:     row["Current Gradeband"],
-			NextBand: row["Next Gradeband"],
-		})
 	}
 
-	for _, row := range tables["Room Parents"] {
+	for i, grade := range gradeOrder {
-		band, email := row["Gradeband"], strings.ToLower(row["Email Address"])
+		g := Grade{Name: grade, Band: gradeBands[grade]}
-		if band == "" || email == "" {
+		if i+1 < len(gradeOrder) {
-			continue
+			g.NextName = gradeOrder[i+1]
+			g.NextBand = gradeBands[g.NextName]
 		}
-		model.RoomParents[band] = append(model.RoomParents[band], email)
+		model.Grades = append(model.Grades, g)
 	}
 
-	type department struct {
+	for band, emails := range roomParents {
-		name  string
+		model.RoomParents[bandLabel(band)] = emails
-		order float64
-	}
-	departments := []department{}
-	for _, row := range tables["Departments"] {
-		if row["Department"] == "" {
-			continue
-		}
-		order, err := strconv.ParseFloat(row["Order"], 64)
-		if err != nil {
-			order = float64(len(departments))
-		}
-		departments = append(departments, department{name: row["Department"], order: order})
-	}
-	sort.SliceStable(departments, func(i, j int) bool { return departments[i].order < departments[j].order })
-	for _, d := range departments {
-		model.Departments = append(model.Departments, d.name)
 	}
 
+	model.Departments = append(model.Departments, departmentOrder...)
+
 	return model, nil
 }
 
-func blobURL(folder, email, kind, source string) string {
+func bandLabel(band string) string {
-	if source == "" {
+	if band == "Hummingbirds" {
-		return ""
+		return "K"
 	}
-	local, _, _ := strings.Cut(email, "@")
+	labels := []string{}
-	return "/blob/" + folder + "/" + local + "-" + kind
+	for _, grade := range gradeOrder {
+		if gradeBands[grade] != band {
+			continue
+		}
+		number := strings.TrimPrefix(grade, "Grade ")
+		switch number {
+		case "1":
+			labels = append(labels, "1st")
+		case "2":
+			labels = append(labels, "2nd")
+		case "3":
+			labels = append(labels, "3rd")
+		default:
+			labels = append(labels, number+"th")
+		}
+	}
+	return strings.Join(labels, " / ")
 }
 
 func surname(fullName string) string {
@@ -209,12 +611,16 @@ func surname(fullName string) string {
 	return fields[len(fields)-1]
 }
 
-func familyName(f Family, byEmail map[string]Person) string {
+func familyNameFor(f Family, people map[string]*Person) string {
 	members := append(append([]string{}, f.KidEmails...), f.AdultEmails...)
 	seen := map[string]bool{}
 	names := []string{}
 	for _, email := range members {
-		s := surname(byEmail[email].FullName)
+		p, ok := people[email]
+		if !ok {
+			continue
+		}
+		s := surname(p.FullName)
 		if s == "" || seen[s] {
 			continue
 		}

internal/directory/model.go

@@ -28,6 +28,7 @@ type Family struct {
 	Key              string   `json:"key"`
 	Name             string   `json:"name,omitempty"`
 	Address          string   `json:"address,omitempty"`
+	Phone            string   `json:"phone,omitempty"`
 	Lat              float64  `json:"lat,omitempty"`
 	Lng              float64  `json:"lng,omitempty"`
 	PhotoURL         string   `json:"photoUrl,omitempty"`

internal/directory/upload.go

@@ -15,7 +15,24 @@ import (
 
 const changeLogTable = "Change Log"
 
-var changeLogHeader = []string{"Timestamp", "Actor", "Target", "Kind", "File", "Archived"}
+var changeLogHeader = append([]string{"Timestamp", "Actor"}, overrideColumns...)
+
+func changeLogRow(actor, email string, previous map[string]string) []string {
+	row := make([]string, len(changeLogHeader))
+	row[0] = time.Now().UTC().Format(time.RFC3339)
+	row[1] = actor
+	for i, column := range changeLogHeader {
+		if column == "Email" {
+			row[i] = email
+		} else if value, ok := previous[column]; ok {
+			if value == "" {
+				value = "-"
+			}
+			row[i] = value
+		}
+	}
+	return row
+}
 
 var photoExtensions = map[string]string{
 	"image/jpeg": "jpg",
@@ -68,11 +85,11 @@ func (u uploader) facts(w http.ResponseWriter, r *http.Request) {
 			break
 		}
 	}
-	if err := u.sheet.SetColumn(appName, "Basic Directory", "Email Lower", key, "Facts", facts); err != nil {
+	if err := u.sheet.Upsert(appName, "Overrides", "Email", key, "Facts", facts); err != nil {
 		serverError(w, fmt.Errorf("update facts for %s: %w", key, err))
 		return
 	}
-	logRow := []string{time.Now().UTC().Format(time.RFC3339), me, key, "person facts", facts, old}
+	logRow := changeLogRow(me, key, map[string]string{"Facts": old})
 	if err := u.sheet.Append(appName, changeLogTable, changeLogHeader, logRow); err != nil {
 		serverError(w, fmt.Errorf("append change log after facts update for %s: %w", key, err))
 		return
@@ -125,12 +142,8 @@ func (u uploader) upload(w http.ResponseWriter, r *http.Request) {
 	}
 
 	folder := "people"
-	keyColumn := "Email Lower"
-	column := map[string]string{"photo": "Primary Photo", "pronunciation": "Pronunciation"}[kind]
 	if target == "family" {
 		folder = "families"
-		keyColumn = "Family Key"
-		column = map[string]string{"photo": "Family Photo", "pronunciation": "Family Pronunciation"}[kind]
 	}
 	local, _, _ := strings.Cut(key, "@")
 	base := local + "-" + kind
@@ -141,15 +154,6 @@ func (u uploader) upload(w http.ResponseWriter, r *http.Request) {
 		serverError(w, err)
 		return
 	}
-	if err := u.sheet.SetColumn(appName, "Basic Directory", keyColumn, key, column, name); err != nil {
-		serverError(w, fmt.Errorf("update sheet after upload of %s/%s: %w", folder, name, err))
-		return
-	}
-	logRow := []string{time.Now().UTC().Format(time.RFC3339), me, key, target + " " + kind, name, archived}
-	if err := u.sheet.Append(appName, changeLogTable, changeLogHeader, logRow); err != nil {
-		serverError(w, fmt.Errorf("append change log after upload of %s/%s: %w", folder, name, err))
-		return
-	}
 	if err := u.cache.Refresh(); err != nil {
 		serverError(w, fmt.Errorf("refresh model after upload: %w", err))
 		return

main.go

@@ -90,18 +90,24 @@ func main() {
 	serverKey := mapsKey("GOOGLE_MAPS_SERVER_KEY", "creds/geocoding.key")
 	browserKey := mapsKey("GOOGLE_MAPS_BROWSER_KEY", "creds/maps.key")
 	source := directorySource()
-	cache, err := directory.NewCache(source, geocode.New(serverKey))
+	var store *blob.Store
+	var blobs directory.BlobChecker
+	if os.Getenv("DIRECTORY_SHEET") != "" {
+		var err error
+		store, err = blob.New()
+		if err != nil {
+			log.Fatalf("[ERROR] blob store: %v", err)
+		}
+		blobs = store
+	}
+	cache, err := directory.NewCache(source, geocode.New(serverKey), blobs)
 	if err != nil {
 		log.Fatalf("[ERROR] load directory data: %v", err)
 	}
 	mux := http.NewServeMux()
 	authn.Register(mux)
 	directory.Register(mux, cache, browserKey)
-	if os.Getenv("DIRECTORY_SHEET") != "" {
+	if store != nil {
-		store, err := blob.New()
-		if err != nil {
-			log.Fatalf("[ERROR] blob store: %v", err)
-		}
 		blob.Register(mux, store)
 		directory.RegisterUpload(mux, cache, source.(*data.Sheet), store)
 	}

tools/columns/main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"sort"
 
 	"heliosian/internal/data"
 )
@@ -22,28 +21,13 @@ func main() {
 		}
 		source = s
 	}
-	for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} {
+	for _, name := range []string{"Veracross Import", "Name to Email", "Overrides", "Change Log"} {
-		rows, err := source.Table("directory", name)
+		header, rows, err := source.Table("directory", name)
 		if err != nil {
 			log.Fatalf("[ERROR] table %s: %v", name, err)
 		}
-		if len(rows) == 0 {
+		fmt.Printf("%s (%d rows):\n", name, len(rows))
-			fmt.Printf("%s: no rows\n", name)
+		for _, column := range header {
-			continue
-		}
-		seen := map[string]bool{}
-		columns := []string{}
-		for _, row := range rows {
-			for column := range row {
-				if !seen[column] {
-					seen[column] = true
-					columns = append(columns, column)
-				}
-			}
-		}
-		sort.Strings(columns)
-		fmt.Printf("%s:\n", name)
-		for _, column := range columns {
 			fmt.Printf("  %s\n", column)
 		}
 	}

tools/importblobs/main.go

@@ -198,7 +198,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("[ERROR] sheet source: %v", err)
 	}
-	model, err := directory.LoadModel(source)
+	model, err := directory.LoadModel(source, nil)
 	if err != nil {
 		log.Fatalf("[ERROR] load model: %v", err)
 	}

tools/loadcheck/main.go

@@ -0,0 +1,88 @@
+// Command loadcheck loads the directory model from a sheet and prints a summary.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+
+	"heliosian/internal/data"
+	"heliosian/internal/directory"
+)
+
+func main() {
+	sheet := flag.String("sheet", "", "spreadsheet id")
+	flag.Parse()
+	if *sheet == "" {
+		log.Fatal("[ERROR] -sheet <spreadsheet id> is required")
+	}
+	source, err := data.NewSheet(map[string]string{"directory": *sheet})
+	if err != nil {
+		log.Fatalf("[ERROR] sheet source: %v", err)
+	}
+	model, err := directory.LoadModel(source, nil)
+	if err != nil {
+		log.Fatalf("[ERROR] load model: %v", err)
+	}
+	students, parents, staff, isNew := 0, 0, 0, 0
+	for _, p := range model.People {
+		if p.IsStudent {
+			students++
+		}
+		if p.IsParent {
+			parents++
+		}
+		if p.IsStaff {
+			staff++
+		}
+		if p.IsNew {
+			isNew++
+		}
+	}
+	fmt.Printf("people: %d (students %d, parents %d, staff %d, new %d)\n",
+		len(model.People), students, parents, staff, isNew)
+	fmt.Printf("families: %d\n", len(model.Families))
+	twoHousehold := map[string]int{}
+	for _, f := range model.Families {
+		for _, kid := range f.KidEmails {
+			twoHousehold[kid]++
+		}
+	}
+	for kid, n := range twoHousehold {
+		if n > 1 {
+			fmt.Printf("  student in %d households: %s\n", n, kid)
+		}
+	}
+	fmt.Println("classrooms:")
+	for _, c := range model.Classrooms {
+		fmt.Printf("  %s (image %v, crews %v)\n", c.Name, c.ImageURL != "", c.HasSections)
+	}
+	fmt.Println("crews:")
+	for _, s := range model.Sections {
+		fmt.Printf("  %s | %s | %s | teachers %v\n", s.Classroom, s.Name, s.GradeBand, s.Teachers)
+	}
+	bands := []string{}
+	for band := range model.RoomParents {
+		bands = append(bands, band)
+	}
+	sort.Strings(bands)
+	fmt.Println("room parents:")
+	for _, band := range bands {
+		fmt.Printf("  %s: %d\n", band, len(model.RoomParents[band]))
+	}
+	fmt.Println("departments:", model.Departments)
+	fmt.Println("grades:")
+	for _, g := range model.Grades {
+		fmt.Printf("  %s -> %s (%s -> %s)\n", g.Name, g.NextName, g.Band, g.NextBand)
+	}
+	for _, email := range []string{"lexi.augenbergs@heliosschool.org", "daren.liang@heliosschool.org", "yeada.li@heliosschool.org", "dog@heliosschool.org", "mike.orlando@heliosschool.org", "evie.weiss@heliosschool.org"} {
+		for _, p := range model.People {
+			if p.Email == email {
+				fmt.Printf("spot %s: full=%q legal=%q pref=%q roles=S%v/P%v/T%v grade=%q class=%q crew=%q band=%q dept=%q title=%q family=%s\n",
+					email, p.FullName, p.LegalName, p.PreferredName, p.IsStudent, p.IsParent, p.IsStaff,
+					p.Grade, p.Classroom, p.Section, p.GradeBand, p.Department, p.JobTitle, p.FamilyKey)
+			}
+		}
+	}
+}