diff --git a/internal/blob/blob.go b/internal/blob/blob.go index 635d894..75cc11f 100644 --- a/internal/blob/blob.go +++ b/internal/blob/blob.go @@ -231,6 +231,13 @@ func (s *Store) Refresh() error { return s.refresh() } +func (s *Store) Has(key string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.entries[key] + return ok +} + func (s *Store) Upload(folder, base, ext, mimeType string, content []byte) (string, error) { folderID, err := s.subfolder(folder) if err != nil { diff --git a/internal/data/data.go b/internal/data/data.go index d469cce..ac736d1 100644 --- a/internal/data/data.go +++ b/internal/data/data.go @@ -3,40 +3,35 @@ package data import ( "encoding/csv" - "fmt" "os" "path/filepath" ) type Source interface { - Table(app, name string) ([]map[string]string, error) + Table(app, name string) ([]string, []map[string]string, error) } type Dir struct { Root string } -func (d Dir) Table(app, name string) ([]map[string]string, error) { +func (d Dir) Table(app, name string) ([]string, []map[string]string, error) { f, err := os.Open(filepath.Join(d.Root, app, name+".csv")) if err != nil { - return nil, err + return nil, nil, err } defer f.Close() rows, err := csv.NewReader(f).ReadAll() if err != nil { - return nil, err + return nil, nil, err } - if len(rows) == 0 { - return nil, fmt.Errorf("table %s/%s has no header row", app, name) - } - header := rows[0] - records := []map[string]string{} - for _, row := range rows[1:] { - record := map[string]string{} - for i, column := range header { - record[column] = row[i] + values := make([][]interface{}, len(rows)) + for i, row := range rows { + cells := make([]interface{}, len(row)) + for j, cell := range row { + cells[j] = cell } - records = append(records, record) + values[i] = cells } - return records, nil + return parseTable(name, values) } diff --git a/internal/data/sheet.go b/internal/data/sheet.go index cd6aa68..0186bb1 100644 --- a/internal/data/sheet.go +++ b/internal/data/sheet.go @@ -26,24 +26,24 @@ func NewSheet(spreadsheets map[string]string) (*Sheet, error) { return &Sheet{service: service, spreadsheets: spreadsheets}, nil } -func (s *Sheet) Table(app, name string) ([]map[string]string, error) { +func (s *Sheet) Table(app, name string) ([]string, []map[string]string, error) { id, ok := s.spreadsheets[app] if !ok { - return nil, fmt.Errorf("no spreadsheet configured for app %q", app) + return nil, nil, fmt.Errorf("no spreadsheet configured for app %q", app) } - resp, err := s.service.Spreadsheets.Values.Get(id, "'"+strings.ReplaceAll(name, "'", "''")+"'").Do() + resp, err := s.service.Spreadsheets.Values.Get(id, quoteTab(name)).Do() if err != nil { - return nil, err + return nil, nil, err } - return toRecords(resp.Values), nil + return parseTable(name, resp.Values) } -func (s *Sheet) SetColumn(app, table, keyColumn, keyValue, column, value string) error { +func (s *Sheet) Upsert(app, table, keyColumn, keyValue, column, value string) error { id, ok := s.spreadsheets[app] if !ok { return fmt.Errorf("no spreadsheet configured for app %q", app) } - quoted := "'" + strings.ReplaceAll(table, "'", "''") + "'" + quoted := quoteTab(table) resp, err := s.service.Spreadsheets.Values.Get(id, quoted).Do() if err != nil { return err @@ -74,7 +74,17 @@ func (s *Sheet) SetColumn(app, table, keyColumn, keyValue, column, value string) }) } if len(ranges) == 0 { - return fmt.Errorf("no row in %s has %s = %q", table, keyColumn, keyValue) + width := max(keyIdx, colIdx) + 1 + row := make([]interface{}, width) + for i := range row { + row[i] = "" + } + row[keyIdx] = keyValue + row[colIdx] = value + _, err := s.service.Spreadsheets.Values.Append(id, quoted, &sheets.ValueRange{ + Values: [][]interface{}{row}, + }).ValueInputOption("RAW").InsertDataOption("INSERT_ROWS").Do() + return err } _, err = s.service.Spreadsheets.Values.BatchUpdate(id, &sheets.BatchUpdateValuesRequest{ ValueInputOption: "RAW", @@ -99,7 +109,7 @@ func (s *Sheet) Append(app, table string, header, row []string) error { break } } - quoted := "'" + strings.ReplaceAll(table, "'", "''") + "'" + quoted := quoteTab(table) if !exists { _, err := s.service.Spreadsheets.BatchUpdate(id, &sheets.BatchUpdateSpreadsheetRequest{ Requests: []*sheets.Request{{AddSheet: &sheets.AddSheetRequest{ @@ -127,6 +137,10 @@ func (s *Sheet) appendRow(id, quotedTable string, row []string) error { return err } +func quoteTab(title string) string { + return "'" + strings.ReplaceAll(title, "'", "''") + "'" +} + func columnName(idx int) string { name := "" for idx >= 0 { @@ -136,13 +150,19 @@ func columnName(idx int) string { return name } -func toRecords(values [][]interface{}) []map[string]string { +func parseTable(name string, values [][]interface{}) ([]string, []map[string]string, error) { if len(values) == 0 { - return nil + return nil, nil, fmt.Errorf("table %s has no header row", name) } header := make([]string, len(values[0])) + seen := map[string]bool{} for i, cell := range values[0] { - header[i] = strings.TrimSpace(fmt.Sprint(cell)) + h := strings.TrimSpace(fmt.Sprint(cell)) + if h != "" && seen[h] { + return nil, nil, fmt.Errorf("table %s has duplicate header %q", name, h) + } + seen[h] = true + header[i] = h } records := []map[string]string{} for _, row := range values[1:] { @@ -161,5 +181,5 @@ func toRecords(values [][]interface{}) []map[string]string { records = append(records, record) } } - return records + return header, records, nil } diff --git a/internal/directory/cache.go b/internal/directory/cache.go index 8697726..e714b8b 100644 --- a/internal/directory/cache.go +++ b/internal/directory/cache.go @@ -14,12 +14,13 @@ const refreshInterval = 5 * time.Minute type Cache struct { source data.Source geocoder *geocode.Client + blobs BlobChecker mu sync.RWMutex model *Model } -func NewCache(source data.Source, geocoder *geocode.Client) (*Cache, error) { - c := &Cache{source: source, geocoder: geocoder} +func NewCache(source data.Source, geocoder *geocode.Client, blobs BlobChecker) (*Cache, error) { + c := &Cache{source: source, geocoder: geocoder, blobs: blobs} if err := c.refresh(); err != nil { return nil, err } @@ -47,7 +48,7 @@ func (c *Cache) refreshLoop() { func (c *Cache) refresh() error { start := time.Now() - model, err := LoadModel(c.source) + model, err := LoadModel(c.source, c.blobs) if err != nil { return err } diff --git a/internal/directory/load.go b/internal/directory/load.go index 2b90d7c..580c407 100644 --- a/internal/directory/load.go +++ b/internal/directory/load.go @@ -1,8 +1,13 @@ package directory import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "regexp" "sort" - "strconv" "strings" "heliosian/internal/data" @@ -10,110 +15,454 @@ import ( const appName = "directory" -func LoadModel(source data.Source) (*Model, error) { - tables := map[string][]map[string]string{} - for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} { - rows, err := source.Table(appName, name) - if err != nil { - return nil, err +var gradeOrder = []string{ + "Kindergarten", "Grade 1", "Grade 2", "Grade 3", "Grade 4", + "Grade 5", "Grade 6", "Grade 7", "Grade 8", +} + +var gradeBands = map[string]string{ + "Kindergarten": "Hummingbirds", + "Grade 1": "Halcons", + "Grade 2": "Halcons", + "Grade 3": "Jayvens", + "Grade 4": "Jayvens", + "Grade 5": "Cospreys", + "Grade 6": "Cospreys", + "Grade 7": "Hegrets", + "Grade 8": "Hegrets", +} + +var departmentOrder = []string{ + "Admin and Office Staff", + "Co-Curriculars and Specialists", + "Classroom Teachers", + "Facilities Staff", +} + +var importColumns = []string{ + "entry_sort_name", "student_full_name", "student_classifications", "student_email", "student_phone_mobile", +} + +var overrideColumns = []string{ + "Email", "Added", "Full Name", "Legal Name", "Preferred Name", + "Is Student", "Is Parent", "Is Staff", "New to Helios", "Pronouns", "Facts", + "Grade", "Classroom", "Crew", "Phone", "Job Title", "Department", "Grade Band", "Room Parent", + "Address", "Family Phone", "Family Photo Caption", +} + +type BlobChecker interface { + Has(key string) bool +} + +type parsedName struct { + display, legal, preferred string +} + +var nameForm = regexp.MustCompile(`^(.+?) \((.+?)\) (.+)$`) + +func parseName(raw string) parsedName { + raw = strings.Join(strings.Fields(raw), " ") + m := nameForm.FindStringSubmatch(raw) + if m == nil { + return parsedName{display: raw, legal: raw} + } + return parsedName{display: m[1] + " " + m[3], legal: m[2] + " " + m[3], preferred: m[1]} +} + +func splitHomeroom(homeroom string) (classroom, crew string) { + fields := strings.Fields(homeroom) + if len(fields) == 1 { + return homeroom, "" + } + return fields[len(fields)-1], strings.Join(fields[:len(fields)-1], " ") +} + +func normName(raw string) string { + return strings.ToLower(strings.Join(strings.Fields(raw), " ")) +} + +type household struct { + adults []string + kids []string + address string + phone string +} + +func requireColumns(table string, header, wanted []string) error { + present := map[string]bool{} + for _, h := range header { + present[h] = true + } + for _, w := range wanted { + if !present[w] { + return fmt.Errorf("table %s is missing column %q", table, w) + } + } + return nil +} + +func familyHash(members []string) string { + sorted := append([]string{}, members...) + sort.Strings(sorted) + sum := sha256.Sum256([]byte(strings.Join(sorted, "\n"))) + return hex.EncodeToString(sum[:])[:16] +} + +func LoadModel(source data.Source, blobs BlobChecker) (*Model, error) { + importHeader, importRows, err := source.Table(appName, "Veracross Import") + if err != nil { + return nil, err + } + if err := requireColumns("Veracross Import", importHeader, importColumns); err != nil { + return nil, err + } + mapHeader, mapRows, err := source.Table(appName, "Name to Email") + if err != nil { + return nil, err + } + if err := requireColumns("Name to Email", mapHeader, []string{"Name", "Email"}); err != nil { + return nil, err + } + overrideHeader, overrideRows, err := source.Table(appName, "Overrides") + if err != nil { + return nil, err + } + if err := requireColumns("Overrides", overrideHeader, overrideColumns); err != nil { + return nil, err + } + + nameToEmail := map[string]string{} + for _, row := range mapRows { + name, email := normName(row["Name"]), strings.ToLower(row["Email"]) + if name == "" || email == "" { + return nil, fmt.Errorf("name to email row %v is incomplete", row) + } + if _, ok := nameToEmail[name]; ok { + return nil, fmt.Errorf("name to email has duplicate name %q", row["Name"]) + } + nameToEmail[name] = email + } + mappingUses := map[string]int{} + + people := map[string]*Person{} + order := []string{} + households := map[string]*household{} + householdOrder := []string{} + personHouseholds := map[string][]string{} + + addAdult := func(rawName, email, phone string) error { + n := parseName(rawName) + if p, ok := people[email]; ok { + if p.FullName != n.display || p.LegalName != n.legal || p.PreferredName != n.preferred { + return fmt.Errorf("adult %s has conflicting names %q and %q", email, p.FullName, rawName) + } + if p.Phone != "" && phone != "" && p.Phone != phone { + return fmt.Errorf("adult %s has conflicting phones %q and %q", email, p.Phone, phone) + } + if p.Phone == "" { + p.Phone = phone + } + p.IsParent = true + return nil + } + people[email] = &Person{ + Email: email, FullName: n.display, LegalName: n.legal, PreferredName: n.preferred, + Phone: phone, IsParent: true, + } + order = append(order, email) + return nil + } + + for _, row := range importRows { + rawName := row["student_full_name"] + if rawName == "" { + return nil, fmt.Errorf("import row %v has no student name", row) + } + var classifications struct { + GradeLevel string `json:"grade_level"` + Homeroom string `json:"homeroom"` + } + if err := json.Unmarshal([]byte(row["student_classifications"]), &classifications); err != nil { + return nil, fmt.Errorf("student %s classifications: %w", rawName, err) + } + if gradeBands[classifications.GradeLevel] == "" { + return nil, fmt.Errorf("student %s has unknown grade %q", rawName, classifications.GradeLevel) + } + if classifications.Homeroom == "" { + return nil, fmt.Errorf("student %s has no homeroom", rawName) + } + classroom, crew := splitHomeroom(classifications.Homeroom) + + email := strings.ToLower(row["student_email"]) + if mapped, ok := nameToEmail[normName(rawName)]; ok { + email = mapped + mappingUses[normName(rawName)]++ + } + if email == "" { + return nil, fmt.Errorf("student %s has no email and no name to email entry", rawName) + } + if _, ok := people[email]; ok { + return nil, fmt.Errorf("student email %s appears twice", email) + } + n := parseName(rawName) + student := &Person{ + Email: email, FullName: n.display, LegalName: n.legal, PreferredName: n.preferred, + IsStudent: true, Grade: classifications.GradeLevel, Classroom: classroom, Section: crew, + Phone: row["student_phone_mobile"], + } + people[email] = student + order = append(order, email) + + for _, hn := range []string{"1", "2"} { + adults := []string{} + for _, pn := range []string{"1", "2"} { + prefix := "household_" + hn + "_person_" + pn + "_" + if row[prefix+"full_name"] == "" { + continue + } + adultEmail := strings.ToLower(row[prefix+"email"]) + if adultEmail == "" { + return nil, fmt.Errorf("adult %q of student %s has no email", row[prefix+"full_name"], rawName) + } + phone := row[prefix+"phone_mobile"] + if phone == "" { + phone = row[prefix+"phone_business"] + } + if err := addAdult(row[prefix+"full_name"], adultEmail, phone); err != nil { + return nil, err + } + adults = append(adults, adultEmail) + } + if len(adults) == 0 { + continue + } + address, phone := row["household_"+hn+"_address"], row["household_"+hn+"_phone"] + setKey := strings.Join(func() []string { + s := append([]string{}, adults...) + sort.Strings(s) + return s + }(), "\n") + hh, ok := households[setKey] + if !ok { + hh = &household{adults: adults, address: address, phone: phone} + households[setKey] = hh + householdOrder = append(householdOrder, setKey) + for _, a := range adults { + if len(personHouseholds[a]) > 0 { + return nil, fmt.Errorf("adult %s belongs to more than one household", a) + } + personHouseholds[a] = append(personHouseholds[a], setKey) + } + } else if hh.address != address || hh.phone != phone { + return nil, fmt.Errorf("household of %v has conflicting address or phone across rows", adults) + } + hh.kids = append(hh.kids, email) + personHouseholds[email] = append(personHouseholds[email], setKey) + student.ParentContactEmails = append(student.ParentContactEmails, adults...) + } + } + + for name := range nameToEmail { + switch mappingUses[name] { + case 0: + return nil, fmt.Errorf("name to email entry %q matches no import row", name) + case 1: + default: + return nil, fmt.Errorf("name to email entry %q matches %d import rows", name, mappingUses[name]) + } + } + + type familyCells struct { + address, phone, caption string + hasAddress, hasPhone, hasCaption bool + } + familyOverrides := map[string]familyCells{} + bandSet := map[string]bool{} + for _, band := range gradeBands { + bandSet[band] = true + } + roomParents := map[string][]string{} + + seenOverride := map[string]bool{} + for _, row := range overrideRows { + email := strings.ToLower(row["Email"]) + if email == "" { + return nil, fmt.Errorf("overrides row %v has no email", row) + } + if seenOverride[email] { + return nil, fmt.Errorf("overrides has duplicate email %s", email) + } + seenOverride[email] = true + added := row["Added"] == "TRUE" + p, exists := people[email] + if added && exists { + return nil, fmt.Errorf("overrides row %s is flagged added but the import covers this person", email) + } + if !added && !exists { + return nil, fmt.Errorf("overrides row %s matches no imported person", email) + } + if added { + p = &Person{Email: email} + people[email] = p + order = append(order, email) + } + + apply := func(cell string, field *string) { + switch cell { + case "": + case "-": + *field = "" + default: + *field = cell + } + } + applyBool := func(column string, field *bool) error { + switch row[column] { + case "": + case "-", "FALSE": + *field = false + case "TRUE": + *field = true + default: + return fmt.Errorf("overrides row %s has invalid %s %q", email, column, row[column]) + } + return nil + } + apply(row["Full Name"], &p.FullName) + apply(row["Legal Name"], &p.LegalName) + apply(row["Preferred Name"], &p.PreferredName) + for column, field := range map[string]*bool{ + "Is Student": &p.IsStudent, "Is Parent": &p.IsParent, "Is Staff": &p.IsStaff, "New to Helios": &p.IsNew, + } { + if err := applyBool(column, field); err != nil { + return nil, err + } + } + apply(row["Pronouns"], &p.Pronouns) + apply(row["Facts"], &p.Facts) + if cell := row["Grade"]; cell != "" && cell != "-" && !added && gradeBands[cell] == "" { + return nil, fmt.Errorf("overrides row %s has unknown grade %q", email, cell) + } + apply(row["Grade"], &p.Grade) + apply(row["Classroom"], &p.Classroom) + apply(row["Crew"], &p.Section) + apply(row["Phone"], &p.Phone) + apply(row["Job Title"], &p.JobTitle) + apply(row["Department"], &p.Department) + if cell := row["Grade Band"]; cell != "" && cell != "-" && !bandSet[cell] { + return nil, fmt.Errorf("overrides row %s has unknown grade band %q", email, cell) + } + apply(row["Grade Band"], &p.GradeBand) + if cell := row["Room Parent"]; cell != "" && cell != "-" { + if !bandSet[cell] { + return nil, fmt.Errorf("overrides row %s has unknown room parent band %q", email, cell) + } + roomParents[cell] = append(roomParents[cell], email) + } + + cells := familyCells{} + if cell := row["Address"]; cell != "" { + cells.hasAddress = true + if cell != "-" { + cells.address = cell + } + } + if cell := row["Family Phone"]; cell != "" { + cells.hasPhone = true + if cell != "-" { + cells.phone = cell + } + } + if cell := row["Family Photo Caption"]; cell != "" { + cells.hasCaption = true + if cell != "-" { + cells.caption = cell + } + } + if cells.hasAddress || cells.hasPhone || cells.hasCaption { + familyOverrides[email] = cells + } + + if added { + if p.FullName == "" { + return nil, fmt.Errorf("added row %s has no full name", email) + } + if !p.IsStudent && !p.IsParent && !p.IsStaff { + return nil, fmt.Errorf("added row %s has no role", email) + } } - tables[name] = rows } model := &Model{Families: map[string]Family{}, RoomParents: map[string][]string{}} - staffByEmail := map[string]map[string]string{} - for _, row := range tables["Staff Details"] { - email := strings.ToLower(row["Email Lower"]) - if email != "" { - staffByEmail[email] = row + familyKeys := map[string]string{} + for _, setKey := range householdOrder { + hh := households[setKey] + members := append(append([]string{}, hh.adults...), hh.kids...) + key := familyHash(members) + familyKeys[setKey] = key + family := Family{ + Key: key, + Address: hh.address, + Phone: hh.phone, + AdultEmails: hh.adults, + KidEmails: hh.kids, } + family.Name = familyNameFor(family, people) + model.Families[key] = family + } + for email, sets := range personHouseholds { + if p := people[email]; p.FamilyKey == "" { + p.FamilyKey = familyKeys[sets[0]] + } + } + for email, cells := range familyOverrides { + p := people[email] + if !p.IsParent { + return nil, fmt.Errorf("overrides row %s has family cells but %s is not a parent", email, email) + } + sets := personHouseholds[email] + if len(sets) != 1 { + return nil, fmt.Errorf("overrides row %s has family cells but %s has no household", email, email) + } + key := familyKeys[sets[0]] + family := model.Families[key] + if cells.hasAddress { + family.Address = cells.address + } + if cells.hasPhone { + family.Phone = cells.phone + } + if cells.hasCaption { + family.PhotoCaption = cells.caption + } + model.Families[key] = family } - type familyAcc struct { - family Family - hasParent bool - hasStudent bool - } - families := map[string]*familyAcc{} - byEmail := map[string]Person{} - for _, row := range tables["Basic Directory"] { - email := strings.ToLower(row["Email Lower"]) - if email == "" { - continue - } - p := Person{ - Email: email, - FullName: row["Full Name"], - LegalName: row["Legal Name"], - PreferredName: row["Preferred Name"], - IsStaff: row["Is Staff?"] == "TRUE", - IsParent: row["Is Parent?"] == "TRUE", - IsStudent: row["Is Student?"] == "TRUE", - IsNew: row["Is Totally New"] == "TRUE", - Pronouns: row["Pronouns"], - Facts: row["Facts"], - PronunciationURL: blobURL("people", email, "pronunciation", row["Pronunciation"]), - PhotoURL: blobURL("people", email, "photo", row["Primary Photo"]), - Grade: row["Grade"], - Classroom: row["Class"], - Section: row["Section"], - Phone: row["Phone Number"], - FamilyKey: strings.ToLower(row["Family Key"]), - } - for _, contact := range strings.Split(row["Parent Contact Emails"], ",") { - if contact = strings.ToLower(strings.TrimSpace(contact)); contact != "" { - p.ParentContactEmails = append(p.ParentContactEmails, contact) + if blobs != nil { + for _, p := range people { + local, _, _ := strings.Cut(p.Email, "@") + if blobs.Has("people/" + local + "-photo") { + p.PhotoURL = "/blob/people/" + local + "-photo" + } + if blobs.Has("people/" + local + "-pronunciation") { + p.PronunciationURL = "/blob/people/" + local + "-pronunciation" } } - if details, ok := staffByEmail[email]; ok { - p.JobTitle = details["Job Title"] - p.Department = details["Department"] - p.GradeBand = details["Grade Band"] - } - model.People = append(model.People, p) - byEmail[email] = p - - if p.FamilyKey == "" { - continue - } - acc, ok := families[p.FamilyKey] - if !ok { - acc = &familyAcc{family: Family{Key: p.FamilyKey}} - families[p.FamilyKey] = acc - } - if acc.family.Address == "" && row["Address 1"] != "" { - acc.family.Address = row["Address 1"] - if row["Address 2"] != "" { - acc.family.Address += ", " + row["Address 2"] + for key, family := range model.Families { + if blobs.Has("families/" + key + "-photo") { + family.PhotoURL = "/blob/families/" + key + "-photo" } + if blobs.Has("families/" + key + "-pronunciation") { + family.PronunciationURL = "/blob/families/" + key + "-pronunciation" + } + model.Families[key] = family } - if acc.family.PhotoURL == "" { - acc.family.PhotoURL = blobURL("families", p.FamilyKey, "photo", row["Family Photo"]) - } - if acc.family.PhotoCaption == "" { - acc.family.PhotoCaption = row["Family Photo Description"] - } - if acc.family.PronunciationURL == "" { - acc.family.PronunciationURL = blobURL("families", p.FamilyKey, "pronunciation", row["Family Pronunciation"]) - } - if p.IsParent { - acc.hasParent = true - acc.family.AdultEmails = append(acc.family.AdultEmails, email) - } - if p.IsStudent { - acc.hasStudent = true - acc.family.KidEmails = append(acc.family.KidEmails, email) - } - } - for key, acc := range families { - if !acc.hasParent && !acc.hasStudent { - continue - } - acc.family.Name = familyName(acc.family, byEmail) - model.Families[key] = acc.family } + for _, email := range order { + model.People = append(model.People, *people[email]) + } sort.Slice(model.People, func(i, j int) bool { si, sj := surname(model.People[i].FullName), surname(model.People[j].FullName) if si != sj { @@ -122,83 +471,136 @@ func LoadModel(source data.Source) (*Model, error) { return model.People[i].FullName < model.People[j].FullName }) - for _, row := range tables["Classrooms"] { - if row["Class"] == "" { - continue - } - imageURL := "" - if row["Classroom Image"] != "" { - imageURL = "/static/brand/classrooms/classroom-" + strings.ToLower(row["Class"]) + ".jpg" - } - model.Classrooms = append(model.Classrooms, Classroom{ - Name: row["Class"], - ImageURL: imageURL, - HasSections: row["Has Sections"] == "TRUE", - }) + type classroomInfo struct { + crews map[string]bool + minGrade int + bands map[string]bool } - - for _, row := range tables["Schedules"] { - if row["Classroom"] == "" { + classrooms := map[string]*classroomInfo{} + for _, p := range model.People { + if p.Classroom == "" || !p.IsStudent || gradeBands[p.Grade] == "" { continue } - section := Section{Classroom: row["Classroom"], Name: row["Section"], GradeBand: row["Grade Band"]} - for _, column := range []string{"Teacher 1", "Teacher 2", "Teacher 3"} { - if row[column] != "" { - section.Teachers = append(section.Teachers, row[column]) + info, ok := classrooms[p.Classroom] + if !ok { + info = &classroomInfo{crews: map[string]bool{}, minGrade: len(gradeOrder), bands: map[string]bool{}} + classrooms[p.Classroom] = info + } + if p.Section != "" { + info.crews[p.Section] = true + } + for i, g := range gradeOrder { + if g == p.Grade && i < info.minGrade { + info.minGrade = i } } - model.Sections = append(model.Sections, section) + info.bands[gradeBands[p.Grade]] = true } - - for _, row := range tables["Grade Lookup"] { - if row["Current Grade"] == "" { - continue + classroomNames := []string{} + for name := range classrooms { + classroomNames = append(classroomNames, name) + } + sort.Slice(classroomNames, func(i, j int) bool { + ci, cj := classrooms[classroomNames[i]], classrooms[classroomNames[j]] + if ci.minGrade != cj.minGrade { + return ci.minGrade < cj.minGrade } - model.Grades = append(model.Grades, Grade{ - Name: row["Current Grade"], - NextName: row["Next Grade"], - Band: row["Current Gradeband"], - NextBand: row["Next Gradeband"], + return classroomNames[i] < classroomNames[j] + }) + for _, name := range classroomNames { + info := classrooms[name] + if len(info.bands) != 1 { + return nil, fmt.Errorf("classroom %s spans multiple grade bands", name) + } + imageURL := "" + imagePath := "web/static/brand/classrooms/classroom-" + strings.ToLower(name) + ".jpg" + if _, err := os.Stat(imagePath); err == nil { + imageURL = "/static/brand/classrooms/classroom-" + strings.ToLower(name) + ".jpg" + } + model.Classrooms = append(model.Classrooms, Classroom{ + Name: name, + ImageURL: imageURL, + HasSections: len(info.crews) > 0, }) } - for _, row := range tables["Room Parents"] { - band, email := row["Gradeband"], strings.ToLower(row["Email Address"]) - if band == "" || email == "" { + for _, p := range model.People { + if !p.IsStaff || p.Classroom == "" { continue } - model.RoomParents[band] = append(model.RoomParents[band], email) + info, ok := classrooms[p.Classroom] + if !ok { + return nil, fmt.Errorf("staff %s is assigned to unknown classroom %q", p.Email, p.Classroom) + } + if p.Section != "" && !info.crews[p.Section] { + return nil, fmt.Errorf("staff %s is assigned to unknown crew %q of %s", p.Email, p.Section, p.Classroom) + } + } + for _, name := range classroomNames { + info := classrooms[name] + band := "" + for b := range info.bands { + band = b + } + crews := []string{} + for crew := range info.crews { + crews = append(crews, crew) + } + sort.Strings(crews) + if len(crews) == 0 { + crews = []string{""} + } + for _, crew := range crews { + section := Section{Classroom: name, Name: crew, GradeBand: band} + for _, p := range model.People { + if p.IsStaff && p.Classroom == name && p.Section == crew { + section.Teachers = append(section.Teachers, p.Email) + } + } + model.Sections = append(model.Sections, section) + } } - type department struct { - name string - order float64 - } - departments := []department{} - for _, row := range tables["Departments"] { - if row["Department"] == "" { - continue + for i, grade := range gradeOrder { + g := Grade{Name: grade, Band: gradeBands[grade]} + if i+1 < len(gradeOrder) { + g.NextName = gradeOrder[i+1] + g.NextBand = gradeBands[g.NextName] } - order, err := strconv.ParseFloat(row["Order"], 64) - if err != nil { - order = float64(len(departments)) - } - departments = append(departments, department{name: row["Department"], order: order}) + model.Grades = append(model.Grades, g) } - sort.SliceStable(departments, func(i, j int) bool { return departments[i].order < departments[j].order }) - for _, d := range departments { - model.Departments = append(model.Departments, d.name) + + for band, emails := range roomParents { + model.RoomParents[bandLabel(band)] = emails } + model.Departments = append(model.Departments, departmentOrder...) + return model, nil } -func blobURL(folder, email, kind, source string) string { - if source == "" { - return "" +func bandLabel(band string) string { + if band == "Hummingbirds" { + return "K" } - local, _, _ := strings.Cut(email, "@") - return "/blob/" + folder + "/" + local + "-" + kind + labels := []string{} + for _, grade := range gradeOrder { + if gradeBands[grade] != band { + continue + } + number := strings.TrimPrefix(grade, "Grade ") + switch number { + case "1": + labels = append(labels, "1st") + case "2": + labels = append(labels, "2nd") + case "3": + labels = append(labels, "3rd") + default: + labels = append(labels, number+"th") + } + } + return strings.Join(labels, " / ") } func surname(fullName string) string { @@ -209,12 +611,16 @@ func surname(fullName string) string { return fields[len(fields)-1] } -func familyName(f Family, byEmail map[string]Person) string { +func familyNameFor(f Family, people map[string]*Person) string { members := append(append([]string{}, f.KidEmails...), f.AdultEmails...) seen := map[string]bool{} names := []string{} for _, email := range members { - s := surname(byEmail[email].FullName) + p, ok := people[email] + if !ok { + continue + } + s := surname(p.FullName) if s == "" || seen[s] { continue } diff --git a/internal/directory/model.go b/internal/directory/model.go index 07e204f..f0b4d6f 100644 --- a/internal/directory/model.go +++ b/internal/directory/model.go @@ -28,6 +28,7 @@ type Family struct { Key string `json:"key"` Name string `json:"name,omitempty"` Address string `json:"address,omitempty"` + Phone string `json:"phone,omitempty"` Lat float64 `json:"lat,omitempty"` Lng float64 `json:"lng,omitempty"` PhotoURL string `json:"photoUrl,omitempty"` diff --git a/internal/directory/upload.go b/internal/directory/upload.go index ad2d2a6..1644a15 100644 --- a/internal/directory/upload.go +++ b/internal/directory/upload.go @@ -15,7 +15,24 @@ import ( const changeLogTable = "Change Log" -var changeLogHeader = []string{"Timestamp", "Actor", "Target", "Kind", "File", "Archived"} +var changeLogHeader = append([]string{"Timestamp", "Actor"}, overrideColumns...) + +func changeLogRow(actor, email string, previous map[string]string) []string { + row := make([]string, len(changeLogHeader)) + row[0] = time.Now().UTC().Format(time.RFC3339) + row[1] = actor + for i, column := range changeLogHeader { + if column == "Email" { + row[i] = email + } else if value, ok := previous[column]; ok { + if value == "" { + value = "-" + } + row[i] = value + } + } + return row +} var photoExtensions = map[string]string{ "image/jpeg": "jpg", @@ -68,11 +85,11 @@ func (u uploader) facts(w http.ResponseWriter, r *http.Request) { break } } - if err := u.sheet.SetColumn(appName, "Basic Directory", "Email Lower", key, "Facts", facts); err != nil { + if err := u.sheet.Upsert(appName, "Overrides", "Email", key, "Facts", facts); err != nil { serverError(w, fmt.Errorf("update facts for %s: %w", key, err)) return } - logRow := []string{time.Now().UTC().Format(time.RFC3339), me, key, "person facts", facts, old} + logRow := changeLogRow(me, key, map[string]string{"Facts": old}) if err := u.sheet.Append(appName, changeLogTable, changeLogHeader, logRow); err != nil { serverError(w, fmt.Errorf("append change log after facts update for %s: %w", key, err)) return @@ -125,12 +142,8 @@ func (u uploader) upload(w http.ResponseWriter, r *http.Request) { } folder := "people" - keyColumn := "Email Lower" - column := map[string]string{"photo": "Primary Photo", "pronunciation": "Pronunciation"}[kind] if target == "family" { folder = "families" - keyColumn = "Family Key" - column = map[string]string{"photo": "Family Photo", "pronunciation": "Family Pronunciation"}[kind] } local, _, _ := strings.Cut(key, "@") base := local + "-" + kind @@ -141,15 +154,6 @@ func (u uploader) upload(w http.ResponseWriter, r *http.Request) { serverError(w, err) return } - if err := u.sheet.SetColumn(appName, "Basic Directory", keyColumn, key, column, name); err != nil { - serverError(w, fmt.Errorf("update sheet after upload of %s/%s: %w", folder, name, err)) - return - } - logRow := []string{time.Now().UTC().Format(time.RFC3339), me, key, target + " " + kind, name, archived} - if err := u.sheet.Append(appName, changeLogTable, changeLogHeader, logRow); err != nil { - serverError(w, fmt.Errorf("append change log after upload of %s/%s: %w", folder, name, err)) - return - } if err := u.cache.Refresh(); err != nil { serverError(w, fmt.Errorf("refresh model after upload: %w", err)) return diff --git a/main.go b/main.go index 6f7a6f3..c102846 100644 --- a/main.go +++ b/main.go @@ -90,18 +90,24 @@ func main() { serverKey := mapsKey("GOOGLE_MAPS_SERVER_KEY", "creds/geocoding.key") browserKey := mapsKey("GOOGLE_MAPS_BROWSER_KEY", "creds/maps.key") source := directorySource() - cache, err := directory.NewCache(source, geocode.New(serverKey)) + var store *blob.Store + var blobs directory.BlobChecker + if os.Getenv("DIRECTORY_SHEET") != "" { + var err error + store, err = blob.New() + if err != nil { + log.Fatalf("[ERROR] blob store: %v", err) + } + blobs = store + } + cache, err := directory.NewCache(source, geocode.New(serverKey), blobs) if err != nil { log.Fatalf("[ERROR] load directory data: %v", err) } mux := http.NewServeMux() authn.Register(mux) directory.Register(mux, cache, browserKey) - if os.Getenv("DIRECTORY_SHEET") != "" { - store, err := blob.New() - if err != nil { - log.Fatalf("[ERROR] blob store: %v", err) - } + if store != nil { blob.Register(mux, store) directory.RegisterUpload(mux, cache, source.(*data.Sheet), store) } diff --git a/tools/columns/main.go b/tools/columns/main.go index 73cbc4f..617f6f0 100644 --- a/tools/columns/main.go +++ b/tools/columns/main.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "sort" "heliosian/internal/data" ) @@ -22,28 +21,13 @@ func main() { } source = s } - for _, name := range []string{"Basic Directory", "Staff Details", "Classrooms", "Schedules", "Grade Lookup", "Room Parents", "Departments"} { - rows, err := source.Table("directory", name) + for _, name := range []string{"Veracross Import", "Name to Email", "Overrides", "Change Log"} { + header, rows, err := source.Table("directory", name) if err != nil { log.Fatalf("[ERROR] table %s: %v", name, err) } - if len(rows) == 0 { - fmt.Printf("%s: no rows\n", name) - continue - } - seen := map[string]bool{} - columns := []string{} - for _, row := range rows { - for column := range row { - if !seen[column] { - seen[column] = true - columns = append(columns, column) - } - } - } - sort.Strings(columns) - fmt.Printf("%s:\n", name) - for _, column := range columns { + fmt.Printf("%s (%d rows):\n", name, len(rows)) + for _, column := range header { fmt.Printf(" %s\n", column) } } diff --git a/tools/importblobs/main.go b/tools/importblobs/main.go index b3a9c84..85c4d3b 100644 --- a/tools/importblobs/main.go +++ b/tools/importblobs/main.go @@ -198,7 +198,7 @@ func main() { if err != nil { log.Fatalf("[ERROR] sheet source: %v", err) } - model, err := directory.LoadModel(source) + model, err := directory.LoadModel(source, nil) if err != nil { log.Fatalf("[ERROR] load model: %v", err) } diff --git a/tools/loadcheck/main.go b/tools/loadcheck/main.go new file mode 100644 index 0000000..f665650 --- /dev/null +++ b/tools/loadcheck/main.go @@ -0,0 +1,88 @@ +// Command loadcheck loads the directory model from a sheet and prints a summary. +package main + +import ( + "flag" + "fmt" + "log" + "sort" + + "heliosian/internal/data" + "heliosian/internal/directory" +) + +func main() { + sheet := flag.String("sheet", "", "spreadsheet id") + flag.Parse() + if *sheet == "" { + log.Fatal("[ERROR] -sheet is required") + } + source, err := data.NewSheet(map[string]string{"directory": *sheet}) + if err != nil { + log.Fatalf("[ERROR] sheet source: %v", err) + } + model, err := directory.LoadModel(source, nil) + if err != nil { + log.Fatalf("[ERROR] load model: %v", err) + } + students, parents, staff, isNew := 0, 0, 0, 0 + for _, p := range model.People { + if p.IsStudent { + students++ + } + if p.IsParent { + parents++ + } + if p.IsStaff { + staff++ + } + if p.IsNew { + isNew++ + } + } + fmt.Printf("people: %d (students %d, parents %d, staff %d, new %d)\n", + len(model.People), students, parents, staff, isNew) + fmt.Printf("families: %d\n", len(model.Families)) + twoHousehold := map[string]int{} + for _, f := range model.Families { + for _, kid := range f.KidEmails { + twoHousehold[kid]++ + } + } + for kid, n := range twoHousehold { + if n > 1 { + fmt.Printf(" student in %d households: %s\n", n, kid) + } + } + fmt.Println("classrooms:") + for _, c := range model.Classrooms { + fmt.Printf(" %s (image %v, crews %v)\n", c.Name, c.ImageURL != "", c.HasSections) + } + fmt.Println("crews:") + for _, s := range model.Sections { + fmt.Printf(" %s | %s | %s | teachers %v\n", s.Classroom, s.Name, s.GradeBand, s.Teachers) + } + bands := []string{} + for band := range model.RoomParents { + bands = append(bands, band) + } + sort.Strings(bands) + fmt.Println("room parents:") + for _, band := range bands { + fmt.Printf(" %s: %d\n", band, len(model.RoomParents[band])) + } + fmt.Println("departments:", model.Departments) + fmt.Println("grades:") + for _, g := range model.Grades { + fmt.Printf(" %s -> %s (%s -> %s)\n", g.Name, g.NextName, g.Band, g.NextBand) + } + for _, email := range []string{"lexi.augenbergs@heliosschool.org", "daren.liang@heliosschool.org", "yeada.li@heliosschool.org", "dog@heliosschool.org", "mike.orlando@heliosschool.org", "evie.weiss@heliosschool.org"} { + for _, p := range model.People { + if p.Email == email { + fmt.Printf("spot %s: full=%q legal=%q pref=%q roles=S%v/P%v/T%v grade=%q class=%q crew=%q band=%q dept=%q title=%q family=%s\n", + email, p.FullName, p.LegalName, p.PreferredName, p.IsStudent, p.IsParent, p.IsStaff, + p.Grade, p.Classroom, p.Section, p.GradeBand, p.Department, p.JobTitle, p.FamilyKey) + } + } + } +}