package parse import ( "fmt" "sort" "strings" "time" ) // Source and item types match the items table CHECK constraints (AGENTS.md ยง7). const ( SourceCompass = "compass" TypeMessage = "message" TypeTask = "task" TypeEvent = "event" ) // Recognised page slugs, matching the .html files stored by the Phase 2 // fetchers under var/raw///. const ( PageHome = "home" PageChildProfile = "child-profile" PageLearningTasks = "learning-tasks" PageCalendar = "calendar" ) // Item is one message, learning task or calendar event read from Compass. // Every field maps to a column of the items table (FR-5). type Item struct { Child string `json:"child"` Source string `json:"source"` Type string `json:"type"` Subject string `json:"subject,omitempty"` Teacher string `json:"teacher,omitempty"` Title string `json:"title"` Body string `json:"body,omitempty"` DueAt *time.Time `json:"due_at,omitempty"` URL string `json:"url,omitempty"` SourcePage string `json:"source_page"` ExternalRef string `json:"external_ref"` Importance string `json:"importance"` } // Page is one stored raw page: its slug, the URL it was fetched from, and the // HTML bytes. type Page struct { Slug string URL string HTML []byte } // Parse turns the stored pages for one child into structured items. It fails, // loudly and explicitly, when a page it recognises cannot be parsed, so a broken // scan can never be shown as "nothing due" (FR-12). // // Pages with an unknown slug are ignored. When no recognised page yields any // structure at all, Parse returns an error. func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { if strings.TrimSpace(child) == "" { return nil, fmt.Errorf("parse: child is empty") } if loc == nil { loc = time.UTC } var items []Item recognised := 0 for _, p := range pagesOf(pages) { switch p.Slug { case PageLearningTasks: got, err := parseLearningTasks(child, p, loc) if err != nil { return nil, err } items = append(items, got...) recognised++ case PageChildProfile: // The React profile page has no server-rendered grid; when it does // carry the legacy grid, use it. got, found, err := extractLearningTasks(child, p, loc) if err != nil { return nil, err } if found { items = append(items, got...) } recognised++ case PageHome: got, err := parseNews(child, p, loc) if err != nil { return nil, err } items = append(items, got...) recognised++ case PageCalendar: got, err := parseCalendar(child, p, loc) if err != nil { return nil, err } items = append(items, got...) recognised++ } } if recognised == 0 { return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages)) } // A task row is the same task wherever it appears (learning tasks and child // profile pages both carry the grid). Keep the first, drop the repeats. seen := map[string]bool{} out := items[:0] for _, it := range items { key := it.Type + "\x1f" + it.ExternalRef if seen[key] { continue } seen[key] = true if it.Importance == "" { it.Importance = importanceNormal } out = append(out, it) } sort.SliceStable(out, func(i, j int) bool { if out[i].Type != out[j].Type { return out[i].Type < out[j].Type } return out[i].ExternalRef < out[j].ExternalRef }) return out, nil } // pagesOf returns the pages with duplicates removed, keeping the last copy of a // slug (the fetcher overwrites a slug file on a re-fetch, so the last is // authoritative). func pagesOf(pages []Page) []Page { bySlug := map[string]Page{} var order []string for _, p := range pages { if _, ok := bySlug[p.Slug]; !ok { order = append(order, p.Slug) } bySlug[p.Slug] = p } out := make([]Page, 0, len(order)) for _, slug := range order { out = append(out, bySlug[slug]) } return out } // dedupeKey joins the identity fields of an item into a stable external // reference. Re-parsing the same page must update the same row, not insert a // second one. func dedupeKey(parts ...string) string { clean := make([]string, 0, len(parts)) for _, p := range parts { clean = append(clean, collapse(p)) } return strings.Join(clean, "|") }