Files
compass_teams_nhs/internal/parse/calendar.go
Sam Rolfe d7d529eb0f Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table
(AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12).

- internal/parse: dependency-free HTML reader plus extractors for the ExtJS
  learning-task grid (subject, title, category, kind, due date, status), the
  home news feed (author, posted time, title, body) and the calendar week grid
  (day column + grid offset give the time; label gives title/room/teacher).
- internal/parse: fail loudly. A challenge page, login page, changed layout or
  unknown due-date format is an explicit error, never an empty day.
- FR-4: structure is parsed deterministically; importance is classified by an
  OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a
  deterministic fallback. The classifier used is printed and recorded.
- internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies).
- internal/store: migration 002 adds source_page, importance and external_ref
  with a unique (child_id, source, type, external_ref) index; UpsertItems,
  CountItemsByType, SampleItems and ChildByName.
- cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan
  row (ok=0 plus the error on failure) and prints counts and samples.
- scripts/parse-compass.sh and make parse run it in a one-off container against
  the service's own volume without restarting the service.
- docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the
  known gaps.

Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event).
Re-running the same raw pages inserts 0 and updates 56.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-15 17:27:25 +11:00

170 lines
5.2 KiB
Go

package parse
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
// calendarPxPerHour is the Compass week-grid scale: the day gutter is 1200px
// tall for 24 hours, so 50px is one hour. The event's top offset is therefore
// the authoritative hour; the printed "6:00" prefix is 12-hour and ambiguous.
const calendarPxPerHour = 50.0
var (
// calendarDayCol matches the day column id, for example
// calendar-manager-week-bd-day-col-20261016.
calendarDayCol = regexp.MustCompile(`calendar-manager-week-bd-day-col-(\d{8})`)
// calendarTop matches the top offset in an event's inline style.
calendarTop = regexp.MustCompile(`top:\s*([0-9.]+)px`)
// clockPrefix matches the leading "8:55: " of an event label.
clockPrefix = regexp.MustCompile(`^(\d{1,2}):(\d{2}):\s*(.*)$`)
// periodLabel matches the period part of a lesson label: "5", "C" or
// "RLE GROUP D".
periodLabel = regexp.MustCompile(`^(\d{1,2}|[A-Z]|RLE GROUP [A-Z])$`)
)
// parseCalendar parses the week calendar into event items. It fails loudly when
// the page has no calendar grid (FR-12).
func parseCalendar(child string, page Page, loc *time.Location) ([]Item, error) {
items, found, err := extractCalendar(child, page, loc)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("parse: %s: no calendar grid found in %s (layout changed or fetch failed)", child, page.Slug)
}
return items, nil
}
// extractCalendar returns the calendar events in one page. found reports whether
// the page carried a week grid at all.
func extractCalendar(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
root := parseHTML(page.HTML)
dayCols := findAll(root, func(n *node) bool {
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
})
events := findAll(root, func(n *node) bool {
return n.Tag == "div" && n.hasClassPart("calendar-manager-week-bd-evt-")
})
found = len(dayCols) > 0 || len(events) > 0
if !found {
return nil, false, nil
}
for _, ev := range events {
it, ok, err := eventItem(child, ev, page, loc)
if err != nil {
return nil, false, err
}
if ok {
items = append(items, it)
}
}
return items, true, nil
}
func eventItem(child string, ev *node, page Page, loc *time.Location) (Item, bool, error) {
labelSpan := find(ev, func(n *node) bool { return n.hasClass("ext-evt-bd") })
if labelSpan == nil {
return Item{}, false, nil // not an event body
}
label := collapse(labelSpan.Text())
if label == "" {
return Item{}, false, nil
}
day := nearest(ev, func(n *node) bool {
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
})
if day == nil {
return Item{}, false, fmt.Errorf("parse: %s: event %q has no day column", page.Slug, label)
}
m := calendarDayCol.FindStringSubmatch(day.Attrs["id"])
date, err := time.ParseInLocation("20060102", m[1], loc)
if err != nil {
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad day column %q", page.Slug, label, day.Attrs["id"])
}
topMatch := calendarTop.FindStringSubmatch(ev.Attrs["style"])
if topMatch == nil {
return Item{}, false, fmt.Errorf("parse: %s: event %q has no top offset", page.Slug, label)
}
topPx, err := strconv.ParseFloat(topMatch[1], 64)
if err != nil {
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad top offset %q", page.Slug, label, topMatch[1])
}
// The hour comes from the grid offset; the minutes come from the label.
hour := int(topPx / calendarPxPerHour)
minute := 0
title := label
if cm := clockPrefix.FindStringSubmatch(label); cm != nil {
if v, err := strconv.Atoi(cm[2]); err == nil {
minute = v
}
title = strings.TrimSpace(cm[3])
}
if hour < 0 || hour > 23 || minute < 0 || minute > 59 {
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad time %02d:%02d", page.Slug, label, hour, minute)
}
at := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc)
subject, teacher, body := splitEvent(label, title)
url := page.URL
if url == "" {
url = compassHost + "/Organise/Calendar/"
}
return Item{
Child: child,
Source: SourceCompass,
Type: TypeEvent,
Subject: subject,
Teacher: teacher,
Title: title,
Body: body,
DueAt: &at,
URL: url,
SourcePage: page.Slug,
ExternalRef: dedupeKey(at.Format(time.RFC3339), title),
Importance: importanceNormal,
}, true, nil
}
// splitEvent breaks a calendar label into subject, teacher and body. A lesson
// label is "5 - 11HLEGB5 - D8 - LHT"; a school event is
// "End of Year Senior Music Concert - HALL (...) - KSH, CFR, ...".
func splitEvent(label, title string) (subject, teacher, body string) {
parts := splitDash(title)
if len(parts) >= 2 && periodLabel.MatchString(parts[0]) {
for _, p := range parts[1:] {
if subject == "" && classCode.MatchString(p) {
subject = p
}
}
if !classCode.MatchString(parts[0]) {
teacher = parts[len(parts)-1]
}
}
body = label
if teacher != "" {
body = label + "\nTeacher(s): " + teacher
}
return subject, teacher, body
}
// splitDash splits on " - " and trims each part.
func splitDash(s string) []string {
raw := strings.Split(s, " - ")
out := make([]string, 0, len(raw))
for _, p := range raw {
out = append(out, strings.TrimSpace(p))
}
return out
}