Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
169
internal/parse/calendar.go
Normal file
169
internal/parse/calendar.go
Normal file
@@ -0,0 +1,169 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// calendarPxPerHour is the Compass week-grid scale: the day gutter is 1200px
|
||||
// tall for 24 hours, so 50px is one hour. The event's top offset is therefore
|
||||
// the authoritative hour; the printed "6:00" prefix is 12-hour and ambiguous.
|
||||
const calendarPxPerHour = 50.0
|
||||
|
||||
var (
|
||||
// calendarDayCol matches the day column id, for example
|
||||
// calendar-manager-week-bd-day-col-20261016.
|
||||
calendarDayCol = regexp.MustCompile(`calendar-manager-week-bd-day-col-(\d{8})`)
|
||||
// calendarTop matches the top offset in an event's inline style.
|
||||
calendarTop = regexp.MustCompile(`top:\s*([0-9.]+)px`)
|
||||
// clockPrefix matches the leading "8:55: " of an event label.
|
||||
clockPrefix = regexp.MustCompile(`^(\d{1,2}):(\d{2}):\s*(.*)$`)
|
||||
// periodLabel matches the period part of a lesson label: "5", "C" or
|
||||
// "RLE GROUP D".
|
||||
periodLabel = regexp.MustCompile(`^(\d{1,2}|[A-Z]|RLE GROUP [A-Z])$`)
|
||||
)
|
||||
|
||||
// parseCalendar parses the week calendar into event items. It fails loudly when
|
||||
// the page has no calendar grid (FR-12).
|
||||
func parseCalendar(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractCalendar(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no calendar grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractCalendar returns the calendar events in one page. found reports whether
|
||||
// the page carried a week grid at all.
|
||||
func extractCalendar(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
|
||||
dayCols := findAll(root, func(n *node) bool {
|
||||
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
||||
})
|
||||
events := findAll(root, func(n *node) bool {
|
||||
return n.Tag == "div" && n.hasClassPart("calendar-manager-week-bd-evt-")
|
||||
})
|
||||
found = len(dayCols) > 0 || len(events) > 0
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
for _, ev := range events {
|
||||
it, ok, err := eventItem(child, ev, page, loc)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if ok {
|
||||
items = append(items, it)
|
||||
}
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
func eventItem(child string, ev *node, page Page, loc *time.Location) (Item, bool, error) {
|
||||
labelSpan := find(ev, func(n *node) bool { return n.hasClass("ext-evt-bd") })
|
||||
if labelSpan == nil {
|
||||
return Item{}, false, nil // not an event body
|
||||
}
|
||||
label := collapse(labelSpan.Text())
|
||||
if label == "" {
|
||||
return Item{}, false, nil
|
||||
}
|
||||
|
||||
day := nearest(ev, func(n *node) bool {
|
||||
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
||||
})
|
||||
if day == nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q has no day column", page.Slug, label)
|
||||
}
|
||||
m := calendarDayCol.FindStringSubmatch(day.Attrs["id"])
|
||||
date, err := time.ParseInLocation("20060102", m[1], loc)
|
||||
if err != nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad day column %q", page.Slug, label, day.Attrs["id"])
|
||||
}
|
||||
|
||||
topMatch := calendarTop.FindStringSubmatch(ev.Attrs["style"])
|
||||
if topMatch == nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q has no top offset", page.Slug, label)
|
||||
}
|
||||
topPx, err := strconv.ParseFloat(topMatch[1], 64)
|
||||
if err != nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad top offset %q", page.Slug, label, topMatch[1])
|
||||
}
|
||||
|
||||
// The hour comes from the grid offset; the minutes come from the label.
|
||||
hour := int(topPx / calendarPxPerHour)
|
||||
minute := 0
|
||||
title := label
|
||||
if cm := clockPrefix.FindStringSubmatch(label); cm != nil {
|
||||
if v, err := strconv.Atoi(cm[2]); err == nil {
|
||||
minute = v
|
||||
}
|
||||
title = strings.TrimSpace(cm[3])
|
||||
}
|
||||
if hour < 0 || hour > 23 || minute < 0 || minute > 59 {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad time %02d:%02d", page.Slug, label, hour, minute)
|
||||
}
|
||||
at := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc)
|
||||
|
||||
subject, teacher, body := splitEvent(label, title)
|
||||
|
||||
url := page.URL
|
||||
if url == "" {
|
||||
url = compassHost + "/Organise/Calendar/"
|
||||
}
|
||||
|
||||
return Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeEvent,
|
||||
Subject: subject,
|
||||
Teacher: teacher,
|
||||
Title: title,
|
||||
Body: body,
|
||||
DueAt: &at,
|
||||
URL: url,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: dedupeKey(at.Format(time.RFC3339), title),
|
||||
Importance: importanceNormal,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
// splitEvent breaks a calendar label into subject, teacher and body. A lesson
|
||||
// label is "5 - 11HLEGB5 - D8 - LHT"; a school event is
|
||||
// "End of Year Senior Music Concert - HALL (...) - KSH, CFR, ...".
|
||||
func splitEvent(label, title string) (subject, teacher, body string) {
|
||||
parts := splitDash(title)
|
||||
if len(parts) >= 2 && periodLabel.MatchString(parts[0]) {
|
||||
for _, p := range parts[1:] {
|
||||
if subject == "" && classCode.MatchString(p) {
|
||||
subject = p
|
||||
}
|
||||
}
|
||||
if !classCode.MatchString(parts[0]) {
|
||||
teacher = parts[len(parts)-1]
|
||||
}
|
||||
}
|
||||
body = label
|
||||
if teacher != "" {
|
||||
body = label + "\nTeacher(s): " + teacher
|
||||
}
|
||||
return subject, teacher, body
|
||||
}
|
||||
|
||||
// splitDash splits on " - " and trims each part.
|
||||
func splitDash(s string) []string {
|
||||
raw := strings.Split(s, " - ")
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
out = append(out, strings.TrimSpace(p))
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user