Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
170 lines
5.2 KiB
Go
170 lines
5.2 KiB
Go
package parse
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// calendarPxPerHour is the Compass week-grid scale: the day gutter is 1200px
|
|
// tall for 24 hours, so 50px is one hour. The event's top offset is therefore
|
|
// the authoritative hour; the printed "6:00" prefix is 12-hour and ambiguous.
|
|
const calendarPxPerHour = 50.0
|
|
|
|
var (
|
|
// calendarDayCol matches the day column id, for example
|
|
// calendar-manager-week-bd-day-col-20261016.
|
|
calendarDayCol = regexp.MustCompile(`calendar-manager-week-bd-day-col-(\d{8})`)
|
|
// calendarTop matches the top offset in an event's inline style.
|
|
calendarTop = regexp.MustCompile(`top:\s*([0-9.]+)px`)
|
|
// clockPrefix matches the leading "8:55: " of an event label.
|
|
clockPrefix = regexp.MustCompile(`^(\d{1,2}):(\d{2}):\s*(.*)$`)
|
|
// periodLabel matches the period part of a lesson label: "5", "C" or
|
|
// "RLE GROUP D".
|
|
periodLabel = regexp.MustCompile(`^(\d{1,2}|[A-Z]|RLE GROUP [A-Z])$`)
|
|
)
|
|
|
|
// parseCalendar parses the week calendar into event items. It fails loudly when
|
|
// the page has no calendar grid (FR-12).
|
|
func parseCalendar(child string, page Page, loc *time.Location) ([]Item, error) {
|
|
items, found, err := extractCalendar(child, page, loc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
return nil, fmt.Errorf("parse: %s: no calendar grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// extractCalendar returns the calendar events in one page. found reports whether
|
|
// the page carried a week grid at all.
|
|
func extractCalendar(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
|
root := parseHTML(page.HTML)
|
|
|
|
dayCols := findAll(root, func(n *node) bool {
|
|
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
|
})
|
|
events := findAll(root, func(n *node) bool {
|
|
return n.Tag == "div" && n.hasClassPart("calendar-manager-week-bd-evt-")
|
|
})
|
|
found = len(dayCols) > 0 || len(events) > 0
|
|
if !found {
|
|
return nil, false, nil
|
|
}
|
|
|
|
for _, ev := range events {
|
|
it, ok, err := eventItem(child, ev, page, loc)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if ok {
|
|
items = append(items, it)
|
|
}
|
|
}
|
|
return items, true, nil
|
|
}
|
|
|
|
func eventItem(child string, ev *node, page Page, loc *time.Location) (Item, bool, error) {
|
|
labelSpan := find(ev, func(n *node) bool { return n.hasClass("ext-evt-bd") })
|
|
if labelSpan == nil {
|
|
return Item{}, false, nil // not an event body
|
|
}
|
|
label := collapse(labelSpan.Text())
|
|
if label == "" {
|
|
return Item{}, false, nil
|
|
}
|
|
|
|
day := nearest(ev, func(n *node) bool {
|
|
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
|
})
|
|
if day == nil {
|
|
return Item{}, false, fmt.Errorf("parse: %s: event %q has no day column", page.Slug, label)
|
|
}
|
|
m := calendarDayCol.FindStringSubmatch(day.Attrs["id"])
|
|
date, err := time.ParseInLocation("20060102", m[1], loc)
|
|
if err != nil {
|
|
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad day column %q", page.Slug, label, day.Attrs["id"])
|
|
}
|
|
|
|
topMatch := calendarTop.FindStringSubmatch(ev.Attrs["style"])
|
|
if topMatch == nil {
|
|
return Item{}, false, fmt.Errorf("parse: %s: event %q has no top offset", page.Slug, label)
|
|
}
|
|
topPx, err := strconv.ParseFloat(topMatch[1], 64)
|
|
if err != nil {
|
|
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad top offset %q", page.Slug, label, topMatch[1])
|
|
}
|
|
|
|
// The hour comes from the grid offset; the minutes come from the label.
|
|
hour := int(topPx / calendarPxPerHour)
|
|
minute := 0
|
|
title := label
|
|
if cm := clockPrefix.FindStringSubmatch(label); cm != nil {
|
|
if v, err := strconv.Atoi(cm[2]); err == nil {
|
|
minute = v
|
|
}
|
|
title = strings.TrimSpace(cm[3])
|
|
}
|
|
if hour < 0 || hour > 23 || minute < 0 || minute > 59 {
|
|
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad time %02d:%02d", page.Slug, label, hour, minute)
|
|
}
|
|
at := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc)
|
|
|
|
subject, teacher, body := splitEvent(label, title)
|
|
|
|
url := page.URL
|
|
if url == "" {
|
|
url = compassHost + "/Organise/Calendar/"
|
|
}
|
|
|
|
return Item{
|
|
Child: child,
|
|
Source: SourceCompass,
|
|
Type: TypeEvent,
|
|
Subject: subject,
|
|
Teacher: teacher,
|
|
Title: title,
|
|
Body: body,
|
|
DueAt: &at,
|
|
URL: url,
|
|
SourcePage: page.Slug,
|
|
ExternalRef: dedupeKey(at.Format(time.RFC3339), title),
|
|
Importance: importanceNormal,
|
|
}, true, nil
|
|
}
|
|
|
|
// splitEvent breaks a calendar label into subject, teacher and body. A lesson
|
|
// label is "5 - 11HLEGB5 - D8 - LHT"; a school event is
|
|
// "End of Year Senior Music Concert - HALL (...) - KSH, CFR, ...".
|
|
func splitEvent(label, title string) (subject, teacher, body string) {
|
|
parts := splitDash(title)
|
|
if len(parts) >= 2 && periodLabel.MatchString(parts[0]) {
|
|
for _, p := range parts[1:] {
|
|
if subject == "" && classCode.MatchString(p) {
|
|
subject = p
|
|
}
|
|
}
|
|
if !classCode.MatchString(parts[0]) {
|
|
teacher = parts[len(parts)-1]
|
|
}
|
|
}
|
|
body = label
|
|
if teacher != "" {
|
|
body = label + "\nTeacher(s): " + teacher
|
|
}
|
|
return subject, teacher, body
|
|
}
|
|
|
|
// splitDash splits on " - " and trims each part.
|
|
func splitDash(s string) []string {
|
|
raw := strings.Split(s, " - ")
|
|
out := make([]string, 0, len(raw))
|
|
for _, p := range raw {
|
|
out = append(out, strings.TrimSpace(p))
|
|
}
|
|
return out
|
|
}
|