Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
263
internal/parse/tasks.go
Normal file
263
internal/parse/tasks.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// compassDueLayouts are the due-date formats Compass writes. The grid prints
|
||||
// "19/10/26 04:00 PM"; the fallback accepts single-digit fields.
|
||||
var compassDueLayouts = []string{"02/01/06 03:04 PM", "2/1/06 3:04 PM"}
|
||||
|
||||
// noDueDate is the literal the grid prints when a task has no due date.
|
||||
const noDueDate = "No due date"
|
||||
|
||||
// taskFieldLabels are the learning-task datafield titles that carry a value.
|
||||
var taskFieldLabels = map[string]bool{
|
||||
"Due date:": true,
|
||||
"Submitted:": true,
|
||||
"Report:": true,
|
||||
}
|
||||
|
||||
// parseLearningTasks parses the legacy ExtJS learning-task grid into task items.
|
||||
// It fails loudly when the grid is missing entirely, because that means the page
|
||||
// changed or the fetch returned something else (FR-12).
|
||||
func parseLearningTasks(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractLearningTasks(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no learning-task grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractLearningTasks returns the task rows in one grid page. found reports
|
||||
// whether the page contained a learning-task grid at all, so a caller can accept
|
||||
// an empty option page (for example the React child-profile page) without
|
||||
// treating it as a failure.
|
||||
func extractLearningTasks(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
if !hasGrid(root) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
subject := ""
|
||||
profileURL := ""
|
||||
var parseErr error
|
||||
stop := false
|
||||
|
||||
walk(root, func(n *node) bool {
|
||||
if stop || parseErr != nil {
|
||||
return false
|
||||
}
|
||||
if n.Tag == "div" && n.hasClass("x-grid-group-title") {
|
||||
if s, u := groupSubject(n); s != "" {
|
||||
subject = s
|
||||
if u != "" {
|
||||
profileURL = u
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if n.Tag == "a" && n.hasClass("task-submission-launcher") {
|
||||
it, err := taskFromAnchor(child, n, subject, profileURL, page, loc)
|
||||
if err != nil {
|
||||
parseErr = err
|
||||
stop = true
|
||||
return false
|
||||
}
|
||||
items = append(items, it)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
if parseErr != nil {
|
||||
return nil, false, parseErr
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
// hasGrid reports whether the document carries an ExtJS grid table.
|
||||
func hasGrid(root *node) bool {
|
||||
return find(root, func(n *node) bool {
|
||||
return n.Tag == "table" && n.hasClass("x-grid-table")
|
||||
}) != nil
|
||||
}
|
||||
|
||||
// groupSubject reads the subject and the child profile URL from an ExtJS group
|
||||
// header such as:
|
||||
//
|
||||
// <a href="/Records/User.aspx?userId=27038">Harrison ROLFE</a> -
|
||||
// <a href="/Organise/Subjects/Subject.aspx?subjectId=5657...">11HLEG ...</a>
|
||||
func groupSubject(title *node) (subject, profileURL string) {
|
||||
for _, a := range findAll(title, func(n *node) bool { return n.Tag == "a" }) {
|
||||
href := a.Attrs["href"]
|
||||
text := collapse(a.Text())
|
||||
switch {
|
||||
case strings.Contains(href, "Subject.aspx"):
|
||||
if subject == "" {
|
||||
subject = text
|
||||
}
|
||||
case strings.Contains(href, "Records/User.aspx"):
|
||||
if profileURL == "" && text != "" {
|
||||
profileURL = href
|
||||
}
|
||||
}
|
||||
}
|
||||
if subject == "" {
|
||||
// Fall back to the text after "NAME - ": "Harrison ROLFE - 11HLEG ...".
|
||||
text := title.Text()
|
||||
if i := strings.Index(text, " - "); i >= 0 {
|
||||
subject = strings.TrimSpace(text[i+3:])
|
||||
}
|
||||
}
|
||||
return subject, profileURL
|
||||
}
|
||||
|
||||
// taskFromAnchor builds one task item from a task-submission-launcher anchor.
|
||||
func taskFromAnchor(child string, a *node, subject, profileURL string, page Page, loc *time.Location) (Item, error) {
|
||||
title := collapse(a.Attrs["data-qtip"])
|
||||
if title == "" {
|
||||
title = collapse(a.Text())
|
||||
}
|
||||
if title == "" {
|
||||
return Item{}, fmt.Errorf("parse: %s: learning-task row with no title", page.Slug)
|
||||
}
|
||||
|
||||
tr := nearest(a, func(n *node) bool { return n.Tag == "tr" })
|
||||
if tr == nil {
|
||||
return Item{}, fmt.Errorf("parse: %s: task %q has no row", page.Slug, title)
|
||||
}
|
||||
|
||||
var category string
|
||||
if c := find(tr, func(n *node) bool { return n.hasClass("task-details-category-label") }); c != nil {
|
||||
category = collapse(c.Text())
|
||||
}
|
||||
|
||||
fields, kind := taskFields(tr)
|
||||
rawDue := fields["Due date:"]
|
||||
|
||||
due, err := parseCompassDue(rawDue, loc)
|
||||
if err != nil {
|
||||
return Item{}, fmt.Errorf("parse: %s: task %q: %w", page.Slug, title, err)
|
||||
}
|
||||
|
||||
submitted := fields["Submitted:"]
|
||||
var body []string
|
||||
if category != "" {
|
||||
body = append(body, "Category: "+category)
|
||||
}
|
||||
if kind != "" {
|
||||
body = append(body, "Task: "+kind)
|
||||
}
|
||||
if submitted != "" {
|
||||
body = append(body, "Submitted: "+submitted)
|
||||
}
|
||||
if rawDue != "" {
|
||||
body = append(body, "Due: "+rawDue)
|
||||
}
|
||||
|
||||
url := page.URL
|
||||
if profileURL != "" {
|
||||
url = absoluteCompassURL(profileURL) + "#learningTasks"
|
||||
} else if page.URL != "" {
|
||||
url = strings.Split(page.URL, "#")[0] + "#learningTasks"
|
||||
}
|
||||
|
||||
return Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeTask,
|
||||
Subject: subject,
|
||||
Title: title,
|
||||
Body: strings.Join(body, "\n"),
|
||||
DueAt: due,
|
||||
URL: url,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: dedupeKey(subject, title, kind),
|
||||
Importance: importanceNormal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// taskFields walks a learning-task row and returns the datafield values by
|
||||
// label, plus the task kind ("Class Task", "Subject-wide Task"). It relies on
|
||||
// document order: each value span follows its title span.
|
||||
func taskFields(tr *node) (fields map[string]string, kind string) {
|
||||
fields = map[string]string{}
|
||||
last := ""
|
||||
walk(tr, func(n *node) bool {
|
||||
if n.Tag != "span" {
|
||||
return true
|
||||
}
|
||||
switch {
|
||||
case n.hasClass("learning-task-datafield-title"):
|
||||
text := collapse(n.Text())
|
||||
if !taskFieldLabels[text] && kind == "" {
|
||||
kind = text
|
||||
}
|
||||
last = text
|
||||
return false
|
||||
case n.hasClass("learning-task-datafield-value"):
|
||||
if last != "" {
|
||||
fields[last] = collapse(n.Text())
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return fields, kind
|
||||
}
|
||||
|
||||
// parseCompassDue parses a Compass due-date string in loc. "No due date" and an
|
||||
// empty value mean no due date; anything else that does not parse is an error,
|
||||
// never a silent zero (FR-12).
|
||||
func parseCompassDue(s string, loc *time.Location) (*time.Time, error) {
|
||||
s = collapse(s)
|
||||
if s == "" || strings.EqualFold(s, noDueDate) {
|
||||
return nil, nil
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
for _, layout := range compassDueLayouts {
|
||||
if t, err := time.ParseInLocation(layout, s, loc); err == nil {
|
||||
return &t, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unrecognised due date %q", s)
|
||||
}
|
||||
|
||||
// absoluteCompassURL turns a site-relative Compass href into a full URL.
|
||||
func absoluteCompassURL(href string) string {
|
||||
href = strings.TrimSpace(href)
|
||||
if href == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
if !strings.HasPrefix(href, "/") {
|
||||
href = "/" + href
|
||||
}
|
||||
return compassHost + href
|
||||
}
|
||||
|
||||
// compassHost is the Northcote High School Compass origin (AGENTS.md §2a).
|
||||
const compassHost = "https://nhs-vic.compass.education"
|
||||
|
||||
// nearest returns the closest ancestor (or the node itself) matching pred.
|
||||
func nearest(n *node, pred func(*node) bool) *node {
|
||||
for cur := n; cur != nil; cur = cur.Parent {
|
||||
if pred(cur) {
|
||||
return cur
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// classCode matches Compass class codes such as 11HLEGB5 or 7CHI2.
|
||||
var classCode = regexp.MustCompile(`^[0-9]{1,2}[A-Z]{2,}[A-Z0-9.]*$`)
|
||||
Reference in New Issue
Block a user