Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
264 lines
7.2 KiB
Go
264 lines
7.2 KiB
Go
package parse
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// compassDueLayouts are the due-date formats Compass writes. The grid prints
|
|
// "19/10/26 04:00 PM"; the fallback accepts single-digit fields.
|
|
var compassDueLayouts = []string{"02/01/06 03:04 PM", "2/1/06 3:04 PM"}
|
|
|
|
// noDueDate is the literal the grid prints when a task has no due date.
|
|
const noDueDate = "No due date"
|
|
|
|
// taskFieldLabels are the learning-task datafield titles that carry a value.
|
|
var taskFieldLabels = map[string]bool{
|
|
"Due date:": true,
|
|
"Submitted:": true,
|
|
"Report:": true,
|
|
}
|
|
|
|
// parseLearningTasks parses the legacy ExtJS learning-task grid into task items.
|
|
// It fails loudly when the grid is missing entirely, because that means the page
|
|
// changed or the fetch returned something else (FR-12).
|
|
func parseLearningTasks(child string, page Page, loc *time.Location) ([]Item, error) {
|
|
items, found, err := extractLearningTasks(child, page, loc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
return nil, fmt.Errorf("parse: %s: no learning-task grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// extractLearningTasks returns the task rows in one grid page. found reports
|
|
// whether the page contained a learning-task grid at all, so a caller can accept
|
|
// an empty option page (for example the React child-profile page) without
|
|
// treating it as a failure.
|
|
func extractLearningTasks(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
|
root := parseHTML(page.HTML)
|
|
if !hasGrid(root) {
|
|
return nil, false, nil
|
|
}
|
|
|
|
subject := ""
|
|
profileURL := ""
|
|
var parseErr error
|
|
stop := false
|
|
|
|
walk(root, func(n *node) bool {
|
|
if stop || parseErr != nil {
|
|
return false
|
|
}
|
|
if n.Tag == "div" && n.hasClass("x-grid-group-title") {
|
|
if s, u := groupSubject(n); s != "" {
|
|
subject = s
|
|
if u != "" {
|
|
profileURL = u
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
if n.Tag == "a" && n.hasClass("task-submission-launcher") {
|
|
it, err := taskFromAnchor(child, n, subject, profileURL, page, loc)
|
|
if err != nil {
|
|
parseErr = err
|
|
stop = true
|
|
return false
|
|
}
|
|
items = append(items, it)
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
if parseErr != nil {
|
|
return nil, false, parseErr
|
|
}
|
|
return items, true, nil
|
|
}
|
|
|
|
// hasGrid reports whether the document carries an ExtJS grid table.
|
|
func hasGrid(root *node) bool {
|
|
return find(root, func(n *node) bool {
|
|
return n.Tag == "table" && n.hasClass("x-grid-table")
|
|
}) != nil
|
|
}
|
|
|
|
// groupSubject reads the subject and the child profile URL from an ExtJS group
|
|
// header such as:
|
|
//
|
|
// <a href="/Records/User.aspx?userId=27038">Harrison ROLFE</a> -
|
|
// <a href="/Organise/Subjects/Subject.aspx?subjectId=5657...">11HLEG ...</a>
|
|
func groupSubject(title *node) (subject, profileURL string) {
|
|
for _, a := range findAll(title, func(n *node) bool { return n.Tag == "a" }) {
|
|
href := a.Attrs["href"]
|
|
text := collapse(a.Text())
|
|
switch {
|
|
case strings.Contains(href, "Subject.aspx"):
|
|
if subject == "" {
|
|
subject = text
|
|
}
|
|
case strings.Contains(href, "Records/User.aspx"):
|
|
if profileURL == "" && text != "" {
|
|
profileURL = href
|
|
}
|
|
}
|
|
}
|
|
if subject == "" {
|
|
// Fall back to the text after "NAME - ": "Harrison ROLFE - 11HLEG ...".
|
|
text := title.Text()
|
|
if i := strings.Index(text, " - "); i >= 0 {
|
|
subject = strings.TrimSpace(text[i+3:])
|
|
}
|
|
}
|
|
return subject, profileURL
|
|
}
|
|
|
|
// taskFromAnchor builds one task item from a task-submission-launcher anchor.
|
|
func taskFromAnchor(child string, a *node, subject, profileURL string, page Page, loc *time.Location) (Item, error) {
|
|
title := collapse(a.Attrs["data-qtip"])
|
|
if title == "" {
|
|
title = collapse(a.Text())
|
|
}
|
|
if title == "" {
|
|
return Item{}, fmt.Errorf("parse: %s: learning-task row with no title", page.Slug)
|
|
}
|
|
|
|
tr := nearest(a, func(n *node) bool { return n.Tag == "tr" })
|
|
if tr == nil {
|
|
return Item{}, fmt.Errorf("parse: %s: task %q has no row", page.Slug, title)
|
|
}
|
|
|
|
var category string
|
|
if c := find(tr, func(n *node) bool { return n.hasClass("task-details-category-label") }); c != nil {
|
|
category = collapse(c.Text())
|
|
}
|
|
|
|
fields, kind := taskFields(tr)
|
|
rawDue := fields["Due date:"]
|
|
|
|
due, err := parseCompassDue(rawDue, loc)
|
|
if err != nil {
|
|
return Item{}, fmt.Errorf("parse: %s: task %q: %w", page.Slug, title, err)
|
|
}
|
|
|
|
submitted := fields["Submitted:"]
|
|
var body []string
|
|
if category != "" {
|
|
body = append(body, "Category: "+category)
|
|
}
|
|
if kind != "" {
|
|
body = append(body, "Task: "+kind)
|
|
}
|
|
if submitted != "" {
|
|
body = append(body, "Submitted: "+submitted)
|
|
}
|
|
if rawDue != "" {
|
|
body = append(body, "Due: "+rawDue)
|
|
}
|
|
|
|
url := page.URL
|
|
if profileURL != "" {
|
|
url = absoluteCompassURL(profileURL) + "#learningTasks"
|
|
} else if page.URL != "" {
|
|
url = strings.Split(page.URL, "#")[0] + "#learningTasks"
|
|
}
|
|
|
|
return Item{
|
|
Child: child,
|
|
Source: SourceCompass,
|
|
Type: TypeTask,
|
|
Subject: subject,
|
|
Title: title,
|
|
Body: strings.Join(body, "\n"),
|
|
DueAt: due,
|
|
URL: url,
|
|
SourcePage: page.Slug,
|
|
ExternalRef: dedupeKey(subject, title, kind),
|
|
Importance: importanceNormal,
|
|
}, nil
|
|
}
|
|
|
|
// taskFields walks a learning-task row and returns the datafield values by
|
|
// label, plus the task kind ("Class Task", "Subject-wide Task"). It relies on
|
|
// document order: each value span follows its title span.
|
|
func taskFields(tr *node) (fields map[string]string, kind string) {
|
|
fields = map[string]string{}
|
|
last := ""
|
|
walk(tr, func(n *node) bool {
|
|
if n.Tag != "span" {
|
|
return true
|
|
}
|
|
switch {
|
|
case n.hasClass("learning-task-datafield-title"):
|
|
text := collapse(n.Text())
|
|
if !taskFieldLabels[text] && kind == "" {
|
|
kind = text
|
|
}
|
|
last = text
|
|
return false
|
|
case n.hasClass("learning-task-datafield-value"):
|
|
if last != "" {
|
|
fields[last] = collapse(n.Text())
|
|
}
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
return fields, kind
|
|
}
|
|
|
|
// parseCompassDue parses a Compass due-date string in loc. "No due date" and an
|
|
// empty value mean no due date; anything else that does not parse is an error,
|
|
// never a silent zero (FR-12).
|
|
func parseCompassDue(s string, loc *time.Location) (*time.Time, error) {
|
|
s = collapse(s)
|
|
if s == "" || strings.EqualFold(s, noDueDate) {
|
|
return nil, nil
|
|
}
|
|
if loc == nil {
|
|
loc = time.UTC
|
|
}
|
|
for _, layout := range compassDueLayouts {
|
|
if t, err := time.ParseInLocation(layout, s, loc); err == nil {
|
|
return &t, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("unrecognised due date %q", s)
|
|
}
|
|
|
|
// absoluteCompassURL turns a site-relative Compass href into a full URL.
|
|
func absoluteCompassURL(href string) string {
|
|
href = strings.TrimSpace(href)
|
|
if href == "" {
|
|
return ""
|
|
}
|
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
|
return href
|
|
}
|
|
if !strings.HasPrefix(href, "/") {
|
|
href = "/" + href
|
|
}
|
|
return compassHost + href
|
|
}
|
|
|
|
// compassHost is the Northcote High School Compass origin (AGENTS.md §2a).
|
|
const compassHost = "https://nhs-vic.compass.education"
|
|
|
|
// nearest returns the closest ancestor (or the node itself) matching pred.
|
|
func nearest(n *node, pred func(*node) bool) *node {
|
|
for cur := n; cur != nil; cur = cur.Parent {
|
|
if pred(cur) {
|
|
return cur
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// classCode matches Compass class codes such as 11HLEGB5 or 7CHI2.
|
|
var classCode = regexp.MustCompile(`^[0-9]{1,2}[A-Z]{2,}[A-Z0-9.]*$`)
|