Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
169
internal/parse/calendar.go
Normal file
169
internal/parse/calendar.go
Normal file
@@ -0,0 +1,169 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// calendarPxPerHour is the Compass week-grid scale: the day gutter is 1200px
|
||||
// tall for 24 hours, so 50px is one hour. The event's top offset is therefore
|
||||
// the authoritative hour; the printed "6:00" prefix is 12-hour and ambiguous.
|
||||
const calendarPxPerHour = 50.0
|
||||
|
||||
var (
|
||||
// calendarDayCol matches the day column id, for example
|
||||
// calendar-manager-week-bd-day-col-20261016.
|
||||
calendarDayCol = regexp.MustCompile(`calendar-manager-week-bd-day-col-(\d{8})`)
|
||||
// calendarTop matches the top offset in an event's inline style.
|
||||
calendarTop = regexp.MustCompile(`top:\s*([0-9.]+)px`)
|
||||
// clockPrefix matches the leading "8:55: " of an event label.
|
||||
clockPrefix = regexp.MustCompile(`^(\d{1,2}):(\d{2}):\s*(.*)$`)
|
||||
// periodLabel matches the period part of a lesson label: "5", "C" or
|
||||
// "RLE GROUP D".
|
||||
periodLabel = regexp.MustCompile(`^(\d{1,2}|[A-Z]|RLE GROUP [A-Z])$`)
|
||||
)
|
||||
|
||||
// parseCalendar parses the week calendar into event items. It fails loudly when
|
||||
// the page has no calendar grid (FR-12).
|
||||
func parseCalendar(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractCalendar(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no calendar grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractCalendar returns the calendar events in one page. found reports whether
|
||||
// the page carried a week grid at all.
|
||||
func extractCalendar(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
|
||||
dayCols := findAll(root, func(n *node) bool {
|
||||
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
||||
})
|
||||
events := findAll(root, func(n *node) bool {
|
||||
return n.Tag == "div" && n.hasClassPart("calendar-manager-week-bd-evt-")
|
||||
})
|
||||
found = len(dayCols) > 0 || len(events) > 0
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
for _, ev := range events {
|
||||
it, ok, err := eventItem(child, ev, page, loc)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if ok {
|
||||
items = append(items, it)
|
||||
}
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
func eventItem(child string, ev *node, page Page, loc *time.Location) (Item, bool, error) {
|
||||
labelSpan := find(ev, func(n *node) bool { return n.hasClass("ext-evt-bd") })
|
||||
if labelSpan == nil {
|
||||
return Item{}, false, nil // not an event body
|
||||
}
|
||||
label := collapse(labelSpan.Text())
|
||||
if label == "" {
|
||||
return Item{}, false, nil
|
||||
}
|
||||
|
||||
day := nearest(ev, func(n *node) bool {
|
||||
return n.Tag == "div" && calendarDayCol.MatchString(n.Attrs["id"])
|
||||
})
|
||||
if day == nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q has no day column", page.Slug, label)
|
||||
}
|
||||
m := calendarDayCol.FindStringSubmatch(day.Attrs["id"])
|
||||
date, err := time.ParseInLocation("20060102", m[1], loc)
|
||||
if err != nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad day column %q", page.Slug, label, day.Attrs["id"])
|
||||
}
|
||||
|
||||
topMatch := calendarTop.FindStringSubmatch(ev.Attrs["style"])
|
||||
if topMatch == nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q has no top offset", page.Slug, label)
|
||||
}
|
||||
topPx, err := strconv.ParseFloat(topMatch[1], 64)
|
||||
if err != nil {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad top offset %q", page.Slug, label, topMatch[1])
|
||||
}
|
||||
|
||||
// The hour comes from the grid offset; the minutes come from the label.
|
||||
hour := int(topPx / calendarPxPerHour)
|
||||
minute := 0
|
||||
title := label
|
||||
if cm := clockPrefix.FindStringSubmatch(label); cm != nil {
|
||||
if v, err := strconv.Atoi(cm[2]); err == nil {
|
||||
minute = v
|
||||
}
|
||||
title = strings.TrimSpace(cm[3])
|
||||
}
|
||||
if hour < 0 || hour > 23 || minute < 0 || minute > 59 {
|
||||
return Item{}, false, fmt.Errorf("parse: %s: event %q: bad time %02d:%02d", page.Slug, label, hour, minute)
|
||||
}
|
||||
at := time.Date(date.Year(), date.Month(), date.Day(), hour, minute, 0, 0, loc)
|
||||
|
||||
subject, teacher, body := splitEvent(label, title)
|
||||
|
||||
url := page.URL
|
||||
if url == "" {
|
||||
url = compassHost + "/Organise/Calendar/"
|
||||
}
|
||||
|
||||
return Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeEvent,
|
||||
Subject: subject,
|
||||
Teacher: teacher,
|
||||
Title: title,
|
||||
Body: body,
|
||||
DueAt: &at,
|
||||
URL: url,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: dedupeKey(at.Format(time.RFC3339), title),
|
||||
Importance: importanceNormal,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
// splitEvent breaks a calendar label into subject, teacher and body. A lesson
|
||||
// label is "5 - 11HLEGB5 - D8 - LHT"; a school event is
|
||||
// "End of Year Senior Music Concert - HALL (...) - KSH, CFR, ...".
|
||||
func splitEvent(label, title string) (subject, teacher, body string) {
|
||||
parts := splitDash(title)
|
||||
if len(parts) >= 2 && periodLabel.MatchString(parts[0]) {
|
||||
for _, p := range parts[1:] {
|
||||
if subject == "" && classCode.MatchString(p) {
|
||||
subject = p
|
||||
}
|
||||
}
|
||||
if !classCode.MatchString(parts[0]) {
|
||||
teacher = parts[len(parts)-1]
|
||||
}
|
||||
}
|
||||
body = label
|
||||
if teacher != "" {
|
||||
body = label + "\nTeacher(s): " + teacher
|
||||
}
|
||||
return subject, teacher, body
|
||||
}
|
||||
|
||||
// splitDash splits on " - " and trims each part.
|
||||
func splitDash(s string) []string {
|
||||
raw := strings.Split(s, " - ")
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
out = append(out, strings.TrimSpace(p))
|
||||
}
|
||||
return out
|
||||
}
|
||||
323
internal/parse/html.go
Normal file
323
internal/parse/html.go
Normal file
@@ -0,0 +1,323 @@
|
||||
// Package parse turns stored Compass pages into structured items (AGENTS.md
|
||||
// §4 FR-3/FR-4/FR-5, §11 Phase 3). It is a deterministic, dependency-free HTML
|
||||
// reader: the Compass pages are machine-generated, so parsing the markup is more
|
||||
// reliable than asking a model to transcribe a table.
|
||||
package parse
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"html"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// voidElements never have children and are therefore never pushed on the stack.
|
||||
var voidElements = map[string]bool{
|
||||
"area": true, "base": true, "br": true, "col": true, "embed": true,
|
||||
"hr": true, "img": true, "input": true, "link": true, "meta": true,
|
||||
"param": true, "source": true, "track": true, "wbr": true,
|
||||
}
|
||||
|
||||
// rawTextElements have text content that must not be tokenised as markup.
|
||||
var rawTextElements = map[string]bool{
|
||||
"script": true, "style": true, "textarea": true,
|
||||
}
|
||||
|
||||
// node is one element in the parsed document. Text content is stored as
|
||||
// "#text" children so that text and elements keep their document order; use
|
||||
// Text for the concatenated descendant text.
|
||||
type node struct {
|
||||
Tag string
|
||||
Attrs map[string]string
|
||||
Kids []*node
|
||||
Parent *node
|
||||
text string
|
||||
}
|
||||
|
||||
// Class returns the element's class attribute.
|
||||
func (n *node) Class() string { return n.Attrs["class"] }
|
||||
|
||||
// hasClass reports whether the element carries the exact class.
|
||||
func (n *node) hasClass(cls string) bool {
|
||||
for _, f := range strings.Fields(n.Class()) {
|
||||
if f == cls {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasClassPart reports whether any class token contains part.
|
||||
func (n *node) hasClassPart(part string) bool {
|
||||
for _, f := range strings.Fields(n.Class()) {
|
||||
if strings.Contains(f, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasAttr reports whether the attribute is present with a non-empty value.
|
||||
func (n *node) hasAttr(key string) bool { return strings.TrimSpace(n.Attrs[key]) != "" }
|
||||
|
||||
// Text returns the concatenated text of the element and all descendants,
|
||||
// whitespace-collapsed.
|
||||
func (n *node) Text() string {
|
||||
var b strings.Builder
|
||||
n.writeText(&b)
|
||||
return collapse(b.String())
|
||||
}
|
||||
|
||||
func (n *node) writeText(b *strings.Builder) {
|
||||
b.WriteString(n.text)
|
||||
for _, k := range n.Kids {
|
||||
k.writeText(b)
|
||||
}
|
||||
}
|
||||
func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// textUntil returns the concatenated text of the node's descendants up to and
|
||||
// including the first matching direct-or-descendant element, exclusive of that
|
||||
// element's subtree.
|
||||
func (n *node) textUntil(pred func(*node) bool) string {
|
||||
var b strings.Builder
|
||||
var w func(x *node) bool
|
||||
w = func(x *node) bool {
|
||||
if pred(x) {
|
||||
return false
|
||||
}
|
||||
b.WriteString(x.text)
|
||||
for _, k := range x.Kids {
|
||||
if !w(k) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
w(n)
|
||||
return collapse(b.String())
|
||||
}
|
||||
|
||||
// parseHTML tokenises and nests the byte slice. It never fails: malformed
|
||||
// markup, stray tags and unclosed elements are tolerated, because ExtJS emits
|
||||
// group headers whose <td> is not closed before the next table starts.
|
||||
func parseHTML(data []byte) *node {
|
||||
root := &node{Tag: "#root", Attrs: map[string]string{}}
|
||||
stack := []*node{root}
|
||||
n := len(data)
|
||||
i := 0
|
||||
|
||||
top := func() *node { return stack[len(stack)-1] }
|
||||
|
||||
for i < n {
|
||||
if data[i] != '<' {
|
||||
j := bytes.IndexByte(data[i:], '<')
|
||||
if j < 0 {
|
||||
j = n - i
|
||||
}
|
||||
appendText(top(), string(data[i:i+j]))
|
||||
i += j
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case bytes.HasPrefix(data[i:], []byte("<!--")):
|
||||
end := bytes.Index(data[i:], []byte("-->"))
|
||||
if end < 0 {
|
||||
i = n
|
||||
continue
|
||||
}
|
||||
i += end + 3
|
||||
case bytes.HasPrefix(data[i:], []byte("<!")) || bytes.HasPrefix(data[i:], []byte("<?")):
|
||||
end := bytes.IndexByte(data[i:], '>')
|
||||
if end < 0 {
|
||||
i = n
|
||||
continue
|
||||
}
|
||||
i += end + 1
|
||||
case bytes.HasPrefix(data[i:], []byte("</")):
|
||||
k := i + 2
|
||||
start := k
|
||||
for k < n && isNameChar(data[k]) {
|
||||
k++
|
||||
}
|
||||
name := strings.ToLower(string(data[start:k]))
|
||||
gt := bytes.IndexByte(data[k:], '>')
|
||||
if gt < 0 {
|
||||
i = n
|
||||
continue
|
||||
}
|
||||
i = k + gt + 1
|
||||
stack = closeTag(stack, name)
|
||||
default:
|
||||
k := i + 1
|
||||
start := k
|
||||
for k < n && isNameChar(data[k]) {
|
||||
k++
|
||||
}
|
||||
name := strings.ToLower(string(data[start:k]))
|
||||
if name == "" { // a bare '<'
|
||||
appendText(top(), "<")
|
||||
i++
|
||||
continue
|
||||
}
|
||||
attrs, selfClose, next := parseAttrs(data, k)
|
||||
i = next
|
||||
|
||||
child := &node{Tag: name, Attrs: attrs, Parent: top()}
|
||||
top().Kids = append(top().Kids, child)
|
||||
|
||||
if selfClose || voidElements[name] {
|
||||
continue
|
||||
}
|
||||
stack = append(stack, child)
|
||||
|
||||
if rawTextElements[name] {
|
||||
// Skip the content; scripts and styles are not parsed as markup.
|
||||
if idx := indexFold(data[i:], "</"+name); idx >= 0 {
|
||||
i += idx
|
||||
} else {
|
||||
i = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// parseAttrs reads tag attributes starting at p (just past the tag name). It
|
||||
// returns the attributes, whether the tag closed with "/>", and the index just
|
||||
// past the closing '>'.
|
||||
func parseAttrs(data []byte, p int) (map[string]string, bool, int) {
|
||||
attrs := map[string]string{}
|
||||
selfClose := false
|
||||
n := len(data)
|
||||
for p < n {
|
||||
for p < n && isSpace(data[p]) {
|
||||
p++
|
||||
}
|
||||
if p >= n {
|
||||
break
|
||||
}
|
||||
if data[p] == '>' {
|
||||
p++
|
||||
break
|
||||
}
|
||||
if data[p] == '/' {
|
||||
selfClose = true
|
||||
p++
|
||||
continue
|
||||
}
|
||||
start := p
|
||||
for p < n && !isSpace(data[p]) && data[p] != '=' && data[p] != '>' && data[p] != '/' {
|
||||
p++
|
||||
}
|
||||
name := strings.ToLower(string(data[start:p]))
|
||||
q := p
|
||||
for q < n && isSpace(data[q]) {
|
||||
q++
|
||||
}
|
||||
var value string
|
||||
if q < n && data[q] == '=' {
|
||||
q++
|
||||
for q < n && isSpace(data[q]) {
|
||||
q++
|
||||
}
|
||||
if q < n && (data[q] == '"' || data[q] == '\'') {
|
||||
quote := data[q]
|
||||
q++
|
||||
v := q
|
||||
for q < n && data[q] != quote {
|
||||
q++
|
||||
}
|
||||
value = string(data[v:q])
|
||||
if q < n {
|
||||
q++
|
||||
}
|
||||
} else {
|
||||
v := q
|
||||
for q < n && !isSpace(data[q]) && data[q] != '>' {
|
||||
q++
|
||||
}
|
||||
value = string(data[v:q])
|
||||
}
|
||||
p = q
|
||||
}
|
||||
if name != "" {
|
||||
attrs[name] = html.UnescapeString(value)
|
||||
}
|
||||
}
|
||||
return attrs, selfClose, p
|
||||
}
|
||||
|
||||
// appendText appends an entity-decoded text node to the parent, keeping its
|
||||
// position in document order.
|
||||
func appendText(n *node, s string) {
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
n.Kids = append(n.Kids, &node{Tag: "#text", text: html.UnescapeString(s), Parent: n})
|
||||
}
|
||||
|
||||
// closeTag pops elements up to and including the nearest matching tag and
|
||||
// returns the remaining stack.
|
||||
func closeTag(stack []*node, name string) []*node {
|
||||
for i := len(stack) - 1; i > 0; i-- {
|
||||
if stack[i].Tag == name {
|
||||
return stack[:i]
|
||||
}
|
||||
}
|
||||
return stack
|
||||
}
|
||||
|
||||
func isNameChar(c byte) bool {
|
||||
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' ||
|
||||
c >= '0' && c <= '9' || c == '-' || c == '_' || c == ':' || c == '.'
|
||||
}
|
||||
|
||||
func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' }
|
||||
|
||||
// indexFold finds needle in haystack, ignoring ASCII case.
|
||||
func indexFold(haystack []byte, needle string) int {
|
||||
if len(needle) == 0 {
|
||||
return 0
|
||||
}
|
||||
lower := bytes.ToLower(haystack)
|
||||
return bytes.Index(lower, []byte(strings.ToLower(needle)))
|
||||
}
|
||||
|
||||
// walk visits n and its descendants in document order. When fn returns false the
|
||||
// node's children are skipped; siblings are still visited. To abort a whole
|
||||
// traversal, the predicate must track its own stop flag.
|
||||
func walk(n *node, fn func(*node) bool) {
|
||||
if !fn(n) {
|
||||
return
|
||||
}
|
||||
for _, k := range n.Kids {
|
||||
walk(k, fn)
|
||||
}
|
||||
}
|
||||
|
||||
// find returns the first descendant (including n) matching pred.
|
||||
func find(n *node, pred func(*node) bool) *node {
|
||||
if pred(n) {
|
||||
return n
|
||||
}
|
||||
for _, k := range n.Kids {
|
||||
if got := find(k, pred); got != nil {
|
||||
return got
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findAll returns every descendant (including n) matching pred, in document order.
|
||||
func findAll(n *node, pred func(*node) bool) []*node {
|
||||
var out []*node
|
||||
walk(n, func(x *node) bool {
|
||||
if pred(x) {
|
||||
out = append(out, x)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return out
|
||||
}
|
||||
204
internal/parse/importance.go
Normal file
204
internal/parse/importance.go
Normal file
@@ -0,0 +1,204 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Importance labels stored in items.importance (FR-4).
|
||||
const (
|
||||
importanceHigh = "high"
|
||||
importanceNormal = "normal"
|
||||
importanceLow = "low"
|
||||
)
|
||||
|
||||
// ImportanceClassifiers names the classifier used for a run, so the choice is
|
||||
// recorded rather than implied.
|
||||
const (
|
||||
ClassifierNone = "deterministic"
|
||||
ClassifierOmniRoute = "omniroute"
|
||||
ClassifierJEV = "jev"
|
||||
classifyBatchSize = 40
|
||||
classifyMaxResponseChar = 16 * 1024
|
||||
)
|
||||
|
||||
// Completer is one text-completion call. internal/llm supplies the OmniRoute
|
||||
// client; tests supply a stub.
|
||||
type Completer interface {
|
||||
Complete(ctx context.Context, system, prompt string) (string, error)
|
||||
}
|
||||
|
||||
// ClassifyDeterministic is the offline importance rule. Overdue work and work
|
||||
// due within the next week is high; everything else is normal. It is the
|
||||
// fallback when no model is configured, and it is deliberately conservative: it
|
||||
// never marks work "done".
|
||||
func ClassifyDeterministic(items []Item, now time.Time) {
|
||||
week := now.AddDate(0, 0, 7)
|
||||
for i := range items {
|
||||
items[i].Importance = deterministicImportance(items[i], week)
|
||||
}
|
||||
}
|
||||
|
||||
func deterministicImportance(it Item, week time.Time) string {
|
||||
// Only work carries a deadline that makes it important. A calendar entry's
|
||||
// due_at is its start time, and a lesson that starts this week is not
|
||||
// overdue work, so events and messages stay normal under the offline rule.
|
||||
if it.Type == TypeTask {
|
||||
if it.DueAt != nil && it.DueAt.Before(week) {
|
||||
return importanceHigh
|
||||
}
|
||||
if strings.Contains(strings.ToLower(it.Body), "overdue") {
|
||||
return importanceHigh
|
||||
}
|
||||
}
|
||||
return importanceNormal
|
||||
}
|
||||
|
||||
// ClassifyModel asks a model to classify importance for every item (FR-4). It
|
||||
// fails, loudly, on a model error or an unparseable answer; it never silently
|
||||
// leaves a wrong label. Items are updated in place. now lets the model judge
|
||||
// whether a due date is overdue.
|
||||
func ClassifyModel(ctx context.Context, c Completer, items []Item, model string, now time.Time) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("classify: no model client configured")
|
||||
}
|
||||
const system = "You classify school items for a parent's morning brief. " +
|
||||
"Reply with JSON only. No prose, no code fences."
|
||||
|
||||
for start := 0; start < len(items); start += classifyBatchSize {
|
||||
end := start + classifyBatchSize
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
|
||||
prompt := classifyPrompt(batch, model, now)
|
||||
raw, err := c.Complete(ctx, system, prompt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("classify: items %d-%d: %w", start, end-1, err)
|
||||
}
|
||||
labels, err := parseLabels(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("classify: items %d-%d: %w", start, end-1, err)
|
||||
}
|
||||
for i, label := range labels {
|
||||
if i >= len(batch) {
|
||||
return fmt.Errorf("classify: items %d-%d: model returned %d labels for %d items", start, end-1, len(labels), len(batch))
|
||||
}
|
||||
switch label {
|
||||
case importanceHigh, importanceNormal, importanceLow:
|
||||
batch[i].Importance = label
|
||||
default:
|
||||
return fmt.Errorf("classify: item %d: invalid importance %q", start+i, label)
|
||||
}
|
||||
}
|
||||
if len(labels) != len(batch) {
|
||||
return fmt.Errorf("classify: items %d-%d: model returned %d labels for %d items", start, end-1, len(labels), len(batch))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// classifyPrompt builds the user message for one batch.
|
||||
func classifyPrompt(batch []Item, model string, now time.Time) string {
|
||||
type row struct {
|
||||
I int `json:"i"`
|
||||
Child string `json:"child"`
|
||||
Type string `json:"type"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Title string `json:"title"`
|
||||
Due string `json:"due,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
rows := make([]row, 0, len(batch))
|
||||
for i, it := range batch {
|
||||
due := ""
|
||||
if it.DueAt != nil {
|
||||
due = it.DueAt.Format(time.RFC3339)
|
||||
}
|
||||
rows = append(rows, row{
|
||||
I: i,
|
||||
Child: it.Child,
|
||||
Type: it.Type,
|
||||
Subject: it.Subject,
|
||||
Title: it.Title,
|
||||
Due: due,
|
||||
Detail: detail(it.Body),
|
||||
})
|
||||
}
|
||||
payload, _ := json.Marshal(rows)
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Model: %s\n", model)
|
||||
fmt.Fprintf(&b, "Now: %s\n", now.Format(time.RFC3339))
|
||||
b.WriteString("Label every item with importance high, normal or low.\n")
|
||||
b.WriteString("high: overdue work, work due today or in the next 48 hours, or a school event that affects the family.\n")
|
||||
b.WriteString("low: routine recurring lessons, past submitted work, or notices with no action.\n")
|
||||
b.WriteString("normal: anything else.\n")
|
||||
b.WriteString("A task with a due date before Now and a submission status of Pending or Overdue is high.\n")
|
||||
b.WriteString("Return a JSON array with one object per item, in order, exactly: " +
|
||||
`{"i":<index>,"importance":"high|normal|low"}` + "\n\nItems:\n")
|
||||
b.Write(payload)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// detail flattens an item body into one line for the model.
|
||||
func detail(s string) string {
|
||||
return collapse(strings.ReplaceAll(s, "\n", "; "))
|
||||
}
|
||||
|
||||
// parseLabels extracts the importance labels from a model reply. It tolerates
|
||||
// code fences and surrounding prose, and returns the labels ordered by index.
|
||||
func parseLabels(raw string) ([]string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if len(raw) > classifyMaxResponseChar {
|
||||
return nil, fmt.Errorf("model reply too large (%d bytes)", len(raw))
|
||||
}
|
||||
raw = strings.TrimPrefix(raw, "```json")
|
||||
raw = strings.TrimPrefix(raw, "```")
|
||||
raw = strings.TrimSuffix(raw, "```")
|
||||
raw = strings.TrimSpace(raw)
|
||||
|
||||
if i := strings.IndexByte(raw, '['); i >= 0 {
|
||||
if j := strings.LastIndexByte(raw, ']'); j > i {
|
||||
raw = raw[i : j+1]
|
||||
}
|
||||
}
|
||||
|
||||
var entries []struct {
|
||||
I *int `json:"i"`
|
||||
Importance string `json:"importance"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &entries); err != nil {
|
||||
return nil, fmt.Errorf("model reply is not JSON: %w", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, fmt.Errorf("model reply has no items")
|
||||
}
|
||||
|
||||
type indexed struct {
|
||||
i int
|
||||
label string
|
||||
}
|
||||
got := make([]indexed, 0, len(entries))
|
||||
for pos, e := range entries {
|
||||
idx := pos
|
||||
if e.I != nil {
|
||||
idx = *e.I
|
||||
}
|
||||
got = append(got, indexed{i: idx, label: strings.ToLower(strings.TrimSpace(e.Importance))})
|
||||
}
|
||||
sort.Slice(got, func(a, b int) bool { return got[a].i < got[b].i })
|
||||
|
||||
out := make([]string, len(got))
|
||||
for k, g := range got {
|
||||
if g.i < 0 || g.i >= len(got) {
|
||||
return nil, fmt.Errorf("model returned out-of-range index %d", g.i)
|
||||
}
|
||||
out[k] = g.label
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
102
internal/parse/news.go
Normal file
102
internal/parse/news.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newsTimeLayout is the aria-label format on the feed timestamp, for example
|
||||
// "14/10/2026 09:00 AM".
|
||||
const newsTimeLayout = "02/01/2006 03:04 PM"
|
||||
|
||||
// parseNews parses the home-page news feed into message items. It fails loudly
|
||||
// when the page carries neither the feed toolbar nor a post, so a login or
|
||||
// challenge page is never mistaken for "no news" (FR-12).
|
||||
func parseNews(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractNews(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no news feed or posts found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractNews returns news posts from the home page. found reports whether the
|
||||
// feed was present at all.
|
||||
func extractNews(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
|
||||
toolbar := find(root, func(n *node) bool {
|
||||
return strings.EqualFold(n.Attrs["data-testid"], "feed-toolbar")
|
||||
})
|
||||
cards := findAll(root, func(n *node) bool { return n.hasClass("MuiCardContent-root") })
|
||||
found = toolbar != nil || len(cards) > 0
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
titleNode := find(card, func(n *node) bool { return n.hasClassPart("postTitle") })
|
||||
if titleNode == nil {
|
||||
continue // not a news post
|
||||
}
|
||||
title := collapse(titleNode.Text())
|
||||
if title == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var body string
|
||||
if b := find(card, func(n *node) bool { return n.hasClassPart("froala-viewer") }); b != nil {
|
||||
body = b.Text()
|
||||
}
|
||||
|
||||
var author string
|
||||
if a := find(card, func(n *node) bool { return n.hasClassPart("MuiTypography-dialogMdStrong") }); a != nil {
|
||||
author = collapse(a.Text())
|
||||
}
|
||||
|
||||
var posted *time.Time
|
||||
if t := find(card, func(n *node) bool {
|
||||
return n.hasClassPart("MuiTypography-dialogSmStrong") && n.hasAttr("aria-label")
|
||||
}); t != nil {
|
||||
raw := strings.TrimSpace(t.Attrs["aria-label"])
|
||||
parsed, perr := time.ParseInLocation(newsTimeLayout, raw, loc)
|
||||
if perr != nil {
|
||||
return nil, false, fmt.Errorf("parse: %s: news %q: unrecognised posted time %q", page.Slug, title, raw)
|
||||
}
|
||||
posted = &parsed
|
||||
}
|
||||
|
||||
pageURL := page.URL
|
||||
if pageURL == "" {
|
||||
pageURL = compassHost + "/"
|
||||
}
|
||||
ref := dedupeKey(author, title, relativeOrISO(posted))
|
||||
|
||||
items = append(items, Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeMessage,
|
||||
Teacher: author, // messages carry the author, not a teacher
|
||||
Title: title,
|
||||
Body: body,
|
||||
DueAt: posted,
|
||||
URL: pageURL,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: ref,
|
||||
Importance: importanceNormal,
|
||||
})
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
// relativeOrISO renders a posted time for the dedupe key.
|
||||
func relativeOrISO(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
163
internal/parse/parse.go
Normal file
163
internal/parse/parse.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Source and item types match the items table CHECK constraints (AGENTS.md §7).
|
||||
const (
|
||||
SourceCompass = "compass"
|
||||
|
||||
TypeMessage = "message"
|
||||
TypeTask = "task"
|
||||
TypeEvent = "event"
|
||||
)
|
||||
|
||||
// Recognised page slugs, matching the <slug>.html files stored by the Phase 2
|
||||
// fetchers under var/raw/<child>/<runId>/.
|
||||
const (
|
||||
PageHome = "home"
|
||||
PageChildProfile = "child-profile"
|
||||
PageLearningTasks = "learning-tasks"
|
||||
PageCalendar = "calendar"
|
||||
)
|
||||
|
||||
// Item is one message, learning task or calendar event read from Compass.
|
||||
// Every field maps to a column of the items table (FR-5).
|
||||
type Item struct {
|
||||
Child string `json:"child"`
|
||||
Source string `json:"source"`
|
||||
Type string `json:"type"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Teacher string `json:"teacher,omitempty"`
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body,omitempty"`
|
||||
DueAt *time.Time `json:"due_at,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
SourcePage string `json:"source_page"`
|
||||
ExternalRef string `json:"external_ref"`
|
||||
Importance string `json:"importance"`
|
||||
}
|
||||
|
||||
// Page is one stored raw page: its slug, the URL it was fetched from, and the
|
||||
// HTML bytes.
|
||||
type Page struct {
|
||||
Slug string
|
||||
URL string
|
||||
HTML []byte
|
||||
}
|
||||
|
||||
// Parse turns the stored pages for one child into structured items. It fails,
|
||||
// loudly and explicitly, when a page it recognises cannot be parsed, so a broken
|
||||
// scan can never be shown as "nothing due" (FR-12).
|
||||
//
|
||||
// Pages with an unknown slug are ignored. When no recognised page yields any
|
||||
// structure at all, Parse returns an error.
|
||||
func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
|
||||
if strings.TrimSpace(child) == "" {
|
||||
return nil, fmt.Errorf("parse: child is empty")
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
|
||||
var items []Item
|
||||
recognised := 0
|
||||
for _, p := range pagesOf(pages) {
|
||||
switch p.Slug {
|
||||
case PageLearningTasks:
|
||||
got, err := parseLearningTasks(child, p, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, got...)
|
||||
recognised++
|
||||
case PageChildProfile:
|
||||
// The React profile page has no server-rendered grid; when it does
|
||||
// carry the legacy grid, use it.
|
||||
got, found, err := extractLearningTasks(child, p, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
items = append(items, got...)
|
||||
}
|
||||
recognised++
|
||||
case PageHome:
|
||||
got, err := parseNews(child, p, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, got...)
|
||||
recognised++
|
||||
case PageCalendar:
|
||||
got, err := parseCalendar(child, p, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, got...)
|
||||
recognised++
|
||||
}
|
||||
}
|
||||
|
||||
if recognised == 0 {
|
||||
return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages))
|
||||
}
|
||||
|
||||
// A task row is the same task wherever it appears (learning tasks and child
|
||||
// profile pages both carry the grid). Keep the first, drop the repeats.
|
||||
seen := map[string]bool{}
|
||||
out := items[:0]
|
||||
for _, it := range items {
|
||||
key := it.Type + "\x1f" + it.ExternalRef
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
if it.Importance == "" {
|
||||
it.Importance = importanceNormal
|
||||
}
|
||||
out = append(out, it)
|
||||
}
|
||||
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if out[i].Type != out[j].Type {
|
||||
return out[i].Type < out[j].Type
|
||||
}
|
||||
return out[i].ExternalRef < out[j].ExternalRef
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pagesOf returns the pages with duplicates removed, keeping the last copy of a
|
||||
// slug (the fetcher overwrites a slug file on a re-fetch, so the last is
|
||||
// authoritative).
|
||||
func pagesOf(pages []Page) []Page {
|
||||
bySlug := map[string]Page{}
|
||||
var order []string
|
||||
for _, p := range pages {
|
||||
if _, ok := bySlug[p.Slug]; !ok {
|
||||
order = append(order, p.Slug)
|
||||
}
|
||||
bySlug[p.Slug] = p
|
||||
}
|
||||
out := make([]Page, 0, len(order))
|
||||
for _, slug := range order {
|
||||
out = append(out, bySlug[slug])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupeKey joins the identity fields of an item into a stable external
|
||||
// reference. Re-parsing the same page must update the same row, not insert a
|
||||
// second one.
|
||||
func dedupeKey(parts ...string) string {
|
||||
clean := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
clean = append(clean, collapse(p))
|
||||
}
|
||||
return strings.Join(clean, "|")
|
||||
}
|
||||
305
internal/parse/parse_test.go
Normal file
305
internal/parse/parse_test.go
Normal file
@@ -0,0 +1,305 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
// Embed the zone database so the tests can use Australia/Melbourne.
|
||||
_ "time/tzdata"
|
||||
)
|
||||
|
||||
// melbourne is the family timezone used in the tests.
|
||||
func melbourne(t *testing.T) *time.Location {
|
||||
t.Helper()
|
||||
loc, err := time.LoadLocation("Australia/Melbourne")
|
||||
if err != nil {
|
||||
t.Fatalf("load location: %v", err)
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
// tasksFixture mirrors the malformed ExtJS grid: the group header <td> is not
|
||||
// closed before the group's inner table, exactly as Compass emits it.
|
||||
const tasksFixture = `<html><body>
|
||||
<table class="x-gridview-1-table x-grid-table"><tbody>
|
||||
<tr class="x-grid-row x-grid-wrap-row x-grid-group-row"><td class="x-group-hd-container" colspan="4">
|
||||
<div class="x-grid-group-hd x-grid-group-hd-collapsible"><div class="x-grid-group-title" style="">
|
||||
<a href="/Records/User.aspx?userId=999">Test Child</a> - <a href="/Organise/Subjects/Subject.aspx?subjectId=1&openLearningTaskTab">11EENG 11 ENGLISH</a>
|
||||
</div></div>
|
||||
<table class="x-gridview-1-table x-grid-table"><tbody>
|
||||
<tr class="x-grid-row x-grid-data-row">
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
|
||||
<span class="task-details-category-label" style="background-color:#F17056">Assessment</span>
|
||||
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Essay draft">Essay draft</a></span></div>
|
||||
<span data-qtip="This is a Class Learning Task."><span class="learning-task-datafield-title">Class Task</span></span> | <span data-qtip="report"><span class="learning-task-datafield-title">Report: </span><span class="learning-task-datafield-value">Yes</span></span>
|
||||
</div></div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner">
|
||||
<span data-qtip="due soon"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">19/10/26 04:00 PM</span></span><br>
|
||||
<span data-qtip="pending"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Pending.png"> Pending</span></span>
|
||||
</div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
|
||||
</tr>
|
||||
<tr class="x-grid-row x-grid-row-alt x-grid-data-row">
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
|
||||
<span class="task-details-category-label">Homework</span>
|
||||
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Chapter 4 questions">Chapter 4 questions</a></span></div>
|
||||
<span data-qtip="Subject-wide"><span class="learning-task-datafield-title">Subject-wide Task</span></span>
|
||||
</div></div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner">
|
||||
<span data-qtip="overdue"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">09/10/26 04:00 PM</span></span><br>
|
||||
<span data-qtip="overdue"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Overdue.png"> Overdue</span></span>
|
||||
</div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</td></tr>
|
||||
</tbody></table>
|
||||
</body></html>`
|
||||
|
||||
func TestParseLearningTasks(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseLearningTasks("Harry", Page{
|
||||
Slug: PageLearningTasks,
|
||||
URL: "https://nhs-vic.compass.education/Records/UserNew.aspx#learningTasks",
|
||||
HTML: []byte(tasksFixture),
|
||||
}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLearningTasks: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("got %d items, want 2", len(items))
|
||||
}
|
||||
|
||||
first := items[0]
|
||||
if first.Type != TypeTask {
|
||||
t.Errorf("type = %q, want %q", first.Type, TypeTask)
|
||||
}
|
||||
if first.Subject != "11EENG 11 ENGLISH" {
|
||||
t.Errorf("subject = %q", first.Subject)
|
||||
}
|
||||
if first.Title != "Essay draft" {
|
||||
t.Errorf("title = %q", first.Title)
|
||||
}
|
||||
if !strings.Contains(first.Body, "Category: Assessment") || !strings.Contains(first.Body, "Task: Class Task") {
|
||||
t.Errorf("body = %q", first.Body)
|
||||
}
|
||||
if first.DueAt == nil || first.DueAt.Format("2006-01-02 15:04") != "2026-10-19 16:00" {
|
||||
t.Errorf("due = %v", first.DueAt)
|
||||
}
|
||||
if !strings.Contains(first.URL, "userId=999") || !strings.HasSuffix(first.URL, "#learningTasks") {
|
||||
t.Errorf("url = %q", first.URL)
|
||||
}
|
||||
if first.SourcePage != PageLearningTasks {
|
||||
t.Errorf("source page = %q", first.SourcePage)
|
||||
}
|
||||
if first.ExternalRef == "" {
|
||||
t.Error("empty external ref")
|
||||
}
|
||||
|
||||
second := items[1]
|
||||
if second.DueAt == nil || second.DueAt.Format("2006-01-02") != "2026-10-09" {
|
||||
t.Errorf("second due = %v", second.DueAt)
|
||||
}
|
||||
if second.ExternalRef == first.ExternalRef {
|
||||
t.Error("two tasks share an external ref")
|
||||
}
|
||||
|
||||
// Re-parsing must produce the same references, so the store de-duplicates.
|
||||
again, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: []byte(tasksFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("re-parse: %v", err)
|
||||
}
|
||||
if again[0].ExternalRef != first.ExternalRef {
|
||||
t.Errorf("external ref not stable: %q vs %q", again[0].ExternalRef, first.ExternalRef)
|
||||
}
|
||||
}
|
||||
|
||||
const newsFixture = `<html><body>
|
||||
<div data-testid="feed-toolbar"><p>News</p></div>
|
||||
<div class="MuiPaper-root"><div class="MuiCardContent-root">
|
||||
<span class="MuiTypography-root MuiTypography-dialogMdStrong css-1">T Smith</span>
|
||||
<span class="MuiTypography-root MuiTypography-dialogSmStrong css-2" aria-label="14/10/2026 09:00 AM">1 day ago</span>
|
||||
<span class="postTitle css-3"><span class="">Health Information</span></span>
|
||||
<div class="froala-viewer"><p>Dear families, there is a notice.</p></div>
|
||||
</div></div>
|
||||
</body></html>`
|
||||
|
||||
func TestParseNews(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseNews("Harry", Page{Slug: PageHome, URL: "https://nhs-vic.compass.education/", HTML: []byte(newsFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseNews: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("got %d items, want 1", len(items))
|
||||
}
|
||||
it := items[0]
|
||||
if it.Type != TypeMessage {
|
||||
t.Errorf("type = %q", it.Type)
|
||||
}
|
||||
if it.Title != "Health Information" {
|
||||
t.Errorf("title = %q", it.Title)
|
||||
}
|
||||
if it.Teacher != "T Smith" {
|
||||
t.Errorf("author = %q", it.Teacher)
|
||||
}
|
||||
if !strings.Contains(it.Body, "Dear families") {
|
||||
t.Errorf("body = %q", it.Body)
|
||||
}
|
||||
if it.DueAt == nil || it.DueAt.Format("2006-01-02 15:04") != "2026-10-14 09:00" {
|
||||
t.Errorf("posted = %v", it.DueAt)
|
||||
}
|
||||
}
|
||||
|
||||
const calendarFixture = `<html><body>
|
||||
<table class="ext-cal"><tbody><tr><td class="ext-cal-day-col"><div class="ext-cal-day-col-inner">
|
||||
<div id="calendar-manager-week-bd-day-col-20261015" class="ext-cal-day-col-gutter" style="height:1200px;">
|
||||
<div class="calendar-manager-week-bd-evt-a calendar-event-1001 custom-event ext-cal-evt activity-type-2 ext-event-no-roll" style="left:0%;width:100%;top:900px;height:149px;background-color:#ccffcc;">
|
||||
<span class="ext-evt-bd" lang="en-GB">6:00: End of Year Concert - HALL - KSH</span>
|
||||
</div>
|
||||
<div class="calendar-manager-week-bd-evt-b calendar-event-1002 custom-event ext-cal-evt activity-type-1 ext-event-no-roll" style="left:0%;width:100%;top:705px;height:48px;">
|
||||
<span class="ext-evt-bd" lang="en-GB">2:06: 5 - 11HLEGB5 - D8 - LHT</span>
|
||||
</div>
|
||||
</div>
|
||||
</div></td></tr></tbody></table>
|
||||
</body></html>`
|
||||
|
||||
func TestParseCalendar(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseCalendar("Harry", Page{Slug: PageCalendar, URL: "https://nhs-vic.compass.education/Organise/Calendar/", HTML: []byte(calendarFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCalendar: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("got %d items, want 2", len(items))
|
||||
}
|
||||
if items[0].Type != TypeEvent {
|
||||
t.Errorf("type = %q", items[0].Type)
|
||||
}
|
||||
if got := items[0].DueAt.Format("2006-01-02 15:04"); got != "2026-10-15 18:00" {
|
||||
t.Errorf("event 1 time = %s", got)
|
||||
}
|
||||
if items[0].Title != "End of Year Concert - HALL - KSH" {
|
||||
t.Errorf("event 1 title = %q", items[0].Title)
|
||||
}
|
||||
if items[1].Subject != "11HLEGB5" || items[1].Teacher != "LHT" {
|
||||
t.Errorf("event 2 subject/teacher = %q/%q", items[1].Subject, items[1].Teacher)
|
||||
}
|
||||
if got := items[1].DueAt.Format("15:04"); got != "14:06" {
|
||||
t.Errorf("event 2 time = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseFailsLoudly proves FR-12: a challenge or login page must not be read
|
||||
// as "nothing due".
|
||||
func TestParseFailsLoudly(t *testing.T) {
|
||||
challenge := []byte(`<html><head><title>Just a moment...</title></head><body><h1>Just a moment...</h1></body></html>`)
|
||||
|
||||
if _, err := parseNews("Harry", Page{Slug: PageHome, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseNews accepted a challenge page")
|
||||
}
|
||||
if _, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseLearningTasks accepted a page with no grid")
|
||||
}
|
||||
if _, err := parseCalendar("Harry", Page{Slug: PageCalendar, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseCalendar accepted a page with no grid")
|
||||
}
|
||||
if _, err := Parse("Harry", []Page{{Slug: "unknown", HTML: challenge}}, time.UTC); err == nil {
|
||||
t.Error("Parse accepted a page set with no recognised page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompassDue(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"19/10/26 04:00 PM", "2026-10-19 16:00"},
|
||||
{"09/10/26 04:00 PM", "2026-10-09 16:00"},
|
||||
{"No due date", ""},
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := parseCompassDue(tc.in, loc)
|
||||
if err != nil {
|
||||
t.Errorf("parseCompassDue(%q): %v", tc.in, err)
|
||||
continue
|
||||
}
|
||||
if tc.want == "" {
|
||||
if got != nil {
|
||||
t.Errorf("parseCompassDue(%q) = %v, want nil", tc.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got == nil || got.Format("2006-01-02 15:04") != tc.want {
|
||||
t.Errorf("parseCompassDue(%q) = %v, want %s", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
if _, err := parseCompassDue("tomorrow", loc); err == nil {
|
||||
t.Error("parseCompassDue accepted an unknown format")
|
||||
}
|
||||
}
|
||||
|
||||
// stubCompleter returns a fixed reply.
|
||||
type stubCompleter struct {
|
||||
reply string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *stubCompleter) Complete(context.Context, string, string) (string, error) {
|
||||
s.calls++
|
||||
if s.err != nil {
|
||||
return "", s.err
|
||||
}
|
||||
return s.reply, nil
|
||||
}
|
||||
|
||||
func TestClassifyModel(t *testing.T) {
|
||||
items := []Item{
|
||||
{Type: TypeTask, Title: "Essay", DueAt: ptr(time.Date(2026, 10, 9, 16, 0, 0, 0, time.UTC))},
|
||||
{Type: TypeMessage, Title: "Notice"},
|
||||
}
|
||||
reply, _ := json.Marshal([]map[string]any{
|
||||
{"i": 0, "importance": "high"},
|
||||
{"i": 1, "importance": "low"},
|
||||
})
|
||||
c := &stubCompleter{reply: "```json\n" + string(reply) + "\n```"}
|
||||
if err := ClassifyModel(context.Background(), c, items, "test-model", time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("ClassifyModel: %v", err)
|
||||
}
|
||||
if items[0].Importance != importanceHigh || items[1].Importance != importanceLow {
|
||||
t.Errorf("labels = %q/%q", items[0].Importance, items[1].Importance)
|
||||
}
|
||||
|
||||
bad := &stubCompleter{reply: `[{"i":0,"importance":"urgent"}]`}
|
||||
if err := ClassifyModel(context.Background(), bad, items[:1], "test-model", time.Now()); err == nil {
|
||||
t.Error("ClassifyModel accepted an invalid label")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDeterministic(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||
overdue := now.Add(-24 * time.Hour)
|
||||
nextWeek := now.Add(48 * time.Hour)
|
||||
items := []Item{
|
||||
{Type: TypeTask, Title: "overdue", DueAt: &overdue},
|
||||
{Type: TypeTask, Title: "soon", DueAt: &nextWeek},
|
||||
{Type: TypeMessage, Title: "plain"},
|
||||
}
|
||||
ClassifyDeterministic(items, now)
|
||||
if items[0].Importance != importanceHigh || items[1].Importance != importanceHigh {
|
||||
t.Errorf("overdue/soon = %q/%q", items[0].Importance, items[1].Importance)
|
||||
}
|
||||
if items[2].Importance != importanceNormal {
|
||||
t.Errorf("plain = %q", items[2].Importance)
|
||||
}
|
||||
}
|
||||
|
||||
func ptr(t time.Time) *time.Time { return &t }
|
||||
263
internal/parse/tasks.go
Normal file
263
internal/parse/tasks.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// compassDueLayouts are the due-date formats Compass writes. The grid prints
|
||||
// "19/10/26 04:00 PM"; the fallback accepts single-digit fields.
|
||||
var compassDueLayouts = []string{"02/01/06 03:04 PM", "2/1/06 3:04 PM"}
|
||||
|
||||
// noDueDate is the literal the grid prints when a task has no due date.
|
||||
const noDueDate = "No due date"
|
||||
|
||||
// taskFieldLabels are the learning-task datafield titles that carry a value.
|
||||
var taskFieldLabels = map[string]bool{
|
||||
"Due date:": true,
|
||||
"Submitted:": true,
|
||||
"Report:": true,
|
||||
}
|
||||
|
||||
// parseLearningTasks parses the legacy ExtJS learning-task grid into task items.
|
||||
// It fails loudly when the grid is missing entirely, because that means the page
|
||||
// changed or the fetch returned something else (FR-12).
|
||||
func parseLearningTasks(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractLearningTasks(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no learning-task grid found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractLearningTasks returns the task rows in one grid page. found reports
|
||||
// whether the page contained a learning-task grid at all, so a caller can accept
|
||||
// an empty option page (for example the React child-profile page) without
|
||||
// treating it as a failure.
|
||||
func extractLearningTasks(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
if !hasGrid(root) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
subject := ""
|
||||
profileURL := ""
|
||||
var parseErr error
|
||||
stop := false
|
||||
|
||||
walk(root, func(n *node) bool {
|
||||
if stop || parseErr != nil {
|
||||
return false
|
||||
}
|
||||
if n.Tag == "div" && n.hasClass("x-grid-group-title") {
|
||||
if s, u := groupSubject(n); s != "" {
|
||||
subject = s
|
||||
if u != "" {
|
||||
profileURL = u
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if n.Tag == "a" && n.hasClass("task-submission-launcher") {
|
||||
it, err := taskFromAnchor(child, n, subject, profileURL, page, loc)
|
||||
if err != nil {
|
||||
parseErr = err
|
||||
stop = true
|
||||
return false
|
||||
}
|
||||
items = append(items, it)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
if parseErr != nil {
|
||||
return nil, false, parseErr
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
// hasGrid reports whether the document carries an ExtJS grid table.
|
||||
func hasGrid(root *node) bool {
|
||||
return find(root, func(n *node) bool {
|
||||
return n.Tag == "table" && n.hasClass("x-grid-table")
|
||||
}) != nil
|
||||
}
|
||||
|
||||
// groupSubject reads the subject and the child profile URL from an ExtJS group
|
||||
// header such as:
|
||||
//
|
||||
// <a href="/Records/User.aspx?userId=27038">Harrison ROLFE</a> -
|
||||
// <a href="/Organise/Subjects/Subject.aspx?subjectId=5657...">11HLEG ...</a>
|
||||
func groupSubject(title *node) (subject, profileURL string) {
|
||||
for _, a := range findAll(title, func(n *node) bool { return n.Tag == "a" }) {
|
||||
href := a.Attrs["href"]
|
||||
text := collapse(a.Text())
|
||||
switch {
|
||||
case strings.Contains(href, "Subject.aspx"):
|
||||
if subject == "" {
|
||||
subject = text
|
||||
}
|
||||
case strings.Contains(href, "Records/User.aspx"):
|
||||
if profileURL == "" && text != "" {
|
||||
profileURL = href
|
||||
}
|
||||
}
|
||||
}
|
||||
if subject == "" {
|
||||
// Fall back to the text after "NAME - ": "Harrison ROLFE - 11HLEG ...".
|
||||
text := title.Text()
|
||||
if i := strings.Index(text, " - "); i >= 0 {
|
||||
subject = strings.TrimSpace(text[i+3:])
|
||||
}
|
||||
}
|
||||
return subject, profileURL
|
||||
}
|
||||
|
||||
// taskFromAnchor builds one task item from a task-submission-launcher anchor.
|
||||
func taskFromAnchor(child string, a *node, subject, profileURL string, page Page, loc *time.Location) (Item, error) {
|
||||
title := collapse(a.Attrs["data-qtip"])
|
||||
if title == "" {
|
||||
title = collapse(a.Text())
|
||||
}
|
||||
if title == "" {
|
||||
return Item{}, fmt.Errorf("parse: %s: learning-task row with no title", page.Slug)
|
||||
}
|
||||
|
||||
tr := nearest(a, func(n *node) bool { return n.Tag == "tr" })
|
||||
if tr == nil {
|
||||
return Item{}, fmt.Errorf("parse: %s: task %q has no row", page.Slug, title)
|
||||
}
|
||||
|
||||
var category string
|
||||
if c := find(tr, func(n *node) bool { return n.hasClass("task-details-category-label") }); c != nil {
|
||||
category = collapse(c.Text())
|
||||
}
|
||||
|
||||
fields, kind := taskFields(tr)
|
||||
rawDue := fields["Due date:"]
|
||||
|
||||
due, err := parseCompassDue(rawDue, loc)
|
||||
if err != nil {
|
||||
return Item{}, fmt.Errorf("parse: %s: task %q: %w", page.Slug, title, err)
|
||||
}
|
||||
|
||||
submitted := fields["Submitted:"]
|
||||
var body []string
|
||||
if category != "" {
|
||||
body = append(body, "Category: "+category)
|
||||
}
|
||||
if kind != "" {
|
||||
body = append(body, "Task: "+kind)
|
||||
}
|
||||
if submitted != "" {
|
||||
body = append(body, "Submitted: "+submitted)
|
||||
}
|
||||
if rawDue != "" {
|
||||
body = append(body, "Due: "+rawDue)
|
||||
}
|
||||
|
||||
url := page.URL
|
||||
if profileURL != "" {
|
||||
url = absoluteCompassURL(profileURL) + "#learningTasks"
|
||||
} else if page.URL != "" {
|
||||
url = strings.Split(page.URL, "#")[0] + "#learningTasks"
|
||||
}
|
||||
|
||||
return Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeTask,
|
||||
Subject: subject,
|
||||
Title: title,
|
||||
Body: strings.Join(body, "\n"),
|
||||
DueAt: due,
|
||||
URL: url,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: dedupeKey(subject, title, kind),
|
||||
Importance: importanceNormal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// taskFields walks a learning-task row and returns the datafield values by
|
||||
// label, plus the task kind ("Class Task", "Subject-wide Task"). It relies on
|
||||
// document order: each value span follows its title span.
|
||||
func taskFields(tr *node) (fields map[string]string, kind string) {
|
||||
fields = map[string]string{}
|
||||
last := ""
|
||||
walk(tr, func(n *node) bool {
|
||||
if n.Tag != "span" {
|
||||
return true
|
||||
}
|
||||
switch {
|
||||
case n.hasClass("learning-task-datafield-title"):
|
||||
text := collapse(n.Text())
|
||||
if !taskFieldLabels[text] && kind == "" {
|
||||
kind = text
|
||||
}
|
||||
last = text
|
||||
return false
|
||||
case n.hasClass("learning-task-datafield-value"):
|
||||
if last != "" {
|
||||
fields[last] = collapse(n.Text())
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return fields, kind
|
||||
}
|
||||
|
||||
// parseCompassDue parses a Compass due-date string in loc. "No due date" and an
|
||||
// empty value mean no due date; anything else that does not parse is an error,
|
||||
// never a silent zero (FR-12).
|
||||
func parseCompassDue(s string, loc *time.Location) (*time.Time, error) {
|
||||
s = collapse(s)
|
||||
if s == "" || strings.EqualFold(s, noDueDate) {
|
||||
return nil, nil
|
||||
}
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
for _, layout := range compassDueLayouts {
|
||||
if t, err := time.ParseInLocation(layout, s, loc); err == nil {
|
||||
return &t, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unrecognised due date %q", s)
|
||||
}
|
||||
|
||||
// absoluteCompassURL turns a site-relative Compass href into a full URL.
|
||||
func absoluteCompassURL(href string) string {
|
||||
href = strings.TrimSpace(href)
|
||||
if href == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||
return href
|
||||
}
|
||||
if !strings.HasPrefix(href, "/") {
|
||||
href = "/" + href
|
||||
}
|
||||
return compassHost + href
|
||||
}
|
||||
|
||||
// compassHost is the Northcote High School Compass origin (AGENTS.md §2a).
|
||||
const compassHost = "https://nhs-vic.compass.education"
|
||||
|
||||
// nearest returns the closest ancestor (or the node itself) matching pred.
|
||||
func nearest(n *node, pred func(*node) bool) *node {
|
||||
for cur := n; cur != nil; cur = cur.Parent {
|
||||
if pred(cur) {
|
||||
return cur
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// classCode matches Compass class codes such as 11HLEGB5 or 7CHI2.
|
||||
var classCode = regexp.MustCompile(`^[0-9]{1,2}[A-Z]{2,}[A-Z0-9.]*$`)
|
||||
Reference in New Issue
Block a user