Files
compass_teams_nhs/internal/parse/parse_test.go
Sam Rolfe d7d529eb0f Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table
(AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12).

- internal/parse: dependency-free HTML reader plus extractors for the ExtJS
  learning-task grid (subject, title, category, kind, due date, status), the
  home news feed (author, posted time, title, body) and the calendar week grid
  (day column + grid offset give the time; label gives title/room/teacher).
- internal/parse: fail loudly. A challenge page, login page, changed layout or
  unknown due-date format is an explicit error, never an empty day.
- FR-4: structure is parsed deterministically; importance is classified by an
  OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a
  deterministic fallback. The classifier used is printed and recorded.
- internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies).
- internal/store: migration 002 adds source_page, importance and external_ref
  with a unique (child_id, source, type, external_ref) index; UpsertItems,
  CountItemsByType, SampleItems and ChildByName.
- cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan
  row (ok=0 plus the error on failure) and prints counts and samples.
- scripts/parse-compass.sh and make parse run it in a one-off container against
  the service's own volume without restarting the service.
- docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the
  known gaps.

Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event).
Re-running the same raw pages inserts 0 and updates 56.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-15 17:27:25 +11:00

306 lines
12 KiB
Go

package parse
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
// Embed the zone database so the tests can use Australia/Melbourne.
_ "time/tzdata"
)
// melbourne is the family timezone used in the tests.
func melbourne(t *testing.T) *time.Location {
t.Helper()
loc, err := time.LoadLocation("Australia/Melbourne")
if err != nil {
t.Fatalf("load location: %v", err)
}
return loc
}
// tasksFixture mirrors the malformed ExtJS grid: the group header <td> is not
// closed before the group's inner table, exactly as Compass emits it.
const tasksFixture = `<html><body>
<table class="x-gridview-1-table x-grid-table"><tbody>
<tr class="x-grid-row x-grid-wrap-row x-grid-group-row"><td class="x-group-hd-container" colspan="4">
<div class="x-grid-group-hd x-grid-group-hd-collapsible"><div class="x-grid-group-title" style="">
<a href="/Records/User.aspx?userId=999">Test Child</a> - <a href="/Organise/Subjects/Subject.aspx?subjectId=1&amp;openLearningTaskTab">11EENG 11 ENGLISH</a>
</div></div>
<table class="x-gridview-1-table x-grid-table"><tbody>
<tr class="x-grid-row x-grid-data-row">
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
<span class="task-details-category-label" style="background-color:#F17056">Assessment</span>
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Essay draft">Essay draft</a></span></div>
<span data-qtip="This is a Class Learning Task."><span class="learning-task-datafield-title">Class Task</span></span> | <span data-qtip="report"><span class="learning-task-datafield-title">Report: </span><span class="learning-task-datafield-value">Yes</span></span>
</div></div></div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner">
<span data-qtip="due soon"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">19/10/26 04:00 PM</span></span><br>
<span data-qtip="pending"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Pending.png"> Pending</span></span>
</div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
</tr>
<tr class="x-grid-row x-grid-row-alt x-grid-data-row">
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
<span class="task-details-category-label">Homework</span>
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Chapter 4 questions">Chapter 4 questions</a></span></div>
<span data-qtip="Subject-wide"><span class="learning-task-datafield-title">Subject-wide Task</span></span>
</div></div></div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner">
<span data-qtip="overdue"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">09/10/26 04:00 PM</span></span><br>
<span data-qtip="overdue"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Overdue.png"> Overdue</span></span>
</div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
</tr>
</tbody></table>
</td></tr>
</tbody></table>
</body></html>`
func TestParseLearningTasks(t *testing.T) {
loc := melbourne(t)
items, err := parseLearningTasks("Harry", Page{
Slug: PageLearningTasks,
URL: "https://nhs-vic.compass.education/Records/UserNew.aspx#learningTasks",
HTML: []byte(tasksFixture),
}, loc)
if err != nil {
t.Fatalf("parseLearningTasks: %v", err)
}
if len(items) != 2 {
t.Fatalf("got %d items, want 2", len(items))
}
first := items[0]
if first.Type != TypeTask {
t.Errorf("type = %q, want %q", first.Type, TypeTask)
}
if first.Subject != "11EENG 11 ENGLISH" {
t.Errorf("subject = %q", first.Subject)
}
if first.Title != "Essay draft" {
t.Errorf("title = %q", first.Title)
}
if !strings.Contains(first.Body, "Category: Assessment") || !strings.Contains(first.Body, "Task: Class Task") {
t.Errorf("body = %q", first.Body)
}
if first.DueAt == nil || first.DueAt.Format("2006-01-02 15:04") != "2026-10-19 16:00" {
t.Errorf("due = %v", first.DueAt)
}
if !strings.Contains(first.URL, "userId=999") || !strings.HasSuffix(first.URL, "#learningTasks") {
t.Errorf("url = %q", first.URL)
}
if first.SourcePage != PageLearningTasks {
t.Errorf("source page = %q", first.SourcePage)
}
if first.ExternalRef == "" {
t.Error("empty external ref")
}
second := items[1]
if second.DueAt == nil || second.DueAt.Format("2006-01-02") != "2026-10-09" {
t.Errorf("second due = %v", second.DueAt)
}
if second.ExternalRef == first.ExternalRef {
t.Error("two tasks share an external ref")
}
// Re-parsing must produce the same references, so the store de-duplicates.
again, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: []byte(tasksFixture)}, loc)
if err != nil {
t.Fatalf("re-parse: %v", err)
}
if again[0].ExternalRef != first.ExternalRef {
t.Errorf("external ref not stable: %q vs %q", again[0].ExternalRef, first.ExternalRef)
}
}
const newsFixture = `<html><body>
<div data-testid="feed-toolbar"><p>News</p></div>
<div class="MuiPaper-root"><div class="MuiCardContent-root">
<span class="MuiTypography-root MuiTypography-dialogMdStrong css-1">T Smith</span>
<span class="MuiTypography-root MuiTypography-dialogSmStrong css-2" aria-label="14/10/2026 09:00 AM">1 day ago</span>
<span class="postTitle css-3"><span class="">Health Information</span></span>
<div class="froala-viewer"><p>Dear families, there is a notice.</p></div>
</div></div>
</body></html>`
func TestParseNews(t *testing.T) {
loc := melbourne(t)
items, err := parseNews("Harry", Page{Slug: PageHome, URL: "https://nhs-vic.compass.education/", HTML: []byte(newsFixture)}, loc)
if err != nil {
t.Fatalf("parseNews: %v", err)
}
if len(items) != 1 {
t.Fatalf("got %d items, want 1", len(items))
}
it := items[0]
if it.Type != TypeMessage {
t.Errorf("type = %q", it.Type)
}
if it.Title != "Health Information" {
t.Errorf("title = %q", it.Title)
}
if it.Teacher != "T Smith" {
t.Errorf("author = %q", it.Teacher)
}
if !strings.Contains(it.Body, "Dear families") {
t.Errorf("body = %q", it.Body)
}
if it.DueAt == nil || it.DueAt.Format("2006-01-02 15:04") != "2026-10-14 09:00" {
t.Errorf("posted = %v", it.DueAt)
}
}
const calendarFixture = `<html><body>
<table class="ext-cal"><tbody><tr><td class="ext-cal-day-col"><div class="ext-cal-day-col-inner">
<div id="calendar-manager-week-bd-day-col-20261015" class="ext-cal-day-col-gutter" style="height:1200px;">
<div class="calendar-manager-week-bd-evt-a calendar-event-1001 custom-event ext-cal-evt activity-type-2 ext-event-no-roll" style="left:0%;width:100%;top:900px;height:149px;background-color:#ccffcc;">
<span class="ext-evt-bd" lang="en-GB">6:00: End of Year Concert - HALL - KSH</span>
</div>
<div class="calendar-manager-week-bd-evt-b calendar-event-1002 custom-event ext-cal-evt activity-type-1 ext-event-no-roll" style="left:0%;width:100%;top:705px;height:48px;">
<span class="ext-evt-bd" lang="en-GB">2:06: 5 - 11HLEGB5 - D8 - LHT</span>
</div>
</div>
</div></td></tr></tbody></table>
</body></html>`
func TestParseCalendar(t *testing.T) {
loc := melbourne(t)
items, err := parseCalendar("Harry", Page{Slug: PageCalendar, URL: "https://nhs-vic.compass.education/Organise/Calendar/", HTML: []byte(calendarFixture)}, loc)
if err != nil {
t.Fatalf("parseCalendar: %v", err)
}
if len(items) != 2 {
t.Fatalf("got %d items, want 2", len(items))
}
if items[0].Type != TypeEvent {
t.Errorf("type = %q", items[0].Type)
}
if got := items[0].DueAt.Format("2006-01-02 15:04"); got != "2026-10-15 18:00" {
t.Errorf("event 1 time = %s", got)
}
if items[0].Title != "End of Year Concert - HALL - KSH" {
t.Errorf("event 1 title = %q", items[0].Title)
}
if items[1].Subject != "11HLEGB5" || items[1].Teacher != "LHT" {
t.Errorf("event 2 subject/teacher = %q/%q", items[1].Subject, items[1].Teacher)
}
if got := items[1].DueAt.Format("15:04"); got != "14:06" {
t.Errorf("event 2 time = %q", got)
}
}
// TestParseFailsLoudly proves FR-12: a challenge or login page must not be read
// as "nothing due".
func TestParseFailsLoudly(t *testing.T) {
challenge := []byte(`<html><head><title>Just a moment...</title></head><body><h1>Just a moment...</h1></body></html>`)
if _, err := parseNews("Harry", Page{Slug: PageHome, HTML: challenge}, time.UTC); err == nil {
t.Error("parseNews accepted a challenge page")
}
if _, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: challenge}, time.UTC); err == nil {
t.Error("parseLearningTasks accepted a page with no grid")
}
if _, err := parseCalendar("Harry", Page{Slug: PageCalendar, HTML: challenge}, time.UTC); err == nil {
t.Error("parseCalendar accepted a page with no grid")
}
if _, err := Parse("Harry", []Page{{Slug: "unknown", HTML: challenge}}, time.UTC); err == nil {
t.Error("Parse accepted a page set with no recognised page")
}
}
func TestParseCompassDue(t *testing.T) {
loc := melbourne(t)
cases := []struct {
in string
want string
}{
{"19/10/26 04:00 PM", "2026-10-19 16:00"},
{"09/10/26 04:00 PM", "2026-10-09 16:00"},
{"No due date", ""},
{"", ""},
}
for _, tc := range cases {
got, err := parseCompassDue(tc.in, loc)
if err != nil {
t.Errorf("parseCompassDue(%q): %v", tc.in, err)
continue
}
if tc.want == "" {
if got != nil {
t.Errorf("parseCompassDue(%q) = %v, want nil", tc.in, got)
}
continue
}
if got == nil || got.Format("2006-01-02 15:04") != tc.want {
t.Errorf("parseCompassDue(%q) = %v, want %s", tc.in, got, tc.want)
}
}
if _, err := parseCompassDue("tomorrow", loc); err == nil {
t.Error("parseCompassDue accepted an unknown format")
}
}
// stubCompleter returns a fixed reply.
type stubCompleter struct {
reply string
err error
calls int
}
func (s *stubCompleter) Complete(context.Context, string, string) (string, error) {
s.calls++
if s.err != nil {
return "", s.err
}
return s.reply, nil
}
func TestClassifyModel(t *testing.T) {
items := []Item{
{Type: TypeTask, Title: "Essay", DueAt: ptr(time.Date(2026, 10, 9, 16, 0, 0, 0, time.UTC))},
{Type: TypeMessage, Title: "Notice"},
}
reply, _ := json.Marshal([]map[string]any{
{"i": 0, "importance": "high"},
{"i": 1, "importance": "low"},
})
c := &stubCompleter{reply: "```json\n" + string(reply) + "\n```"}
if err := ClassifyModel(context.Background(), c, items, "test-model", time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("ClassifyModel: %v", err)
}
if items[0].Importance != importanceHigh || items[1].Importance != importanceLow {
t.Errorf("labels = %q/%q", items[0].Importance, items[1].Importance)
}
bad := &stubCompleter{reply: `[{"i":0,"importance":"urgent"}]`}
if err := ClassifyModel(context.Background(), bad, items[:1], "test-model", time.Now()); err == nil {
t.Error("ClassifyModel accepted an invalid label")
}
}
func TestClassifyDeterministic(t *testing.T) {
now := time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
overdue := now.Add(-24 * time.Hour)
nextWeek := now.Add(48 * time.Hour)
items := []Item{
{Type: TypeTask, Title: "overdue", DueAt: &overdue},
{Type: TypeTask, Title: "soon", DueAt: &nextWeek},
{Type: TypeMessage, Title: "plain"},
}
ClassifyDeterministic(items, now)
if items[0].Importance != importanceHigh || items[1].Importance != importanceHigh {
t.Errorf("overdue/soon = %q/%q", items[0].Importance, items[1].Importance)
}
if items[2].Importance != importanceNormal {
t.Errorf("plain = %q", items[2].Importance)
}
}
func ptr(t time.Time) *time.Time { return &t }