Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
305
internal/parse/parse_test.go
Normal file
305
internal/parse/parse_test.go
Normal file
@@ -0,0 +1,305 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
// Embed the zone database so the tests can use Australia/Melbourne.
|
||||
_ "time/tzdata"
|
||||
)
|
||||
|
||||
// melbourne is the family timezone used in the tests.
|
||||
func melbourne(t *testing.T) *time.Location {
|
||||
t.Helper()
|
||||
loc, err := time.LoadLocation("Australia/Melbourne")
|
||||
if err != nil {
|
||||
t.Fatalf("load location: %v", err)
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
// tasksFixture mirrors the malformed ExtJS grid: the group header <td> is not
|
||||
// closed before the group's inner table, exactly as Compass emits it.
|
||||
const tasksFixture = `<html><body>
|
||||
<table class="x-gridview-1-table x-grid-table"><tbody>
|
||||
<tr class="x-grid-row x-grid-wrap-row x-grid-group-row"><td class="x-group-hd-container" colspan="4">
|
||||
<div class="x-grid-group-hd x-grid-group-hd-collapsible"><div class="x-grid-group-title" style="">
|
||||
<a href="/Records/User.aspx?userId=999">Test Child</a> - <a href="/Organise/Subjects/Subject.aspx?subjectId=1&openLearningTaskTab">11EENG 11 ENGLISH</a>
|
||||
</div></div>
|
||||
<table class="x-gridview-1-table x-grid-table"><tbody>
|
||||
<tr class="x-grid-row x-grid-data-row">
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
|
||||
<span class="task-details-category-label" style="background-color:#F17056">Assessment</span>
|
||||
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Essay draft">Essay draft</a></span></div>
|
||||
<span data-qtip="This is a Class Learning Task."><span class="learning-task-datafield-title">Class Task</span></span> | <span data-qtip="report"><span class="learning-task-datafield-title">Report: </span><span class="learning-task-datafield-value">Yes</span></span>
|
||||
</div></div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner">
|
||||
<span data-qtip="due soon"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">19/10/26 04:00 PM</span></span><br>
|
||||
<span data-qtip="pending"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Pending.png"> Pending</span></span>
|
||||
</div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
|
||||
</tr>
|
||||
<tr class="x-grid-row x-grid-row-alt x-grid-data-row">
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div><div><div style="display:flex;">
|
||||
<span class="task-details-category-label">Homework</span>
|
||||
<span class="learning-task-title"><a class="task-submission-launcher" data-qtip="Chapter 4 questions">Chapter 4 questions</a></span></div>
|
||||
<span data-qtip="Subject-wide"><span class="learning-task-datafield-title">Subject-wide Task</span></span>
|
||||
</div></div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner">
|
||||
<span data-qtip="overdue"><span class="learning-task-datafield-title">Due date: </span><span class="learning-task-datafield-value">09/10/26 04:00 PM</span></span><br>
|
||||
<span data-qtip="overdue"><span class="learning-task-datafield-title">Submitted: </span><span class="learning-task-datafield-value"><img src="/Assets/Overdue.png"> Overdue</span></span>
|
||||
</div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><div class="learning-task-row-value-large">-</div></div></td>
|
||||
<td class="x-grid-cell"><div class="x-grid-cell-inner"><img src="/Assets/tools.png"></div></td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
</td></tr>
|
||||
</tbody></table>
|
||||
</body></html>`
|
||||
|
||||
func TestParseLearningTasks(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseLearningTasks("Harry", Page{
|
||||
Slug: PageLearningTasks,
|
||||
URL: "https://nhs-vic.compass.education/Records/UserNew.aspx#learningTasks",
|
||||
HTML: []byte(tasksFixture),
|
||||
}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLearningTasks: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("got %d items, want 2", len(items))
|
||||
}
|
||||
|
||||
first := items[0]
|
||||
if first.Type != TypeTask {
|
||||
t.Errorf("type = %q, want %q", first.Type, TypeTask)
|
||||
}
|
||||
if first.Subject != "11EENG 11 ENGLISH" {
|
||||
t.Errorf("subject = %q", first.Subject)
|
||||
}
|
||||
if first.Title != "Essay draft" {
|
||||
t.Errorf("title = %q", first.Title)
|
||||
}
|
||||
if !strings.Contains(first.Body, "Category: Assessment") || !strings.Contains(first.Body, "Task: Class Task") {
|
||||
t.Errorf("body = %q", first.Body)
|
||||
}
|
||||
if first.DueAt == nil || first.DueAt.Format("2006-01-02 15:04") != "2026-10-19 16:00" {
|
||||
t.Errorf("due = %v", first.DueAt)
|
||||
}
|
||||
if !strings.Contains(first.URL, "userId=999") || !strings.HasSuffix(first.URL, "#learningTasks") {
|
||||
t.Errorf("url = %q", first.URL)
|
||||
}
|
||||
if first.SourcePage != PageLearningTasks {
|
||||
t.Errorf("source page = %q", first.SourcePage)
|
||||
}
|
||||
if first.ExternalRef == "" {
|
||||
t.Error("empty external ref")
|
||||
}
|
||||
|
||||
second := items[1]
|
||||
if second.DueAt == nil || second.DueAt.Format("2006-01-02") != "2026-10-09" {
|
||||
t.Errorf("second due = %v", second.DueAt)
|
||||
}
|
||||
if second.ExternalRef == first.ExternalRef {
|
||||
t.Error("two tasks share an external ref")
|
||||
}
|
||||
|
||||
// Re-parsing must produce the same references, so the store de-duplicates.
|
||||
again, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: []byte(tasksFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("re-parse: %v", err)
|
||||
}
|
||||
if again[0].ExternalRef != first.ExternalRef {
|
||||
t.Errorf("external ref not stable: %q vs %q", again[0].ExternalRef, first.ExternalRef)
|
||||
}
|
||||
}
|
||||
|
||||
const newsFixture = `<html><body>
|
||||
<div data-testid="feed-toolbar"><p>News</p></div>
|
||||
<div class="MuiPaper-root"><div class="MuiCardContent-root">
|
||||
<span class="MuiTypography-root MuiTypography-dialogMdStrong css-1">T Smith</span>
|
||||
<span class="MuiTypography-root MuiTypography-dialogSmStrong css-2" aria-label="14/10/2026 09:00 AM">1 day ago</span>
|
||||
<span class="postTitle css-3"><span class="">Health Information</span></span>
|
||||
<div class="froala-viewer"><p>Dear families, there is a notice.</p></div>
|
||||
</div></div>
|
||||
</body></html>`
|
||||
|
||||
func TestParseNews(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseNews("Harry", Page{Slug: PageHome, URL: "https://nhs-vic.compass.education/", HTML: []byte(newsFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseNews: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("got %d items, want 1", len(items))
|
||||
}
|
||||
it := items[0]
|
||||
if it.Type != TypeMessage {
|
||||
t.Errorf("type = %q", it.Type)
|
||||
}
|
||||
if it.Title != "Health Information" {
|
||||
t.Errorf("title = %q", it.Title)
|
||||
}
|
||||
if it.Teacher != "T Smith" {
|
||||
t.Errorf("author = %q", it.Teacher)
|
||||
}
|
||||
if !strings.Contains(it.Body, "Dear families") {
|
||||
t.Errorf("body = %q", it.Body)
|
||||
}
|
||||
if it.DueAt == nil || it.DueAt.Format("2006-01-02 15:04") != "2026-10-14 09:00" {
|
||||
t.Errorf("posted = %v", it.DueAt)
|
||||
}
|
||||
}
|
||||
|
||||
const calendarFixture = `<html><body>
|
||||
<table class="ext-cal"><tbody><tr><td class="ext-cal-day-col"><div class="ext-cal-day-col-inner">
|
||||
<div id="calendar-manager-week-bd-day-col-20261015" class="ext-cal-day-col-gutter" style="height:1200px;">
|
||||
<div class="calendar-manager-week-bd-evt-a calendar-event-1001 custom-event ext-cal-evt activity-type-2 ext-event-no-roll" style="left:0%;width:100%;top:900px;height:149px;background-color:#ccffcc;">
|
||||
<span class="ext-evt-bd" lang="en-GB">6:00: End of Year Concert - HALL - KSH</span>
|
||||
</div>
|
||||
<div class="calendar-manager-week-bd-evt-b calendar-event-1002 custom-event ext-cal-evt activity-type-1 ext-event-no-roll" style="left:0%;width:100%;top:705px;height:48px;">
|
||||
<span class="ext-evt-bd" lang="en-GB">2:06: 5 - 11HLEGB5 - D8 - LHT</span>
|
||||
</div>
|
||||
</div>
|
||||
</div></td></tr></tbody></table>
|
||||
</body></html>`
|
||||
|
||||
func TestParseCalendar(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
items, err := parseCalendar("Harry", Page{Slug: PageCalendar, URL: "https://nhs-vic.compass.education/Organise/Calendar/", HTML: []byte(calendarFixture)}, loc)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCalendar: %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("got %d items, want 2", len(items))
|
||||
}
|
||||
if items[0].Type != TypeEvent {
|
||||
t.Errorf("type = %q", items[0].Type)
|
||||
}
|
||||
if got := items[0].DueAt.Format("2006-01-02 15:04"); got != "2026-10-15 18:00" {
|
||||
t.Errorf("event 1 time = %s", got)
|
||||
}
|
||||
if items[0].Title != "End of Year Concert - HALL - KSH" {
|
||||
t.Errorf("event 1 title = %q", items[0].Title)
|
||||
}
|
||||
if items[1].Subject != "11HLEGB5" || items[1].Teacher != "LHT" {
|
||||
t.Errorf("event 2 subject/teacher = %q/%q", items[1].Subject, items[1].Teacher)
|
||||
}
|
||||
if got := items[1].DueAt.Format("15:04"); got != "14:06" {
|
||||
t.Errorf("event 2 time = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseFailsLoudly proves FR-12: a challenge or login page must not be read
|
||||
// as "nothing due".
|
||||
func TestParseFailsLoudly(t *testing.T) {
|
||||
challenge := []byte(`<html><head><title>Just a moment...</title></head><body><h1>Just a moment...</h1></body></html>`)
|
||||
|
||||
if _, err := parseNews("Harry", Page{Slug: PageHome, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseNews accepted a challenge page")
|
||||
}
|
||||
if _, err := parseLearningTasks("Harry", Page{Slug: PageLearningTasks, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseLearningTasks accepted a page with no grid")
|
||||
}
|
||||
if _, err := parseCalendar("Harry", Page{Slug: PageCalendar, HTML: challenge}, time.UTC); err == nil {
|
||||
t.Error("parseCalendar accepted a page with no grid")
|
||||
}
|
||||
if _, err := Parse("Harry", []Page{{Slug: "unknown", HTML: challenge}}, time.UTC); err == nil {
|
||||
t.Error("Parse accepted a page set with no recognised page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompassDue(t *testing.T) {
|
||||
loc := melbourne(t)
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"19/10/26 04:00 PM", "2026-10-19 16:00"},
|
||||
{"09/10/26 04:00 PM", "2026-10-09 16:00"},
|
||||
{"No due date", ""},
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got, err := parseCompassDue(tc.in, loc)
|
||||
if err != nil {
|
||||
t.Errorf("parseCompassDue(%q): %v", tc.in, err)
|
||||
continue
|
||||
}
|
||||
if tc.want == "" {
|
||||
if got != nil {
|
||||
t.Errorf("parseCompassDue(%q) = %v, want nil", tc.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got == nil || got.Format("2006-01-02 15:04") != tc.want {
|
||||
t.Errorf("parseCompassDue(%q) = %v, want %s", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
if _, err := parseCompassDue("tomorrow", loc); err == nil {
|
||||
t.Error("parseCompassDue accepted an unknown format")
|
||||
}
|
||||
}
|
||||
|
||||
// stubCompleter returns a fixed reply.
|
||||
type stubCompleter struct {
|
||||
reply string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *stubCompleter) Complete(context.Context, string, string) (string, error) {
|
||||
s.calls++
|
||||
if s.err != nil {
|
||||
return "", s.err
|
||||
}
|
||||
return s.reply, nil
|
||||
}
|
||||
|
||||
func TestClassifyModel(t *testing.T) {
|
||||
items := []Item{
|
||||
{Type: TypeTask, Title: "Essay", DueAt: ptr(time.Date(2026, 10, 9, 16, 0, 0, 0, time.UTC))},
|
||||
{Type: TypeMessage, Title: "Notice"},
|
||||
}
|
||||
reply, _ := json.Marshal([]map[string]any{
|
||||
{"i": 0, "importance": "high"},
|
||||
{"i": 1, "importance": "low"},
|
||||
})
|
||||
c := &stubCompleter{reply: "```json\n" + string(reply) + "\n```"}
|
||||
if err := ClassifyModel(context.Background(), c, items, "test-model", time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("ClassifyModel: %v", err)
|
||||
}
|
||||
if items[0].Importance != importanceHigh || items[1].Importance != importanceLow {
|
||||
t.Errorf("labels = %q/%q", items[0].Importance, items[1].Importance)
|
||||
}
|
||||
|
||||
bad := &stubCompleter{reply: `[{"i":0,"importance":"urgent"}]`}
|
||||
if err := ClassifyModel(context.Background(), bad, items[:1], "test-model", time.Now()); err == nil {
|
||||
t.Error("ClassifyModel accepted an invalid label")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyDeterministic(t *testing.T) {
|
||||
now := time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||
overdue := now.Add(-24 * time.Hour)
|
||||
nextWeek := now.Add(48 * time.Hour)
|
||||
items := []Item{
|
||||
{Type: TypeTask, Title: "overdue", DueAt: &overdue},
|
||||
{Type: TypeTask, Title: "soon", DueAt: &nextWeek},
|
||||
{Type: TypeMessage, Title: "plain"},
|
||||
}
|
||||
ClassifyDeterministic(items, now)
|
||||
if items[0].Importance != importanceHigh || items[1].Importance != importanceHigh {
|
||||
t.Errorf("overdue/soon = %q/%q", items[0].Importance, items[1].Importance)
|
||||
}
|
||||
if items[2].Importance != importanceNormal {
|
||||
t.Errorf("plain = %q", items[2].Importance)
|
||||
}
|
||||
}
|
||||
|
||||
func ptr(t time.Time) *time.Time { return &t }
|
||||
Reference in New Issue
Block a user