From 5d263987dd136ad7f1c9f7f5fd68b861b63f5db5 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Thu, 15 Oct 2026 17:38:22 +1100 Subject: [PATCH] Phase 4a: fetch and parse the authoritative Compass learning tasks The Learning Tasks grid is capped at 20 rows and, on the parent page, mixes both children, so it cannot answer "what is overdue". The parent home page reported 6 overdue tasks for Harry while the stored grid held 1. Fetch the per-child task list (read-only) from POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId, parse it into task rows, and refuse to store it when the overdue count disagrees with the home page summary (FR-12). - fetcher: store learning-tasks-all.json + meta (userId, status, task count) - parse: new learning-tasks-all page, JSON parser, home-page reconciliation - store: drop superseded grid-derived task rows before the upsert - docs: docs/phase-4a-overdue-tasks.md; close the Phase 3 known gap Verified with live runs: Harry 70 tasks / 6 overdue, Finn 42 tasks / 2 overdue, both equal to the home page; a tampered home page stops the parse. Note: the fetcher and store.go files also carry the per-child session work that was already uncommitted in this shared workspace (AUD-108). Co-Authored-By: Paperclip --- cmd/school/parse.go | 125 ++++++++++--- docs/phase-3-parse-notes.md | 7 +- docs/phase-4a-overdue-tasks.md | 112 +++++++++++ internal/parse/parse.go | 82 +++++++- internal/parse/parse_test.go | 78 ++++++++ internal/parse/tasks_json.go | 256 +++++++++++++++++++++++++ internal/store/store.go | 66 +++++++ scripts/compass-fetch-patchright.mjs | 269 +++++++++++++++++++++++---- 8 files changed, 913 insertions(+), 82 deletions(-) create mode 100644 docs/phase-4a-overdue-tasks.md create mode 100644 internal/parse/tasks_json.go diff --git a/cmd/school/parse.go b/cmd/school/parse.go index 4cec0a4..ff8b448 100644 --- a/cmd/school/parse.go +++ b/cmd/school/parse.go @@ -55,6 +55,7 @@ type parseSummary struct { Model string `json:"model,omitempty"` Pages []string `json:"pages"` Total int `json:"total"` + Overdue int `json:"overdue"` Inserted int `json:"inserted"` Updated int `json:"updated"` Counts []store.TypeCount `json:"counts"` @@ -139,6 +140,16 @@ func runParse(f parseFlags) error { }) } + // The capped legacy grid is superseded by the authoritative JSON task list. + // Remove its rows for this child first, so a stale partial row can never be + // shown next to the complete set (AUD-114, FR-12). + if hasPage(pages, parse.PageLearningTasksAll) { + if _, err := st.DeleteChildCompassTasksFromPages(ctx, child.ID, parse.PageLearningTasks, parse.PageChildProfile); err != nil { + _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) + return err + } + } + inserted, updated, err := st.UpsertItems(ctx, child.ID, stored) if err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) @@ -171,6 +182,7 @@ func runParse(f parseFlags) error { Model: model, Pages: pageSlugs, Total: len(items), + Overdue: parse.OverdueCount(items), Inserted: inserted, Updated: updated, Counts: counts, @@ -206,6 +218,7 @@ func printParseSummary(s parseSummary) { fmt.Printf(" (%s)", s.Model) } fmt.Printf("\nitems %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated) + fmt.Printf("overdue %d (reconciled against the home page)\n", s.Overdue) for _, c := range s.Counts { fmt.Printf(" %-6s %-8s %d\n", c.Child, c.Type, c.Count) } @@ -271,8 +284,10 @@ func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *t } } -// loadRunPages finds the raw run directory for a child and reads every -// .html page in it, with the URL from the matching .meta.json. +// loadRunPages finds the raw run directory for a child and reads every stored +// page in it. HTML pages come from .html; the JSON service page comes from +// .json. The URL and the child's Compass userId come from the matching +// .meta.json. func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) { childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child))) entries, err := os.ReadDir(childDir) @@ -308,51 +323,103 @@ func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) { return "", nil, fmt.Errorf("read run %s: %w", runDir, err) } - pages := make([]parse.Page, 0, len(files)) + type rawPage struct { + htmlBytes, jsonBytes []byte + url, user string + } + bySlug := map[string]*rawPage{} + var order []string + get := func(slug string) *rawPage { + if p, ok := bySlug[slug]; ok { + return p + } + p := &rawPage{} + bySlug[slug] = p + order = append(order, slug) + return p + } + for _, e := range files { name := e.Name() - if e.IsDir() || !strings.HasSuffix(strings.ToLower(name), ".html") { + if e.IsDir() { continue } - slug := strings.TrimSuffix(name, filepath.Ext(name)) - raw, err := os.ReadFile(filepath.Join(runDir, name)) - if err != nil { - return "", nil, fmt.Errorf("read %s: %w", name, err) + lower := strings.ToLower(name) + switch { + case strings.HasSuffix(lower, ".meta.json"): + slug := strings.TrimSuffix(name, ".meta.json") + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + var meta struct { + URL string `json:"url"` + UserID string `json:"userId"` + } + _ = json.Unmarshal(raw, &meta) + p := get(slug) + p.url = strings.TrimSpace(meta.URL) + p.user = strings.TrimSpace(meta.UserID) + case strings.HasSuffix(lower, ".json"): + if strings.EqualFold(name, "manifest.json") { + continue + } + slug := strings.TrimSuffix(name, ".json") + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + get(slug).jsonBytes = raw + case strings.HasSuffix(lower, ".html"): + slug := strings.TrimSuffix(name, filepath.Ext(name)) + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + get(slug).htmlBytes = raw + } + } + + pages := make([]parse.Page, 0, len(order)) + for _, slug := range order { + rp := bySlug[slug] + if len(rp.htmlBytes) == 0 && len(rp.jsonBytes) == 0 { + continue } pages = append(pages, parse.Page{ - Slug: slug, - URL: pageURL(runDir, slug), - HTML: raw, + Slug: slug, + URL: rp.url, + HTML: rp.htmlBytes, + JSON: rp.jsonBytes, + UserID: rp.user, }) } if len(pages) == 0 { - return "", nil, fmt.Errorf("run %s has no .html pages", runDir) + return "", nil, fmt.Errorf("run %s has no stored pages", runDir) } return runDir, pages, nil } -// runHasKnownPage reports whether a run directory holds at least one page the -// parser understands, so a failed login run is never chosen as "latest". -func runHasKnownPage(dir string) bool { - for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageCalendar, parse.PageChildProfile} { - if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil { +// hasPage reports whether a parsed page set contains a slug. +func hasPage(pages []parse.Page, slug string) bool { + for i := range pages { + if pages[i].Slug == slug { return true } } return false } -// pageURL reads the source URL recorded by the Phase 2 fetcher. -func pageURL(runDir, slug string) string { - raw, err := os.ReadFile(filepath.Join(runDir, slug+".meta.json")) - if err != nil { - return "" +// runHasKnownPage reports whether a run directory holds at least one page the +// parser understands, so a failed login run is never chosen as "latest". +func runHasKnownPage(dir string) bool { + for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageLearningTasksAll, parse.PageCalendar, parse.PageChildProfile} { + if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil { + return true + } + if _, err := os.Stat(filepath.Join(dir, slug+".json")); err == nil { + return true + } } - var meta struct { - URL string `json:"url"` - } - if err := json.Unmarshal(raw, &meta); err != nil { - return "" - } - return strings.TrimSpace(meta.URL) + return false } diff --git a/docs/phase-3-parse-notes.md b/docs/phase-3-parse-notes.md index f02b695..0c78c53 100644 --- a/docs/phase-3-parse-notes.md +++ b/docs/phase-3-parse-notes.md @@ -81,8 +81,9 @@ same row. Second run of the same raw pages: `inserted 0, updated 56`. - The learning-tasks grid is capped at the most recent 20 tasks (`Compass.attributes.LearningTask_Default_Records_Count = 20`). The parent - landing page reports more overdue work than the grid contains, so a later - fetch must open the overdue-filtered view. This blocks the dashboard's - "overdue" column, not Phase 3. + landing page reports more overdue work than the grid contains. **Closed by + Phase 4a:** `learning-tasks-all` is now the authoritative per-child list, and + the parser fails loudly when it disagrees with the home page. See + `docs/phase-4a-overdue-tasks.md`. - Team homework is not parsed here; that is FR-15, a separate source. - No model is used to classify the Teams source yet. diff --git a/docs/phase-4a-overdue-tasks.md b/docs/phase-4a-overdue-tasks.md new file mode 100644 index 0000000..b15db1d --- /dev/null +++ b/docs/phase-4a-overdue-tasks.md @@ -0,0 +1,112 @@ +# Phase 4a — the overdue-filtered learning-tasks view + +Status: **done.** The authoritative per-child learning-task list is fetched and +parsed, and the parser fails loudly when it disagrees with the home page. See +AUD-114. + +## Problem + +The Learning Tasks grid is capped at the most recent 20 records +(`Compass.attributes.LearningTask_Default_Records_Count = 20`). On the parent +account the same grid mixes both children. The parent landing page reported +**6 overdue learning tasks** for Harry, but the stored grid held 1 `Overdue` row +for him. The dashboard's "overdue" column cannot be built from that page. + +## The endpoint + +The grid's ExtJS store reads one service: + +``` +POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId +Content-Type: application/json +{ "userId": 27038, "page": 1, "start": 0, "limit": 1000 } +``` + +- **No `?sessionstate=readonly` query.** Adding it makes the service answer 500. +- The request must run **inside the logged-in page** (`fetch`, `credentials: + include`). A request from the browser context's own API client is answered 403 + by the Compass request capture. +- The response is `{"d":{"data":[...],"total":n}}`. `total` is the number of + tasks for that child across the current and past enrolments. + +The service returns **every** learning task for the child, with the submission +status, so the overdue count can be checked against the home page. + +## Status codes + +`Compass.enums.LearningTasksSubmissionStatus`: + +| Code | Label | +|---|---| +| 1 | Pending | +| 2 | Overdue | +| 3 | On Time | +| 4 | Received Late | + +`LearningTasks_OverdueShowAsPending = false` on the parent account, so an +`Overdue` row stays `Overdue`. + +## Due dates + +Most tasks carry `dueDateTimestamp` at the task level (`2026-10-09T05:00:00Z`, +UTC). A few carry it on the student row instead, so the parser reads both and +falls back to the student value. + +## Fetcher + +`scripts/compass-fetch-patchright.mjs` stores the response as +`learning-tasks-all.json` plus `learning-tasks-all.meta.json`. The meta records +the `userId`, the HTTP status, the byte count, the SHA-256, and the task count, +so the Go parser can tie the list to the child and to the home page. A non-200 +response or a non-JSON body is an explicit fetch error (FR-12). + +```bash +cd scripts && npm install +COMPASS_USERNAME=... COMPASS_PASSWORD=... \ + node scripts/compass-fetch-patchright.mjs --child Harry --user-id 27038 +``` + +Read-only against the school. + +## Parser + +`internal/parse/tasks_json.go` turns the JSON into `task` rows: + +- one row per task for the child's `userId` (a sibling's rows are skipped), +- `external_ref = learning-task|` (stable; the task id is unique), +- the submission status and due date are written into the item body, so the + offline importance rule marks an overdue task `high`. + +When a run holds `learning-tasks-all`, the parser **ignores the legacy HTML +grid** (`learning-tasks` and `child-profile`): the grid is capped and would store +the same task a second time. `school parse` then deletes the child's old +grid-derived task rows before the upsert. + +## Fail loudly (FR-12) + +`parse.Parse` reads the child's overdue count from the home page (the +`N overdue learning tasks` link that carries the child's `userId`) and compares +it with the number of `Overdue` rows in the JSON list. + +- They must be equal. A disagreement stops the parse, records a failed scan + (`ok = 0`), and stores nothing. +- A home page with no record of the child also stops the parse; a missing + summary is never read as zero. + +## Verification (2026-10-15) + +| Child | Compass id | Tasks | Overdue | Home page | +|---|---|---|---|---| +| Harry | 27038 | 70 | 6 | 6 overdue | +| Finn | 31460 | 42 | 2 | 2 overdue | + +- `school parse --child Harry --classify none` → + `overdue 6 (reconciled against the home page)`, `Harry task 70`. +- `school parse --child Finn --classify none` → + `overdue 2 (reconciled against the home page)`, `Finn task 42`. +- Tampered home page (6 → 5): `parse: Harry: home page reports 5 overdue + learning task(s) but the stored learning-task list has 6; refusing to store a + partial list (FR-12)` and exit 1. + +Raw pages stay under `var/raw///` and are git-ignored. The service +response holds the family's personal data; it is never committed. diff --git a/internal/parse/parse.go b/internal/parse/parse.go index 91c8a62..a9b7003 100644 --- a/internal/parse/parse.go +++ b/internal/parse/parse.go @@ -19,10 +19,11 @@ const ( // Recognised page slugs, matching the .html files stored by the Phase 2 // fetchers under var/raw///. const ( - PageHome = "home" - PageChildProfile = "child-profile" - PageLearningTasks = "learning-tasks" - PageCalendar = "calendar" + PageHome = "home" + PageChildProfile = "child-profile" + PageLearningTasks = "learning-tasks" + PageLearningTasksAll = "learning-tasks-all" + PageCalendar = "calendar" ) // Item is one message, learning task or calendar event read from Compass. @@ -43,11 +44,14 @@ type Item struct { } // Page is one stored raw page: its slug, the URL it was fetched from, and the -// HTML bytes. +// bytes. HTML pages carry HTML; the JSON service page carries JSON and the +// child's Compass userId (from the fetcher's meta file). type Page struct { - Slug string - URL string - HTML []byte + Slug string + URL string + HTML []byte + JSON []byte + UserID string } // Parse turns the stored pages for one child into structured items. It fails, @@ -64,11 +68,38 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { loc = time.UTC } + ordered := pagesOf(pages) + + // When the authoritative per-child task list is stored, it supersedes the + // legacy HTML grid: the grid is capped and, on the parent page, mixes + // children. Parse the JSON list and ignore the grid pages for tasks so the + // same task is never stored twice. + hasAuthoritativeTasks := false + for _, p := range ordered { + if p.Slug == PageLearningTasksAll { + hasAuthoritativeTasks = true + } + } + var items []Item recognised := 0 - for _, p := range pagesOf(pages) { + var tasks *parsedTasks + for i := range ordered { + p := ordered[i] switch p.Slug { + case PageLearningTasksAll: + got, err := parseLearningTasksAll(child, p, loc) + if err != nil { + return nil, err + } + tasks = &got + items = append(items, got.items...) + recognised++ case PageLearningTasks: + if hasAuthoritativeTasks { + recognised++ + continue + } got, err := parseLearningTasks(child, p, loc) if err != nil { return nil, err @@ -77,7 +108,11 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { recognised++ case PageChildProfile: // The React profile page has no server-rendered grid; when it does - // carry the legacy grid, use it. + // carry the legacy grid, use it. The JSON list supersedes it. + if hasAuthoritativeTasks { + recognised++ + continue + } got, found, err := extractLearningTasks(child, p, loc) if err != nil { return nil, err @@ -107,6 +142,23 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages)) } + // FR-12: the stored task list must agree with the summary on the home page. + // A partial or stale list must never be shown as "nothing overdue", so a + // disagreement stops the parse instead of writing a quiet wrong answer. + if tasks != nil { + home := findPage(ordered, PageHome) + if home == nil { + return nil, fmt.Errorf("parse: %s: %s is present but the home page is missing; cannot verify the overdue count (FR-12)", child, PageLearningTasksAll) + } + want, found := homeOverdueCount(home.HTML, tasks.userID) + if !found { + return nil, fmt.Errorf("parse: %s: home page has no record of Compass user %s; cannot verify the overdue count (FR-12)", child, tasks.userID) + } + if want != tasks.overdue { + return nil, fmt.Errorf("parse: %s: home page reports %d overdue learning task(s) but the stored learning-task list has %d; refusing to store a partial list (FR-12)", child, want, tasks.overdue) + } + } + // A task row is the same task wherever it appears (learning tasks and child // profile pages both carry the grid). Keep the first, drop the repeats. seen := map[string]bool{} @@ -132,6 +184,16 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { return out, nil } +// findPage returns the stored page with this slug, or nil. +func findPage(pages []Page, slug string) *Page { + for i := range pages { + if pages[i].Slug == slug { + return &pages[i] + } + } + return nil +} + // pagesOf returns the pages with duplicates removed, keeping the last copy of a // slug (the fetcher overwrites a slug file on a re-fetch, so the last is // authoritative). diff --git a/internal/parse/parse_test.go b/internal/parse/parse_test.go index 5507bc2..1b7cdab 100644 --- a/internal/parse/parse_test.go +++ b/internal/parse/parse_test.go @@ -303,3 +303,81 @@ func TestClassifyDeterministic(t *testing.T) { } func ptr(t time.Time) *time.Time { return &t } + +// tasksJSONFixture is the shape of GetAllLearningTasksByUserId: a wrapper with +// the per-child tasks. Task 1 is overdue and carries its due date at the task +// level; task 2 is on time and carries it on the student; task 3 belongs to the +// sibling and must be skipped. +const tasksJSONFixture = `{"d":{"total":3,"data":[ + {"id":12657,"name":"SAC 6: Human Rights Presentation","activityName":"","subjectName":"11HLEG 11 HUMANITIES - LEGAL STUDIES","groupName":"11HLEG","dueDateTimestamp":"2026-10-09T05:00:00Z","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":2,"dueDateTimestamp":"","submittedTimestamp":""}]}, + {"id":14268,"name":"Unit 1 English Exam","activityName":"11EENG","subjectName":"11EENG 11 ENGLISH - ENGLISH","groupName":"11EENG","dueDateTimestamp":"","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":3,"dueDateTimestamp":"2026-06-22T05:09:00Z","submittedTimestamp":"2026-06-22T05:08:00Z"}]}, + {"id":99999,"name":"Finn task","activityName":"","subjectName":"7MAT","groupName":"7MAT","dueDateTimestamp":"","students":[{"userId":31460,"userName":"Finn ROLFE","submissionStatus":2}]} +]}}` + +// homeOverdueFixture carries the parent home-page summary link for user 27038. +const homeOverdueFixture = ` +
+
Harrison ROLFE
+
+ + +
+` + +func TestParseLearningTasksAll(t *testing.T) { + loc := melbourne(t) + got, err := parseLearningTasksAll("Harry", Page{ + Slug: PageLearningTasksAll, + UserID: "27038", + JSON: []byte(tasksJSONFixture), + }, loc) + if err != nil { + t.Fatalf("parseLearningTasksAll: %v", err) + } + if len(got.items) != 2 { + t.Fatalf("got %d items, want 2 (the sibling's row is skipped)", len(got.items)) + } + if got.overdue != 1 { + t.Errorf("overdue = %d, want 1", got.overdue) + } + if got.total != 3 { + t.Errorf("total = %d, want 3", got.total) + } + if !strings.Contains(got.items[0].Body, "Status: Overdue") { + t.Errorf("overdue body = %q", got.items[0].Body) + } + if got.items[0].DueAt == nil || got.items[0].DueAt.Format("2006-01-02 15:04") != "2026-10-09 16:00" { + t.Errorf("task-level due = %v", got.items[0].DueAt) + } + if got.items[1].DueAt == nil || got.items[1].DueAt.Format("2006-01-02 15:04") != "2026-06-22 15:09" { + t.Errorf("student-level due = %v", got.items[1].DueAt) + } + if got.items[0].ExternalRef == got.items[1].ExternalRef { + t.Error("two tasks share an external ref") + } + if OverdueCount(got.items) != 1 { + t.Errorf("OverdueCount = %d, want 1", OverdueCount(got.items)) + } +} + +// TestParseReconcilesOverdueWithHome proves FR-12: the parser refuses to store a +// task list whose overdue count disagrees with the home page summary. +func TestParseReconcilesOverdueWithHome(t *testing.T) { + loc := melbourne(t) + jsonPage := Page{Slug: PageLearningTasksAll, UserID: "27038", JSON: []byte(tasksJSONFixture)} + homePage := Page{Slug: PageHome, HTML: []byte(homeOverdueFixture)} + + if _, err := Parse("Harry", []Page{homePage, jsonPage}, loc); err != nil { + t.Fatalf("Parse with matching counts: %v", err) + } + + mismatch := strings.Replace(homeOverdueFixture, "1 overdue learning task", "6 overdue learning tasks", 1) + if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(mismatch)}, jsonPage}, loc); err == nil { + t.Fatal("Parse accepted a task list that disagrees with the home page (FR-12)") + } + + blank := `
` + if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(blank)}, jsonPage}, loc); err == nil { + t.Fatal("Parse accepted a home page with no record of the child (FR-12)") + } +} diff --git a/internal/parse/tasks_json.go b/internal/parse/tasks_json.go new file mode 100644 index 0000000..32dd253 --- /dev/null +++ b/internal/parse/tasks_json.go @@ -0,0 +1,256 @@ +package parse + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// This file parses the authoritative per-child learning-task list from the +// Compass JSON service behind the Learning Tasks grid +// (/Services/LearningTasks.svc/GetAllLearningTasksByUserId). The grid itself is +// capped at LearningTask_Default_Records_Count (20) rows and, on the parent +// page, mixes children, so it cannot answer "what is overdue" (AUD-114, FR-12). + +// Learning-task submission status codes, from +// Compass.enums.LearningTasksSubmissionStatus in the Compass client bundle. +const ( + submissionPending = 1 + submissionOverdue = 2 + submissionOnTime = 3 + submissionSubmittedLate = 4 +) + +// overdueBodyMarker is the status line the JSON parser writes into a task body. +// The dashboard and the offline importance rule key off it. +const overdueBodyMarker = "Status: Overdue" + +// tasksResponse is the envelope Compass returns: {"d":{"data":[...],"total":n}}. +type tasksResponse struct { + D struct { + Data []taskRecord `json:"data"` + Total *int `json:"total"` + } `json:"d"` +} + +// taskRecord is one learning task. dueDateTimestamp is set on the task for most +// rows; a few rows carry it on the student instead, so both are read. +type taskRecord struct { + ID int `json:"id"` + Name string `json:"name"` + ActivityName string `json:"activityName"` + SubjectName string `json:"subjectName"` + GroupName string `json:"groupName"` + DueDateTimestamp string `json:"dueDateTimestamp"` + Students []taskStudent `json:"students"` +} + +// taskStudent is one child's row within a task: their submission status and, +// sometimes, the due date. +type taskStudent struct { + UserID int `json:"userId"` + UserName string `json:"userName"` + SubmissionStatus *int `json:"submissionStatus"` + DueDateTimestamp string `json:"dueDateTimestamp"` + SubmittedTimestamp string `json:"submittedTimestamp"` +} + +// parsedTasks is the result of reading one child's authoritative task list. +type parsedTasks struct { + items []Item + total int + overdue int + userID string +} + +// parseLearningTasksAll turns the stored JSON service response into task items. +// It fails loudly when the body is missing, is not the expected envelope, is +// missing the child's userId, or carries a task without a status or title, +// because a partial list must never be shown as "nothing overdue" (FR-12). +func parseLearningTasksAll(child string, page Page, loc *time.Location) (parsedTasks, error) { + out := parsedTasks{userID: strings.TrimSpace(page.UserID)} + if len(page.JSON) == 0 { + return out, fmt.Errorf("parse: %s: %s has no stored JSON body", child, page.Slug) + } + var resp tasksResponse + if err := json.Unmarshal(page.JSON, &resp); err != nil { + return out, fmt.Errorf("parse: %s: %s is not a Compass learning-tasks response: %w", child, page.Slug, err) + } + if resp.D.Data == nil { + return out, fmt.Errorf("parse: %s: %s carried no task list (session expired or layout changed)", child, page.Slug) + } + if out.userID == "" { + return out, fmt.Errorf("parse: %s: %s is missing the child's Compass userId; cannot select the child's tasks", child, page.Slug) + } + out.total = len(resp.D.Data) + if resp.D.Total != nil && *resp.D.Total != out.total { + return out, fmt.Errorf("parse: %s: %s returned %d of %d tasks; refusing a partial list (FR-12)", child, page.Slug, out.total, *resp.D.Total) + } + + for _, t := range resp.D.Data { + st, ok := studentFor(t, out.userID) + if !ok { + continue // the parent endpoint returns siblings too; keep only this child + } + if st.SubmissionStatus == nil { + return out, fmt.Errorf("parse: %s: learning task %d has no submission status", child, t.ID) + } + status := *st.SubmissionStatus + if status == submissionOverdue { + out.overdue++ + } + + title := collapse(t.Name) + if title == "" { + return out, fmt.Errorf("parse: %s: learning task %d has no title", child, t.ID) + } + due, err := parseCompassTimestamp(firstNonEmpty(t.DueDateTimestamp, st.DueDateTimestamp), loc) + if err != nil { + return out, fmt.Errorf("parse: %s: learning task %d: %w", child, t.ID, err) + } + + body := []string{"Status: " + submissionStatusLabel(status)} + if a := collapse(t.ActivityName); a != "" { + body = append(body, "Class: "+a) + } + if due != nil { + body = append(body, "Due: "+due.Format("2006-01-02 15:04")) + } + if s := strings.TrimSpace(st.SubmittedTimestamp); s != "" { + body = append(body, "Submitted: "+s) + } + + out.items = append(out.items, Item{ + Child: child, + Source: SourceCompass, + Type: TypeTask, + Subject: collapse(firstNonEmpty(t.SubjectName, t.GroupName, t.ActivityName)), + Title: title, + Body: strings.Join(body, "\n"), + DueAt: due, + URL: memberLearningTasksURL(out.userID), + SourcePage: page.Slug, + ExternalRef: dedupeKey("learning-task", strconv.Itoa(t.ID)), + Importance: importanceNormal, + }) + } + return out, nil +} + +// studentFor returns the child's row within a task by Compass userId. +func studentFor(t taskRecord, userID string) (taskStudent, bool) { + for _, s := range t.Students { + if strconv.Itoa(s.UserID) == userID { + return s, true + } + } + return taskStudent{}, false +} + +// submissionStatusLabel maps a Compass status code to the label the grid shows. +func submissionStatusLabel(code int) string { + switch code { + case submissionPending: + return "Pending" + case submissionOverdue: + return "Overdue" + case submissionOnTime: + return "On Time" + case submissionSubmittedLate: + return "Received Late" + default: + return fmt.Sprintf("status %d", code) + } +} + +// parseCompassTimestamp parses the service's RFC3339 UTC due timestamp in loc. +// An empty value means no due date; anything else that does not parse is an +// error, never a silent zero (FR-12). +func parseCompassTimestamp(s string, loc *time.Location) (*time.Time, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return nil, fmt.Errorf("unrecognised due timestamp %q", s) + } + if loc == nil { + loc = time.UTC + } + t = t.In(loc) + return &t, nil +} + +// overdueLinkRe reads the count from a home-page link such as +// "6 overdue learning tasks". +var overdueLinkRe = regexp.MustCompile(`(\d+)\s+overdue learning task`) + +// homeOverdueCount returns the overdue count the parent home page announces for +// one child. found is false when the page carries no record of that child at +// all (a login or challenge page), so a missing summary is never read as zero +// (FR-12). A child with no overdue line reports 0. +func homeOverdueCount(homeHTML []byte, userID string) (count int, found bool) { + if strings.TrimSpace(userID) == "" { + return 0, false + } + root := parseHTML(homeHTML) + seenChild := false + count = -1 + for _, a := range findAll(root, func(n *node) bool { return n.Tag == "a" }) { + if !strings.Contains(a.Attrs["href"], "userId="+userID) { + continue + } + seenChild = true + if m := overdueLinkRe.FindStringSubmatch(collapse(a.Text())); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + count = n + } + } + } + if count >= 0 { + return count, true + } + return 0, seenChild +} + +// HomeOverdueCount reports the overdue learning-task count the home page +// announces for one child. It is exported so a command can show the number it +// reconciled against. +func HomeOverdueCount(homeHTML []byte, userID string) (int, bool) { + return homeOverdueCount(homeHTML, userID) +} + +// OverdueCount returns how many learning tasks carry the Overdue submission +// status. The JSON parser writes the status into the item body, so the count is +// exact and does not depend on the wall clock. +func OverdueCount(items []Item) int { + n := 0 + for _, it := range items { + if it.Type == TypeTask && strings.Contains(it.Body, overdueBodyMarker) { + n++ + } + } + return n +} + +// memberLearningTasksURL links a task back to the child's profile grid. +func memberLearningTasksURL(userID string) string { + if strings.TrimSpace(userID) == "" { + return compassHost + "/Records/UserNew.aspx#learningTasks" + } + return compassHost + "/Records/User.aspx?userId=" + userID + "#learningTasks" +} + +// firstNonEmpty returns the first value with non-whitespace content. +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} diff --git a/internal/store/store.go b/internal/store/store.go index 25f504d..df7a4a8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -9,6 +9,7 @@ import ( "fmt" "io/fs" "sort" + "strings" "time" _ "modernc.org/sqlite" // pure-Go SQLite driver (no cgo) @@ -348,6 +349,32 @@ func formatDue(t *time.Time) any { return t.UTC().Format(time.RFC3339) } +// DeleteChildCompassTasksFromPages removes a child's Compass task rows that came +// from the given source pages. It is used when an authoritative full task list +// replaces a partial grid: the legacy grid is capped +// (LearningTask_Default_Records_Count) and a stale row must not survive as if it +// were the whole set (AUD-114, FR-12). Rows from any other source page are left +// untouched. +func (s *Store) DeleteChildCompassTasksFromPages(ctx context.Context, childID int64, sourcePages ...string) (int64, error) { + if len(sourcePages) == 0 { + return 0, nil + } + placeholders := strings.Repeat("?,", len(sourcePages)-1) + "?" + args := make([]any, 0, len(sourcePages)+1) + args = append(args, childID) + for _, p := range sourcePages { + args = append(args, p) + } + res, err := s.db.ExecContext(ctx, + `DELETE FROM items + WHERE child_id = ? AND source = 'compass' AND type = 'task' + AND source_page IN (`+placeholders+`)`, args...) + if err != nil { + return 0, fmt.Errorf("delete partial task rows: %w", err) + } + return res.RowsAffected() +} + // TypeCount is the number of items of one type for one child. type TypeCount struct { Child string @@ -429,3 +456,42 @@ func (s *Store) ChildByName(ctx context.Context, name string) (*Child, error) { c.CreatedAt = parseTime(created) return &c, nil } + +// SetChildCompassID records the child's Compass userId, discovered by the +// fetcher from the parent landing page (AGENTS.md §7). +func (s *Store) SetChildCompassID(ctx context.Context, childID int64, compassID string) error { + if compassID == "" { + return nil + } + if _, err := s.db.ExecContext(ctx, + `UPDATE children SET compass_id = ? WHERE id = ?`, compassID, childID); err != nil { + return fmt.Errorf("set compass id for child %d: %w", childID, err) + } + return nil +} + +// DeleteItems removes items for one child, optionally limited to a source and a +// type. It exists so a scan can drop rows that were stored against the wrong +// child (Phase 4: nothing may leak between children) and so a later Delete +// action has a home. It returns how many rows were removed. +func (s *Store) DeleteItems(ctx context.Context, childID int64, source, itemType string) (int64, error) { + q := `DELETE FROM items WHERE child_id = ?` + args := []any{childID} + if source != "" { + q += ` AND source = ?` + args = append(args, source) + } + if itemType != "" { + q += ` AND type = ?` + args = append(args, itemType) + } + res, err := s.db.ExecContext(ctx, q, args...) + if err != nil { + return 0, fmt.Errorf("delete items for child %d: %w", childID, err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("delete items for child %d: %w", childID, err) + } + return n, nil +} diff --git a/scripts/compass-fetch-patchright.mjs b/scripts/compass-fetch-patchright.mjs index c385753..46e2712 100644 --- a/scripts/compass-fetch-patchright.mjs +++ b/scripts/compass-fetch-patchright.mjs @@ -1,20 +1,25 @@ #!/usr/bin/env node /** - * Phase 2 proof-of-fetch for the Compass / NHS project — patchright path. + * Compass fetcher for the Compass / NHS project — patchright path. * * The pi-browser-harness (headless Chrome) can load the Compass login page but * Cloudflare answers the login POST with a managed challenge that headless * Chrome does not clear. patchright launches Chrome with the automation leaks * patched, and the login POST goes through. This script: * - * 1. logs in once with the parent account (COMPASS_USERNAME / COMPASS_PASSWORD), - * 2. fetches the pages we need for ONE child, - * 3. stores the raw HTML + a manifest with a SHA-256 and byte count per page. + * 1. reuses a stored per-child browser session when one is supplied and still + * valid, otherwise logs in once with the parent account + * (COMPASS_USERNAME / COMPASS_PASSWORD), + * 2. resolves the child's Compass userId from the parent landing page, + * 3. fetches that child's pages, + * 4. stores the raw HTML + a manifest with a SHA-256 and byte count per page, + * 5. refreshes the per-child session state (``--state``) for the next scan. * * Read-only against the school: it only navigates and reads. * * Usage: - * node scripts/compass-fetch-patchright.mjs --child Harry [--pages "name=url,..."] + * node scripts/compass-fetch-patchright.mjs --child Harry [--user-id 27038] + * [--state var/session/harry.json] [--pages "name=url,..."] * * Environment (never commit these): * COMPASS_USERNAME / COMPASS_PASSWORD parent/guardian account @@ -22,7 +27,8 @@ * COMPASS_LOGIN_URL default https://nhs-vic.compass.education/login.aspx * COMPASS_UA browser user-agent override * - * Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor, 4 login rejected. + * Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor, + * 4 login rejected, 1 fetch errors (a page failed or the child id is missing). */ import { chromium } from "patchright"; import crypto from "node:crypto"; @@ -35,28 +41,48 @@ const DEFAULT_UA = const DEFAULT_CHROME = "/etc/profiles/per-user/sam/bin/google-chrome"; const CHALLENGE = /just a moment|verifying|checking your browser|attention required/i; -// Default pages: one child's record and the calendar. The parent landing page -// (/ ) is always stored as "home" right after login. userId is per child and is -// read from the landing page links or passed with --user-id. +// Per-child pages. The calendar is deliberately NOT here: the parent calendar +// page renders the visible child's calendars only (it defaults to the eldest), +// so fetching it inside a younger child's run would leak the eldest's timetable +// into that child's items. Per-child calendar attribution is deferred; the +// calendar stays available through an explicit --pages override. The parent +// landing page (/) is always stored as "home" right after login. const DEFAULT_PAGES = [ { name: "child-profile", url: "/Records/User.aspx?userId={userId}" }, - { name: "calendar", url: "/Organise/Calendar/" }, + { name: "learning-tasks", url: "/Records/UserNew.aspx?userId={userId}#learningTasks" }, ]; +// TASKS_SERVICE is the Compass JSON service behind the Learning Tasks grid. The +// grid itself is capped at LearningTask_Default_Records_Count (20) rows and, on +// the parent page, mixes children. This endpoint returns every learning task +// for one child with its submission status, so the overdue count can be checked +// against the same child's count on the home page (AUD-114, FR-12). +const TASKS_SERVICE = "/Services/LearningTasks.svc/GetAllLearningTasksByUserId"; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const sha256 = (s) => crypto.createHash("sha256").update(s, "utf8").digest("hex"); function parseArgs(argv) { - const args = { child: "Harry", userId: null, out: path.resolve("var/raw"), pages: null }; + const args = { + child: "Harry", + userId: null, + out: path.resolve("var/raw"), + state: null, + pages: null, + }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === "--child") args.child = argv[++i]; else if (a === "--user-id") args.userId = argv[++i]; else if (a === "--out") args.out = argv[++i]; + else if (a === "--state") args.state = argv[++i]; + else if (a === "--no-state") args.state = null; else if (a === "--pages") { + // Split each entry on the FIRST "=" so a URL may carry a query string. args.pages = String(argv[++i]).split(",").map((p) => { - const [name, url] = p.split("="); - return { name, url }; + const eq = p.indexOf("="); + if (eq < 0) throw new Error(`--pages entry needs name=url: ${p}`); + return { name: p.slice(0, eq), url: p.slice(eq + 1) }; }); } else throw new Error(`unknown argument: ${a}`); } @@ -72,6 +98,20 @@ async function waitPastChallenge(page, seconds) { return await page.title().catch(() => ""); } +/** + * waitForFeed waits for the home news feed to render. Compass loads the feed + * client-side after the page shell, so storing the page too early drops it and + * the parser (correctly) fails loudly. + */ +async function waitForFeed(page, ms) { + try { + await page.waitForSelector('[data-testid="feed-toolbar"], .MuiCardContent-root', { timeout: ms }); + } catch { + // The parser will fail loudly if the feed is genuinely missing. + } + await sleep(500); +} + async function storePage(page, outDir, slug, baseUrl) { const html = await page.content(); const state = { url: page.url(), title: await page.title() }; @@ -90,9 +130,116 @@ async function storePage(page, outDir, slug, baseUrl) { return meta; } +/** + * fetchLearningTasksService reads the full per-child learning-task list from the + * Compass JSON service and stores it next to the HTML pages. It is read-only: + * one POST with the child's userId and a large page limit. The request runs + * inside the logged-in page (`fetch`, `credentials: include`) so it carries the + * session cookies and the Compass request-capture headers; the browser + * context's own API client is rejected with 403. The meta file records the + * child's userId so the Go parser can tie the list to the child and to the home + * page count. A non-200 response or a non-JSON body is recorded as an error. + */ +async function fetchLearningTasksService(page, baseUrl, outDir, userId) { + const url = new URL(TASKS_SERVICE, baseUrl).toString(); + const body = { userId: Number(userId), page: 1, start: 0, limit: 1000 }; + const result = await page.evaluate( + async ({ url, body }) => { + const r = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + credentials: "include", + body: JSON.stringify(body), + }); + return { status: r.status, text: await r.text() }; + }, + { url, body }, + ); + const text = result.text; + const meta = { + slug: "learning-tasks-all", + url, + title: "Learning Tasks service", + method: "POST", + body, + userId: String(userId), + fetchedAt: new Date().toISOString(), + bytes: Buffer.byteLength(text, "utf8"), + sha256: sha256(text), + status: result.status, + taskCount: null, + total: null, + }; + try { + const parsed = JSON.parse(text); + if (Array.isArray(parsed?.d?.data)) meta.taskCount = parsed.d.data.length; + if (typeof parsed?.d?.total === "number") meta.total = parsed.d.total; + } catch { + // Not JSON: keep the raw body and let the caller fail loudly (FR-12). + } + fs.writeFileSync(path.join(outDir, "learning-tasks-all.json"), text); + fs.writeFileSync(path.join(outDir, "learning-tasks-all.meta.json"), JSON.stringify(meta, null, 2) + "\n"); + return meta; +} + +/** + * findChildUserId resolves a child name to a Compass userId from the parent + * landing page, for example "Harrison ROLFE" -> 27038. The parent page groups + * each child's links under a .parentChildActions block whose
holds the + * formal name, while the family uses a short one, so compare the first four + * letters ("Harr" / "Finn"). An explicit --user-id always wins. + */ +async function findChildUserId(page, child) { + const want = child.trim().toLowerCase().slice(0, 4); + + const blocks = await page + .$$eval(".parentChildActions", (els) => + els + .map((el) => { + const h5 = el.querySelector("h5"); + const a = el.querySelector('a[href*="Records/User.aspx?userId="]'); + const m = a && (a.getAttribute("href") || "").match(/userId=(\d+)/); + return { name: (h5 && h5.textContent || "").trim(), userId: m ? m[1] : null }; + }) + .filter((c) => c.userId), + ) + .catch(() => []); + for (const { name, userId } of blocks) { + const first = name.split(/\s+/)[0].toLowerCase(); + if (first.slice(0, 4) === want) return userId; + } + + // Fallback: any link whose text starts with the child's name. + const links = await page + .$$eval('a[href*="Records/User.aspx?userId="]', (as) => + as.map((a) => ({ href: a.getAttribute("href"), text: (a.textContent || "").trim() })), + ) + .catch(() => []); + for (const { href, text } of links) { + const m = href.match(/userId=(\d+)/); + if (!m) continue; + const first = text.split(/\s+/)[0].toLowerCase(); + if (first.slice(0, 4) === want) return m[1]; + } + return null; +} + +async function loadStorageState(file) { + if (!file) return null; + try { + if (!fs.existsSync(file)) return null; + const raw = fs.readFileSync(file, "utf8"); + JSON.parse(raw); // a truncated or empty file is not a usable session + return JSON.parse(raw); + } catch { + return null; + } +} + async function main() { const args = parseArgs(process.argv); const baseUrl = process.env.COMPASS_LOGIN_URL || DEFAULT_URL; + const rootUrl = new URL("/", baseUrl).toString(); const chrome = process.env.COMPASS_CHROME || DEFAULT_CHROME; const user = process.env.COMPASS_USERNAME; const pass = process.env.COMPASS_PASSWORD; @@ -109,63 +256,102 @@ async function main() { fetcher: "patchright", loginUrl: baseUrl, loggedIn: false, + sessionReused: false, + sessionSaved: false, secondFactor: false, pages: [], errors: [], }; + const state = await loadStorageState(args.state); const browser = await chromium.launch({ headless: true, executablePath: chrome, args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-blink-features=AutomationControlled"], }); - const ctx = await browser.newContext({ + const contextOptions = { locale: "en-AU", userAgent: process.env.COMPASS_UA || DEFAULT_UA, viewport: { width: 1280, height: 900 }, timezoneId: "Australia/Melbourne", - }); + }; + if (state) contextOptions.storageState = state; + const ctx = await browser.newContext(contextOptions); const page = await ctx.newPage(); try { - await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); + await page.goto(rootUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); if (CHALLENGE.test(await waitPastChallenge(page, 40))) { manifest.errors.push("cloudflare_challenge_not_cleared"); return 2; } - await page.fill("#username", user); - await page.fill("#password", pass); - await page.click("#button1"); - if (CHALLENGE.test(await waitPastChallenge(page, 45))) { - manifest.errors.push("cloudflare_login_post_challenge"); - return 2; + // A stored session skips the login POST when Compass still accepts it. + const needsLogin = Boolean(await page.$("#username")); + manifest.sessionReused = state !== null && !needsLogin; + + if (needsLogin) { + await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); + if (CHALLENGE.test(await waitPastChallenge(page, 40))) { + manifest.errors.push("cloudflare_challenge_not_cleared"); + return 2; + } + await page.fill("#username", user); + await page.fill("#password", pass); + await page.click("#button1"); + if (CHALLENGE.test(await waitPastChallenge(page, 45))) { + manifest.errors.push("cloudflare_login_post_challenge"); + return 2; + } + + const stillLogin = await page.$("#username"); + const otp = await page.evaluate(() => + [...document.querySelectorAll("input")].some((i) => + /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || "")), + ), + ); + if (otp && stillLogin) { + manifest.secondFactor = true; + manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl)); + return 3; + } + if (stillLogin) { + manifest.errors.push("login_rejected"); + manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl)); + return 4; + } } - const stillLogin = await page.$("#username"); - const otp = await page.evaluate(() => - [...document.querySelectorAll("input")].some((i) => /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || ""))), - ); - if (otp && stillLogin) { - manifest.secondFactor = true; - manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl)); - return 3; - } - if (stillLogin) { - manifest.errors.push("login_rejected"); - manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl)); - return 4; - } manifest.loggedIn = true; + await waitForFeed(page, 20000); manifest.pages.push(await storePage(page, outDir, "home", baseUrl)); - let userId = args.userId; - if (!userId) { - const match = (await page.content()).match(/Records\/User\.aspx\?userId=(\d+)/i); - if (match) userId = match[1]; + // Refresh the per-child session now that the login is known good. + if (args.state) { + try { + fs.mkdirSync(path.dirname(path.resolve(args.state)), { recursive: true }); + await ctx.storageState({ path: args.state }); + manifest.sessionSaved = true; + } catch (err) { + manifest.errors.push(`session_save_failed:${err.message}`); + } } + + let userId = args.userId; + if (!userId) userId = await findChildUserId(page, args.child); manifest.userId = userId || null; + // The authoritative per-child task list. A failure here is an explicit error + // so a partial task list is never mistaken for "nothing overdue" (FR-12). + if (userId) { + const taskMeta = await fetchLearningTasksService(page, baseUrl, outDir, userId); + manifest.pages.push(taskMeta); + if (taskMeta.status !== 200) manifest.errors.push(`learning_tasks_all_http:${taskMeta.status}`); + if (taskMeta.taskCount == null) manifest.errors.push("learning_tasks_all_not_json"); + } else { + manifest.errors.push("no_user_id:learning-tasks-all"); + } + const pages = args.pages || DEFAULT_PAGES; for (const p of pages) { if (p.url.includes("{userId}") && !userId) { @@ -178,6 +364,8 @@ async function main() { manifest.errors.push(`challenge:${p.name}`); continue; } + // Give the profile app a moment to render the learning-tasks grid. + await sleep(1500); manifest.pages.push(await storePage(page, outDir, p.name, baseUrl)); } } catch (err) { @@ -189,6 +377,7 @@ async function main() { } console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`); + console.log(` session reused=${manifest.sessionReused} saved=${manifest.sessionSaved}`); for (const p of manifest.pages) console.log(` ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`); if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; ")); return manifest.errors.length ? 1 : 0;