diff --git a/cmd/school/parse.go b/cmd/school/parse.go index 4cec0a4..ff8b448 100644 --- a/cmd/school/parse.go +++ b/cmd/school/parse.go @@ -55,6 +55,7 @@ type parseSummary struct { Model string `json:"model,omitempty"` Pages []string `json:"pages"` Total int `json:"total"` + Overdue int `json:"overdue"` Inserted int `json:"inserted"` Updated int `json:"updated"` Counts []store.TypeCount `json:"counts"` @@ -139,6 +140,16 @@ func runParse(f parseFlags) error { }) } + // The capped legacy grid is superseded by the authoritative JSON task list. + // Remove its rows for this child first, so a stale partial row can never be + // shown next to the complete set (AUD-114, FR-12). + if hasPage(pages, parse.PageLearningTasksAll) { + if _, err := st.DeleteChildCompassTasksFromPages(ctx, child.ID, parse.PageLearningTasks, parse.PageChildProfile); err != nil { + _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) + return err + } + } + inserted, updated, err := st.UpsertItems(ctx, child.ID, stored) if err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) @@ -171,6 +182,7 @@ func runParse(f parseFlags) error { Model: model, Pages: pageSlugs, Total: len(items), + Overdue: parse.OverdueCount(items), Inserted: inserted, Updated: updated, Counts: counts, @@ -206,6 +218,7 @@ func printParseSummary(s parseSummary) { fmt.Printf(" (%s)", s.Model) } fmt.Printf("\nitems %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated) + fmt.Printf("overdue %d (reconciled against the home page)\n", s.Overdue) for _, c := range s.Counts { fmt.Printf(" %-6s %-8s %d\n", c.Child, c.Type, c.Count) } @@ -271,8 +284,10 @@ func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *t } } -// loadRunPages finds the raw run directory for a child and reads every -// .html page in it, with the URL from the matching .meta.json. +// loadRunPages finds the raw run directory for a child and reads every stored +// page in it. HTML pages come from .html; the JSON service page comes from +// .json. The URL and the child's Compass userId come from the matching +// .meta.json. func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) { childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child))) entries, err := os.ReadDir(childDir) @@ -308,51 +323,103 @@ func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) { return "", nil, fmt.Errorf("read run %s: %w", runDir, err) } - pages := make([]parse.Page, 0, len(files)) + type rawPage struct { + htmlBytes, jsonBytes []byte + url, user string + } + bySlug := map[string]*rawPage{} + var order []string + get := func(slug string) *rawPage { + if p, ok := bySlug[slug]; ok { + return p + } + p := &rawPage{} + bySlug[slug] = p + order = append(order, slug) + return p + } + for _, e := range files { name := e.Name() - if e.IsDir() || !strings.HasSuffix(strings.ToLower(name), ".html") { + if e.IsDir() { continue } - slug := strings.TrimSuffix(name, filepath.Ext(name)) - raw, err := os.ReadFile(filepath.Join(runDir, name)) - if err != nil { - return "", nil, fmt.Errorf("read %s: %w", name, err) + lower := strings.ToLower(name) + switch { + case strings.HasSuffix(lower, ".meta.json"): + slug := strings.TrimSuffix(name, ".meta.json") + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + var meta struct { + URL string `json:"url"` + UserID string `json:"userId"` + } + _ = json.Unmarshal(raw, &meta) + p := get(slug) + p.url = strings.TrimSpace(meta.URL) + p.user = strings.TrimSpace(meta.UserID) + case strings.HasSuffix(lower, ".json"): + if strings.EqualFold(name, "manifest.json") { + continue + } + slug := strings.TrimSuffix(name, ".json") + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + get(slug).jsonBytes = raw + case strings.HasSuffix(lower, ".html"): + slug := strings.TrimSuffix(name, filepath.Ext(name)) + raw, err := os.ReadFile(filepath.Join(runDir, name)) + if err != nil { + return "", nil, fmt.Errorf("read %s: %w", name, err) + } + get(slug).htmlBytes = raw + } + } + + pages := make([]parse.Page, 0, len(order)) + for _, slug := range order { + rp := bySlug[slug] + if len(rp.htmlBytes) == 0 && len(rp.jsonBytes) == 0 { + continue } pages = append(pages, parse.Page{ - Slug: slug, - URL: pageURL(runDir, slug), - HTML: raw, + Slug: slug, + URL: rp.url, + HTML: rp.htmlBytes, + JSON: rp.jsonBytes, + UserID: rp.user, }) } if len(pages) == 0 { - return "", nil, fmt.Errorf("run %s has no .html pages", runDir) + return "", nil, fmt.Errorf("run %s has no stored pages", runDir) } return runDir, pages, nil } -// runHasKnownPage reports whether a run directory holds at least one page the -// parser understands, so a failed login run is never chosen as "latest". -func runHasKnownPage(dir string) bool { - for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageCalendar, parse.PageChildProfile} { - if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil { +// hasPage reports whether a parsed page set contains a slug. +func hasPage(pages []parse.Page, slug string) bool { + for i := range pages { + if pages[i].Slug == slug { return true } } return false } -// pageURL reads the source URL recorded by the Phase 2 fetcher. -func pageURL(runDir, slug string) string { - raw, err := os.ReadFile(filepath.Join(runDir, slug+".meta.json")) - if err != nil { - return "" +// runHasKnownPage reports whether a run directory holds at least one page the +// parser understands, so a failed login run is never chosen as "latest". +func runHasKnownPage(dir string) bool { + for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageLearningTasksAll, parse.PageCalendar, parse.PageChildProfile} { + if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil { + return true + } + if _, err := os.Stat(filepath.Join(dir, slug+".json")); err == nil { + return true + } } - var meta struct { - URL string `json:"url"` - } - if err := json.Unmarshal(raw, &meta); err != nil { - return "" - } - return strings.TrimSpace(meta.URL) + return false } diff --git a/docs/phase-3-parse-notes.md b/docs/phase-3-parse-notes.md index f02b695..0c78c53 100644 --- a/docs/phase-3-parse-notes.md +++ b/docs/phase-3-parse-notes.md @@ -81,8 +81,9 @@ same row. Second run of the same raw pages: `inserted 0, updated 56`. - The learning-tasks grid is capped at the most recent 20 tasks (`Compass.attributes.LearningTask_Default_Records_Count = 20`). The parent - landing page reports more overdue work than the grid contains, so a later - fetch must open the overdue-filtered view. This blocks the dashboard's - "overdue" column, not Phase 3. + landing page reports more overdue work than the grid contains. **Closed by + Phase 4a:** `learning-tasks-all` is now the authoritative per-child list, and + the parser fails loudly when it disagrees with the home page. See + `docs/phase-4a-overdue-tasks.md`. - Team homework is not parsed here; that is FR-15, a separate source. - No model is used to classify the Teams source yet. diff --git a/docs/phase-4a-overdue-tasks.md b/docs/phase-4a-overdue-tasks.md new file mode 100644 index 0000000..b15db1d --- /dev/null +++ b/docs/phase-4a-overdue-tasks.md @@ -0,0 +1,112 @@ +# Phase 4a — the overdue-filtered learning-tasks view + +Status: **done.** The authoritative per-child learning-task list is fetched and +parsed, and the parser fails loudly when it disagrees with the home page. See +AUD-114. + +## Problem + +The Learning Tasks grid is capped at the most recent 20 records +(`Compass.attributes.LearningTask_Default_Records_Count = 20`). On the parent +account the same grid mixes both children. The parent landing page reported +**6 overdue learning tasks** for Harry, but the stored grid held 1 `Overdue` row +for him. The dashboard's "overdue" column cannot be built from that page. + +## The endpoint + +The grid's ExtJS store reads one service: + +``` +POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId +Content-Type: application/json +{ "userId": 27038, "page": 1, "start": 0, "limit": 1000 } +``` + +- **No `?sessionstate=readonly` query.** Adding it makes the service answer 500. +- The request must run **inside the logged-in page** (`fetch`, `credentials: + include`). A request from the browser context's own API client is answered 403 + by the Compass request capture. +- The response is `{"d":{"data":[...],"total":n}}`. `total` is the number of + tasks for that child across the current and past enrolments. + +The service returns **every** learning task for the child, with the submission +status, so the overdue count can be checked against the home page. + +## Status codes + +`Compass.enums.LearningTasksSubmissionStatus`: + +| Code | Label | +|---|---| +| 1 | Pending | +| 2 | Overdue | +| 3 | On Time | +| 4 | Received Late | + +`LearningTasks_OverdueShowAsPending = false` on the parent account, so an +`Overdue` row stays `Overdue`. + +## Due dates + +Most tasks carry `dueDateTimestamp` at the task level (`2026-10-09T05:00:00Z`, +UTC). A few carry it on the student row instead, so the parser reads both and +falls back to the student value. + +## Fetcher + +`scripts/compass-fetch-patchright.mjs` stores the response as +`learning-tasks-all.json` plus `learning-tasks-all.meta.json`. The meta records +the `userId`, the HTTP status, the byte count, the SHA-256, and the task count, +so the Go parser can tie the list to the child and to the home page. A non-200 +response or a non-JSON body is an explicit fetch error (FR-12). + +```bash +cd scripts && npm install +COMPASS_USERNAME=... COMPASS_PASSWORD=... \ + node scripts/compass-fetch-patchright.mjs --child Harry --user-id 27038 +``` + +Read-only against the school. + +## Parser + +`internal/parse/tasks_json.go` turns the JSON into `task` rows: + +- one row per task for the child's `userId` (a sibling's rows are skipped), +- `external_ref = learning-task|` (stable; the task id is unique), +- the submission status and due date are written into the item body, so the + offline importance rule marks an overdue task `high`. + +When a run holds `learning-tasks-all`, the parser **ignores the legacy HTML +grid** (`learning-tasks` and `child-profile`): the grid is capped and would store +the same task a second time. `school parse` then deletes the child's old +grid-derived task rows before the upsert. + +## Fail loudly (FR-12) + +`parse.Parse` reads the child's overdue count from the home page (the +`N overdue learning tasks` link that carries the child's `userId`) and compares +it with the number of `Overdue` rows in the JSON list. + +- They must be equal. A disagreement stops the parse, records a failed scan + (`ok = 0`), and stores nothing. +- A home page with no record of the child also stops the parse; a missing + summary is never read as zero. + +## Verification (2026-10-15) + +| Child | Compass id | Tasks | Overdue | Home page | +|---|---|---|---|---| +| Harry | 27038 | 70 | 6 | 6 overdue | +| Finn | 31460 | 42 | 2 | 2 overdue | + +- `school parse --child Harry --classify none` → + `overdue 6 (reconciled against the home page)`, `Harry task 70`. +- `school parse --child Finn --classify none` → + `overdue 2 (reconciled against the home page)`, `Finn task 42`. +- Tampered home page (6 → 5): `parse: Harry: home page reports 5 overdue + learning task(s) but the stored learning-task list has 6; refusing to store a + partial list (FR-12)` and exit 1. + +Raw pages stay under `var/raw///` and are git-ignored. The service +response holds the family's personal data; it is never committed. diff --git a/internal/parse/parse.go b/internal/parse/parse.go index 91c8a62..a9b7003 100644 --- a/internal/parse/parse.go +++ b/internal/parse/parse.go @@ -19,10 +19,11 @@ const ( // Recognised page slugs, matching the .html files stored by the Phase 2 // fetchers under var/raw///. const ( - PageHome = "home" - PageChildProfile = "child-profile" - PageLearningTasks = "learning-tasks" - PageCalendar = "calendar" + PageHome = "home" + PageChildProfile = "child-profile" + PageLearningTasks = "learning-tasks" + PageLearningTasksAll = "learning-tasks-all" + PageCalendar = "calendar" ) // Item is one message, learning task or calendar event read from Compass. @@ -43,11 +44,14 @@ type Item struct { } // Page is one stored raw page: its slug, the URL it was fetched from, and the -// HTML bytes. +// bytes. HTML pages carry HTML; the JSON service page carries JSON and the +// child's Compass userId (from the fetcher's meta file). type Page struct { - Slug string - URL string - HTML []byte + Slug string + URL string + HTML []byte + JSON []byte + UserID string } // Parse turns the stored pages for one child into structured items. It fails, @@ -64,11 +68,38 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { loc = time.UTC } + ordered := pagesOf(pages) + + // When the authoritative per-child task list is stored, it supersedes the + // legacy HTML grid: the grid is capped and, on the parent page, mixes + // children. Parse the JSON list and ignore the grid pages for tasks so the + // same task is never stored twice. + hasAuthoritativeTasks := false + for _, p := range ordered { + if p.Slug == PageLearningTasksAll { + hasAuthoritativeTasks = true + } + } + var items []Item recognised := 0 - for _, p := range pagesOf(pages) { + var tasks *parsedTasks + for i := range ordered { + p := ordered[i] switch p.Slug { + case PageLearningTasksAll: + got, err := parseLearningTasksAll(child, p, loc) + if err != nil { + return nil, err + } + tasks = &got + items = append(items, got.items...) + recognised++ case PageLearningTasks: + if hasAuthoritativeTasks { + recognised++ + continue + } got, err := parseLearningTasks(child, p, loc) if err != nil { return nil, err @@ -77,7 +108,11 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { recognised++ case PageChildProfile: // The React profile page has no server-rendered grid; when it does - // carry the legacy grid, use it. + // carry the legacy grid, use it. The JSON list supersedes it. + if hasAuthoritativeTasks { + recognised++ + continue + } got, found, err := extractLearningTasks(child, p, loc) if err != nil { return nil, err @@ -107,6 +142,23 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages)) } + // FR-12: the stored task list must agree with the summary on the home page. + // A partial or stale list must never be shown as "nothing overdue", so a + // disagreement stops the parse instead of writing a quiet wrong answer. + if tasks != nil { + home := findPage(ordered, PageHome) + if home == nil { + return nil, fmt.Errorf("parse: %s: %s is present but the home page is missing; cannot verify the overdue count (FR-12)", child, PageLearningTasksAll) + } + want, found := homeOverdueCount(home.HTML, tasks.userID) + if !found { + return nil, fmt.Errorf("parse: %s: home page has no record of Compass user %s; cannot verify the overdue count (FR-12)", child, tasks.userID) + } + if want != tasks.overdue { + return nil, fmt.Errorf("parse: %s: home page reports %d overdue learning task(s) but the stored learning-task list has %d; refusing to store a partial list (FR-12)", child, want, tasks.overdue) + } + } + // A task row is the same task wherever it appears (learning tasks and child // profile pages both carry the grid). Keep the first, drop the repeats. seen := map[string]bool{} @@ -132,6 +184,16 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) { return out, nil } +// findPage returns the stored page with this slug, or nil. +func findPage(pages []Page, slug string) *Page { + for i := range pages { + if pages[i].Slug == slug { + return &pages[i] + } + } + return nil +} + // pagesOf returns the pages with duplicates removed, keeping the last copy of a // slug (the fetcher overwrites a slug file on a re-fetch, so the last is // authoritative). diff --git a/internal/parse/parse_test.go b/internal/parse/parse_test.go index 5507bc2..1b7cdab 100644 --- a/internal/parse/parse_test.go +++ b/internal/parse/parse_test.go @@ -303,3 +303,81 @@ func TestClassifyDeterministic(t *testing.T) { } func ptr(t time.Time) *time.Time { return &t } + +// tasksJSONFixture is the shape of GetAllLearningTasksByUserId: a wrapper with +// the per-child tasks. Task 1 is overdue and carries its due date at the task +// level; task 2 is on time and carries it on the student; task 3 belongs to the +// sibling and must be skipped. +const tasksJSONFixture = `{"d":{"total":3,"data":[ + {"id":12657,"name":"SAC 6: Human Rights Presentation","activityName":"","subjectName":"11HLEG 11 HUMANITIES - LEGAL STUDIES","groupName":"11HLEG","dueDateTimestamp":"2026-10-09T05:00:00Z","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":2,"dueDateTimestamp":"","submittedTimestamp":""}]}, + {"id":14268,"name":"Unit 1 English Exam","activityName":"11EENG","subjectName":"11EENG 11 ENGLISH - ENGLISH","groupName":"11EENG","dueDateTimestamp":"","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":3,"dueDateTimestamp":"2026-06-22T05:09:00Z","submittedTimestamp":"2026-06-22T05:08:00Z"}]}, + {"id":99999,"name":"Finn task","activityName":"","subjectName":"7MAT","groupName":"7MAT","dueDateTimestamp":"","students":[{"userId":31460,"userName":"Finn ROLFE","submissionStatus":2}]} +]}}` + +// homeOverdueFixture carries the parent home-page summary link for user 27038. +const homeOverdueFixture = ` +
+
Harrison ROLFE
+ +` + +func TestParseLearningTasksAll(t *testing.T) { + loc := melbourne(t) + got, err := parseLearningTasksAll("Harry", Page{ + Slug: PageLearningTasksAll, + UserID: "27038", + JSON: []byte(tasksJSONFixture), + }, loc) + if err != nil { + t.Fatalf("parseLearningTasksAll: %v", err) + } + if len(got.items) != 2 { + t.Fatalf("got %d items, want 2 (the sibling's row is skipped)", len(got.items)) + } + if got.overdue != 1 { + t.Errorf("overdue = %d, want 1", got.overdue) + } + if got.total != 3 { + t.Errorf("total = %d, want 3", got.total) + } + if !strings.Contains(got.items[0].Body, "Status: Overdue") { + t.Errorf("overdue body = %q", got.items[0].Body) + } + if got.items[0].DueAt == nil || got.items[0].DueAt.Format("2006-01-02 15:04") != "2026-10-09 16:00" { + t.Errorf("task-level due = %v", got.items[0].DueAt) + } + if got.items[1].DueAt == nil || got.items[1].DueAt.Format("2006-01-02 15:04") != "2026-06-22 15:09" { + t.Errorf("student-level due = %v", got.items[1].DueAt) + } + if got.items[0].ExternalRef == got.items[1].ExternalRef { + t.Error("two tasks share an external ref") + } + if OverdueCount(got.items) != 1 { + t.Errorf("OverdueCount = %d, want 1", OverdueCount(got.items)) + } +} + +// TestParseReconcilesOverdueWithHome proves FR-12: the parser refuses to store a +// task list whose overdue count disagrees with the home page summary. +func TestParseReconcilesOverdueWithHome(t *testing.T) { + loc := melbourne(t) + jsonPage := Page{Slug: PageLearningTasksAll, UserID: "27038", JSON: []byte(tasksJSONFixture)} + homePage := Page{Slug: PageHome, HTML: []byte(homeOverdueFixture)} + + if _, err := Parse("Harry", []Page{homePage, jsonPage}, loc); err != nil { + t.Fatalf("Parse with matching counts: %v", err) + } + + mismatch := strings.Replace(homeOverdueFixture, "1 overdue learning task", "6 overdue learning tasks", 1) + if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(mismatch)}, jsonPage}, loc); err == nil { + t.Fatal("Parse accepted a task list that disagrees with the home page (FR-12)") + } + + blank := `
` + if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(blank)}, jsonPage}, loc); err == nil { + t.Fatal("Parse accepted a home page with no record of the child (FR-12)") + } +} diff --git a/internal/parse/tasks_json.go b/internal/parse/tasks_json.go new file mode 100644 index 0000000..32dd253 --- /dev/null +++ b/internal/parse/tasks_json.go @@ -0,0 +1,256 @@ +package parse + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "time" +) + +// This file parses the authoritative per-child learning-task list from the +// Compass JSON service behind the Learning Tasks grid +// (/Services/LearningTasks.svc/GetAllLearningTasksByUserId). The grid itself is +// capped at LearningTask_Default_Records_Count (20) rows and, on the parent +// page, mixes children, so it cannot answer "what is overdue" (AUD-114, FR-12). + +// Learning-task submission status codes, from +// Compass.enums.LearningTasksSubmissionStatus in the Compass client bundle. +const ( + submissionPending = 1 + submissionOverdue = 2 + submissionOnTime = 3 + submissionSubmittedLate = 4 +) + +// overdueBodyMarker is the status line the JSON parser writes into a task body. +// The dashboard and the offline importance rule key off it. +const overdueBodyMarker = "Status: Overdue" + +// tasksResponse is the envelope Compass returns: {"d":{"data":[...],"total":n}}. +type tasksResponse struct { + D struct { + Data []taskRecord `json:"data"` + Total *int `json:"total"` + } `json:"d"` +} + +// taskRecord is one learning task. dueDateTimestamp is set on the task for most +// rows; a few rows carry it on the student instead, so both are read. +type taskRecord struct { + ID int `json:"id"` + Name string `json:"name"` + ActivityName string `json:"activityName"` + SubjectName string `json:"subjectName"` + GroupName string `json:"groupName"` + DueDateTimestamp string `json:"dueDateTimestamp"` + Students []taskStudent `json:"students"` +} + +// taskStudent is one child's row within a task: their submission status and, +// sometimes, the due date. +type taskStudent struct { + UserID int `json:"userId"` + UserName string `json:"userName"` + SubmissionStatus *int `json:"submissionStatus"` + DueDateTimestamp string `json:"dueDateTimestamp"` + SubmittedTimestamp string `json:"submittedTimestamp"` +} + +// parsedTasks is the result of reading one child's authoritative task list. +type parsedTasks struct { + items []Item + total int + overdue int + userID string +} + +// parseLearningTasksAll turns the stored JSON service response into task items. +// It fails loudly when the body is missing, is not the expected envelope, is +// missing the child's userId, or carries a task without a status or title, +// because a partial list must never be shown as "nothing overdue" (FR-12). +func parseLearningTasksAll(child string, page Page, loc *time.Location) (parsedTasks, error) { + out := parsedTasks{userID: strings.TrimSpace(page.UserID)} + if len(page.JSON) == 0 { + return out, fmt.Errorf("parse: %s: %s has no stored JSON body", child, page.Slug) + } + var resp tasksResponse + if err := json.Unmarshal(page.JSON, &resp); err != nil { + return out, fmt.Errorf("parse: %s: %s is not a Compass learning-tasks response: %w", child, page.Slug, err) + } + if resp.D.Data == nil { + return out, fmt.Errorf("parse: %s: %s carried no task list (session expired or layout changed)", child, page.Slug) + } + if out.userID == "" { + return out, fmt.Errorf("parse: %s: %s is missing the child's Compass userId; cannot select the child's tasks", child, page.Slug) + } + out.total = len(resp.D.Data) + if resp.D.Total != nil && *resp.D.Total != out.total { + return out, fmt.Errorf("parse: %s: %s returned %d of %d tasks; refusing a partial list (FR-12)", child, page.Slug, out.total, *resp.D.Total) + } + + for _, t := range resp.D.Data { + st, ok := studentFor(t, out.userID) + if !ok { + continue // the parent endpoint returns siblings too; keep only this child + } + if st.SubmissionStatus == nil { + return out, fmt.Errorf("parse: %s: learning task %d has no submission status", child, t.ID) + } + status := *st.SubmissionStatus + if status == submissionOverdue { + out.overdue++ + } + + title := collapse(t.Name) + if title == "" { + return out, fmt.Errorf("parse: %s: learning task %d has no title", child, t.ID) + } + due, err := parseCompassTimestamp(firstNonEmpty(t.DueDateTimestamp, st.DueDateTimestamp), loc) + if err != nil { + return out, fmt.Errorf("parse: %s: learning task %d: %w", child, t.ID, err) + } + + body := []string{"Status: " + submissionStatusLabel(status)} + if a := collapse(t.ActivityName); a != "" { + body = append(body, "Class: "+a) + } + if due != nil { + body = append(body, "Due: "+due.Format("2006-01-02 15:04")) + } + if s := strings.TrimSpace(st.SubmittedTimestamp); s != "" { + body = append(body, "Submitted: "+s) + } + + out.items = append(out.items, Item{ + Child: child, + Source: SourceCompass, + Type: TypeTask, + Subject: collapse(firstNonEmpty(t.SubjectName, t.GroupName, t.ActivityName)), + Title: title, + Body: strings.Join(body, "\n"), + DueAt: due, + URL: memberLearningTasksURL(out.userID), + SourcePage: page.Slug, + ExternalRef: dedupeKey("learning-task", strconv.Itoa(t.ID)), + Importance: importanceNormal, + }) + } + return out, nil +} + +// studentFor returns the child's row within a task by Compass userId. +func studentFor(t taskRecord, userID string) (taskStudent, bool) { + for _, s := range t.Students { + if strconv.Itoa(s.UserID) == userID { + return s, true + } + } + return taskStudent{}, false +} + +// submissionStatusLabel maps a Compass status code to the label the grid shows. +func submissionStatusLabel(code int) string { + switch code { + case submissionPending: + return "Pending" + case submissionOverdue: + return "Overdue" + case submissionOnTime: + return "On Time" + case submissionSubmittedLate: + return "Received Late" + default: + return fmt.Sprintf("status %d", code) + } +} + +// parseCompassTimestamp parses the service's RFC3339 UTC due timestamp in loc. +// An empty value means no due date; anything else that does not parse is an +// error, never a silent zero (FR-12). +func parseCompassTimestamp(s string, loc *time.Location) (*time.Time, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return nil, fmt.Errorf("unrecognised due timestamp %q", s) + } + if loc == nil { + loc = time.UTC + } + t = t.In(loc) + return &t, nil +} + +// overdueLinkRe reads the count from a home-page link such as +// "6 overdue learning tasks". +var overdueLinkRe = regexp.MustCompile(`(\d+)\s+overdue learning task`) + +// homeOverdueCount returns the overdue count the parent home page announces for +// one child. found is false when the page carries no record of that child at +// all (a login or challenge page), so a missing summary is never read as zero +// (FR-12). A child with no overdue line reports 0. +func homeOverdueCount(homeHTML []byte, userID string) (count int, found bool) { + if strings.TrimSpace(userID) == "" { + return 0, false + } + root := parseHTML(homeHTML) + seenChild := false + count = -1 + for _, a := range findAll(root, func(n *node) bool { return n.Tag == "a" }) { + if !strings.Contains(a.Attrs["href"], "userId="+userID) { + continue + } + seenChild = true + if m := overdueLinkRe.FindStringSubmatch(collapse(a.Text())); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + count = n + } + } + } + if count >= 0 { + return count, true + } + return 0, seenChild +} + +// HomeOverdueCount reports the overdue learning-task count the home page +// announces for one child. It is exported so a command can show the number it +// reconciled against. +func HomeOverdueCount(homeHTML []byte, userID string) (int, bool) { + return homeOverdueCount(homeHTML, userID) +} + +// OverdueCount returns how many learning tasks carry the Overdue submission +// status. The JSON parser writes the status into the item body, so the count is +// exact and does not depend on the wall clock. +func OverdueCount(items []Item) int { + n := 0 + for _, it := range items { + if it.Type == TypeTask && strings.Contains(it.Body, overdueBodyMarker) { + n++ + } + } + return n +} + +// memberLearningTasksURL links a task back to the child's profile grid. +func memberLearningTasksURL(userID string) string { + if strings.TrimSpace(userID) == "" { + return compassHost + "/Records/UserNew.aspx#learningTasks" + } + return compassHost + "/Records/User.aspx?userId=" + userID + "#learningTasks" +} + +// firstNonEmpty returns the first value with non-whitespace content. +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} diff --git a/internal/store/store.go b/internal/store/store.go index 25f504d..df7a4a8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -9,6 +9,7 @@ import ( "fmt" "io/fs" "sort" + "strings" "time" _ "modernc.org/sqlite" // pure-Go SQLite driver (no cgo) @@ -348,6 +349,32 @@ func formatDue(t *time.Time) any { return t.UTC().Format(time.RFC3339) } +// DeleteChildCompassTasksFromPages removes a child's Compass task rows that came +// from the given source pages. It is used when an authoritative full task list +// replaces a partial grid: the legacy grid is capped +// (LearningTask_Default_Records_Count) and a stale row must not survive as if it +// were the whole set (AUD-114, FR-12). Rows from any other source page are left +// untouched. +func (s *Store) DeleteChildCompassTasksFromPages(ctx context.Context, childID int64, sourcePages ...string) (int64, error) { + if len(sourcePages) == 0 { + return 0, nil + } + placeholders := strings.Repeat("?,", len(sourcePages)-1) + "?" + args := make([]any, 0, len(sourcePages)+1) + args = append(args, childID) + for _, p := range sourcePages { + args = append(args, p) + } + res, err := s.db.ExecContext(ctx, + `DELETE FROM items + WHERE child_id = ? AND source = 'compass' AND type = 'task' + AND source_page IN (`+placeholders+`)`, args...) + if err != nil { + return 0, fmt.Errorf("delete partial task rows: %w", err) + } + return res.RowsAffected() +} + // TypeCount is the number of items of one type for one child. type TypeCount struct { Child string @@ -429,3 +456,42 @@ func (s *Store) ChildByName(ctx context.Context, name string) (*Child, error) { c.CreatedAt = parseTime(created) return &c, nil } + +// SetChildCompassID records the child's Compass userId, discovered by the +// fetcher from the parent landing page (AGENTS.md §7). +func (s *Store) SetChildCompassID(ctx context.Context, childID int64, compassID string) error { + if compassID == "" { + return nil + } + if _, err := s.db.ExecContext(ctx, + `UPDATE children SET compass_id = ? WHERE id = ?`, compassID, childID); err != nil { + return fmt.Errorf("set compass id for child %d: %w", childID, err) + } + return nil +} + +// DeleteItems removes items for one child, optionally limited to a source and a +// type. It exists so a scan can drop rows that were stored against the wrong +// child (Phase 4: nothing may leak between children) and so a later Delete +// action has a home. It returns how many rows were removed. +func (s *Store) DeleteItems(ctx context.Context, childID int64, source, itemType string) (int64, error) { + q := `DELETE FROM items WHERE child_id = ?` + args := []any{childID} + if source != "" { + q += ` AND source = ?` + args = append(args, source) + } + if itemType != "" { + q += ` AND type = ?` + args = append(args, itemType) + } + res, err := s.db.ExecContext(ctx, q, args...) + if err != nil { + return 0, fmt.Errorf("delete items for child %d: %w", childID, err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("delete items for child %d: %w", childID, err) + } + return n, nil +} diff --git a/scripts/compass-fetch-patchright.mjs b/scripts/compass-fetch-patchright.mjs index c385753..46e2712 100644 --- a/scripts/compass-fetch-patchright.mjs +++ b/scripts/compass-fetch-patchright.mjs @@ -1,20 +1,25 @@ #!/usr/bin/env node /** - * Phase 2 proof-of-fetch for the Compass / NHS project — patchright path. + * Compass fetcher for the Compass / NHS project — patchright path. * * The pi-browser-harness (headless Chrome) can load the Compass login page but * Cloudflare answers the login POST with a managed challenge that headless * Chrome does not clear. patchright launches Chrome with the automation leaks * patched, and the login POST goes through. This script: * - * 1. logs in once with the parent account (COMPASS_USERNAME / COMPASS_PASSWORD), - * 2. fetches the pages we need for ONE child, - * 3. stores the raw HTML + a manifest with a SHA-256 and byte count per page. + * 1. reuses a stored per-child browser session when one is supplied and still + * valid, otherwise logs in once with the parent account + * (COMPASS_USERNAME / COMPASS_PASSWORD), + * 2. resolves the child's Compass userId from the parent landing page, + * 3. fetches that child's pages, + * 4. stores the raw HTML + a manifest with a SHA-256 and byte count per page, + * 5. refreshes the per-child session state (``--state``) for the next scan. * * Read-only against the school: it only navigates and reads. * * Usage: - * node scripts/compass-fetch-patchright.mjs --child Harry [--pages "name=url,..."] + * node scripts/compass-fetch-patchright.mjs --child Harry [--user-id 27038] + * [--state var/session/harry.json] [--pages "name=url,..."] * * Environment (never commit these): * COMPASS_USERNAME / COMPASS_PASSWORD parent/guardian account @@ -22,7 +27,8 @@ * COMPASS_LOGIN_URL default https://nhs-vic.compass.education/login.aspx * COMPASS_UA browser user-agent override * - * Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor, 4 login rejected. + * Exit codes: 0 ok, 2 Cloudflare challenge not cleared, 3 second factor, + * 4 login rejected, 1 fetch errors (a page failed or the child id is missing). */ import { chromium } from "patchright"; import crypto from "node:crypto"; @@ -35,28 +41,48 @@ const DEFAULT_UA = const DEFAULT_CHROME = "/etc/profiles/per-user/sam/bin/google-chrome"; const CHALLENGE = /just a moment|verifying|checking your browser|attention required/i; -// Default pages: one child's record and the calendar. The parent landing page -// (/ ) is always stored as "home" right after login. userId is per child and is -// read from the landing page links or passed with --user-id. +// Per-child pages. The calendar is deliberately NOT here: the parent calendar +// page renders the visible child's calendars only (it defaults to the eldest), +// so fetching it inside a younger child's run would leak the eldest's timetable +// into that child's items. Per-child calendar attribution is deferred; the +// calendar stays available through an explicit --pages override. The parent +// landing page (/) is always stored as "home" right after login. const DEFAULT_PAGES = [ { name: "child-profile", url: "/Records/User.aspx?userId={userId}" }, - { name: "calendar", url: "/Organise/Calendar/" }, + { name: "learning-tasks", url: "/Records/UserNew.aspx?userId={userId}#learningTasks" }, ]; +// TASKS_SERVICE is the Compass JSON service behind the Learning Tasks grid. The +// grid itself is capped at LearningTask_Default_Records_Count (20) rows and, on +// the parent page, mixes children. This endpoint returns every learning task +// for one child with its submission status, so the overdue count can be checked +// against the same child's count on the home page (AUD-114, FR-12). +const TASKS_SERVICE = "/Services/LearningTasks.svc/GetAllLearningTasksByUserId"; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const sha256 = (s) => crypto.createHash("sha256").update(s, "utf8").digest("hex"); function parseArgs(argv) { - const args = { child: "Harry", userId: null, out: path.resolve("var/raw"), pages: null }; + const args = { + child: "Harry", + userId: null, + out: path.resolve("var/raw"), + state: null, + pages: null, + }; for (let i = 2; i < argv.length; i++) { const a = argv[i]; if (a === "--child") args.child = argv[++i]; else if (a === "--user-id") args.userId = argv[++i]; else if (a === "--out") args.out = argv[++i]; + else if (a === "--state") args.state = argv[++i]; + else if (a === "--no-state") args.state = null; else if (a === "--pages") { + // Split each entry on the FIRST "=" so a URL may carry a query string. args.pages = String(argv[++i]).split(",").map((p) => { - const [name, url] = p.split("="); - return { name, url }; + const eq = p.indexOf("="); + if (eq < 0) throw new Error(`--pages entry needs name=url: ${p}`); + return { name: p.slice(0, eq), url: p.slice(eq + 1) }; }); } else throw new Error(`unknown argument: ${a}`); } @@ -72,6 +98,20 @@ async function waitPastChallenge(page, seconds) { return await page.title().catch(() => ""); } +/** + * waitForFeed waits for the home news feed to render. Compass loads the feed + * client-side after the page shell, so storing the page too early drops it and + * the parser (correctly) fails loudly. + */ +async function waitForFeed(page, ms) { + try { + await page.waitForSelector('[data-testid="feed-toolbar"], .MuiCardContent-root', { timeout: ms }); + } catch { + // The parser will fail loudly if the feed is genuinely missing. + } + await sleep(500); +} + async function storePage(page, outDir, slug, baseUrl) { const html = await page.content(); const state = { url: page.url(), title: await page.title() }; @@ -90,9 +130,116 @@ async function storePage(page, outDir, slug, baseUrl) { return meta; } +/** + * fetchLearningTasksService reads the full per-child learning-task list from the + * Compass JSON service and stores it next to the HTML pages. It is read-only: + * one POST with the child's userId and a large page limit. The request runs + * inside the logged-in page (`fetch`, `credentials: include`) so it carries the + * session cookies and the Compass request-capture headers; the browser + * context's own API client is rejected with 403. The meta file records the + * child's userId so the Go parser can tie the list to the child and to the home + * page count. A non-200 response or a non-JSON body is recorded as an error. + */ +async function fetchLearningTasksService(page, baseUrl, outDir, userId) { + const url = new URL(TASKS_SERVICE, baseUrl).toString(); + const body = { userId: Number(userId), page: 1, start: 0, limit: 1000 }; + const result = await page.evaluate( + async ({ url, body }) => { + const r = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + credentials: "include", + body: JSON.stringify(body), + }); + return { status: r.status, text: await r.text() }; + }, + { url, body }, + ); + const text = result.text; + const meta = { + slug: "learning-tasks-all", + url, + title: "Learning Tasks service", + method: "POST", + body, + userId: String(userId), + fetchedAt: new Date().toISOString(), + bytes: Buffer.byteLength(text, "utf8"), + sha256: sha256(text), + status: result.status, + taskCount: null, + total: null, + }; + try { + const parsed = JSON.parse(text); + if (Array.isArray(parsed?.d?.data)) meta.taskCount = parsed.d.data.length; + if (typeof parsed?.d?.total === "number") meta.total = parsed.d.total; + } catch { + // Not JSON: keep the raw body and let the caller fail loudly (FR-12). + } + fs.writeFileSync(path.join(outDir, "learning-tasks-all.json"), text); + fs.writeFileSync(path.join(outDir, "learning-tasks-all.meta.json"), JSON.stringify(meta, null, 2) + "\n"); + return meta; +} + +/** + * findChildUserId resolves a child name to a Compass userId from the parent + * landing page, for example "Harrison ROLFE" -> 27038. The parent page groups + * each child's links under a .parentChildActions block whose
holds the + * formal name, while the family uses a short one, so compare the first four + * letters ("Harr" / "Finn"). An explicit --user-id always wins. + */ +async function findChildUserId(page, child) { + const want = child.trim().toLowerCase().slice(0, 4); + + const blocks = await page + .$$eval(".parentChildActions", (els) => + els + .map((el) => { + const h5 = el.querySelector("h5"); + const a = el.querySelector('a[href*="Records/User.aspx?userId="]'); + const m = a && (a.getAttribute("href") || "").match(/userId=(\d+)/); + return { name: (h5 && h5.textContent || "").trim(), userId: m ? m[1] : null }; + }) + .filter((c) => c.userId), + ) + .catch(() => []); + for (const { name, userId } of blocks) { + const first = name.split(/\s+/)[0].toLowerCase(); + if (first.slice(0, 4) === want) return userId; + } + + // Fallback: any link whose text starts with the child's name. + const links = await page + .$$eval('a[href*="Records/User.aspx?userId="]', (as) => + as.map((a) => ({ href: a.getAttribute("href"), text: (a.textContent || "").trim() })), + ) + .catch(() => []); + for (const { href, text } of links) { + const m = href.match(/userId=(\d+)/); + if (!m) continue; + const first = text.split(/\s+/)[0].toLowerCase(); + if (first.slice(0, 4) === want) return m[1]; + } + return null; +} + +async function loadStorageState(file) { + if (!file) return null; + try { + if (!fs.existsSync(file)) return null; + const raw = fs.readFileSync(file, "utf8"); + JSON.parse(raw); // a truncated or empty file is not a usable session + return JSON.parse(raw); + } catch { + return null; + } +} + async function main() { const args = parseArgs(process.argv); const baseUrl = process.env.COMPASS_LOGIN_URL || DEFAULT_URL; + const rootUrl = new URL("/", baseUrl).toString(); const chrome = process.env.COMPASS_CHROME || DEFAULT_CHROME; const user = process.env.COMPASS_USERNAME; const pass = process.env.COMPASS_PASSWORD; @@ -109,63 +256,102 @@ async function main() { fetcher: "patchright", loginUrl: baseUrl, loggedIn: false, + sessionReused: false, + sessionSaved: false, secondFactor: false, pages: [], errors: [], }; + const state = await loadStorageState(args.state); const browser = await chromium.launch({ headless: true, executablePath: chrome, args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-blink-features=AutomationControlled"], }); - const ctx = await browser.newContext({ + const contextOptions = { locale: "en-AU", userAgent: process.env.COMPASS_UA || DEFAULT_UA, viewport: { width: 1280, height: 900 }, timezoneId: "Australia/Melbourne", - }); + }; + if (state) contextOptions.storageState = state; + const ctx = await browser.newContext(contextOptions); const page = await ctx.newPage(); try { - await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); + await page.goto(rootUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); if (CHALLENGE.test(await waitPastChallenge(page, 40))) { manifest.errors.push("cloudflare_challenge_not_cleared"); return 2; } - await page.fill("#username", user); - await page.fill("#password", pass); - await page.click("#button1"); - if (CHALLENGE.test(await waitPastChallenge(page, 45))) { - manifest.errors.push("cloudflare_login_post_challenge"); - return 2; + // A stored session skips the login POST when Compass still accepts it. + const needsLogin = Boolean(await page.$("#username")); + manifest.sessionReused = state !== null && !needsLogin; + + if (needsLogin) { + await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 60000 }); + if (CHALLENGE.test(await waitPastChallenge(page, 40))) { + manifest.errors.push("cloudflare_challenge_not_cleared"); + return 2; + } + await page.fill("#username", user); + await page.fill("#password", pass); + await page.click("#button1"); + if (CHALLENGE.test(await waitPastChallenge(page, 45))) { + manifest.errors.push("cloudflare_login_post_challenge"); + return 2; + } + + const stillLogin = await page.$("#username"); + const otp = await page.evaluate(() => + [...document.querySelectorAll("input")].some((i) => + /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || "")), + ), + ); + if (otp && stillLogin) { + manifest.secondFactor = true; + manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl)); + return 3; + } + if (stillLogin) { + manifest.errors.push("login_rejected"); + manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl)); + return 4; + } } - const stillLogin = await page.$("#username"); - const otp = await page.evaluate(() => - [...document.querySelectorAll("input")].some((i) => /otp|one-?time|passcode|verification|code/i.test((i.name || "") + " " + (i.id || ""))), - ); - if (otp && stillLogin) { - manifest.secondFactor = true; - manifest.pages.push(await storePage(page, outDir, "second-factor-required", baseUrl)); - return 3; - } - if (stillLogin) { - manifest.errors.push("login_rejected"); - manifest.pages.push(await storePage(page, outDir, "login-rejected", baseUrl)); - return 4; - } manifest.loggedIn = true; + await waitForFeed(page, 20000); manifest.pages.push(await storePage(page, outDir, "home", baseUrl)); - let userId = args.userId; - if (!userId) { - const match = (await page.content()).match(/Records\/User\.aspx\?userId=(\d+)/i); - if (match) userId = match[1]; + // Refresh the per-child session now that the login is known good. + if (args.state) { + try { + fs.mkdirSync(path.dirname(path.resolve(args.state)), { recursive: true }); + await ctx.storageState({ path: args.state }); + manifest.sessionSaved = true; + } catch (err) { + manifest.errors.push(`session_save_failed:${err.message}`); + } } + + let userId = args.userId; + if (!userId) userId = await findChildUserId(page, args.child); manifest.userId = userId || null; + // The authoritative per-child task list. A failure here is an explicit error + // so a partial task list is never mistaken for "nothing overdue" (FR-12). + if (userId) { + const taskMeta = await fetchLearningTasksService(page, baseUrl, outDir, userId); + manifest.pages.push(taskMeta); + if (taskMeta.status !== 200) manifest.errors.push(`learning_tasks_all_http:${taskMeta.status}`); + if (taskMeta.taskCount == null) manifest.errors.push("learning_tasks_all_not_json"); + } else { + manifest.errors.push("no_user_id:learning-tasks-all"); + } + const pages = args.pages || DEFAULT_PAGES; for (const p of pages) { if (p.url.includes("{userId}") && !userId) { @@ -178,6 +364,8 @@ async function main() { manifest.errors.push(`challenge:${p.name}`); continue; } + // Give the profile app a moment to render the learning-tasks grid. + await sleep(1500); manifest.pages.push(await storePage(page, outDir, p.name, baseUrl)); } } catch (err) { @@ -189,6 +377,7 @@ async function main() { } console.log(`stored ${manifest.pages.length} page(s) to ${outDir}`); + console.log(` session reused=${manifest.sessionReused} saved=${manifest.sessionSaved}`); for (const p of manifest.pages) console.log(` ${p.slug}: ${p.bytes} bytes sha256=${p.sha256} (${p.title})`); if (manifest.errors.length) console.error("errors: " + manifest.errors.join("; ")); return manifest.errors.length ? 1 : 0;