Phase 4a: fetch and parse the authoritative Compass learning tasks

The Learning Tasks grid is capped at 20 rows and, on the parent page, mixes both children, so it cannot answer "what is overdue". The parent home page reported 6 overdue tasks for Harry while the stored grid held 1. Fetch the per-child task list (read-only) from POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId, parse it into task rows, and refuse to store it when the overdue count disagrees with the home page summary (FR-12). - fetcher: store learning-tasks-all.json + meta (userId, status, task count) - parse: new learning-tasks-all page, JSON parser, home-page reconciliation - store: drop superseded grid-derived task rows before the upsert - docs: docs/phase-4a-overdue-tasks.md; close the Phase 3 known gap Verified with live runs: Harry 70 tasks / 6 overdue, Finn 42 tasks / 2 overdue, both equal to the home page; a tampered home page stops the parse. Note: the fetcher and store.go files also carry the per-child session work that was already uncommitted in this shared workspace (AUD-108). Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-15 17:38:22 +11:00
parent d7d529eb0f
commit 5d263987dd
8 changed files with 913 additions and 82 deletions
--- a/internal/parse/parse.go
+++ b/internal/parse/parse.go
@@ -19,10 +19,11 @@ const (
 // Recognised page slugs, matching the <slug>.html files stored by the Phase 2
 // fetchers under var/raw/<child>/<runId>/.
 const (
-	PageHome          = "home"
-	PageChildProfile  = "child-profile"
-	PageLearningTasks = "learning-tasks"
-	PageCalendar      = "calendar"
+	PageHome             = "home"
+	PageChildProfile     = "child-profile"
+	PageLearningTasks    = "learning-tasks"
+	PageLearningTasksAll = "learning-tasks-all"
+	PageCalendar         = "calendar"
 )

 // Item is one message, learning task or calendar event read from Compass.
@@ -43,11 +44,14 @@ type Item struct {
 }

 // Page is one stored raw page: its slug, the URL it was fetched from, and the
-// HTML bytes.
+// bytes. HTML pages carry HTML; the JSON service page carries JSON and the
+// child's Compass userId (from the fetcher's meta file).
 type Page struct {
-	Slug string
-	URL  string
-	HTML []byte
+	Slug   string
+	URL    string
+	HTML   []byte
+	JSON   []byte
+	UserID string
 }

 // Parse turns the stored pages for one child into structured items. It fails,
@@ -64,11 +68,38 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
 		loc = time.UTC
 	}

+	ordered := pagesOf(pages)
+
+	// When the authoritative per-child task list is stored, it supersedes the
+	// legacy HTML grid: the grid is capped and, on the parent page, mixes
+	// children. Parse the JSON list and ignore the grid pages for tasks so the
+	// same task is never stored twice.
+	hasAuthoritativeTasks := false
+	for _, p := range ordered {
+		if p.Slug == PageLearningTasksAll {
+			hasAuthoritativeTasks = true
+		}
+	}
+
 	var items []Item
 	recognised := 0
-	for _, p := range pagesOf(pages) {
+	var tasks *parsedTasks
+	for i := range ordered {
+		p := ordered[i]
 		switch p.Slug {
+		case PageLearningTasksAll:
+			got, err := parseLearningTasksAll(child, p, loc)
+			if err != nil {
+				return nil, err
+			}
+			tasks = &got
+			items = append(items, got.items...)
+			recognised++
 		case PageLearningTasks:
+			if hasAuthoritativeTasks {
+				recognised++
+				continue
+			}
 			got, err := parseLearningTasks(child, p, loc)
 			if err != nil {
 				return nil, err
@@ -77,7 +108,11 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
 			recognised++
 		case PageChildProfile:
 			// The React profile page has no server-rendered grid; when it does
-			// carry the legacy grid, use it.
+			// carry the legacy grid, use it. The JSON list supersedes it.
+			if hasAuthoritativeTasks {
+				recognised++
+				continue
+			}
 			got, found, err := extractLearningTasks(child, p, loc)
 			if err != nil {
 				return nil, err
@@ -107,6 +142,23 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
 		return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages))
 	}

+	// FR-12: the stored task list must agree with the summary on the home page.
+	// A partial or stale list must never be shown as "nothing overdue", so a
+	// disagreement stops the parse instead of writing a quiet wrong answer.
+	if tasks != nil {
+		home := findPage(ordered, PageHome)
+		if home == nil {
+			return nil, fmt.Errorf("parse: %s: %s is present but the home page is missing; cannot verify the overdue count (FR-12)", child, PageLearningTasksAll)
+		}
+		want, found := homeOverdueCount(home.HTML, tasks.userID)
+		if !found {
+			return nil, fmt.Errorf("parse: %s: home page has no record of Compass user %s; cannot verify the overdue count (FR-12)", child, tasks.userID)
+		}
+		if want != tasks.overdue {
+			return nil, fmt.Errorf("parse: %s: home page reports %d overdue learning task(s) but the stored learning-task list has %d; refusing to store a partial list (FR-12)", child, want, tasks.overdue)
+		}
+	}
+
 	// A task row is the same task wherever it appears (learning tasks and child
 	// profile pages both carry the grid). Keep the first, drop the repeats.
 	seen := map[string]bool{}
@@ -132,6 +184,16 @@ func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
 	return out, nil
 }

+// findPage returns the stored page with this slug, or nil.
+func findPage(pages []Page, slug string) *Page {
+	for i := range pages {
+		if pages[i].Slug == slug {
+			return &pages[i]
+		}
+	}
+	return nil
+}
+
 // pagesOf returns the pages with duplicates removed, keeping the last copy of a
 // slug (the fetcher overwrites a slug file on a re-fetch, so the last is
 // authoritative).
--- a/internal/parse/parse_test.go
+++ b/internal/parse/parse_test.go
@@ -303,3 +303,81 @@ func TestClassifyDeterministic(t *testing.T) {
 }

 func ptr(t time.Time) *time.Time { return &t }
+
+// tasksJSONFixture is the shape of GetAllLearningTasksByUserId: a wrapper with
+// the per-child tasks. Task 1 is overdue and carries its due date at the task
+// level; task 2 is on time and carries it on the student; task 3 belongs to the
+// sibling and must be skipped.
+const tasksJSONFixture = `{"d":{"total":3,"data":[
+  {"id":12657,"name":"SAC 6: Human Rights Presentation","activityName":"","subjectName":"11HLEG 11 HUMANITIES - LEGAL STUDIES","groupName":"11HLEG","dueDateTimestamp":"2026-10-09T05:00:00Z","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":2,"dueDateTimestamp":"","submittedTimestamp":""}]},
+  {"id":14268,"name":"Unit 1 English Exam","activityName":"11EENG","subjectName":"11EENG 11 ENGLISH - ENGLISH","groupName":"11EENG","dueDateTimestamp":"","students":[{"userId":27038,"userName":"Harrison ROLFE","submissionStatus":3,"dueDateTimestamp":"2026-06-22T05:09:00Z","submittedTimestamp":"2026-06-22T05:08:00Z"}]},
+  {"id":99999,"name":"Finn task","activityName":"","subjectName":"7MAT","groupName":"7MAT","dueDateTimestamp":"","students":[{"userId":31460,"userName":"Finn ROLFE","submissionStatus":2}]}
+]}}`
+
+// homeOverdueFixture carries the parent home-page summary link for user 27038.
+const homeOverdueFixture = `<html><body>
+<div data-testid="feed-toolbar"></div>
+<h5>Harrison ROLFE</h5>
+<div class="parentChildActions-learningTaskItems">
+  <div><a href="Records/User.aspx?userId=27038#learningTasks">1 overdue learning task</a></div>
+  <div><a href="Records/User.aspx?userId=27038#learningTasks">1 learning task due in the next week</a></div>
+</div>
+</body></html>`
+
+func TestParseLearningTasksAll(t *testing.T) {
+	loc := melbourne(t)
+	got, err := parseLearningTasksAll("Harry", Page{
+		Slug:   PageLearningTasksAll,
+		UserID: "27038",
+		JSON:   []byte(tasksJSONFixture),
+	}, loc)
+	if err != nil {
+		t.Fatalf("parseLearningTasksAll: %v", err)
+	}
+	if len(got.items) != 2 {
+		t.Fatalf("got %d items, want 2 (the sibling's row is skipped)", len(got.items))
+	}
+	if got.overdue != 1 {
+		t.Errorf("overdue = %d, want 1", got.overdue)
+	}
+	if got.total != 3 {
+		t.Errorf("total = %d, want 3", got.total)
+	}
+	if !strings.Contains(got.items[0].Body, "Status: Overdue") {
+		t.Errorf("overdue body = %q", got.items[0].Body)
+	}
+	if got.items[0].DueAt == nil || got.items[0].DueAt.Format("2006-01-02 15:04") != "2026-10-09 16:00" {
+		t.Errorf("task-level due = %v", got.items[0].DueAt)
+	}
+	if got.items[1].DueAt == nil || got.items[1].DueAt.Format("2006-01-02 15:04") != "2026-06-22 15:09" {
+		t.Errorf("student-level due = %v", got.items[1].DueAt)
+	}
+	if got.items[0].ExternalRef == got.items[1].ExternalRef {
+		t.Error("two tasks share an external ref")
+	}
+	if OverdueCount(got.items) != 1 {
+		t.Errorf("OverdueCount = %d, want 1", OverdueCount(got.items))
+	}
+}
+
+// TestParseReconcilesOverdueWithHome proves FR-12: the parser refuses to store a
+// task list whose overdue count disagrees with the home page summary.
+func TestParseReconcilesOverdueWithHome(t *testing.T) {
+	loc := melbourne(t)
+	jsonPage := Page{Slug: PageLearningTasksAll, UserID: "27038", JSON: []byte(tasksJSONFixture)}
+	homePage := Page{Slug: PageHome, HTML: []byte(homeOverdueFixture)}
+
+	if _, err := Parse("Harry", []Page{homePage, jsonPage}, loc); err != nil {
+		t.Fatalf("Parse with matching counts: %v", err)
+	}
+
+	mismatch := strings.Replace(homeOverdueFixture, "1 overdue learning task", "6 overdue learning tasks", 1)
+	if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(mismatch)}, jsonPage}, loc); err == nil {
+		t.Fatal("Parse accepted a task list that disagrees with the home page (FR-12)")
+	}
+
+	blank := `<html><body><div data-testid="feed-toolbar"></div></body></html>`
+	if _, err := Parse("Harry", []Page{{Slug: PageHome, HTML: []byte(blank)}, jsonPage}, loc); err == nil {
+		t.Fatal("Parse accepted a home page with no record of the child (FR-12)")
+	}
+}
--- a/internal/parse/tasks_json.go
+++ b/internal/parse/tasks_json.go
@@ -0,0 +1,256 @@
+package parse
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// This file parses the authoritative per-child learning-task list from the
+// Compass JSON service behind the Learning Tasks grid
+// (/Services/LearningTasks.svc/GetAllLearningTasksByUserId). The grid itself is
+// capped at LearningTask_Default_Records_Count (20) rows and, on the parent
+// page, mixes children, so it cannot answer "what is overdue" (AUD-114, FR-12).
+
+// Learning-task submission status codes, from
+// Compass.enums.LearningTasksSubmissionStatus in the Compass client bundle.
+const (
+	submissionPending       = 1
+	submissionOverdue       = 2
+	submissionOnTime        = 3
+	submissionSubmittedLate = 4
+)
+
+// overdueBodyMarker is the status line the JSON parser writes into a task body.
+// The dashboard and the offline importance rule key off it.
+const overdueBodyMarker = "Status: Overdue"
+
+// tasksResponse is the envelope Compass returns: {"d":{"data":[...],"total":n}}.
+type tasksResponse struct {
+	D struct {
+		Data  []taskRecord `json:"data"`
+		Total *int         `json:"total"`
+	} `json:"d"`
+}
+
+// taskRecord is one learning task. dueDateTimestamp is set on the task for most
+// rows; a few rows carry it on the student instead, so both are read.
+type taskRecord struct {
+	ID               int           `json:"id"`
+	Name             string        `json:"name"`
+	ActivityName     string        `json:"activityName"`
+	SubjectName      string        `json:"subjectName"`
+	GroupName        string        `json:"groupName"`
+	DueDateTimestamp string        `json:"dueDateTimestamp"`
+	Students         []taskStudent `json:"students"`
+}
+
+// taskStudent is one child's row within a task: their submission status and,
+// sometimes, the due date.
+type taskStudent struct {
+	UserID             int    `json:"userId"`
+	UserName           string `json:"userName"`
+	SubmissionStatus   *int   `json:"submissionStatus"`
+	DueDateTimestamp   string `json:"dueDateTimestamp"`
+	SubmittedTimestamp string `json:"submittedTimestamp"`
+}
+
+// parsedTasks is the result of reading one child's authoritative task list.
+type parsedTasks struct {
+	items   []Item
+	total   int
+	overdue int
+	userID  string
+}
+
+// parseLearningTasksAll turns the stored JSON service response into task items.
+// It fails loudly when the body is missing, is not the expected envelope, is
+// missing the child's userId, or carries a task without a status or title,
+// because a partial list must never be shown as "nothing overdue" (FR-12).
+func parseLearningTasksAll(child string, page Page, loc *time.Location) (parsedTasks, error) {
+	out := parsedTasks{userID: strings.TrimSpace(page.UserID)}
+	if len(page.JSON) == 0 {
+		return out, fmt.Errorf("parse: %s: %s has no stored JSON body", child, page.Slug)
+	}
+	var resp tasksResponse
+	if err := json.Unmarshal(page.JSON, &resp); err != nil {
+		return out, fmt.Errorf("parse: %s: %s is not a Compass learning-tasks response: %w", child, page.Slug, err)
+	}
+	if resp.D.Data == nil {
+		return out, fmt.Errorf("parse: %s: %s carried no task list (session expired or layout changed)", child, page.Slug)
+	}
+	if out.userID == "" {
+		return out, fmt.Errorf("parse: %s: %s is missing the child's Compass userId; cannot select the child's tasks", child, page.Slug)
+	}
+	out.total = len(resp.D.Data)
+	if resp.D.Total != nil && *resp.D.Total != out.total {
+		return out, fmt.Errorf("parse: %s: %s returned %d of %d tasks; refusing a partial list (FR-12)", child, page.Slug, out.total, *resp.D.Total)
+	}
+
+	for _, t := range resp.D.Data {
+		st, ok := studentFor(t, out.userID)
+		if !ok {
+			continue // the parent endpoint returns siblings too; keep only this child
+		}
+		if st.SubmissionStatus == nil {
+			return out, fmt.Errorf("parse: %s: learning task %d has no submission status", child, t.ID)
+		}
+		status := *st.SubmissionStatus
+		if status == submissionOverdue {
+			out.overdue++
+		}
+
+		title := collapse(t.Name)
+		if title == "" {
+			return out, fmt.Errorf("parse: %s: learning task %d has no title", child, t.ID)
+		}
+		due, err := parseCompassTimestamp(firstNonEmpty(t.DueDateTimestamp, st.DueDateTimestamp), loc)
+		if err != nil {
+			return out, fmt.Errorf("parse: %s: learning task %d: %w", child, t.ID, err)
+		}
+
+		body := []string{"Status: " + submissionStatusLabel(status)}
+		if a := collapse(t.ActivityName); a != "" {
+			body = append(body, "Class: "+a)
+		}
+		if due != nil {
+			body = append(body, "Due: "+due.Format("2006-01-02 15:04"))
+		}
+		if s := strings.TrimSpace(st.SubmittedTimestamp); s != "" {
+			body = append(body, "Submitted: "+s)
+		}
+
+		out.items = append(out.items, Item{
+			Child:       child,
+			Source:      SourceCompass,
+			Type:        TypeTask,
+			Subject:     collapse(firstNonEmpty(t.SubjectName, t.GroupName, t.ActivityName)),
+			Title:       title,
+			Body:        strings.Join(body, "\n"),
+			DueAt:       due,
+			URL:         memberLearningTasksURL(out.userID),
+			SourcePage:  page.Slug,
+			ExternalRef: dedupeKey("learning-task", strconv.Itoa(t.ID)),
+			Importance:  importanceNormal,
+		})
+	}
+	return out, nil
+}
+
+// studentFor returns the child's row within a task by Compass userId.
+func studentFor(t taskRecord, userID string) (taskStudent, bool) {
+	for _, s := range t.Students {
+		if strconv.Itoa(s.UserID) == userID {
+			return s, true
+		}
+	}
+	return taskStudent{}, false
+}
+
+// submissionStatusLabel maps a Compass status code to the label the grid shows.
+func submissionStatusLabel(code int) string {
+	switch code {
+	case submissionPending:
+		return "Pending"
+	case submissionOverdue:
+		return "Overdue"
+	case submissionOnTime:
+		return "On Time"
+	case submissionSubmittedLate:
+		return "Received Late"
+	default:
+		return fmt.Sprintf("status %d", code)
+	}
+}
+
+// parseCompassTimestamp parses the service's RFC3339 UTC due timestamp in loc.
+// An empty value means no due date; anything else that does not parse is an
+// error, never a silent zero (FR-12).
+func parseCompassTimestamp(s string, loc *time.Location) (*time.Time, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil, nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return nil, fmt.Errorf("unrecognised due timestamp %q", s)
+	}
+	if loc == nil {
+		loc = time.UTC
+	}
+	t = t.In(loc)
+	return &t, nil
+}
+
+// overdueLinkRe reads the count from a home-page link such as
+// "6 overdue learning tasks".
+var overdueLinkRe = regexp.MustCompile(`(\d+)\s+overdue learning task`)
+
+// homeOverdueCount returns the overdue count the parent home page announces for
+// one child. found is false when the page carries no record of that child at
+// all (a login or challenge page), so a missing summary is never read as zero
+// (FR-12). A child with no overdue line reports 0.
+func homeOverdueCount(homeHTML []byte, userID string) (count int, found bool) {
+	if strings.TrimSpace(userID) == "" {
+		return 0, false
+	}
+	root := parseHTML(homeHTML)
+	seenChild := false
+	count = -1
+	for _, a := range findAll(root, func(n *node) bool { return n.Tag == "a" }) {
+		if !strings.Contains(a.Attrs["href"], "userId="+userID) {
+			continue
+		}
+		seenChild = true
+		if m := overdueLinkRe.FindStringSubmatch(collapse(a.Text())); m != nil {
+			if n, err := strconv.Atoi(m[1]); err == nil {
+				count = n
+			}
+		}
+	}
+	if count >= 0 {
+		return count, true
+	}
+	return 0, seenChild
+}
+
+// HomeOverdueCount reports the overdue learning-task count the home page
+// announces for one child. It is exported so a command can show the number it
+// reconciled against.
+func HomeOverdueCount(homeHTML []byte, userID string) (int, bool) {
+	return homeOverdueCount(homeHTML, userID)
+}
+
+// OverdueCount returns how many learning tasks carry the Overdue submission
+// status. The JSON parser writes the status into the item body, so the count is
+// exact and does not depend on the wall clock.
+func OverdueCount(items []Item) int {
+	n := 0
+	for _, it := range items {
+		if it.Type == TypeTask && strings.Contains(it.Body, overdueBodyMarker) {
+			n++
+		}
+	}
+	return n
+}
+
+// memberLearningTasksURL links a task back to the child's profile grid.
+func memberLearningTasksURL(userID string) string {
+	if strings.TrimSpace(userID) == "" {
+		return compassHost + "/Records/UserNew.aspx#learningTasks"
+	}
+	return compassHost + "/Records/User.aspx?userId=" + userID + "#learningTasks"
+}
+
+// firstNonEmpty returns the first value with non-whitespace content.
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if strings.TrimSpace(v) != "" {
+			return v
+		}
+	}
+	return ""
+}