Phase 4a: fetch and parse the authoritative Compass learning tasks

The Learning Tasks grid is capped at 20 rows and, on the parent page, mixes
both children, so it cannot answer "what is overdue". The parent home page
reported 6 overdue tasks for Harry while the stored grid held 1.

Fetch the per-child task list (read-only) from
POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId, parse it into
task rows, and refuse to store it when the overdue count disagrees with the
home page summary (FR-12).

- fetcher: store learning-tasks-all.json + meta (userId, status, task count)
- parse: new learning-tasks-all page, JSON parser, home-page reconciliation
- store: drop superseded grid-derived task rows before the upsert
- docs: docs/phase-4a-overdue-tasks.md; close the Phase 3 known gap

Verified with live runs: Harry 70 tasks / 6 overdue, Finn 42 tasks / 2 overdue,
both equal to the home page; a tampered home page stops the parse.

Note: the fetcher and store.go files also carry the per-child session work
that was already uncommitted in this shared workspace (AUD-108).

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cmd/school/parse.go

@@ -55,6 +55,7 @@ type parseSummary struct {
 	Model      string            `json:"model,omitempty"`
 	Pages      []string          `json:"pages"`
 	Total      int               `json:"total"`
+	Overdue    int               `json:"overdue"`
 	Inserted   int               `json:"inserted"`
 	Updated    int               `json:"updated"`
 	Counts     []store.TypeCount `json:"counts"`
@@ -139,6 +140,16 @@ func runParse(f parseFlags) error {
 		})
 	}
 
+	// The capped legacy grid is superseded by the authoritative JSON task list.
+	// Remove its rows for this child first, so a stale partial row can never be
+	// shown next to the complete set (AUD-114, FR-12).
+	if hasPage(pages, parse.PageLearningTasksAll) {
+		if _, err := st.DeleteChildCompassTasksFromPages(ctx, child.ID, parse.PageLearningTasks, parse.PageChildProfile); err != nil {
+			_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
+			return err
+		}
+	}
+
 	inserted, updated, err := st.UpsertItems(ctx, child.ID, stored)
 	if err != nil {
 		_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
@@ -171,6 +182,7 @@ func runParse(f parseFlags) error {
 		Model:      model,
 		Pages:      pageSlugs,
 		Total:      len(items),
+		Overdue:    parse.OverdueCount(items),
 		Inserted:   inserted,
 		Updated:    updated,
 		Counts:     counts,
@@ -206,6 +218,7 @@ func printParseSummary(s parseSummary) {
 		fmt.Printf(" (%s)", s.Model)
 	}
 	fmt.Printf("\nitems        %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated)
+	fmt.Printf("overdue      %d (reconciled against the home page)\n", s.Overdue)
 	for _, c := range s.Counts {
 		fmt.Printf("  %-6s %-8s %d\n", c.Child, c.Type, c.Count)
 	}
@@ -271,8 +284,10 @@ func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *t
 	}
 }
 
-// loadRunPages finds the raw run directory for a child and reads every
-// <slug>.html page in it, with the URL from the matching <slug>.meta.json.
+// loadRunPages finds the raw run directory for a child and reads every stored
+// page in it. HTML pages come from <slug>.html; the JSON service page comes from
+// <slug>.json. The URL and the child's Compass userId come from the matching
+// <slug>.meta.json.
 func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) {
 	childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child)))
 	entries, err := os.ReadDir(childDir)
@@ -308,51 +323,103 @@ func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) {
 		return "", nil, fmt.Errorf("read run %s: %w", runDir, err)
 	}
 
-	pages := make([]parse.Page, 0, len(files))
+	type rawPage struct {
+		htmlBytes, jsonBytes []byte
+		url, user            string
+	}
+	bySlug := map[string]*rawPage{}
+	var order []string
+	get := func(slug string) *rawPage {
+		if p, ok := bySlug[slug]; ok {
+			return p
+		}
+		p := &rawPage{}
+		bySlug[slug] = p
+		order = append(order, slug)
+		return p
+	}
+
 	for _, e := range files {
 		name := e.Name()
-		if e.IsDir() || !strings.HasSuffix(strings.ToLower(name), ".html") {
+		if e.IsDir() {
 			continue
 		}
-		slug := strings.TrimSuffix(name, filepath.Ext(name))
-		raw, err := os.ReadFile(filepath.Join(runDir, name))
-		if err != nil {
-			return "", nil, fmt.Errorf("read %s: %w", name, err)
+		lower := strings.ToLower(name)
+		switch {
+		case strings.HasSuffix(lower, ".meta.json"):
+			slug := strings.TrimSuffix(name, ".meta.json")
+			raw, err := os.ReadFile(filepath.Join(runDir, name))
+			if err != nil {
+				return "", nil, fmt.Errorf("read %s: %w", name, err)
+			}
+			var meta struct {
+				URL    string `json:"url"`
+				UserID string `json:"userId"`
+			}
+			_ = json.Unmarshal(raw, &meta)
+			p := get(slug)
+			p.url = strings.TrimSpace(meta.URL)
+			p.user = strings.TrimSpace(meta.UserID)
+		case strings.HasSuffix(lower, ".json"):
+			if strings.EqualFold(name, "manifest.json") {
+				continue
+			}
+			slug := strings.TrimSuffix(name, ".json")
+			raw, err := os.ReadFile(filepath.Join(runDir, name))
+			if err != nil {
+				return "", nil, fmt.Errorf("read %s: %w", name, err)
+			}
+			get(slug).jsonBytes = raw
+		case strings.HasSuffix(lower, ".html"):
+			slug := strings.TrimSuffix(name, filepath.Ext(name))
+			raw, err := os.ReadFile(filepath.Join(runDir, name))
+			if err != nil {
+				return "", nil, fmt.Errorf("read %s: %w", name, err)
+			}
+			get(slug).htmlBytes = raw
+		}
+	}
+
+	pages := make([]parse.Page, 0, len(order))
+	for _, slug := range order {
+		rp := bySlug[slug]
+		if len(rp.htmlBytes) == 0 && len(rp.jsonBytes) == 0 {
+			continue
 		}
 		pages = append(pages, parse.Page{
-			Slug: slug,
-			URL:  pageURL(runDir, slug),
-			HTML: raw,
+			Slug:   slug,
+			URL:    rp.url,
+			HTML:   rp.htmlBytes,
+			JSON:   rp.jsonBytes,
+			UserID: rp.user,
 		})
 	}
 	if len(pages) == 0 {
-		return "", nil, fmt.Errorf("run %s has no .html pages", runDir)
+		return "", nil, fmt.Errorf("run %s has no stored pages", runDir)
 	}
 	return runDir, pages, nil
 }
 
-// runHasKnownPage reports whether a run directory holds at least one page the
-// parser understands, so a failed login run is never chosen as "latest".
-func runHasKnownPage(dir string) bool {
-	for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageCalendar, parse.PageChildProfile} {
-		if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil {
+// hasPage reports whether a parsed page set contains a slug.
+func hasPage(pages []parse.Page, slug string) bool {
+	for i := range pages {
+		if pages[i].Slug == slug {
 			return true
 		}
 	}
 	return false
 }
 
-// pageURL reads the source URL recorded by the Phase 2 fetcher.
-func pageURL(runDir, slug string) string {
-	raw, err := os.ReadFile(filepath.Join(runDir, slug+".meta.json"))
-	if err != nil {
-		return ""
+// runHasKnownPage reports whether a run directory holds at least one page the
+// parser understands, so a failed login run is never chosen as "latest".
+func runHasKnownPage(dir string) bool {
+	for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageLearningTasksAll, parse.PageCalendar, parse.PageChildProfile} {
+		if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil {
+			return true
+		}
+		if _, err := os.Stat(filepath.Join(dir, slug+".json")); err == nil {
+			return true
+		}
 	}
-	var meta struct {
-		URL string `json:"url"`
-	}
-	if err := json.Unmarshal(raw, &meta); err != nil {
-		return ""
-	}
-	return strings.TrimSpace(meta.URL)
+	return false
 }