Phase 4a: fetch and parse the authoritative Compass learning tasks

The Learning Tasks grid is capped at 20 rows and, on the parent page, mixes both children, so it cannot answer "what is overdue". The parent home page reported 6 overdue tasks for Harry while the stored grid held 1. Fetch the per-child task list (read-only) from POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId, parse it into task rows, and refuse to store it when the overdue count disagrees with the home page summary (FR-12). - fetcher: store learning-tasks-all.json + meta (userId, status, task count) - parse: new learning-tasks-all page, JSON parser, home-page reconciliation - store: drop superseded grid-derived task rows before the upsert - docs: docs/phase-4a-overdue-tasks.md; close the Phase 3 known gap Verified with live runs: Harry 70 tasks / 6 overdue, Finn 42 tasks / 2 overdue, both equal to the home page; a tampered home page stops the parse. Note: the fetcher and store.go files also carry the per-child session work that was already uncommitted in this shared workspace (AUD-108). Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-15 17:38:22 +11:00
parent d7d529eb0f
commit 5d263987dd
8 changed files with 913 additions and 82 deletions
--- a/internal/parse/tasks_json.go
+++ b/internal/parse/tasks_json.go
@@ -0,0 +1,256 @@
+package parse
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// This file parses the authoritative per-child learning-task list from the
+// Compass JSON service behind the Learning Tasks grid
+// (/Services/LearningTasks.svc/GetAllLearningTasksByUserId). The grid itself is
+// capped at LearningTask_Default_Records_Count (20) rows and, on the parent
+// page, mixes children, so it cannot answer "what is overdue" (AUD-114, FR-12).
+
+// Learning-task submission status codes, from
+// Compass.enums.LearningTasksSubmissionStatus in the Compass client bundle.
+const (
+	submissionPending       = 1
+	submissionOverdue       = 2
+	submissionOnTime        = 3
+	submissionSubmittedLate = 4
+)
+
+// overdueBodyMarker is the status line the JSON parser writes into a task body.
+// The dashboard and the offline importance rule key off it.
+const overdueBodyMarker = "Status: Overdue"
+
+// tasksResponse is the envelope Compass returns: {"d":{"data":[...],"total":n}}.
+type tasksResponse struct {
+	D struct {
+		Data  []taskRecord `json:"data"`
+		Total *int         `json:"total"`
+	} `json:"d"`
+}
+
+// taskRecord is one learning task. dueDateTimestamp is set on the task for most
+// rows; a few rows carry it on the student instead, so both are read.
+type taskRecord struct {
+	ID               int           `json:"id"`
+	Name             string        `json:"name"`
+	ActivityName     string        `json:"activityName"`
+	SubjectName      string        `json:"subjectName"`
+	GroupName        string        `json:"groupName"`
+	DueDateTimestamp string        `json:"dueDateTimestamp"`
+	Students         []taskStudent `json:"students"`
+}
+
+// taskStudent is one child's row within a task: their submission status and,
+// sometimes, the due date.
+type taskStudent struct {
+	UserID             int    `json:"userId"`
+	UserName           string `json:"userName"`
+	SubmissionStatus   *int   `json:"submissionStatus"`
+	DueDateTimestamp   string `json:"dueDateTimestamp"`
+	SubmittedTimestamp string `json:"submittedTimestamp"`
+}
+
+// parsedTasks is the result of reading one child's authoritative task list.
+type parsedTasks struct {
+	items   []Item
+	total   int
+	overdue int
+	userID  string
+}
+
+// parseLearningTasksAll turns the stored JSON service response into task items.
+// It fails loudly when the body is missing, is not the expected envelope, is
+// missing the child's userId, or carries a task without a status or title,
+// because a partial list must never be shown as "nothing overdue" (FR-12).
+func parseLearningTasksAll(child string, page Page, loc *time.Location) (parsedTasks, error) {
+	out := parsedTasks{userID: strings.TrimSpace(page.UserID)}
+	if len(page.JSON) == 0 {
+		return out, fmt.Errorf("parse: %s: %s has no stored JSON body", child, page.Slug)
+	}
+	var resp tasksResponse
+	if err := json.Unmarshal(page.JSON, &resp); err != nil {
+		return out, fmt.Errorf("parse: %s: %s is not a Compass learning-tasks response: %w", child, page.Slug, err)
+	}
+	if resp.D.Data == nil {
+		return out, fmt.Errorf("parse: %s: %s carried no task list (session expired or layout changed)", child, page.Slug)
+	}
+	if out.userID == "" {
+		return out, fmt.Errorf("parse: %s: %s is missing the child's Compass userId; cannot select the child's tasks", child, page.Slug)
+	}
+	out.total = len(resp.D.Data)
+	if resp.D.Total != nil && *resp.D.Total != out.total {
+		return out, fmt.Errorf("parse: %s: %s returned %d of %d tasks; refusing a partial list (FR-12)", child, page.Slug, out.total, *resp.D.Total)
+	}
+
+	for _, t := range resp.D.Data {
+		st, ok := studentFor(t, out.userID)
+		if !ok {
+			continue // the parent endpoint returns siblings too; keep only this child
+		}
+		if st.SubmissionStatus == nil {
+			return out, fmt.Errorf("parse: %s: learning task %d has no submission status", child, t.ID)
+		}
+		status := *st.SubmissionStatus
+		if status == submissionOverdue {
+			out.overdue++
+		}
+
+		title := collapse(t.Name)
+		if title == "" {
+			return out, fmt.Errorf("parse: %s: learning task %d has no title", child, t.ID)
+		}
+		due, err := parseCompassTimestamp(firstNonEmpty(t.DueDateTimestamp, st.DueDateTimestamp), loc)
+		if err != nil {
+			return out, fmt.Errorf("parse: %s: learning task %d: %w", child, t.ID, err)
+		}
+
+		body := []string{"Status: " + submissionStatusLabel(status)}
+		if a := collapse(t.ActivityName); a != "" {
+			body = append(body, "Class: "+a)
+		}
+		if due != nil {
+			body = append(body, "Due: "+due.Format("2006-01-02 15:04"))
+		}
+		if s := strings.TrimSpace(st.SubmittedTimestamp); s != "" {
+			body = append(body, "Submitted: "+s)
+		}
+
+		out.items = append(out.items, Item{
+			Child:       child,
+			Source:      SourceCompass,
+			Type:        TypeTask,
+			Subject:     collapse(firstNonEmpty(t.SubjectName, t.GroupName, t.ActivityName)),
+			Title:       title,
+			Body:        strings.Join(body, "\n"),
+			DueAt:       due,
+			URL:         memberLearningTasksURL(out.userID),
+			SourcePage:  page.Slug,
+			ExternalRef: dedupeKey("learning-task", strconv.Itoa(t.ID)),
+			Importance:  importanceNormal,
+		})
+	}
+	return out, nil
+}
+
+// studentFor returns the child's row within a task by Compass userId.
+func studentFor(t taskRecord, userID string) (taskStudent, bool) {
+	for _, s := range t.Students {
+		if strconv.Itoa(s.UserID) == userID {
+			return s, true
+		}
+	}
+	return taskStudent{}, false
+}
+
+// submissionStatusLabel maps a Compass status code to the label the grid shows.
+func submissionStatusLabel(code int) string {
+	switch code {
+	case submissionPending:
+		return "Pending"
+	case submissionOverdue:
+		return "Overdue"
+	case submissionOnTime:
+		return "On Time"
+	case submissionSubmittedLate:
+		return "Received Late"
+	default:
+		return fmt.Sprintf("status %d", code)
+	}
+}
+
+// parseCompassTimestamp parses the service's RFC3339 UTC due timestamp in loc.
+// An empty value means no due date; anything else that does not parse is an
+// error, never a silent zero (FR-12).
+func parseCompassTimestamp(s string, loc *time.Location) (*time.Time, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil, nil
+	}
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return nil, fmt.Errorf("unrecognised due timestamp %q", s)
+	}
+	if loc == nil {
+		loc = time.UTC
+	}
+	t = t.In(loc)
+	return &t, nil
+}
+
+// overdueLinkRe reads the count from a home-page link such as
+// "6 overdue learning tasks".
+var overdueLinkRe = regexp.MustCompile(`(\d+)\s+overdue learning task`)
+
+// homeOverdueCount returns the overdue count the parent home page announces for
+// one child. found is false when the page carries no record of that child at
+// all (a login or challenge page), so a missing summary is never read as zero
+// (FR-12). A child with no overdue line reports 0.
+func homeOverdueCount(homeHTML []byte, userID string) (count int, found bool) {
+	if strings.TrimSpace(userID) == "" {
+		return 0, false
+	}
+	root := parseHTML(homeHTML)
+	seenChild := false
+	count = -1
+	for _, a := range findAll(root, func(n *node) bool { return n.Tag == "a" }) {
+		if !strings.Contains(a.Attrs["href"], "userId="+userID) {
+			continue
+		}
+		seenChild = true
+		if m := overdueLinkRe.FindStringSubmatch(collapse(a.Text())); m != nil {
+			if n, err := strconv.Atoi(m[1]); err == nil {
+				count = n
+			}
+		}
+	}
+	if count >= 0 {
+		return count, true
+	}
+	return 0, seenChild
+}
+
+// HomeOverdueCount reports the overdue learning-task count the home page
+// announces for one child. It is exported so a command can show the number it
+// reconciled against.
+func HomeOverdueCount(homeHTML []byte, userID string) (int, bool) {
+	return homeOverdueCount(homeHTML, userID)
+}
+
+// OverdueCount returns how many learning tasks carry the Overdue submission
+// status. The JSON parser writes the status into the item body, so the count is
+// exact and does not depend on the wall clock.
+func OverdueCount(items []Item) int {
+	n := 0
+	for _, it := range items {
+		if it.Type == TypeTask && strings.Contains(it.Body, overdueBodyMarker) {
+			n++
+		}
+	}
+	return n
+}
+
+// memberLearningTasksURL links a task back to the child's profile grid.
+func memberLearningTasksURL(userID string) string {
+	if strings.TrimSpace(userID) == "" {
+		return compassHost + "/Records/UserNew.aspx#learningTasks"
+	}
+	return compassHost + "/Records/User.aspx?userId=" + userID + "#learningTasks"
+}
+
+// firstNonEmpty returns the first value with non-whitespace content.
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if strings.TrimSpace(v) != "" {
+			return v
+		}
+	}
+	return ""
+}