compass_teams_nhs/internal/parse/parse.go

package parse

import (
	"fmt"
	"sort"
	"strings"
	"time"
)

// Source and item types match the items table CHECK constraints (AGENTS.md §7).
const (
	SourceCompass = "compass"

	TypeMessage = "message"
	TypeTask    = "task"
	TypeEvent   = "event"
)

// Recognised page slugs, matching the <slug>.html files stored by the Phase 2
// fetchers under var/raw/<child>/<runId>/.
const (
	PageHome             = "home"
	PageChildProfile     = "child-profile"
	PageLearningTasks    = "learning-tasks"
	PageLearningTasksAll = "learning-tasks-all"
	PageCalendar         = "calendar"
)

// Item is one message, learning task or calendar event read from Compass.
// Every field maps to a column of the items table (FR-5).
type Item struct {
	Child       string     `json:"child"`
	Source      string     `json:"source"`
	Type        string     `json:"type"`
	Subject     string     `json:"subject,omitempty"`
	Teacher     string     `json:"teacher,omitempty"`
	Title       string     `json:"title"`
	Body        string     `json:"body,omitempty"`
	DueAt       *time.Time `json:"due_at,omitempty"`
	URL         string     `json:"url,omitempty"`
	SourcePage  string     `json:"source_page"`
	ExternalRef string     `json:"external_ref"`
	Importance  string     `json:"importance"`
}

// Page is one stored raw page: its slug, the URL it was fetched from, and the
// bytes. HTML pages carry HTML; the JSON service page carries JSON and the
// child's Compass userId (from the fetcher's meta file).
type Page struct {
	Slug   string
	URL    string
	HTML   []byte
	JSON   []byte
	UserID string
}

// Parse turns the stored pages for one child into structured items. It fails,
// loudly and explicitly, when a page it recognises cannot be parsed, so a broken
// scan can never be shown as "nothing due" (FR-12).
//
// Pages with an unknown slug are ignored. When no recognised page yields any
// structure at all, Parse returns an error.
func Parse(child string, pages []Page, loc *time.Location) ([]Item, error) {
	if strings.TrimSpace(child) == "" {
		return nil, fmt.Errorf("parse: child is empty")
	}
	if loc == nil {
		loc = time.UTC
	}

	ordered := pagesOf(pages)

	// When the authoritative per-child task list is stored, it supersedes the
	// legacy HTML grid: the grid is capped and, on the parent page, mixes
	// children. Parse the JSON list and ignore the grid pages for tasks so the
	// same task is never stored twice.
	hasAuthoritativeTasks := false
	for _, p := range ordered {
		if p.Slug == PageLearningTasksAll {
			hasAuthoritativeTasks = true
		}
	}

	var items []Item
	recognised := 0
	var tasks *parsedTasks
	for i := range ordered {
		p := ordered[i]
		switch p.Slug {
		case PageLearningTasksAll:
			got, err := parseLearningTasksAll(child, p, loc)
			if err != nil {
				return nil, err
			}
			tasks = &got
			items = append(items, got.items...)
			recognised++
		case PageLearningTasks:
			if hasAuthoritativeTasks {
				recognised++
				continue
			}
			got, err := parseLearningTasks(child, p, loc)
			if err != nil {
				return nil, err
			}
			items = append(items, got...)
			recognised++
		case PageChildProfile:
			// The React profile page has no server-rendered grid; when it does
			// carry the legacy grid, use it. The JSON list supersedes it.
			if hasAuthoritativeTasks {
				recognised++
				continue
			}
			got, found, err := extractLearningTasks(child, p, loc)
			if err != nil {
				return nil, err
			}
			if found {
				items = append(items, got...)
			}
			recognised++
		case PageHome:
			got, err := parseNews(child, p, loc)
			if err != nil {
				return nil, err
			}
			items = append(items, got...)
			recognised++
		case PageCalendar:
			got, err := parseCalendar(child, p, loc)
			if err != nil {
				return nil, err
			}
			items = append(items, got...)
			recognised++
		}
	}

	if recognised == 0 {
		return nil, fmt.Errorf("parse: no recognised Compass page in %d stored page(s)", len(pages))
	}

	// FR-12: the stored task list must agree with the summary on the home page.
	// A partial or stale list must never be shown as "nothing overdue", so a
	// disagreement stops the parse instead of writing a quiet wrong answer.
	if tasks != nil {
		home := findPage(ordered, PageHome)
		if home == nil {
			return nil, fmt.Errorf("parse: %s: %s is present but the home page is missing; cannot verify the overdue count (FR-12)", child, PageLearningTasksAll)
		}
		want, found := homeOverdueCount(home.HTML, tasks.userID)
		if !found {
			return nil, fmt.Errorf("parse: %s: home page has no record of Compass user %s; cannot verify the overdue count (FR-12)", child, tasks.userID)
		}
		if want != tasks.overdue {
			return nil, fmt.Errorf("parse: %s: home page reports %d overdue learning task(s) but the stored learning-task list has %d; refusing to store a partial list (FR-12)", child, want, tasks.overdue)
		}
	}

	// A task row is the same task wherever it appears (learning tasks and child
	// profile pages both carry the grid). Keep the first, drop the repeats.
	seen := map[string]bool{}
	out := items[:0]
	for _, it := range items {
		key := it.Type + "\x1f" + it.ExternalRef
		if seen[key] {
			continue
		}
		seen[key] = true
		if it.Importance == "" {
			it.Importance = importanceNormal
		}
		out = append(out, it)
	}

	sort.SliceStable(out, func(i, j int) bool {
		if out[i].Type != out[j].Type {
			return out[i].Type < out[j].Type
		}
		return out[i].ExternalRef < out[j].ExternalRef
	})
	return out, nil
}

// findPage returns the stored page with this slug, or nil.
func findPage(pages []Page, slug string) *Page {
	for i := range pages {
		if pages[i].Slug == slug {
			return &pages[i]
		}
	}
	return nil
}

// pagesOf returns the pages with duplicates removed, keeping the last copy of a
// slug (the fetcher overwrites a slug file on a re-fetch, so the last is
// authoritative).
func pagesOf(pages []Page) []Page {
	bySlug := map[string]Page{}
	var order []string
	for _, p := range pages {
		if _, ok := bySlug[p.Slug]; !ok {
			order = append(order, p.Slug)
		}
		bySlug[p.Slug] = p
	}
	out := make([]Page, 0, len(order))
	for _, slug := range order {
		out = append(out, bySlug[slug])
	}
	return out
}

// dedupeKey joins the identity fields of an item into a stable external
// reference. Re-parsing the same page must update the same row, not insert a
// second one.
func dedupeKey(parts ...string) string {
	clean := make([]string, 0, len(parts))
	for _, p := range parts {
		clean = append(clean, collapse(p))
	}
	return strings.Join(clean, "|")
}