compass_teams_nhs/cmd/school/parse.go

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config"
	"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/llm"
	"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/parse"
	"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store"
)

// parseFlags holds the `school parse` options.
type parseFlags struct {
	child    string
	run      string
	rawRoot  string
	classify string
	model    string
	asJSON   bool
}

// parseCommand is the `school parse` entry point. It reads the stored raw
// Compass pages for one child, turns them into items and writes them to SQLite
// (AGENTS.md §11 Phase 3).
func parseCommand(args []string) {
	fs := flag.NewFlagSet("parse", flag.ContinueOnError)
	f := parseFlags{}
	fs.StringVar(&f.child, "child", "Harry", "child name (Harry or Finn)")
	fs.StringVar(&f.run, "run", "latest", "raw run id under var/raw/<child>/, or 'latest'")
	fs.StringVar(&f.rawRoot, "raw-root", "var/raw", "root directory of the stored raw pages")
	fs.StringVar(&f.classify, "classify", "auto", "importance classifier: auto, none or omniroute")
	fs.StringVar(&f.model, "model", "", "model id for importance (default SCHOOL_LLM_MODEL or OmniRoute default)")
	fs.BoolVar(&f.asJSON, "json", false, "print the summary as JSON")
	_ = fs.Parse(args)

	if err := runParse(f); err != nil {
		fmt.Fprintln(os.Stderr, "school parse:", err)
		os.Exit(1)
	}
}

// parseSummary is the machine-readable result of one parse run.
type parseSummary struct {
	Child      string            `json:"child"`
	RunDir     string            `json:"run_dir"`
	Classifier string            `json:"classifier"`
	Model      string            `json:"model,omitempty"`
	Pages      []string          `json:"pages"`
	Total      int               `json:"total"`
	Inserted   int               `json:"inserted"`
	Updated    int               `json:"updated"`
	Counts     []store.TypeCount `json:"counts"`
	Samples    []itemSample      `json:"samples"`
	ScannedAt  string            `json:"scanned_at"`
}

type itemSample struct {
	Type       string `json:"type"`
	Subject    string `json:"subject,omitempty"`
	Title      string `json:"title"`
	DueAt      string `json:"due_at,omitempty"`
	SourcePage string `json:"source_page"`
	Importance string `json:"importance"`
	URL        string `json:"url,omitempty"`
}

func runParse(f parseFlags) error {
	ctx := context.Background()

	cfg, err := config.LoadDB()
	if err != nil {
		return err
	}

	st, err := store.Open(cfg.DBPath)
	if err != nil {
		return err
	}
	defer st.Close()

	if err := st.Migrate(ctx); err != nil {
		return err
	}
	if err := st.SeedChildren(ctx, seedChildren...); err != nil {
		return err
	}

	child, err := st.ChildByName(ctx, f.child)
	if err != nil {
		return err
	}
	if child == nil {
		return fmt.Errorf("unknown child %q", f.child)
	}

	runDir, pages, err := loadRunPages(f.rawRoot, f.child, f.run)
	if err != nil {
		// FR-12: a failed load is an explicit failed scan, never an empty day.
		_, _ = st.RecordScan(ctx, &child.ID, 0, 0, false, err.Error())
		return err
	}

	items, err := parse.Parse(f.child, pages, cfg.Location)
	if err != nil {
		_, _ = st.RecordScan(ctx, &child.ID, len(pages), 0, false, err.Error())
		return err
	}

	classifyCtx, cancelClassify := context.WithTimeout(ctx, 3*time.Minute)
	classifier, model, err := classifyItems(classifyCtx, f, items, cfg.Location)
	cancelClassify()
	if err != nil {
		_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
		return err
	}

	stored := make([]store.Item, 0, len(items))
	for _, it := range items {
		stored = append(stored, store.Item{
			Source:      it.Source,
			Type:        it.Type,
			Subject:     it.Subject,
			Teacher:     it.Teacher,
			Title:       it.Title,
			Body:        it.Body,
			DueAt:       it.DueAt,
			URL:         it.URL,
			SourcePage:  it.SourcePage,
			Importance:  it.Importance,
			ExternalRef: it.ExternalRef,
		})
	}

	inserted, updated, err := st.UpsertItems(ctx, child.ID, stored)
	if err != nil {
		_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
		return err
	}

	if _, err := st.RecordScan(ctx, &child.ID, len(pages), len(items), true, ""); err != nil {
		return err
	}

	counts, err := st.CountItemsByType(ctx)
	if err != nil {
		return err
	}
	samples, err := st.SampleItems(ctx, child.ID, 12)
	if err != nil {
		return err
	}

	pageSlugs := make([]string, 0, len(pages))
	for _, p := range pages {
		pageSlugs = append(pageSlugs, p.Slug)
	}
	sort.Strings(pageSlugs)

	summary := parseSummary{
		Child:      f.child,
		RunDir:     runDir,
		Classifier: classifier,
		Model:      model,
		Pages:      pageSlugs,
		Total:      len(items),
		Inserted:   inserted,
		Updated:    updated,
		Counts:     counts,
		ScannedAt:  time.Now().In(cfg.Location).Format(time.RFC3339),
	}
	for _, s := range samples {
		summary.Samples = append(summary.Samples, itemSample{
			Type:       s.Type,
			Subject:    s.Subject,
			Title:      s.Title,
			DueAt:      dueString(s.DueAt, cfg.Location),
			SourcePage: s.SourcePage,
			Importance: s.Importance,
			URL:        s.URL,
		})
	}

	if f.asJSON {
		enc := json.NewEncoder(os.Stdout)
		enc.SetIndent("", "  ")
		return enc.Encode(summary)
	}
	printParseSummary(summary)
	return nil
}

func printParseSummary(s parseSummary) {
	fmt.Printf("child        %s\n", s.Child)
	fmt.Printf("run          %s\n", s.RunDir)
	fmt.Printf("pages        %s\n", strings.Join(s.Pages, ", "))
	fmt.Printf("classifier   %s", s.Classifier)
	if s.Model != "" {
		fmt.Printf(" (%s)", s.Model)
	}
	fmt.Printf("\nitems        %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated)
	for _, c := range s.Counts {
		fmt.Printf("  %-6s %-8s %d\n", c.Child, c.Type, c.Count)
	}
	fmt.Println("samples:")
	for _, x := range s.Samples {
		due := x.DueAt
		if due == "" {
			due = "-"
		}
		fmt.Printf("  [%s] %-8s %-28s %-45s due=%s\n", x.Importance, x.Type, x.Subject, truncate(x.Title, 45), due)
	}
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n-1] + "…"
}

func dueString(t *time.Time, loc *time.Location) string {
	if t == nil {
		return ""
	}
	return t.In(loc).Format("2006-01-02 15:04")
}

// classifyItems applies the configured importance classifier and returns its
// name and model. It records which classifier was used (AGENTS.md §4 FR-4).
func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *time.Location) (string, string, error) {
	mode := strings.ToLower(strings.TrimSpace(f.classify))
	switch mode {
	case "none", "":
		parse.ClassifyDeterministic(items, time.Now())
		return parse.ClassifierNone, "", nil
	case "auto", "omniroute", "llm":
		key := strings.TrimSpace(os.Getenv("OMNIROUTE_API_KEY"))
		if key == "" {
			if mode == "auto" {
				parse.ClassifyDeterministic(items, time.Now())
				return parse.ClassifierNone, "", nil
			}
			return "", "", fmt.Errorf("classify: OMNIROUTE_API_KEY is not set")
		}
		model := strings.TrimSpace(f.model)
		if model == "" {
			model = strings.TrimSpace(os.Getenv("SCHOOL_LLM_MODEL"))
		}
		if model == "" {
			model = llm.DefaultModel
		}
		base := strings.TrimSpace(os.Getenv("OMNIROUTE_BASE_URL"))
		if base == "" {
			base = llm.DefaultBaseURL
		}
		client := llm.New(base, key, model)
		if err := parse.ClassifyModel(ctx, client, items, model, time.Now().In(loc)); err != nil {
			return "", "", err
		}
		return parse.ClassifierOmniRoute, model, nil
	default:
		return "", "", fmt.Errorf("unknown --classify %q (want auto, none or omniroute)", f.classify)
	}
}

// loadRunPages finds the raw run directory for a child and reads every
// <slug>.html page in it, with the URL from the matching <slug>.meta.json.
func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) {
	childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child)))
	entries, err := os.ReadDir(childDir)
	if err != nil {
		return "", nil, fmt.Errorf("read raw pages for %s at %s: %w", child, childDir, err)
	}

	var candidates []string
	for _, e := range entries {
		if e.IsDir() {
			candidates = append(candidates, e.Name())
		}
	}
	sort.Sort(sort.Reverse(sort.StringSlice(candidates)))

	runDir := ""
	if strings.TrimSpace(run) == "latest" {
		for _, name := range candidates {
			if runHasKnownPage(filepath.Join(childDir, name)) {
				runDir = filepath.Join(childDir, name)
				break
			}
		}
		if runDir == "" {
			return "", nil, fmt.Errorf("no parseable Compass run for %s under %s", child, childDir)
		}
	} else {
		runDir = filepath.Join(childDir, run)
	}

	files, err := os.ReadDir(runDir)
	if err != nil {
		return "", nil, fmt.Errorf("read run %s: %w", runDir, err)
	}

	pages := make([]parse.Page, 0, len(files))
	for _, e := range files {
		name := e.Name()
		if e.IsDir() || !strings.HasSuffix(strings.ToLower(name), ".html") {
			continue
		}
		slug := strings.TrimSuffix(name, filepath.Ext(name))
		raw, err := os.ReadFile(filepath.Join(runDir, name))
		if err != nil {
			return "", nil, fmt.Errorf("read %s: %w", name, err)
		}
		pages = append(pages, parse.Page{
			Slug: slug,
			URL:  pageURL(runDir, slug),
			HTML: raw,
		})
	}
	if len(pages) == 0 {
		return "", nil, fmt.Errorf("run %s has no .html pages", runDir)
	}
	return runDir, pages, nil
}

// runHasKnownPage reports whether a run directory holds at least one page the
// parser understands, so a failed login run is never chosen as "latest".
func runHasKnownPage(dir string) bool {
	for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageCalendar, parse.PageChildProfile} {
		if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil {
			return true
		}
	}
	return false
}

// pageURL reads the source URL recorded by the Phase 2 fetcher.
func pageURL(runDir, slug string) string {
	raw, err := os.ReadFile(filepath.Join(runDir, slug+".meta.json"))
	if err != nil {
		return ""
	}
	var meta struct {
		URL string `json:"url"`
	}
	if err := json.Unmarshal(raw, &meta); err != nil {
		return ""
	}
	return strings.TrimSpace(meta.URL)
}