package main import ( "context" "encoding/json" "flag" "fmt" "os" "path/filepath" "sort" "strings" "time" "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config" "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/llm" "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/parse" "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store" ) // parseFlags holds the `school parse` options. type parseFlags struct { child string run string rawRoot string classify string model string asJSON bool } // parseCommand is the `school parse` entry point. It reads the stored raw // Compass pages for one child, turns them into items and writes them to SQLite // (AGENTS.md §11 Phase 3). func parseCommand(args []string) { fs := flag.NewFlagSet("parse", flag.ContinueOnError) f := parseFlags{} fs.StringVar(&f.child, "child", "Harry", "child name (Harry or Finn)") fs.StringVar(&f.run, "run", "latest", "raw run id under var/raw//, or 'latest'") fs.StringVar(&f.rawRoot, "raw-root", "var/raw", "root directory of the stored raw pages") fs.StringVar(&f.classify, "classify", "auto", "importance classifier: auto, none or omniroute") fs.StringVar(&f.model, "model", "", "model id for importance (default SCHOOL_LLM_MODEL or OmniRoute default)") fs.BoolVar(&f.asJSON, "json", false, "print the summary as JSON") _ = fs.Parse(args) if err := runParse(f); err != nil { fmt.Fprintln(os.Stderr, "school parse:", err) os.Exit(1) } } // parseSummary is the machine-readable result of one parse run. type parseSummary struct { Child string `json:"child"` RunDir string `json:"run_dir"` Classifier string `json:"classifier"` Model string `json:"model,omitempty"` Pages []string `json:"pages"` Total int `json:"total"` Overdue int `json:"overdue"` Inserted int `json:"inserted"` Updated int `json:"updated"` Counts []store.TypeCount `json:"counts"` Samples []itemSample `json:"samples"` ScannedAt string `json:"scanned_at"` } type itemSample struct { Type string `json:"type"` Subject string `json:"subject,omitempty"` Title string `json:"title"` DueAt string `json:"due_at,omitempty"` SourcePage string `json:"source_page"` Importance string `json:"importance"` URL string `json:"url,omitempty"` } func runParse(f parseFlags) error { ctx := context.Background() cfg, err := config.LoadDB() if err != nil { return err } st, err := store.Open(cfg.DBPath) if err != nil { return err } defer st.Close() if err := st.Migrate(ctx); err != nil { return err } if err := st.SeedChildren(ctx, seedChildren...); err != nil { return err } child, err := st.ChildByName(ctx, f.child) if err != nil { return err } if child == nil { return fmt.Errorf("unknown child %q", f.child) } runDir, pages, err := loadRunPages(f.rawRoot, f.child, f.run) if err != nil { // FR-12: a failed load is an explicit failed scan, never an empty day. _, _ = st.RecordScan(ctx, &child.ID, 0, 0, false, err.Error()) return err } items, err := parse.Parse(f.child, pages, cfg.Location) if err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), 0, false, err.Error()) return err } classifyCtx, cancelClassify := context.WithTimeout(ctx, 3*time.Minute) classifier, model, err := classifyItems(classifyCtx, f, items, cfg.Location) cancelClassify() if err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) return err } stored := make([]store.Item, 0, len(items)) for _, it := range items { stored = append(stored, store.Item{ Source: it.Source, Type: it.Type, Subject: it.Subject, Teacher: it.Teacher, Title: it.Title, Body: it.Body, DueAt: it.DueAt, URL: it.URL, SourcePage: it.SourcePage, Importance: it.Importance, ExternalRef: it.ExternalRef, }) } // The capped legacy grid is superseded by the authoritative JSON task list. // Remove its rows for this child first, so a stale partial row can never be // shown next to the complete set (AUD-114, FR-12). if hasPage(pages, parse.PageLearningTasksAll) { if _, err := st.DeleteChildCompassTasksFromPages(ctx, child.ID, parse.PageLearningTasks, parse.PageChildProfile); err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) return err } } inserted, updated, err := st.UpsertItems(ctx, child.ID, stored) if err != nil { _, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error()) return err } if _, err := st.RecordScan(ctx, &child.ID, len(pages), len(items), true, ""); err != nil { return err } counts, err := st.CountItemsByType(ctx) if err != nil { return err } samples, err := st.SampleItems(ctx, child.ID, 12) if err != nil { return err } pageSlugs := make([]string, 0, len(pages)) for _, p := range pages { pageSlugs = append(pageSlugs, p.Slug) } sort.Strings(pageSlugs) summary := parseSummary{ Child: f.child, RunDir: runDir, Classifier: classifier, Model: model, Pages: pageSlugs, Total: len(items), Overdue: parse.OverdueCount(items), Inserted: inserted, Updated: updated, Counts: counts, ScannedAt: time.Now().In(cfg.Location).Format(time.RFC3339), } for _, s := range samples { summary.Samples = append(summary.Samples, itemSample{ Type: s.Type, Subject: s.Subject, Title: s.Title, DueAt: dueString(s.DueAt, cfg.Location), SourcePage: s.SourcePage, Importance: s.Importance, URL: s.URL, }) } if f.asJSON { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(summary) } printParseSummary(summary) return nil } func printParseSummary(s parseSummary) { fmt.Printf("child %s\n", s.Child) fmt.Printf("run %s\n", s.RunDir) fmt.Printf("pages %s\n", strings.Join(s.Pages, ", ")) fmt.Printf("classifier %s", s.Classifier) if s.Model != "" { fmt.Printf(" (%s)", s.Model) } fmt.Printf("\nitems %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated) fmt.Printf("overdue %d (reconciled against the home page)\n", s.Overdue) for _, c := range s.Counts { fmt.Printf(" %-6s %-8s %d\n", c.Child, c.Type, c.Count) } fmt.Println("samples:") for _, x := range s.Samples { due := x.DueAt if due == "" { due = "-" } fmt.Printf(" [%s] %-8s %-28s %-45s due=%s\n", x.Importance, x.Type, x.Subject, truncate(x.Title, 45), due) } } func truncate(s string, n int) string { if len(s) <= n { return s } return s[:n-1] + "…" } func dueString(t *time.Time, loc *time.Location) string { if t == nil { return "" } return t.In(loc).Format("2006-01-02 15:04") } // classifyItems applies the configured importance classifier and returns its // name and model. It records which classifier was used (AGENTS.md §4 FR-4). func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *time.Location) (string, string, error) { mode := strings.ToLower(strings.TrimSpace(f.classify)) switch mode { case "none", "": parse.ClassifyDeterministic(items, time.Now()) return parse.ClassifierNone, "", nil case "auto", "omniroute", "llm": key := strings.TrimSpace(os.Getenv("OMNIROUTE_API_KEY")) if key == "" { if mode == "auto" { parse.ClassifyDeterministic(items, time.Now()) return parse.ClassifierNone, "", nil } return "", "", fmt.Errorf("classify: OMNIROUTE_API_KEY is not set") } model := strings.TrimSpace(f.model) if model == "" { model = strings.TrimSpace(os.Getenv("SCHOOL_LLM_MODEL")) } if model == "" { model = llm.DefaultModel } base := strings.TrimSpace(os.Getenv("OMNIROUTE_BASE_URL")) if base == "" { base = llm.DefaultBaseURL } client := llm.New(base, key, model) if err := parse.ClassifyModel(ctx, client, items, model, time.Now().In(loc)); err != nil { return "", "", err } return parse.ClassifierOmniRoute, model, nil default: return "", "", fmt.Errorf("unknown --classify %q (want auto, none or omniroute)", f.classify) } } // loadRunPages finds the raw run directory for a child and reads every stored // page in it. HTML pages come from .html; the JSON service page comes from // .json. The URL and the child's Compass userId come from the matching // .meta.json. func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) { childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child))) entries, err := os.ReadDir(childDir) if err != nil { return "", nil, fmt.Errorf("read raw pages for %s at %s: %w", child, childDir, err) } var candidates []string for _, e := range entries { if e.IsDir() { candidates = append(candidates, e.Name()) } } sort.Sort(sort.Reverse(sort.StringSlice(candidates))) runDir := "" if strings.TrimSpace(run) == "latest" { for _, name := range candidates { if runHasKnownPage(filepath.Join(childDir, name)) { runDir = filepath.Join(childDir, name) break } } if runDir == "" { return "", nil, fmt.Errorf("no parseable Compass run for %s under %s", child, childDir) } } else { runDir = filepath.Join(childDir, run) } files, err := os.ReadDir(runDir) if err != nil { return "", nil, fmt.Errorf("read run %s: %w", runDir, err) } type rawPage struct { htmlBytes, jsonBytes []byte url, user string } bySlug := map[string]*rawPage{} var order []string get := func(slug string) *rawPage { if p, ok := bySlug[slug]; ok { return p } p := &rawPage{} bySlug[slug] = p order = append(order, slug) return p } for _, e := range files { name := e.Name() if e.IsDir() { continue } lower := strings.ToLower(name) switch { case strings.HasSuffix(lower, ".meta.json"): slug := strings.TrimSuffix(name, ".meta.json") raw, err := os.ReadFile(filepath.Join(runDir, name)) if err != nil { return "", nil, fmt.Errorf("read %s: %w", name, err) } var meta struct { URL string `json:"url"` UserID string `json:"userId"` } _ = json.Unmarshal(raw, &meta) p := get(slug) p.url = strings.TrimSpace(meta.URL) p.user = strings.TrimSpace(meta.UserID) case strings.HasSuffix(lower, ".json"): if strings.EqualFold(name, "manifest.json") { continue } slug := strings.TrimSuffix(name, ".json") raw, err := os.ReadFile(filepath.Join(runDir, name)) if err != nil { return "", nil, fmt.Errorf("read %s: %w", name, err) } get(slug).jsonBytes = raw case strings.HasSuffix(lower, ".html"): slug := strings.TrimSuffix(name, filepath.Ext(name)) raw, err := os.ReadFile(filepath.Join(runDir, name)) if err != nil { return "", nil, fmt.Errorf("read %s: %w", name, err) } get(slug).htmlBytes = raw } } pages := make([]parse.Page, 0, len(order)) for _, slug := range order { rp := bySlug[slug] if len(rp.htmlBytes) == 0 && len(rp.jsonBytes) == 0 { continue } pages = append(pages, parse.Page{ Slug: slug, URL: rp.url, HTML: rp.htmlBytes, JSON: rp.jsonBytes, UserID: rp.user, }) } if len(pages) == 0 { return "", nil, fmt.Errorf("run %s has no stored pages", runDir) } return runDir, pages, nil } // hasPage reports whether a parsed page set contains a slug. func hasPage(pages []parse.Page, slug string) bool { for i := range pages { if pages[i].Slug == slug { return true } } return false } // runHasKnownPage reports whether a run directory holds at least one page the // parser understands, so a failed login run is never chosen as "latest". func runHasKnownPage(dir string) bool { for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageLearningTasksAll, parse.PageCalendar, parse.PageChildProfile} { if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil { return true } if _, err := os.Stat(filepath.Join(dir, slug+".json")); err == nil { return true } } return false }