Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -41,6 +41,9 @@ func main() {
|
||||
case "gen-secret":
|
||||
genSecret()
|
||||
return
|
||||
case "parse":
|
||||
parseCommand(os.Args[2:])
|
||||
return
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return
|
||||
|
||||
358
cmd/school/parse.go
Normal file
358
cmd/school/parse.go
Normal file
@@ -0,0 +1,358 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config"
|
||||
"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/llm"
|
||||
"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/parse"
|
||||
"gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store"
|
||||
)
|
||||
|
||||
// parseFlags holds the `school parse` options.
|
||||
type parseFlags struct {
|
||||
child string
|
||||
run string
|
||||
rawRoot string
|
||||
classify string
|
||||
model string
|
||||
asJSON bool
|
||||
}
|
||||
|
||||
// parseCommand is the `school parse` entry point. It reads the stored raw
|
||||
// Compass pages for one child, turns them into items and writes them to SQLite
|
||||
// (AGENTS.md §11 Phase 3).
|
||||
func parseCommand(args []string) {
|
||||
fs := flag.NewFlagSet("parse", flag.ContinueOnError)
|
||||
f := parseFlags{}
|
||||
fs.StringVar(&f.child, "child", "Harry", "child name (Harry or Finn)")
|
||||
fs.StringVar(&f.run, "run", "latest", "raw run id under var/raw/<child>/, or 'latest'")
|
||||
fs.StringVar(&f.rawRoot, "raw-root", "var/raw", "root directory of the stored raw pages")
|
||||
fs.StringVar(&f.classify, "classify", "auto", "importance classifier: auto, none or omniroute")
|
||||
fs.StringVar(&f.model, "model", "", "model id for importance (default SCHOOL_LLM_MODEL or OmniRoute default)")
|
||||
fs.BoolVar(&f.asJSON, "json", false, "print the summary as JSON")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
if err := runParse(f); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "school parse:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// parseSummary is the machine-readable result of one parse run.
|
||||
type parseSummary struct {
|
||||
Child string `json:"child"`
|
||||
RunDir string `json:"run_dir"`
|
||||
Classifier string `json:"classifier"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Pages []string `json:"pages"`
|
||||
Total int `json:"total"`
|
||||
Inserted int `json:"inserted"`
|
||||
Updated int `json:"updated"`
|
||||
Counts []store.TypeCount `json:"counts"`
|
||||
Samples []itemSample `json:"samples"`
|
||||
ScannedAt string `json:"scanned_at"`
|
||||
}
|
||||
|
||||
type itemSample struct {
|
||||
Type string `json:"type"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Title string `json:"title"`
|
||||
DueAt string `json:"due_at,omitempty"`
|
||||
SourcePage string `json:"source_page"`
|
||||
Importance string `json:"importance"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
func runParse(f parseFlags) error {
|
||||
ctx := context.Background()
|
||||
|
||||
cfg, err := config.LoadDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
st, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := st.SeedChildren(ctx, seedChildren...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
child, err := st.ChildByName(ctx, f.child)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if child == nil {
|
||||
return fmt.Errorf("unknown child %q", f.child)
|
||||
}
|
||||
|
||||
runDir, pages, err := loadRunPages(f.rawRoot, f.child, f.run)
|
||||
if err != nil {
|
||||
// FR-12: a failed load is an explicit failed scan, never an empty day.
|
||||
_, _ = st.RecordScan(ctx, &child.ID, 0, 0, false, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
items, err := parse.Parse(f.child, pages, cfg.Location)
|
||||
if err != nil {
|
||||
_, _ = st.RecordScan(ctx, &child.ID, len(pages), 0, false, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
classifyCtx, cancelClassify := context.WithTimeout(ctx, 3*time.Minute)
|
||||
classifier, model, err := classifyItems(classifyCtx, f, items, cfg.Location)
|
||||
cancelClassify()
|
||||
if err != nil {
|
||||
_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
stored := make([]store.Item, 0, len(items))
|
||||
for _, it := range items {
|
||||
stored = append(stored, store.Item{
|
||||
Source: it.Source,
|
||||
Type: it.Type,
|
||||
Subject: it.Subject,
|
||||
Teacher: it.Teacher,
|
||||
Title: it.Title,
|
||||
Body: it.Body,
|
||||
DueAt: it.DueAt,
|
||||
URL: it.URL,
|
||||
SourcePage: it.SourcePage,
|
||||
Importance: it.Importance,
|
||||
ExternalRef: it.ExternalRef,
|
||||
})
|
||||
}
|
||||
|
||||
inserted, updated, err := st.UpsertItems(ctx, child.ID, stored)
|
||||
if err != nil {
|
||||
_, _ = st.RecordScan(ctx, &child.ID, len(pages), len(items), false, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := st.RecordScan(ctx, &child.ID, len(pages), len(items), true, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
counts, err := st.CountItemsByType(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
samples, err := st.SampleItems(ctx, child.ID, 12)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pageSlugs := make([]string, 0, len(pages))
|
||||
for _, p := range pages {
|
||||
pageSlugs = append(pageSlugs, p.Slug)
|
||||
}
|
||||
sort.Strings(pageSlugs)
|
||||
|
||||
summary := parseSummary{
|
||||
Child: f.child,
|
||||
RunDir: runDir,
|
||||
Classifier: classifier,
|
||||
Model: model,
|
||||
Pages: pageSlugs,
|
||||
Total: len(items),
|
||||
Inserted: inserted,
|
||||
Updated: updated,
|
||||
Counts: counts,
|
||||
ScannedAt: time.Now().In(cfg.Location).Format(time.RFC3339),
|
||||
}
|
||||
for _, s := range samples {
|
||||
summary.Samples = append(summary.Samples, itemSample{
|
||||
Type: s.Type,
|
||||
Subject: s.Subject,
|
||||
Title: s.Title,
|
||||
DueAt: dueString(s.DueAt, cfg.Location),
|
||||
SourcePage: s.SourcePage,
|
||||
Importance: s.Importance,
|
||||
URL: s.URL,
|
||||
})
|
||||
}
|
||||
|
||||
if f.asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(summary)
|
||||
}
|
||||
printParseSummary(summary)
|
||||
return nil
|
||||
}
|
||||
|
||||
func printParseSummary(s parseSummary) {
|
||||
fmt.Printf("child %s\n", s.Child)
|
||||
fmt.Printf("run %s\n", s.RunDir)
|
||||
fmt.Printf("pages %s\n", strings.Join(s.Pages, ", "))
|
||||
fmt.Printf("classifier %s", s.Classifier)
|
||||
if s.Model != "" {
|
||||
fmt.Printf(" (%s)", s.Model)
|
||||
}
|
||||
fmt.Printf("\nitems %d (inserted %d, updated %d)\n", s.Total, s.Inserted, s.Updated)
|
||||
for _, c := range s.Counts {
|
||||
fmt.Printf(" %-6s %-8s %d\n", c.Child, c.Type, c.Count)
|
||||
}
|
||||
fmt.Println("samples:")
|
||||
for _, x := range s.Samples {
|
||||
due := x.DueAt
|
||||
if due == "" {
|
||||
due = "-"
|
||||
}
|
||||
fmt.Printf(" [%s] %-8s %-28s %-45s due=%s\n", x.Importance, x.Type, x.Subject, truncate(x.Title, 45), due)
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n-1] + "…"
|
||||
}
|
||||
|
||||
func dueString(t *time.Time, loc *time.Location) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.In(loc).Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
// classifyItems applies the configured importance classifier and returns its
|
||||
// name and model. It records which classifier was used (AGENTS.md §4 FR-4).
|
||||
func classifyItems(ctx context.Context, f parseFlags, items []parse.Item, loc *time.Location) (string, string, error) {
|
||||
mode := strings.ToLower(strings.TrimSpace(f.classify))
|
||||
switch mode {
|
||||
case "none", "":
|
||||
parse.ClassifyDeterministic(items, time.Now())
|
||||
return parse.ClassifierNone, "", nil
|
||||
case "auto", "omniroute", "llm":
|
||||
key := strings.TrimSpace(os.Getenv("OMNIROUTE_API_KEY"))
|
||||
if key == "" {
|
||||
if mode == "auto" {
|
||||
parse.ClassifyDeterministic(items, time.Now())
|
||||
return parse.ClassifierNone, "", nil
|
||||
}
|
||||
return "", "", fmt.Errorf("classify: OMNIROUTE_API_KEY is not set")
|
||||
}
|
||||
model := strings.TrimSpace(f.model)
|
||||
if model == "" {
|
||||
model = strings.TrimSpace(os.Getenv("SCHOOL_LLM_MODEL"))
|
||||
}
|
||||
if model == "" {
|
||||
model = llm.DefaultModel
|
||||
}
|
||||
base := strings.TrimSpace(os.Getenv("OMNIROUTE_BASE_URL"))
|
||||
if base == "" {
|
||||
base = llm.DefaultBaseURL
|
||||
}
|
||||
client := llm.New(base, key, model)
|
||||
if err := parse.ClassifyModel(ctx, client, items, model, time.Now().In(loc)); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return parse.ClassifierOmniRoute, model, nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("unknown --classify %q (want auto, none or omniroute)", f.classify)
|
||||
}
|
||||
}
|
||||
|
||||
// loadRunPages finds the raw run directory for a child and reads every
|
||||
// <slug>.html page in it, with the URL from the matching <slug>.meta.json.
|
||||
func loadRunPages(rawRoot, child, run string) (string, []parse.Page, error) {
|
||||
childDir := filepath.Join(rawRoot, strings.ToLower(strings.TrimSpace(child)))
|
||||
entries, err := os.ReadDir(childDir)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read raw pages for %s at %s: %w", child, childDir, err)
|
||||
}
|
||||
|
||||
var candidates []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
candidates = append(candidates, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(candidates)))
|
||||
|
||||
runDir := ""
|
||||
if strings.TrimSpace(run) == "latest" {
|
||||
for _, name := range candidates {
|
||||
if runHasKnownPage(filepath.Join(childDir, name)) {
|
||||
runDir = filepath.Join(childDir, name)
|
||||
break
|
||||
}
|
||||
}
|
||||
if runDir == "" {
|
||||
return "", nil, fmt.Errorf("no parseable Compass run for %s under %s", child, childDir)
|
||||
}
|
||||
} else {
|
||||
runDir = filepath.Join(childDir, run)
|
||||
}
|
||||
|
||||
files, err := os.ReadDir(runDir)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read run %s: %w", runDir, err)
|
||||
}
|
||||
|
||||
pages := make([]parse.Page, 0, len(files))
|
||||
for _, e := range files {
|
||||
name := e.Name()
|
||||
if e.IsDir() || !strings.HasSuffix(strings.ToLower(name), ".html") {
|
||||
continue
|
||||
}
|
||||
slug := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
raw, err := os.ReadFile(filepath.Join(runDir, name))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read %s: %w", name, err)
|
||||
}
|
||||
pages = append(pages, parse.Page{
|
||||
Slug: slug,
|
||||
URL: pageURL(runDir, slug),
|
||||
HTML: raw,
|
||||
})
|
||||
}
|
||||
if len(pages) == 0 {
|
||||
return "", nil, fmt.Errorf("run %s has no .html pages", runDir)
|
||||
}
|
||||
return runDir, pages, nil
|
||||
}
|
||||
|
||||
// runHasKnownPage reports whether a run directory holds at least one page the
|
||||
// parser understands, so a failed login run is never chosen as "latest".
|
||||
func runHasKnownPage(dir string) bool {
|
||||
for _, slug := range []string{parse.PageHome, parse.PageLearningTasks, parse.PageCalendar, parse.PageChildProfile} {
|
||||
if _, err := os.Stat(filepath.Join(dir, slug+".html")); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// pageURL reads the source URL recorded by the Phase 2 fetcher.
|
||||
func pageURL(runDir, slug string) string {
|
||||
raw, err := os.ReadFile(filepath.Join(runDir, slug+".meta.json"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var meta struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(meta.URL)
|
||||
}
|
||||
Reference in New Issue
Block a user