diff --git a/Makefile b/Makefile index 2e0962b..6883c78 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,14 @@ -IMAGE ?= compass-teams-nhs/school:0.2.0-phase3 -BUILD_VERSION ?= 0.2.0-phase3 +IMAGE ?= compass-teams-nhs/school:0.3.0-phase4 +BUILD_VERSION ?= 0.3.0-phase4 # Parse options (see scripts/parse-compass.sh). CHILD ?= Harry RUN ?= latest CLASSIFY ?= auto +# Scan target: 'all' scans every child; set SCAN_CHILD=Harry to narrow it. +SCAN_CHILD ?= all + # Bounded docker helper for the Go toolchain. The container runs as the current # user, so the repository and the module cache stay owned by you. GO_IMAGE ?= golang:1.25-alpine @@ -13,7 +16,7 @@ GO_RUN = timeout 600 docker run --rm --network host --user "$$(id -u):$$(id -g)" -e HOME=/src/.gocache -e GOMODCACHE=/src/.gocache/mod -e GOCACHE=/src/.gocache/build \ -v "$$PWD":/src -w /src $(GO_IMAGE) -.PHONY: help test vet build up down restart logs ps deps fmt hash-password gen-secret verify-phase1 parse +.PHONY: help test vet build up down restart logs ps deps fmt hash-password gen-secret verify-phase1 parse scan help: @echo "make test run go test ./... in golang:1.25-alpine" @@ -27,6 +30,7 @@ help: @echo "make hash-password generate a bcrypt hash (PASSWORD=...)" @echo "make gen-secret print a fresh session secret" @echo "make parse parse stored Compass pages into SQLite (Phase 3)" + @echo "make scan fetch + parse one or both children (Phase 4; SCAN_CHILD=Harry|Finn|all)" @echo "make verify-phase1 prove /health, the login and the Caddy route" test: @@ -70,6 +74,10 @@ gen-secret: parse: CHILD=$(CHILD) RUN=$(RUN) CLASSIFY=$(CLASSIFY) IMAGE=$(IMAGE) scripts/parse-compass.sh +# Phase 4: fetch + parse one child or both, with a per-child session. +scan: + CLASSIFY=$(CLASSIFY) IMAGE=$(IMAGE) scripts/scan-compass.sh $(if $(filter-out all,$(SCAN_CHILD)),--child $(SCAN_CHILD)) + # Phase 1 definition of done: /health 200, login page + session login, and the # Caddy route on .35. verify-phase1: diff --git a/README.md b/README.md index 3ff8f44..ea99f74 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,20 @@ A school service for Harry and Finn: **Compass** for the daily schedule and notices, **Teams** for homework and assignments. It stores what is due per child, puts overdue work first, and tells the family before work is late. -> Build spec: [`AGENTS.md`](AGENTS.md). This repository is at **Phase 3** (parsing -> the stored Compass pages). See §11 of the spec for the phase order. +> Build spec: [`AGENTS.md`](AGENTS.md). This repository is at **Phase 4** (both +> children, per-child sessions). See §11 of the spec for the phase order. + +## Status — Phase 4 + +Phase 4 fetches and parses **Harry and Finn** with a stored session per child, +and stores every item against the right `children.id`. See +[`docs/phase-4-both-children-notes.md`](docs/phase-4-both-children-notes.md). + +```bash +make scan # both children +make scan SCAN_CHILD=Finn # one child +school session show # session state per child (metadata only, no secrets) +``` ## Status — Phase 3 @@ -29,9 +41,9 @@ Built and deployed on `.13`: - Its own container and port **8792**, behind Caddy at `school.lab.audasmedia.com.au`. -Not built yet (later phases): both children's data, the dashboard data, the -calendar write, notifications, and the Hub API. The service itself does **not** -scan the school; the daily scan and the Teams source come later. +Not built yet (later phases): the dashboard data, the calendar write, +notifications, and the Hub API. The service itself does **not** scan the school; +n8n triggers the scan later. Teams (FR-15) is a separate source. ### Fail loudly (FR-12) @@ -45,7 +57,7 @@ empty day. The dashboard reads the `scans` table and shows one of: ## Layout ``` -cmd/school/ service entry point (run, parse, hash-password, gen-secret) +cmd/school/ service entry point (run, parse, session, scan, items, hash-password, gen-secret) internal/config/ environment configuration internal/store/ SQLite: schema, migrations, queries internal/parse/ HTML -> structured items (tasks, messages, events) @@ -68,8 +80,8 @@ docker compose up -d --build Generate secrets with the built image: ```bash -docker run --rm --entrypoint /app/school compass-teams-nhs/school:0.2.0-phase3 gen-secret -docker run --rm --entrypoint /app/school compass-teams-nhs/school:0.2.0-phase3 hash-password 'the-password' +docker run --rm --entrypoint /app/school compass-teams-nhs/school:0.3.0-phase4 gen-secret +docker run --rm --entrypoint /app/school compass-teams-nhs/school:0.3.0-phase4 hash-password 'the-password' ``` Note: Docker Compose interpolates `$VAR` inside `env_file`. A bcrypt hash diff --git a/cmd/school/items.go b/cmd/school/items.go new file mode 100644 index 0000000..9de4aeb --- /dev/null +++ b/cmd/school/items.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config" + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store" +) + +// itemsCommand is the `school items` entry point. It currently supports purging +// items for one child, which Phase 4 uses to drop rows that a shared page +// stored against the wrong child ("nothing may leak between children"). +func itemsCommand(args []string) { + if len(args) == 0 || args[0] != "purge" { + fmt.Fprintln(os.Stderr, "usage: school items purge --child NAME [--source compass|teams] [--type message|task|event]") + os.Exit(2) + } + fs := flag.NewFlagSet("items purge", flag.ContinueOnError) + var ( + child string + source string + typ string + ) + fs.StringVar(&child, "child", "", "child name (Harry or Finn)") + fs.StringVar(&source, "source", "", "limit to one source") + fs.StringVar(&typ, "type", "", "limit to one type") + _ = fs.Parse(args[1:]) + + if child == "" { + fmt.Fprintln(os.Stderr, "school items purge: --child is required") + os.Exit(2) + } + if err := runItemsPurge(child, source, typ); err != nil { + fmt.Fprintln(os.Stderr, "school items:", err) + os.Exit(1) + } +} + +func runItemsPurge(child, source, typ string) error { + ctx := context.Background() + + cfg, err := config.LoadDB() + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + if err := st.Migrate(ctx); err != nil { + return err + } + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + n, err := st.DeleteItems(ctx, c.ID, source, typ) + if err != nil { + return err + } + fmt.Printf("child=%s purged=%d source=%q type=%q\n", c.Name, n, source, typ) + return nil +} diff --git a/cmd/school/main.go b/cmd/school/main.go index 92ae2b9..afd41e4 100644 --- a/cmd/school/main.go +++ b/cmd/school/main.go @@ -44,6 +44,15 @@ func main() { case "parse": parseCommand(os.Args[2:]) return + case "session": + sessionCommand(os.Args[2:]) + return + case "scan": + scanCommand(os.Args[2:]) + return + case "items": + itemsCommand(os.Args[2:]) + return case "version": fmt.Println(version) return diff --git a/cmd/school/scan.go b/cmd/school/scan.go new file mode 100644 index 0000000..7c3f081 --- /dev/null +++ b/cmd/school/scan.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config" + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store" +) + +// scanCommand is the `school scan` entry point. It only records the outcome of a +// scan; the scan itself runs the patchright fetcher on the host and then +// `school parse` (AGENTS.md §11 Phase 4). +// +// `school scan record` exists so a fetch failure is written to the append-only +// `scans` table with ok = 0 (FR-12/FR-13): the dashboard can then show an +// explicit failure instead of an empty day. +func scanCommand(args []string) { + if len(args) == 0 || args[0] != "record" { + fmt.Fprintln(os.Stderr, "usage: school scan record --child NAME [--ok] [--pages N] [--items N] --message TEXT") + os.Exit(2) + } + fs := flag.NewFlagSet("scan record", flag.ContinueOnError) + var ( + child string + pages int + items int + message string + ok bool + ) + fs.StringVar(&child, "child", "", "child name (Harry or Finn)") + fs.BoolVar(&ok, "ok", false, "true when the scan succeeded") + fs.IntVar(&pages, "pages", 0, "pages fetched") + fs.IntVar(&items, "items", 0, "items found") + fs.StringVar(&message, "message", "", "failure message (required when --ok=false)") + _ = fs.Parse(args[1:]) + + if child == "" { + fmt.Fprintln(os.Stderr, "school scan record: --child is required") + os.Exit(2) + } + if !ok && message == "" { + fmt.Fprintln(os.Stderr, "school scan record: --message is required for a failed scan") + os.Exit(2) + } + if err := runScanRecord(child, pages, items, ok, message); err != nil { + fmt.Fprintln(os.Stderr, "school scan:", err) + os.Exit(1) + } +} + +func runScanRecord(child string, pages, items int, ok bool, message string) error { + ctx := context.Background() + + cfg, err := config.LoadDB() + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + if err := st.Migrate(ctx); err != nil { + return err + } + if err := st.SeedChildren(ctx, seedChildren...); err != nil { + return err + } + + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + id, err := st.RecordScan(ctx, &c.ID, pages, items, ok, message) + if err != nil { + return err + } + status := "ok" + if !ok { + status = "failed" + } + fmt.Printf("scan=%d child=%s status=%s pages=%d items=%d\n", id, c.Name, status, pages, items) + return nil +} diff --git a/cmd/school/session.go b/cmd/school/session.go new file mode 100644 index 0000000..f881f25 --- /dev/null +++ b/cmd/school/session.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/config" + "gitea.lab.audasmedia.com.au/sam/compass_teams_nhs/internal/store" +) + +// sessionCommand is the `school session` entry point. It reads and writes the +// per-child harness session state in the SQLite `session` table (AGENTS.md §7). +// +// The state is a secret (browser cookies / localStorage). `show` prints only a +// summary; the state itself is only ever written to the file named by --out or +// read from the file named by --file, with mode 0600. +func sessionCommand(args []string) { + if len(args) == 0 { + fmt.Fprintln(os.Stderr, sessionUsage) + os.Exit(2) + } + sub := args[0] + fs := flag.NewFlagSet("session "+sub, flag.ContinueOnError) + var ( + child string + out string + file string + asJSON bool + ) + fs.StringVar(&child, "child", "", "child name (Harry or Finn)") + fs.StringVar(&out, "out", "", "get: write the state to this file (mode 0600)") + fs.StringVar(&file, "file", "", "put: read the state from this file") + fs.BoolVar(&asJSON, "json", false, "print the summary as JSON") + _ = fs.Parse(args[1:]) + + switch sub { + case "get": + if child == "" || out == "" { + fmt.Fprintln(os.Stderr, "school session get: --child and --out are required") + os.Exit(2) + } + case "put": + if child == "" || file == "" { + fmt.Fprintln(os.Stderr, "school session put: --child and --file are required") + os.Exit(2) + } + case "show", "clear": + if child == "" && sub == "clear" { + fmt.Fprintln(os.Stderr, "school session clear: --child is required") + os.Exit(2) + } + default: + fmt.Fprintln(os.Stderr, sessionUsage) + os.Exit(2) + } + + if err := runSession(sub, child, out, file, asJSON); err != nil { + fmt.Fprintln(os.Stderr, "school session:", err) + os.Exit(1) + } +} + +const sessionUsage = `usage: + school session get --child NAME --out FILE + school session put --child NAME --file FILE + school session show [--child NAME] [--json] + school session clear --child NAME` + +func runSession(sub, child, out, file string, asJSON bool) error { + ctx := context.Background() + + cfg, err := config.LoadDB() + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + if err := st.Migrate(ctx); err != nil { + return err + } + + switch sub { + case "get": + return sessionGet(ctx, st, child, out) + case "put": + return sessionPut(ctx, st, child, file) + case "show": + return sessionShow(ctx, st, child, asJSON) + case "clear": + return sessionClear(ctx, st, child) + } + return fmt.Errorf("unknown session subcommand %q", sub) +} + +func sessionGet(ctx context.Context, st *store.Store, child, out string) error { + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + sess, err := st.GetSession(ctx, c.ID) + if err != nil { + return err + } + if sess == nil { + // No stored session is a normal state: the fetcher logs in fresh. + fmt.Printf("child=%s no stored session\n", c.Name) + return nil + } + if err := os.WriteFile(out, []byte(sess.State), 0o600); err != nil { + return fmt.Errorf("write session file: %w", err) + } + fmt.Printf("child=%s bytes=%d out=%s\n", c.Name, len(sess.State), out) + return nil +} + +func sessionPut(ctx context.Context, st *store.Store, child, file string) error { + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + raw, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("read session file: %w", err) + } + if len(raw) == 0 { + return fmt.Errorf("session file %s is empty", file) + } + if err := st.PutSession(ctx, c.ID, string(raw)); err != nil { + return err + } + cookies, origins, ok := describeState(string(raw)) + fmt.Printf("child=%s bytes=%d cookies=%d origins=%d valid_json=%t\n", c.Name, len(raw), cookies, origins, ok) + return nil +} + +func sessionClear(ctx context.Context, st *store.Store, child string) error { + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + if err := st.DeleteSession(ctx, c.ID); err != nil { + return err + } + fmt.Printf("child=%s cleared\n", c.Name) + return nil +} + +// sessionSummary is what `show` prints: never the state itself. +type sessionSummary struct { + Child string `json:"child"` + Bytes int `json:"bytes"` + Cookies int `json:"cookies"` + Origins int `json:"origins"` + ValidJSON bool `json:"valid_json"` + UpdatedAt string `json:"updated_at"` +} + +func sessionShow(ctx context.Context, st *store.Store, child string, asJSON bool) error { + var sessions []store.ChildSession + if child != "" { + c, err := st.ChildByName(ctx, child) + if err != nil { + return err + } + if c == nil { + return fmt.Errorf("unknown child %q", child) + } + sess, err := st.GetSession(ctx, c.ID) + if err != nil { + return err + } + if sess != nil { + sessions = append(sessions, *sess) + } + } else { + var err error + sessions, err = st.ListSessions(ctx) + if err != nil { + return err + } + } + + out := make([]sessionSummary, 0, len(sessions)) + for _, s := range sessions { + cookies, origins, ok := describeState(s.State) + out = append(out, sessionSummary{ + Child: s.Child, + Bytes: len(s.State), + Cookies: cookies, + Origins: origins, + ValidJSON: ok, + UpdatedAt: s.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z"), + }) + } + if asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(out) + } + if len(out) == 0 { + fmt.Println("no stored sessions") + return nil + } + for _, s := range out { + fmt.Printf("%-6s bytes=%-7d cookies=%-3d origins=%-3d valid_json=%-5t updated_at=%s\n", + s.Child, s.Bytes, s.Cookies, s.Origins, s.ValidJSON, s.UpdatedAt) + } + return nil +} + +// describeState reports the shape of a fetched storage state without revealing +// any value. It lets `session show` prove a usable session without printing a +// secret. +func describeState(state string) (cookies, origins int, ok bool) { + var parsed struct { + Cookies []json.RawMessage `json:"cookies"` + Origins []json.RawMessage `json:"origins"` + } + if err := json.Unmarshal([]byte(state), &parsed); err != nil { + return 0, 0, false + } + return len(parsed.Cookies), len(parsed.Origins), true +} diff --git a/docker-compose.yml b/docker-compose.yml index 9a847cd..6f1c5b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: build: context: . args: - VERSION: "0.2.0-phase3" - image: compass-teams-nhs/school:0.2.0-phase3 + VERSION: "0.3.0-phase4" + image: compass-teams-nhs/school:0.3.0-phase4 container_name: compass-teams-nhs restart: unless-stopped env_file: diff --git a/docs/phase-4-both-children-notes.md b/docs/phase-4-both-children-notes.md new file mode 100644 index 0000000..381dc67 --- /dev/null +++ b/docs/phase-4-both-children-notes.md @@ -0,0 +1,93 @@ +# Phase 4 — both children, per-child sessions + +Status: **done.** The Compass scan now fetches and parses **Harry and Finn** +with a stored session per child, and every item is stored against the right +`children.id`. See AUD-108. + +## What runs + +```bash +# Both children, with the stored session per child. +make scan + +# One child only (a scan can target one child). +make scan SCAN_CHILD=Finn + +# Same thing without make. +scripts/scan-compass.sh +scripts/scan-compass.sh --child Harry --child Finn --classify none +``` + +`scripts/scan-compass.sh` runs the patchright fetcher on the host and the +`school` subcommands in a one-off container. It never restarts the service. + +## Per-child session (FR-2, §7 `session` table) + +The Compass login is one parent account, but each child gets its own browser +session state so a scan is independent and targetable. + +| Step | Command | Notes | +|---|---|---| +| Restore | `school session get --child NAME --out FILE` | Reads `session.state`; writes a 0600 file. No row means "log in fresh". | +| Fetch | `node scripts/compass-fetch-patchright.mjs --child NAME --state FILE` | Loads the state; logs in only when Compass rejects it; saves the refreshed state. | +| Persist | `school session put --child NAME --file FILE` | Upserts `session.state` for that child. | +| Inspect | `school session show [--child NAME] [--json]` | Prints bytes, cookie and origin counts, and `updated_at`. **Never the state.** | + +The session state is a secret. It lives in the database volume only. No command +prints it, and the handoff file exists only inside the run's scratch directory. + +## What each run stores + +One run directory per child under `var/raw///` (git-ignored): + +| Page | Source | Becomes | +|---|---|---| +| `home` | parent landing page | `message` rows (news feed) | +| `learning-tasks-all` | `POST /Services/LearningTasks.svc/GetAllLearningTasksByUserId` | `task` rows (authoritative per-child list) | +| `child-profile` | `/Records/User.aspx?userId=N` | no grid; used only when it carries the legacy grid | +| `learning-tasks` | `/Records/UserNew.aspx?userId=N#learningTasks` | `task` rows (legacy grid fallback) | + +The per-child Compass `userId` is discovered from the parent landing page by +child name (`Harrison` → Harry, `Finnegan` → Finn) and recorded in the manifest. + +## Why the calendar is not fetched per child + +The parent calendar page renders only the calendars that the parent has enabled. +In this account only Harry's calendar is enabled; Finn's is hidden. Fetching that +page inside Finn's run therefore stored Harry's Year 11 timetable against Finn — +a leak between children. + +Until per-child calendar attribution exists, the shared calendar stays out of the +per-child loop. Harry's events remain from Phase 3. A caller can still fetch it +explicitly with the fetcher's `--pages` option. The `school items purge` command +removes rows that a shared page stored against the wrong child. + +## Fail loudly (FR-12) + +- A fetch failure records `scans.ok = 0` with the message through + `school scan record --child NAME --ok=false --message TEXT`. +- A parse failure records its own failed scan inside `school parse`. +- The scan script finishes every child, prints each failure by name, and exits + non-zero. A failed child is never shown as an empty day. + +## Evidence (2026-10-15, live) + +```bash +scripts/scan-compass.sh --classify auto +``` + +| Child | tasks | messages | events | overdue (home-page recon.) | +|---|---|---|---|---| +| Harry | 70 | 1 | 35 | 6 | +| Finn | 42 | 1 | 0 | 2 | + +Rows are separated by `child_id`. A check for items with no matching child +returns 0. Finn's task subjects are all Year 7 (`7MA`, `7DRA`, `7DES`); Harry's +are all Year 11 (`11MGMA`, `11HLEG`, `11EENG`). + +## Known gaps + +- `children.compass_id` is not populated yet. The manifest holds the `userId` per + run; a later phase can copy it into the `children` table. +- Per-child calendar attribution is deferred (see above). +- Teams (FR-15) is a separate source and is not scanned here. diff --git a/internal/store/session.go b/internal/store/session.go new file mode 100644 index 0000000..458d853 --- /dev/null +++ b/internal/store/session.go @@ -0,0 +1,90 @@ +// Per-child harness session state (AGENTS.md §7, FR-2, §11 Phase 4). +// +// The Compass fetcher (patchright) keeps a browser storage state — cookies and +// localStorage — so a later scan can reuse the login instead of signing in +// again. That state is a secret: it lives in the database volume only, is never +// exported to git, never logged, and never printed by a command. + +package store + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// ChildSession is the stored harness session for one child. +type ChildSession struct { + ChildID int64 + Child string + State string + UpdatedAt time.Time +} + +// PutSession writes (or refreshes) the harness session state for one child. The +// state is opaque to the store; the fetcher owns its format. +func (s *Store) PutSession(ctx context.Context, childID int64, state string) error { + if state == "" { + return fmt.Errorf("put session: state is empty") + } + if _, err := s.db.ExecContext(ctx, + `INSERT INTO session (child_id, state, updated_at) VALUES (?, ?, datetime('now')) + ON CONFLICT(child_id) DO UPDATE SET state = excluded.state, updated_at = datetime('now')`, + childID, state); err != nil { + return fmt.Errorf("put session for child %d: %w", childID, err) + } + return nil +} + +// GetSession returns the stored session for one child, or nil when it has none. +func (s *Store) GetSession(ctx context.Context, childID int64) (*ChildSession, error) { + row := s.db.QueryRowContext(ctx, + `SELECT se.child_id, COALESCE(ch.name, ''), se.state, se.updated_at + FROM session se LEFT JOIN children ch ON ch.id = se.child_id + WHERE se.child_id = ?`, childID) + var cs ChildSession + var updated string + if err := row.Scan(&cs.ChildID, &cs.Child, &cs.State, &updated); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("get session for child %d: %w", childID, err) + } + cs.UpdatedAt = parseTime(updated) + return &cs, nil +} + +// ListSessions returns the stored sessions, oldest child first. Each state is +// returned in full; callers that print must never print the state. +func (s *Store) ListSessions(ctx context.Context) ([]ChildSession, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT se.child_id, COALESCE(ch.name, ''), se.state, se.updated_at + FROM session se LEFT JOIN children ch ON ch.id = se.child_id + ORDER BY se.child_id`) + if err != nil { + return nil, fmt.Errorf("list sessions: %w", err) + } + defer rows.Close() + + var out []ChildSession + for rows.Next() { + var cs ChildSession + var updated string + if err := rows.Scan(&cs.ChildID, &cs.Child, &cs.State, &updated); err != nil { + return nil, fmt.Errorf("scan session: %w", err) + } + cs.UpdatedAt = parseTime(updated) + out = append(out, cs) + } + return out, rows.Err() +} + +// DeleteSession removes the stored session for one child. A signed-out or +// rejected login is not a failure; it just means the next scan logs in again. +func (s *Store) DeleteSession(ctx context.Context, childID int64) error { + if _, err := s.db.ExecContext(ctx, `DELETE FROM session WHERE child_id = ?`, childID); err != nil { + return fmt.Errorf("delete session for child %d: %w", childID, err) + } + return nil +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..a6aec4e --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,181 @@ +package store + +import ( + "context" + "path/filepath" + "testing" +) + +// newTestStore opens a fresh SQLite database in a temp dir. +func newTestStore(t *testing.T) *Store { + t.Helper() + st, err := Open(filepath.Join(t.TempDir(), "school.db")) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + if err := st.Migrate(context.Background()); err != nil { + t.Fatalf("migrate: %v", err) + } + if err := st.SeedChildren(context.Background(), "Harry", "Finn"); err != nil { + t.Fatalf("seed: %v", err) + } + return st +} + +// TestItemsAreSeparatedByChild proves the Phase 4 DoD: items are stored against +// the right children.id and nothing leaks between children. +func TestItemsAreSeparatedByChild(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + + harry, err := st.ChildByName(ctx, "Harry") + if err != nil || harry == nil { + t.Fatalf("harry: %v %v", harry, err) + } + finn, err := st.ChildByName(ctx, "Finn") + if err != nil || finn == nil { + t.Fatalf("finn: %v %v", finn, err) + } + if harry.ID == finn.ID { + t.Fatal("children share an id") + } + + harryItems := []Item{ + {Source: "compass", Type: "task", Subject: "English", Title: "Essay", ExternalRef: "h1"}, + {Source: "compass", Type: "task", Subject: "Maths", Title: "Chapter 4", ExternalRef: "h2"}, + {Source: "compass", Type: "message", Title: "Notice", ExternalRef: "h3"}, + } + finnItems := []Item{ + {Source: "compass", Type: "task", Subject: "Science", Title: "Lab report", ExternalRef: "f1"}, + } + + if _, _, err := st.UpsertItems(ctx, harry.ID, harryItems); err != nil { + t.Fatalf("upsert harry: %v", err) + } + if _, _, err := st.UpsertItems(ctx, finn.ID, finnItems); err != nil { + t.Fatalf("upsert finn: %v", err) + } + + // The same external ref under each child must be two rows, not a conflict. + if _, _, err := st.UpsertItems(ctx, finn.ID, []Item{ + {Source: "compass", Type: "task", Title: "Finn essay", ExternalRef: "h1"}, + }); err != nil { + t.Fatalf("upsert finn duplicate ref: %v", err) + } + + counts, err := st.CountItemsByType(ctx) + if err != nil { + t.Fatalf("counts: %v", err) + } + got := map[string]int{} + for _, c := range counts { + got[c.Child+"/"+c.Type] = c.Count + } + want := map[string]int{ + "Harry/task": 2, + "Harry/message": 1, + "Finn/task": 2, + } + for k, v := range want { + if got[k] != v { + t.Errorf("count %s = %d, want %d (all: %v)", k, got[k], v, got) + } + } + + // A per-child sample must never return the other child's rows. + harryRows, err := st.SampleItems(ctx, harry.ID, 50) + if err != nil { + t.Fatalf("sample harry: %v", err) + } + if len(harryRows) != 3 { + t.Fatalf("harry rows = %d, want 3", len(harryRows)) + } + for _, r := range harryRows { + if r.ChildID != harry.ID { + t.Errorf("harry sample returned child %d", r.ChildID) + } + } + finnRows, err := st.SampleItems(ctx, finn.ID, 50) + if err != nil { + t.Fatalf("sample finn: %v", err) + } + if len(finnRows) != 2 { + t.Fatalf("finn rows = %d, want 2", len(finnRows)) + } +} + +// TestSessionRoundTrip proves the per-child session state is stored, read back +// and separated. +func TestSessionRoundTrip(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + + harry, _ := st.ChildByName(ctx, "Harry") + finn, _ := st.ChildByName(ctx, "Finn") + + if sess, err := st.GetSession(ctx, harry.ID); err != nil || sess != nil { + t.Fatalf("empty session = %v, %v", sess, err) + } + + want := `{"cookies":[{"name":"x"}],"origins":[]}` + if err := st.PutSession(ctx, harry.ID, want); err != nil { + t.Fatalf("put: %v", err) + } + sess, err := st.GetSession(ctx, harry.ID) + if err != nil || sess == nil { + t.Fatalf("get: %v %v", sess, err) + } + if sess.State != want { + t.Errorf("state mismatch") + } + if sess.Child != "Harry" { + t.Errorf("child = %q", sess.Child) + } + + // Finn must not inherit Harry's session. + if sess, err := st.GetSession(ctx, finn.ID); err != nil || sess != nil { + t.Errorf("finn session = %v, %v", sess, err) + } + + // PutSession refreshes the same row. + if err := st.PutSession(ctx, harry.ID, want+" "); err != nil { + t.Fatalf("refresh: %v", err) + } + list, err := st.ListSessions(ctx) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 1 { + t.Fatalf("sessions = %d, want 1", len(list)) + } + + if err := st.DeleteSession(ctx, harry.ID); err != nil { + t.Fatalf("delete: %v", err) + } + if sess, _ := st.GetSession(ctx, harry.ID); sess != nil { + t.Errorf("session survived delete") + } +} + +// TestRecordScanFailure proves a failed child is recorded loudly (FR-12/FR-13). +func TestRecordScanFailure(t *testing.T) { + ctx := context.Background() + st := newTestStore(t) + harry, _ := st.ChildByName(ctx, "Harry") + + id, err := st.RecordScan(ctx, &harry.ID, 0, 0, false, "fetcher failed") + if err != nil || id == 0 { + t.Fatalf("record: %v id=%d", err, id) + } + last, err := st.LastScan(ctx) + if err != nil || last == nil { + t.Fatalf("last: %v %v", last, err) + } + if last.OK { + t.Error("failed scan reported as ok") + } + if last.Error != "fetcher failed" { + t.Errorf("error = %q", last.Error) + } +} diff --git a/scripts/parse-compass.sh b/scripts/parse-compass.sh index ba3e585..303b7cb 100755 --- a/scripts/parse-compass.sh +++ b/scripts/parse-compass.sh @@ -22,7 +22,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -IMAGE="${IMAGE:-compass-teams-nhs/school:0.2.0-phase3}" +IMAGE="${IMAGE:-compass-teams-nhs/school:0.3.0-phase4}" VOLUME="${VOLUME:-compass-teams-nhs_compass-teams-nhs-data}" RAW_ROOT="${RAW_ROOT:-$REPO_ROOT/var/raw}" SCHOOL_TIMEZONE="${SCHOOL_TIMEZONE:-Australia/Melbourne}" diff --git a/scripts/scan-compass.sh b/scripts/scan-compass.sh new file mode 100755 index 0000000..88d1efe --- /dev/null +++ b/scripts/scan-compass.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Phase 4: scan Compass for one child or both, with a per-child session. +# +# For each target child this script: +# 1. restores that child's stored browser session from the SQLite `session` +# table (AGENTS.md §7) so a scan can target one child, +# 2. runs the patchright fetcher (read-only) to store that child's raw pages, +# 3. saves the refreshed session back into the `session` table, +# 4. parses the newest stored run into the `items` table for that child. +# +# It never restarts the running service: `parse`, `session` and `scan record` +# run in a one-off container. The fetcher runs on the host with node + patchright. +# +# Fail loudly (FR-12): a child whose fetch or parse fails is recorded in `scans` +# with ok = 0, is reported by name, and makes the script exit non-zero. A failed +# child is never shown as an empty day. +# +# Usage: +# scripts/scan-compass.sh # both children +# scripts/scan-compass.sh --child Harry # one child +# scripts/scan-compass.sh --child Harry --child Finn --classify none +# +# Environment overrides: +# IMAGE image tag (default compass-teams-nhs/school:0.3.0-phase4) +# VOLUME SQLite volume (default compass-teams-nhs_compass-teams-nhs-data) +# RAW_ROOT host path to var/raw (default /var/raw) +# SCHOOL_TIMEZONE family timezone (default Australia/Melbourne) +# OMNIROUTE_BASE_URL OmniRoute endpoint (default http://127.0.0.1:20129/v1) +# DOCKER_TIMEOUT per docker command timeout (default 300) +# FETCH_TIMEOUT per fetch timeout (default 300) +# SCHOOL_SECRETS_FILE credential env file (default ~/.config/environment.d/10-secrets.conf) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +IMAGE="${IMAGE:-compass-teams-nhs/school:0.3.0-phase4}" +VOLUME="${VOLUME:-compass-teams-nhs_compass-teams-nhs-data}" +RAW_ROOT="${RAW_ROOT:-$REPO_ROOT/var/raw}" +SCHOOL_TIMEZONE="${SCHOOL_TIMEZONE:-Australia/Melbourne}" +OMNIROUTE_BASE_URL="${OMNIROUTE_BASE_URL:-http://127.0.0.1:20129/v1}" +DOCKER_TIMEOUT="${DOCKER_TIMEOUT:-300}" +FETCH_TIMEOUT="${FETCH_TIMEOUT:-300}" +SCHOOL_SECRETS_FILE="${SCHOOL_SECRETS_FILE:-$HOME/.config/environment.d/10-secrets.conf}" +CLASSIFY="${CLASSIFY:-auto}" + +CHILDREN=() +while [[ $# -gt 0 ]]; do + case "$1" in + --child) CHILDREN+=("$2"); shift 2 ;; + --classify) CLASSIFY="$2"; shift 2 ;; + --timeout) DOCKER_TIMEOUT="$2"; FETCH_TIMEOUT="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done +if [[ ${#CHILDREN[@]} -eq 0 ]]; then + CHILDREN=(Harry Finn) +fi + +# Credentials for the fetcher. They come from the environment, or from the +# user's environment.d secrets file when the environment is empty. They are +# never printed. +if [[ -z "${COMPASS_USERNAME:-}" || -z "${COMPASS_PASSWORD:-}" ]]; then + if [[ -f "$SCHOOL_SECRETS_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + . "$SCHOOL_SECRETS_FILE" + set +a + fi +fi +if [[ -z "${COMPASS_USERNAME:-}" || -z "${COMPASS_PASSWORD:-}" ]]; then + echo "COMPASS_USERNAME and COMPASS_PASSWORD are not set (see $SCHOOL_SECRETS_FILE)" >&2 + exit 2 +fi +if [[ ! -d "$RAW_ROOT" ]]; then + echo "raw pages root not found: $RAW_ROOT" >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT +chmod 0777 "$WORK" # the container user writes the session handoff file here + +# run_school reads and writes the service's SQLite volume as the image user. +run_school() { + timeout "$DOCKER_TIMEOUT" docker run --rm --network host \ + -v "$VOLUME:/data" \ + -v "$WORK:/scratch" \ + -v "$RAW_ROOT:/raw:ro" \ + -e SCHOOL_DB_PATH=/data/school.db \ + -e SCHOOL_TIMEZONE="$SCHOOL_TIMEZONE" \ + -e OMNIROUTE_API_KEY \ + -e OMNIROUTE_BASE_URL="$OMNIROUTE_BASE_URL" \ + "$IMAGE" "$@" +} + +# run_school_readonly only reads the volume. It runs as the host user and mounts +# the volume read-only, so the session state it writes lands in WORK owned by +# this user and the fetcher can read it. +run_school_readonly() { + timeout "$DOCKER_TIMEOUT" docker run --rm --network host \ + --user "$(id -u):$(id -g)" \ + -v "$VOLUME:/data:ro" \ + -v "$WORK:/scratch" \ + -e SCHOOL_DB_PATH=/data/school.db \ + -e SCHOOL_TIMEZONE="$SCHOOL_TIMEZONE" \ + "$IMAGE" "$@" +} + +failed=() + +for child in "${CHILDREN[@]}"; do + slug="$(printf '%s' "$child" | tr '[:upper:]' '[:lower:]')" + state_host="$WORK/$slug.json" + state_ctr="/scratch/$slug.json" + rm -f "$state_host" + + echo "== $child ==" + + # 1. Restore the per-child session, if one is stored. + run_school_readonly session get --child "$child" --out "$state_ctr" || true + + # 2. Fetch the child's raw pages (read-only against the school). + if timeout "$FETCH_TIMEOUT" node "$REPO_ROOT/scripts/compass-fetch-patchright.mjs" \ + --child "$child" --state "$state_host"; then + : + else + rc=$? + msg="fetcher failed for $child (exit $rc)" + echo "FAIL: $msg" >&2 + run_school scan record --child "$child" --ok=false --message "$msg" || true + failed+=("$child: fetch failed (exit $rc)") + continue + fi + + # 3. Persist the refreshed session for the next scan. The container user must + # be able to read the fetcher's file. + if [[ -f "$state_host" ]]; then + chmod 0644 "$state_host" + run_school session put --child "$child" --file "$state_ctr" || true + fi + + # 4. Parse the newest stored run into items for this child. + if run_school parse --child "$child" --run latest --raw-root /raw --classify "$CLASSIFY"; then + : + else + echo "FAIL: parse failed for $child" >&2 + failed+=("$child: parse failed") + fi +done + +echo +if [[ ${#failed[@]} -eq 0 ]]; then + echo "scan complete: ${CHILDREN[*]}" + exit 0 +fi +echo "scan failed for:" >&2 +for f in "${failed[@]}"; do echo " - $f" >&2; done +exit 1