Files
compass_teams_nhs/internal/parse/news.go
Sam Rolfe d7d529eb0f Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table
(AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12).

- internal/parse: dependency-free HTML reader plus extractors for the ExtJS
  learning-task grid (subject, title, category, kind, due date, status), the
  home news feed (author, posted time, title, body) and the calendar week grid
  (day column + grid offset give the time; label gives title/room/teacher).
- internal/parse: fail loudly. A challenge page, login page, changed layout or
  unknown due-date format is an explicit error, never an empty day.
- FR-4: structure is parsed deterministically; importance is classified by an
  OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a
  deterministic fallback. The classifier used is printed and recorded.
- internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies).
- internal/store: migration 002 adds source_page, importance and external_ref
  with a unique (child_id, source, type, external_ref) index; UpsertItems,
  CountItemsByType, SampleItems and ChildByName.
- cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan
  row (ok=0 plus the error on failure) and prints counts and samples.
- scripts/parse-compass.sh and make parse run it in a one-off container against
  the service's own volume without restarting the service.
- docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the
  known gaps.

Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event).
Re-running the same raw pages inserts 0 and updates 56.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-15 17:27:25 +11:00

103 lines
2.9 KiB
Go

package parse
import (
"fmt"
"strings"
"time"
)
// newsTimeLayout is the aria-label format on the feed timestamp, for example
// "14/10/2026 09:00 AM".
const newsTimeLayout = "02/01/2006 03:04 PM"
// parseNews parses the home-page news feed into message items. It fails loudly
// when the page carries neither the feed toolbar nor a post, so a login or
// challenge page is never mistaken for "no news" (FR-12).
func parseNews(child string, page Page, loc *time.Location) ([]Item, error) {
items, found, err := extractNews(child, page, loc)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("parse: %s: no news feed or posts found in %s (layout changed or fetch failed)", child, page.Slug)
}
return items, nil
}
// extractNews returns news posts from the home page. found reports whether the
// feed was present at all.
func extractNews(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
root := parseHTML(page.HTML)
toolbar := find(root, func(n *node) bool {
return strings.EqualFold(n.Attrs["data-testid"], "feed-toolbar")
})
cards := findAll(root, func(n *node) bool { return n.hasClass("MuiCardContent-root") })
found = toolbar != nil || len(cards) > 0
if !found {
return nil, false, nil
}
for _, card := range cards {
titleNode := find(card, func(n *node) bool { return n.hasClassPart("postTitle") })
if titleNode == nil {
continue // not a news post
}
title := collapse(titleNode.Text())
if title == "" {
continue
}
var body string
if b := find(card, func(n *node) bool { return n.hasClassPart("froala-viewer") }); b != nil {
body = b.Text()
}
var author string
if a := find(card, func(n *node) bool { return n.hasClassPart("MuiTypography-dialogMdStrong") }); a != nil {
author = collapse(a.Text())
}
var posted *time.Time
if t := find(card, func(n *node) bool {
return n.hasClassPart("MuiTypography-dialogSmStrong") && n.hasAttr("aria-label")
}); t != nil {
raw := strings.TrimSpace(t.Attrs["aria-label"])
parsed, perr := time.ParseInLocation(newsTimeLayout, raw, loc)
if perr != nil {
return nil, false, fmt.Errorf("parse: %s: news %q: unrecognised posted time %q", page.Slug, title, raw)
}
posted = &parsed
}
pageURL := page.URL
if pageURL == "" {
pageURL = compassHost + "/"
}
ref := dedupeKey(author, title, relativeOrISO(posted))
items = append(items, Item{
Child: child,
Source: SourceCompass,
Type: TypeMessage,
Teacher: author, // messages carry the author, not a teacher
Title: title,
Body: body,
DueAt: posted,
URL: pageURL,
SourcePage: page.Slug,
ExternalRef: ref,
Importance: importanceNormal,
})
}
return items, true, nil
}
// relativeOrISO renders a posted time for the dedupe key.
func relativeOrISO(t *time.Time) string {
if t == nil {
return ""
}
return t.Format(time.RFC3339)
}