Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
102
internal/parse/news.go
Normal file
102
internal/parse/news.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newsTimeLayout is the aria-label format on the feed timestamp, for example
|
||||
// "14/10/2026 09:00 AM".
|
||||
const newsTimeLayout = "02/01/2006 03:04 PM"
|
||||
|
||||
// parseNews parses the home-page news feed into message items. It fails loudly
|
||||
// when the page carries neither the feed toolbar nor a post, so a login or
|
||||
// challenge page is never mistaken for "no news" (FR-12).
|
||||
func parseNews(child string, page Page, loc *time.Location) ([]Item, error) {
|
||||
items, found, err := extractNews(child, page, loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("parse: %s: no news feed or posts found in %s (layout changed or fetch failed)", child, page.Slug)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractNews returns news posts from the home page. found reports whether the
|
||||
// feed was present at all.
|
||||
func extractNews(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
||||
root := parseHTML(page.HTML)
|
||||
|
||||
toolbar := find(root, func(n *node) bool {
|
||||
return strings.EqualFold(n.Attrs["data-testid"], "feed-toolbar")
|
||||
})
|
||||
cards := findAll(root, func(n *node) bool { return n.hasClass("MuiCardContent-root") })
|
||||
found = toolbar != nil || len(cards) > 0
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
titleNode := find(card, func(n *node) bool { return n.hasClassPart("postTitle") })
|
||||
if titleNode == nil {
|
||||
continue // not a news post
|
||||
}
|
||||
title := collapse(titleNode.Text())
|
||||
if title == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var body string
|
||||
if b := find(card, func(n *node) bool { return n.hasClassPart("froala-viewer") }); b != nil {
|
||||
body = b.Text()
|
||||
}
|
||||
|
||||
var author string
|
||||
if a := find(card, func(n *node) bool { return n.hasClassPart("MuiTypography-dialogMdStrong") }); a != nil {
|
||||
author = collapse(a.Text())
|
||||
}
|
||||
|
||||
var posted *time.Time
|
||||
if t := find(card, func(n *node) bool {
|
||||
return n.hasClassPart("MuiTypography-dialogSmStrong") && n.hasAttr("aria-label")
|
||||
}); t != nil {
|
||||
raw := strings.TrimSpace(t.Attrs["aria-label"])
|
||||
parsed, perr := time.ParseInLocation(newsTimeLayout, raw, loc)
|
||||
if perr != nil {
|
||||
return nil, false, fmt.Errorf("parse: %s: news %q: unrecognised posted time %q", page.Slug, title, raw)
|
||||
}
|
||||
posted = &parsed
|
||||
}
|
||||
|
||||
pageURL := page.URL
|
||||
if pageURL == "" {
|
||||
pageURL = compassHost + "/"
|
||||
}
|
||||
ref := dedupeKey(author, title, relativeOrISO(posted))
|
||||
|
||||
items = append(items, Item{
|
||||
Child: child,
|
||||
Source: SourceCompass,
|
||||
Type: TypeMessage,
|
||||
Teacher: author, // messages carry the author, not a teacher
|
||||
Title: title,
|
||||
Body: body,
|
||||
DueAt: posted,
|
||||
URL: pageURL,
|
||||
SourcePage: page.Slug,
|
||||
ExternalRef: ref,
|
||||
Importance: importanceNormal,
|
||||
})
|
||||
}
|
||||
return items, true, nil
|
||||
}
|
||||
|
||||
// relativeOrISO renders a posted time for the dedupe key.
|
||||
func relativeOrISO(t *time.Time) string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
Reference in New Issue
Block a user