Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
103 lines
2.9 KiB
Go
103 lines
2.9 KiB
Go
package parse
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// newsTimeLayout is the aria-label format on the feed timestamp, for example
|
|
// "14/10/2026 09:00 AM".
|
|
const newsTimeLayout = "02/01/2006 03:04 PM"
|
|
|
|
// parseNews parses the home-page news feed into message items. It fails loudly
|
|
// when the page carries neither the feed toolbar nor a post, so a login or
|
|
// challenge page is never mistaken for "no news" (FR-12).
|
|
func parseNews(child string, page Page, loc *time.Location) ([]Item, error) {
|
|
items, found, err := extractNews(child, page, loc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
return nil, fmt.Errorf("parse: %s: no news feed or posts found in %s (layout changed or fetch failed)", child, page.Slug)
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// extractNews returns news posts from the home page. found reports whether the
|
|
// feed was present at all.
|
|
func extractNews(child string, page Page, loc *time.Location) (items []Item, found bool, err error) {
|
|
root := parseHTML(page.HTML)
|
|
|
|
toolbar := find(root, func(n *node) bool {
|
|
return strings.EqualFold(n.Attrs["data-testid"], "feed-toolbar")
|
|
})
|
|
cards := findAll(root, func(n *node) bool { return n.hasClass("MuiCardContent-root") })
|
|
found = toolbar != nil || len(cards) > 0
|
|
if !found {
|
|
return nil, false, nil
|
|
}
|
|
|
|
for _, card := range cards {
|
|
titleNode := find(card, func(n *node) bool { return n.hasClassPart("postTitle") })
|
|
if titleNode == nil {
|
|
continue // not a news post
|
|
}
|
|
title := collapse(titleNode.Text())
|
|
if title == "" {
|
|
continue
|
|
}
|
|
|
|
var body string
|
|
if b := find(card, func(n *node) bool { return n.hasClassPart("froala-viewer") }); b != nil {
|
|
body = b.Text()
|
|
}
|
|
|
|
var author string
|
|
if a := find(card, func(n *node) bool { return n.hasClassPart("MuiTypography-dialogMdStrong") }); a != nil {
|
|
author = collapse(a.Text())
|
|
}
|
|
|
|
var posted *time.Time
|
|
if t := find(card, func(n *node) bool {
|
|
return n.hasClassPart("MuiTypography-dialogSmStrong") && n.hasAttr("aria-label")
|
|
}); t != nil {
|
|
raw := strings.TrimSpace(t.Attrs["aria-label"])
|
|
parsed, perr := time.ParseInLocation(newsTimeLayout, raw, loc)
|
|
if perr != nil {
|
|
return nil, false, fmt.Errorf("parse: %s: news %q: unrecognised posted time %q", page.Slug, title, raw)
|
|
}
|
|
posted = &parsed
|
|
}
|
|
|
|
pageURL := page.URL
|
|
if pageURL == "" {
|
|
pageURL = compassHost + "/"
|
|
}
|
|
ref := dedupeKey(author, title, relativeOrISO(posted))
|
|
|
|
items = append(items, Item{
|
|
Child: child,
|
|
Source: SourceCompass,
|
|
Type: TypeMessage,
|
|
Teacher: author, // messages carry the author, not a teacher
|
|
Title: title,
|
|
Body: body,
|
|
DueAt: posted,
|
|
URL: pageURL,
|
|
SourcePage: page.Slug,
|
|
ExternalRef: ref,
|
|
Importance: importanceNormal,
|
|
})
|
|
}
|
|
return items, true, nil
|
|
}
|
|
|
|
// relativeOrISO renders a posted time for the dedupe key.
|
|
func relativeOrISO(t *time.Time) string {
|
|
if t == nil {
|
|
return ""
|
|
}
|
|
return t.Format(time.RFC3339)
|
|
}
|