Phase 3: parse stored Compass pages into structured items
Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -269,3 +269,163 @@ func (s *Store) RecordScan(ctx context.Context, childID *int64, pages, items int
|
||||
}
|
||||
return res.LastInsertId()
|
||||
}
|
||||
|
||||
// Item is one row of the items table (AGENTS.md §7). It mirrors parse.Item
|
||||
// without importing the parse package, so the store stays independent of the
|
||||
// scraper.
|
||||
type Item struct {
|
||||
ID int64
|
||||
ChildID int64
|
||||
Child string
|
||||
Source string
|
||||
Type string
|
||||
Subject string
|
||||
Teacher string
|
||||
Title string
|
||||
Body string
|
||||
DueAt *time.Time
|
||||
URL string
|
||||
SourcePage string
|
||||
Importance string
|
||||
ExternalRef string
|
||||
}
|
||||
|
||||
// UpsertItems writes every item for one child, de-duplicating on
|
||||
// (child_id, source, type, external_ref). It returns how many rows were
|
||||
// inserted and how many were refreshed, so a re-parse never duplicates an item.
|
||||
func (s *Store) UpsertItems(ctx context.Context, childID int64, items []Item) (inserted, updated int, err error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("upsert items: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
for _, it := range items {
|
||||
if it.ExternalRef == "" {
|
||||
return 0, 0, fmt.Errorf("upsert items: %q has no external reference", it.Title)
|
||||
}
|
||||
var existing int64
|
||||
err := tx.QueryRowContext(ctx,
|
||||
`SELECT id FROM items WHERE child_id = ? AND source = ? AND type = ? AND external_ref = ?`,
|
||||
childID, it.Source, it.Type, it.ExternalRef).Scan(&existing)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO items (child_id, source, type, subject, teacher, title, body,
|
||||
due_at, url, source_page, importance, external_ref)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
childID, it.Source, it.Type, it.Subject, it.Teacher, it.Title, it.Body,
|
||||
formatDue(it.DueAt), it.URL, it.SourcePage, it.Importance, it.ExternalRef); err != nil {
|
||||
return 0, 0, fmt.Errorf("insert item %q: %w", it.Title, err)
|
||||
}
|
||||
inserted++
|
||||
case err != nil:
|
||||
return 0, 0, fmt.Errorf("look up item %q: %w", it.Title, err)
|
||||
default:
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`UPDATE items SET subject = ?, teacher = ?, title = ?, body = ?, due_at = ?,
|
||||
url = ?, source_page = ?, importance = ?, updated_at = datetime('now')
|
||||
WHERE id = ?`,
|
||||
it.Subject, it.Teacher, it.Title, it.Body, formatDue(it.DueAt),
|
||||
it.URL, it.SourcePage, it.Importance, existing); err != nil {
|
||||
return 0, 0, fmt.Errorf("update item %q: %w", it.Title, err)
|
||||
}
|
||||
updated++
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, fmt.Errorf("upsert items: commit: %w", err)
|
||||
}
|
||||
return inserted, updated, nil
|
||||
}
|
||||
|
||||
// formatDue stores a due time in UTC RFC3339 so ordering is DST-safe, or NULL
|
||||
// when there is no due date.
|
||||
func formatDue(t *time.Time) any {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// TypeCount is the number of items of one type for one child.
|
||||
type TypeCount struct {
|
||||
Child string
|
||||
Type string
|
||||
Count int
|
||||
}
|
||||
|
||||
// CountItemsByType returns per-child, per-type row counts, ordered by child then
|
||||
// type. It is the evidence query for Phase 3.
|
||||
func (s *Store) CountItemsByType(ctx context.Context) ([]TypeCount, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT COALESCE(ch.name, '(unknown)'), i.type, COUNT(*)
|
||||
FROM items i LEFT JOIN children ch ON ch.id = i.child_id
|
||||
GROUP BY ch.name, i.type
|
||||
ORDER BY ch.name, i.type`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []TypeCount
|
||||
for rows.Next() {
|
||||
var tc TypeCount
|
||||
if err := rows.Scan(&tc.Child, &tc.Type, &tc.Count); err != nil {
|
||||
return nil, fmt.Errorf("scan item count: %w", err)
|
||||
}
|
||||
out = append(out, tc)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SampleItems returns up to limit items for a child, newest due date first.
|
||||
func (s *Store) SampleItems(ctx context.Context, childID int64, limit int) ([]Item, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT i.id, i.child_id, COALESCE(ch.name, ''), i.source, i.type,
|
||||
COALESCE(i.subject, ''), COALESCE(i.teacher, ''), i.title,
|
||||
COALESCE(i.body, ''), i.due_at, COALESCE(i.url, ''),
|
||||
COALESCE(i.source_page, ''), COALESCE(i.importance, ''), COALESCE(i.external_ref, '')
|
||||
FROM items i LEFT JOIN children ch ON ch.id = i.child_id
|
||||
WHERE i.child_id = ?
|
||||
ORDER BY i.type, i.due_at DESC, i.id
|
||||
LIMIT ?`, childID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sample items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Item
|
||||
for rows.Next() {
|
||||
var it Item
|
||||
var due sql.NullString
|
||||
if err := rows.Scan(&it.ID, &it.ChildID, &it.Child, &it.Source, &it.Type,
|
||||
&it.Subject, &it.Teacher, &it.Title, &it.Body, &due, &it.URL,
|
||||
&it.SourcePage, &it.Importance, &it.ExternalRef); err != nil {
|
||||
return nil, fmt.Errorf("scan item: %w", err)
|
||||
}
|
||||
if due.Valid {
|
||||
t := parseTime(due.String)
|
||||
it.DueAt = &t
|
||||
}
|
||||
out = append(out, it)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ChildByName returns one child by name.
|
||||
func (s *Store) ChildByName(ctx context.Context, name string) (*Child, error) {
|
||||
row := s.db.QueryRowContext(ctx,
|
||||
`SELECT id, name, COALESCE(year, ''), COALESCE(compass_id, ''), created_at
|
||||
FROM children WHERE name = ?`, name)
|
||||
var c Child
|
||||
var created string
|
||||
if err := row.Scan(&c.ID, &c.Name, &c.Year, &c.CompassID, &created); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("child %q: %w", name, err)
|
||||
}
|
||||
c.CreatedAt = parseTime(created)
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user