Turn the raw pages stored by Phase 2 into rows in the SQLite items table (AGENTS.md §11 Phase 3, FR-3/FR-4/FR-5/FR-12). - internal/parse: dependency-free HTML reader plus extractors for the ExtJS learning-task grid (subject, title, category, kind, due date, status), the home news feed (author, posted time, title, body) and the calendar week grid (day column + grid offset give the time; label gives title/room/teacher). - internal/parse: fail loudly. A challenge page, login page, changed layout or unknown due-date format is an explicit error, never an empty day. - FR-4: structure is parsed deterministically; importance is classified by an OmniRoute model (default openrouter/google/gemini-3.5-flash-lite) with a deterministic fallback. The classifier used is printed and recorded. - internal/llm: small OpenAI-compatible OmniRoute client (handles SSE replies). - internal/store: migration 002 adds source_page, importance and external_ref with a unique (child_id, source, type, external_ref) index; UpsertItems, CountItemsByType, SampleItems and ChildByName. - cmd/school parse: reads var/raw/<child>/<run>, upserts items, records the scan row (ok=0 plus the error on failure) and prints counts and samples. - scripts/parse-compass.sh and make parse run it in a one-off container against the service's own volume without restarting the service. - docs/phase-3-parse-notes.md records the approach, the FR-4 decision and the known gaps. Evidence on the live database: 56 rows for Harry (20 task, 1 message, 35 event). Re-running the same raw pages inserts 0 and updates 56. Co-Authored-By: Paperclip <noreply@paperclip.ing>
205 lines
6.4 KiB
Go
205 lines
6.4 KiB
Go
package parse
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Importance labels stored in items.importance (FR-4).
|
|
const (
|
|
importanceHigh = "high"
|
|
importanceNormal = "normal"
|
|
importanceLow = "low"
|
|
)
|
|
|
|
// ImportanceClassifiers names the classifier used for a run, so the choice is
|
|
// recorded rather than implied.
|
|
const (
|
|
ClassifierNone = "deterministic"
|
|
ClassifierOmniRoute = "omniroute"
|
|
ClassifierJEV = "jev"
|
|
classifyBatchSize = 40
|
|
classifyMaxResponseChar = 16 * 1024
|
|
)
|
|
|
|
// Completer is one text-completion call. internal/llm supplies the OmniRoute
|
|
// client; tests supply a stub.
|
|
type Completer interface {
|
|
Complete(ctx context.Context, system, prompt string) (string, error)
|
|
}
|
|
|
|
// ClassifyDeterministic is the offline importance rule. Overdue work and work
|
|
// due within the next week is high; everything else is normal. It is the
|
|
// fallback when no model is configured, and it is deliberately conservative: it
|
|
// never marks work "done".
|
|
func ClassifyDeterministic(items []Item, now time.Time) {
|
|
week := now.AddDate(0, 0, 7)
|
|
for i := range items {
|
|
items[i].Importance = deterministicImportance(items[i], week)
|
|
}
|
|
}
|
|
|
|
func deterministicImportance(it Item, week time.Time) string {
|
|
// Only work carries a deadline that makes it important. A calendar entry's
|
|
// due_at is its start time, and a lesson that starts this week is not
|
|
// overdue work, so events and messages stay normal under the offline rule.
|
|
if it.Type == TypeTask {
|
|
if it.DueAt != nil && it.DueAt.Before(week) {
|
|
return importanceHigh
|
|
}
|
|
if strings.Contains(strings.ToLower(it.Body), "overdue") {
|
|
return importanceHigh
|
|
}
|
|
}
|
|
return importanceNormal
|
|
}
|
|
|
|
// ClassifyModel asks a model to classify importance for every item (FR-4). It
|
|
// fails, loudly, on a model error or an unparseable answer; it never silently
|
|
// leaves a wrong label. Items are updated in place. now lets the model judge
|
|
// whether a due date is overdue.
|
|
func ClassifyModel(ctx context.Context, c Completer, items []Item, model string, now time.Time) error {
|
|
if c == nil {
|
|
return fmt.Errorf("classify: no model client configured")
|
|
}
|
|
const system = "You classify school items for a parent's morning brief. " +
|
|
"Reply with JSON only. No prose, no code fences."
|
|
|
|
for start := 0; start < len(items); start += classifyBatchSize {
|
|
end := start + classifyBatchSize
|
|
if end > len(items) {
|
|
end = len(items)
|
|
}
|
|
batch := items[start:end]
|
|
|
|
prompt := classifyPrompt(batch, model, now)
|
|
raw, err := c.Complete(ctx, system, prompt)
|
|
if err != nil {
|
|
return fmt.Errorf("classify: items %d-%d: %w", start, end-1, err)
|
|
}
|
|
labels, err := parseLabels(raw)
|
|
if err != nil {
|
|
return fmt.Errorf("classify: items %d-%d: %w", start, end-1, err)
|
|
}
|
|
for i, label := range labels {
|
|
if i >= len(batch) {
|
|
return fmt.Errorf("classify: items %d-%d: model returned %d labels for %d items", start, end-1, len(labels), len(batch))
|
|
}
|
|
switch label {
|
|
case importanceHigh, importanceNormal, importanceLow:
|
|
batch[i].Importance = label
|
|
default:
|
|
return fmt.Errorf("classify: item %d: invalid importance %q", start+i, label)
|
|
}
|
|
}
|
|
if len(labels) != len(batch) {
|
|
return fmt.Errorf("classify: items %d-%d: model returned %d labels for %d items", start, end-1, len(labels), len(batch))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// classifyPrompt builds the user message for one batch.
|
|
func classifyPrompt(batch []Item, model string, now time.Time) string {
|
|
type row struct {
|
|
I int `json:"i"`
|
|
Child string `json:"child"`
|
|
Type string `json:"type"`
|
|
Subject string `json:"subject,omitempty"`
|
|
Title string `json:"title"`
|
|
Due string `json:"due,omitempty"`
|
|
Detail string `json:"detail,omitempty"`
|
|
}
|
|
rows := make([]row, 0, len(batch))
|
|
for i, it := range batch {
|
|
due := ""
|
|
if it.DueAt != nil {
|
|
due = it.DueAt.Format(time.RFC3339)
|
|
}
|
|
rows = append(rows, row{
|
|
I: i,
|
|
Child: it.Child,
|
|
Type: it.Type,
|
|
Subject: it.Subject,
|
|
Title: it.Title,
|
|
Due: due,
|
|
Detail: detail(it.Body),
|
|
})
|
|
}
|
|
payload, _ := json.Marshal(rows)
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "Model: %s\n", model)
|
|
fmt.Fprintf(&b, "Now: %s\n", now.Format(time.RFC3339))
|
|
b.WriteString("Label every item with importance high, normal or low.\n")
|
|
b.WriteString("high: overdue work, work due today or in the next 48 hours, or a school event that affects the family.\n")
|
|
b.WriteString("low: routine recurring lessons, past submitted work, or notices with no action.\n")
|
|
b.WriteString("normal: anything else.\n")
|
|
b.WriteString("A task with a due date before Now and a submission status of Pending or Overdue is high.\n")
|
|
b.WriteString("Return a JSON array with one object per item, in order, exactly: " +
|
|
`{"i":<index>,"importance":"high|normal|low"}` + "\n\nItems:\n")
|
|
b.Write(payload)
|
|
return b.String()
|
|
}
|
|
|
|
// detail flattens an item body into one line for the model.
|
|
func detail(s string) string {
|
|
return collapse(strings.ReplaceAll(s, "\n", "; "))
|
|
}
|
|
|
|
// parseLabels extracts the importance labels from a model reply. It tolerates
|
|
// code fences and surrounding prose, and returns the labels ordered by index.
|
|
func parseLabels(raw string) ([]string, error) {
|
|
raw = strings.TrimSpace(raw)
|
|
if len(raw) > classifyMaxResponseChar {
|
|
return nil, fmt.Errorf("model reply too large (%d bytes)", len(raw))
|
|
}
|
|
raw = strings.TrimPrefix(raw, "```json")
|
|
raw = strings.TrimPrefix(raw, "```")
|
|
raw = strings.TrimSuffix(raw, "```")
|
|
raw = strings.TrimSpace(raw)
|
|
|
|
if i := strings.IndexByte(raw, '['); i >= 0 {
|
|
if j := strings.LastIndexByte(raw, ']'); j > i {
|
|
raw = raw[i : j+1]
|
|
}
|
|
}
|
|
|
|
var entries []struct {
|
|
I *int `json:"i"`
|
|
Importance string `json:"importance"`
|
|
}
|
|
if err := json.Unmarshal([]byte(raw), &entries); err != nil {
|
|
return nil, fmt.Errorf("model reply is not JSON: %w", err)
|
|
}
|
|
if len(entries) == 0 {
|
|
return nil, fmt.Errorf("model reply has no items")
|
|
}
|
|
|
|
type indexed struct {
|
|
i int
|
|
label string
|
|
}
|
|
got := make([]indexed, 0, len(entries))
|
|
for pos, e := range entries {
|
|
idx := pos
|
|
if e.I != nil {
|
|
idx = *e.I
|
|
}
|
|
got = append(got, indexed{i: idx, label: strings.ToLower(strings.TrimSpace(e.Importance))})
|
|
}
|
|
sort.Slice(got, func(a, b int) bool { return got[a].i < got[b].i })
|
|
|
|
out := make([]string, len(got))
|
|
for k, g := range got {
|
|
if g.i < 0 || g.i >= len(got) {
|
|
return nil, fmt.Errorf("model returned out-of-range index %d", g.i)
|
|
}
|
|
out[k] = g.label
|
|
}
|
|
return out, nil
|
|
}
|