Phase 7: telemetry and usage (daily rollup + week dashboard)

Build on Phases 1–6. Telemetry already arrived on restrict/<device>/telemetry; this phase stores it in a daily rollup and shows it. - store: migration 003 adds telemetry_daily (device, day, app seconds, site seconds). InsertTelemetry maintains the affected days in the same transaction; RebuildRollups backfills and repairs it. UsageRange and TopUsage serve the week grid and top lists. CapAudit enforces the size-capped append-only audit log from AGENTS.md §7. - web: GET /devices/{id}/usage — Monday-to-Sunday totals plus the top 10 apps and top 10 sites, using CSP-safe native <progress> bars. - theme.js: light/dark from the OS preference, with a nav toggle, loaded as an external script so the strict CSP still holds. - cmd/control: `rollup` subcommand; rollup rebuild and audit cap at start-up and every 15 minutes. - Version 0.5.0-phase7 (compose, Makefile, docs). Tests: go test ./... and go vet ./... pass in golang:1.25-alpine. Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-13 16:18:56 +11:00
parent 9af543c394
commit b58993d994
20 changed files with 1077 additions and 20 deletions
--- a/internal/store/migrations/003_usage_rollup.sql
+++ b/internal/store/migrations/003_usage_rollup.sql
@@ -0,0 +1,24 @@
+-- Migration 003 — Phase 7. Daily usage rollup (AGENTS.md §7 retention).
+--
+-- telemetry holds the per-day detail: one row per (device, day, app, domain).
+-- This table is the daily rollup the dashboard reads for a week and that is
+-- kept long. Apps and domains are summed into separate columns on purpose: a
+-- report can carry both for the same session, so adding them together would
+-- double-count the time.
+--
+-- The rollup is written whenever a telemetry report lands and can be rebuilt
+-- from telemetry at any time (store.RebuildRollups), so it is derived data,
+-- never a second source of truth.
+
+CREATE TABLE IF NOT EXISTS telemetry_daily (
+    device_id      INTEGER NOT NULL REFERENCES devices(id),
+    day            TEXT    NOT NULL,          -- YYYY-MM-DD, local day
+    app_seconds    INTEGER NOT NULL DEFAULT 0,
+    domain_seconds INTEGER NOT NULL DEFAULT 0,
+    app_count      INTEGER NOT NULL DEFAULT 0,
+    domain_count   INTEGER NOT NULL DEFAULT 0,
+    updated_at     TEXT    NOT NULL DEFAULT (datetime('now')),
+    PRIMARY KEY (device_id, day)
+);
+
+CREATE INDEX IF NOT EXISTS idx_telemetry_daily_day ON telemetry_daily(day);
--- a/internal/store/reports.go
+++ b/internal/store/reports.go
@@ -9,6 +9,7 @@ package store

 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 )
@@ -70,7 +71,8 @@ type TelemetryEntry struct {

 // InsertTelemetry upserts a rolling daily usage total per (device, day, app,
 // domain). Reports are repeats, not deltas: a later report for the same key
-// replaces the earlier total (migration 002). It returns the number of rows
+// replaces the earlier total (migration 002). The affected days are rolled up
+// in the same transaction (migration 003). It returns the number of rows
 // written.
 func (s *Store) InsertTelemetry(ctx context.Context, id int64, day string, entries []TelemetryEntry) (int, error) {
 	if len(entries) == 0 {
@@ -84,6 +86,7 @@ func (s *Store) InsertTelemetry(ctx context.Context, id int64, day string, entri
 	defer func() { _ = tx.Rollback() }()

 	n := 0
+	days := make(map[string]struct{}, 1)
 	for _, e := range entries {
 		if e.Seconds < 0 {
 			continue
@@ -97,14 +100,187 @@ func (s *Store) InsertTelemetry(ctx context.Context, id int64, day string, entri
 			id, e.App, e.Domain, e.Seconds, day); err != nil {
 			return n, fmt.Errorf("insert telemetry: %w", err)
 		}
+		days[day] = struct{}{}
 		n++
 	}
+	for d := range days {
+		if err := rollupDay(ctx, tx, id, d); err != nil {
+			return n, err
+		}
+	}
 	if err := tx.Commit(); err != nil {
 		return n, fmt.Errorf("commit telemetry: %w", err)
 	}
 	return n, nil
 }

+// rollupDay recomputes one device/day row of telemetry_daily from the detail
+// table. Apps and domains are summed separately so a report that carries both
+// is not double-counted.
+func rollupDay(ctx context.Context, ex interface {
+	ExecContext(context.Context, string, ...any) (sql.Result, error)
+}, id int64, day string) error {
+	if _, err := ex.ExecContext(ctx, `
+		INSERT INTO telemetry_daily
+			(device_id, day, app_seconds, domain_seconds, app_count, domain_count, updated_at)
+		SELECT ?, ?,
+		       COALESCE(SUM(CASE WHEN app <> '' THEN seconds END), 0),
+		       COALESCE(SUM(CASE WHEN domain <> '' THEN seconds END), 0),
+		       COALESCE(SUM(CASE WHEN app <> '' THEN 1 END), 0),
+		       COALESCE(SUM(CASE WHEN domain <> '' THEN 1 END), 0),
+		       datetime('now')
+		FROM telemetry WHERE device_id = ? AND day = ?
+		ON CONFLICT(device_id, day) DO UPDATE SET
+			app_seconds = excluded.app_seconds,
+			domain_seconds = excluded.domain_seconds,
+			app_count = excluded.app_count,
+			domain_count = excluded.domain_count,
+			updated_at = datetime('now')`,
+		id, day, id, day); err != nil {
+		return fmt.Errorf("rollup day: %w", err)
+	}
+	return nil
+}
+
+// RebuildRollups recomputes every telemetry_daily row from the detail table. It
+// is idempotent and is how the rollup is backfilled after an upgrade or
+// repaired after an import. It returns the number of rows inserted or updated.
+func (s *Store) RebuildRollups(ctx context.Context) (int, error) {
+	res, err := s.db.ExecContext(ctx, `
+		INSERT INTO telemetry_daily
+			(device_id, day, app_seconds, domain_seconds, app_count, domain_count, updated_at)
+		SELECT device_id, day,
+		       COALESCE(SUM(CASE WHEN app <> '' THEN seconds END), 0),
+		       COALESCE(SUM(CASE WHEN domain <> '' THEN seconds END), 0),
+		       COALESCE(SUM(CASE WHEN app <> '' THEN 1 END), 0),
+		       COALESCE(SUM(CASE WHEN domain <> '' THEN 1 END), 0),
+		       datetime('now')
+		FROM telemetry
+		GROUP BY device_id, day
+		ON CONFLICT(device_id, day) DO UPDATE SET
+			app_seconds = excluded.app_seconds,
+			domain_seconds = excluded.domain_seconds,
+			app_count = excluded.app_count,
+			domain_count = excluded.domain_count,
+			updated_at = datetime('now')`)
+	if err != nil {
+		return 0, fmt.Errorf("rebuild rollups: %w", err)
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return 0, fmt.Errorf("rebuild rollups: %w", err)
+	}
+	return int(n), nil
+}
+
+// DayUsage is one day of rolled-up usage for a device. AppSeconds and
+// DomainSeconds are kept apart because a report may carry both.
+type DayUsage struct {
+	Day           string
+	AppSeconds    int
+	DomainSeconds int
+	AppCount      int
+	DomainCount   int
+}
+
+// UsageRange returns the daily rollup for a device in [from, to] inclusive,
+// oldest first. Days without a report are absent; the caller fills the gaps.
+func (s *Store) UsageRange(ctx context.Context, id int64, from, to string) ([]DayUsage, error) {
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT day, app_seconds, domain_seconds, app_count, domain_count
+		FROM telemetry_daily
+		WHERE device_id = ? AND day BETWEEN ? AND ?
+		ORDER BY day`, id, from, to)
+	if err != nil {
+		return nil, fmt.Errorf("usage range: %w", err)
+	}
+	defer rows.Close()
+
+	out := make([]DayUsage, 0, 8)
+	for rows.Next() {
+		var d DayUsage
+		if err := rows.Scan(&d.Day, &d.AppSeconds, &d.DomainSeconds, &d.AppCount, &d.DomainCount); err != nil {
+			return nil, fmt.Errorf("scan usage day: %w", err)
+		}
+		out = append(out, d)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("usage range: %w", err)
+	}
+	return out, nil
+}
+
+// UsageTotal is one app or domain total over a range.
+type UsageTotal struct {
+	Name    string
+	Seconds int
+}
+
+// TopUsage returns the biggest apps or domains for a device in [from, to]
+// inclusive. kind is "app" or "domain"; anything else returns no rows. The
+// column is chosen by a switch, never interpolated, so the argument can never
+// reach SQL as text.
+func (s *Store) TopUsage(ctx context.Context, id int64, from, to, kind string, limit int) ([]UsageTotal, error) {
+	column := ""
+	switch kind {
+	case "app":
+		column = "app"
+	case "domain":
+		column = "domain"
+	default:
+		return nil, nil
+	}
+	if limit <= 0 || limit > 100 {
+		limit = 10
+	}
+
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT `+column+`, SUM(seconds)
+		FROM telemetry
+		WHERE device_id = ? AND day BETWEEN ? AND ? AND `+column+` <> ''
+		GROUP BY `+column+`
+		ORDER BY SUM(seconds) DESC, `+column+` ASC
+		LIMIT ?`, id, from, to, limit)
+	if err != nil {
+		return nil, fmt.Errorf("top %s: %w", column, err)
+	}
+	defer rows.Close()
+
+	out := make([]UsageTotal, 0, limit)
+	for rows.Next() {
+		var t UsageTotal
+		if err := rows.Scan(&t.Name, &t.Seconds); err != nil {
+			return nil, fmt.Errorf("scan top %s: %w", column, err)
+		}
+		out = append(out, t)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("top %s: %w", column, err)
+	}
+	return out, nil
+}
+
+// CapAudit keeps the newest max audit rows and deletes the oldest beyond that,
+// so the append-only log cannot grow without bound (AGENTS.md §7 retention). It
+// returns how many rows it removed. Nothing in the audit log is ever edited.
+func (s *Store) CapAudit(ctx context.Context, max int) (int, error) {
+	if max <= 0 {
+		return 0, nil
+	}
+	res, err := s.db.ExecContext(ctx, `
+		DELETE FROM audit WHERE id NOT IN (
+			SELECT id FROM audit ORDER BY id DESC LIMIT ?
+		)`, max)
+	if err != nil {
+		return 0, fmt.Errorf("cap audit: %w", err)
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return 0, fmt.Errorf("cap audit: %w", err)
+	}
+	return int(n), nil
+}
+
 // TelemetryForDay returns one device's usage rows for a day, biggest first.
 func (s *Store) TelemetryForDay(ctx context.Context, id int64, day string) ([]TelemetryEntry, error) {
 	rows, err := s.db.QueryContext(ctx, `
--- a/internal/store/reports_test.go
+++ b/internal/store/reports_test.go
@@ -185,3 +185,137 @@ func TestDeviceListsAndTopicLookup(t *testing.T) {
 		t.Errorf("unknown topic error = %v, want ErrNotFound", err)
 	}
 }
+
+func TestTelemetryRollupAndWeekQueries(t *testing.T) {
+	st, dev := newTestDevice(t)
+	ctx := context.Background()
+
+	// Three days of usage. Apps and domains are reported for the same day on
+	// purpose, to prove the rollup keeps them apart instead of double-counting.
+	days := []struct {
+		day     string
+		app     string
+		appSecs int
+		domain  string
+		webSecs int
+	}{
+		{"2026-10-12", "com.spotify.music", 600, "youtube.com", 120},
+		{"2026-10-13", "com.spotify.music", 900, "youtube.com", 300},
+		{"2026-10-13", "com.roblox.client", 1800, "roblox.com", 1800},
+		{"2026-10-14", "com.roblox.client", 3600, "roblox.com", 3600},
+	}
+	for _, d := range days {
+		if _, err := st.InsertTelemetry(ctx, dev.ID, d.day, []TelemetryEntry{
+			{App: d.app, Seconds: d.appSecs},
+			{Domain: d.domain, Seconds: d.webSecs},
+		}); err != nil {
+			t.Fatalf("InsertTelemetry %s: %v", d.day, err)
+		}
+	}
+
+	usage, err := st.UsageRange(ctx, dev.ID, "2026-10-12", "2026-10-14")
+	if err != nil {
+		t.Fatalf("UsageRange: %v", err)
+	}
+	if len(usage) != 3 {
+		t.Fatalf("usage rows = %d, want 3", len(usage))
+	}
+	if usage[0].Day != "2026-10-12" || usage[0].AppSeconds != 600 || usage[0].DomainSeconds != 120 {
+		t.Errorf("day 1 = %+v", usage[0])
+	}
+	if usage[1].AppSeconds != 2700 || usage[1].DomainSeconds != 2100 {
+		t.Errorf("day 2 = %+v, want apps 2700 and domains 2100 (no cross-counting)", usage[1])
+	}
+	if usage[2].AppSeconds != 3600 || usage[2].DomainSeconds != 3600 {
+		t.Errorf("day 3 = %+v", usage[2])
+	}
+
+	// A repeated report replaces the day's total, and the rollup follows.
+	if _, err := st.InsertTelemetry(ctx, dev.ID, "2026-10-13", []TelemetryEntry{
+		{App: "com.roblox.client", Seconds: 200},
+	}); err != nil {
+		t.Fatalf("InsertTelemetry replace: %v", err)
+	}
+	usage, _ = st.UsageRange(ctx, dev.ID, "2026-10-13", "2026-10-13")
+	if len(usage) != 1 || usage[0].AppSeconds != 1100 {
+		t.Errorf("day 2 after replace = %+v, want apps 1100 (spotify 900 + roblox 200)", usage)
+	}
+
+	// Top lists are ordered by total seconds.
+	topApps, err := st.TopUsage(ctx, dev.ID, "2026-10-12", "2026-10-14", "app", 10)
+	if err != nil {
+		t.Fatalf("TopUsage app: %v", err)
+	}
+	if len(topApps) != 2 || topApps[0].Name != "com.roblox.client" || topApps[0].Seconds != 3800 {
+		t.Errorf("top apps = %+v, want roblox first with 3800", topApps)
+	}
+	topDomains, err := st.TopUsage(ctx, dev.ID, "2026-10-12", "2026-10-14", "domain", 10)
+	if err != nil {
+		t.Fatalf("TopUsage domain: %v", err)
+	}
+	if len(topDomains) != 2 || topDomains[0].Name != "roblox.com" || topDomains[0].Seconds != 5400 {
+		t.Errorf("top domains = %+v", topDomains)
+	}
+
+	// An unknown kind and a range with no data are both empty, not errors.
+	if got, err := st.TopUsage(ctx, dev.ID, "2026-10-12", "2026-10-14", "nonsense", 10); err != nil || len(got) != 0 {
+		t.Errorf("TopUsage nonsense = (%+v, %v), want empty", got, err)
+	}
+	if got, err := st.UsageRange(ctx, dev.ID, "2020-01-01", "2020-01-07"); err != nil || len(got) != 0 {
+		t.Errorf("UsageRange empty = (%+v, %v), want empty", got, err)
+	}
+}
+
+func TestRebuildRollupsAndCapAudit(t *testing.T) {
+	st, dev := newTestDevice(t)
+	ctx := context.Background()
+
+	// Simulate pre-Phase-7 data: telemetry written before the rollup existed.
+	if _, err := st.DB().ExecContext(ctx,
+		`INSERT INTO telemetry (device_id, app, domain, seconds, day) VALUES (?, '', 'example.com', 300, '2026-10-13')`,
+		dev.ID); err != nil {
+		t.Fatalf("seed telemetry: %v", err)
+	}
+	var count int
+	if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM telemetry_daily`).Scan(&count); err != nil {
+		t.Fatalf("count rollups: %v", err)
+	}
+	if count != 0 {
+		t.Fatalf("telemetry_daily has %d rows before rebuild, want 0", count)
+	}
+	n, err := st.RebuildRollups(ctx)
+	if err != nil {
+		t.Fatalf("RebuildRollups: %v", err)
+	}
+	if n != 1 {
+		t.Errorf("RebuildRollups = %d rows, want 1", n)
+	}
+	usage, err := st.UsageRange(ctx, dev.ID, "2026-10-13", "2026-10-13")
+	if err != nil || len(usage) != 1 || usage[0].DomainSeconds != 300 {
+		t.Fatalf("usage after rebuild = (%+v, %v)", usage, err)
+	}
+
+	// The audit cap keeps the newest rows and never edits the rest.
+	for i := 0; i < 25; i++ {
+		if err := st.Audit(ctx, "sam", "test", nil, "row"); err != nil {
+			t.Fatalf("Audit: %v", err)
+		}
+	}
+	removed, err := st.CapAudit(ctx, 10)
+	if err != nil {
+		t.Fatalf("CapAudit: %v", err)
+	}
+	if removed == 0 {
+		t.Error("CapAudit removed nothing, want the oldest rows gone")
+	}
+	if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM audit`).Scan(&count); err != nil {
+		t.Fatalf("count audit: %v", err)
+	}
+	if count != 10 {
+		t.Errorf("audit rows = %d, want 10", count)
+	}
+	// A no-op cap leaves the log alone.
+	if n, err := st.CapAudit(ctx, 100); err != nil || n != 0 {
+		t.Errorf("CapAudit(100) = (%d, %v), want (0, nil)", n, err)
+	}
+}