Phase 3: MQTT transport — retained config, status/telemetry, LWT presence, TLS/ACL design

- internal/mqtt: publish each device's desired state retained on restrict/<device>/config; subscribe to restrict/+/status and restrict/+/telemetry and write reports to SQLite; publish on every state change, republish on connect and every 15 minutes from SQLite, so the broker is repaired from the source of truth. - store: presence (devices.online and last_seen, audited only on transitions), telemetry upsert per (device, day, app, domain) via migration 002, allow and block lists read into the config payload, device lookup by topic. - config: MQTT settings. TLS plus per-device credentials are required; the pre-TLS shared broker needs an explicit CONTROL_MQTT_INSECURE escape hatch. - web: republish config after lock, unlock, expiry, create and edit; broker card on the status partial. - deploy/mosquitto: TLS listener, topic ACL, credential provisioning and a verifier that proves the policy on a throwaway broker. Not applied to the shared Mosquitto on .13 — that change needs Sam's approval. - cmd/control: mqtt-clients prints the MQTT identity of every device. - paho.mqtt.golang v1.5.1, which pulls golang.org/x/net and with it the golang.org/x/crypto bump to v0.42.0. Verified on .13: retained config delivered to a fresh subscriber; a client killed without DISCONNECT drove the broker's retained will and the service flipped the device offline; telemetry landed in SQLite; make mqtt-verify passed 13/13; the service connected over TLS with credentials and a CA. Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-13 14:37:43 +11:00
parent 7c88d3f52b
commit 14b304e0b0
31 changed files with 2624 additions and 47 deletions
--- a/internal/store/lists.go
+++ b/internal/store/lists.go
@@ -0,0 +1,74 @@
+// Per-device allow and block lists, and device lookup by MQTT topic
+// (AGENTS.md §7 blocklist/allowlist, §11 Phase 3).
+//
+// The editors for these lists arrive in Phase 6. Phase 3 already publishes them
+// in the device config, so a device that is onboarded later gets the real lists
+// without another protocol change. Migration 001 created both tables.
+package store
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// DeviceLists returns the site/app values that stay allowed and the values that
+// are blocked for one device. A row with a NULL device_id applies to every
+// device (§7). Rows are returned in insertion order, de-duplicated.
+func (s *Store) DeviceLists(ctx context.Context, id int64) (allow, block []string, err error) {
+	if allow, err = s.listValues(ctx, "allowlist", id); err != nil {
+		return nil, nil, err
+	}
+	if block, err = s.listValues(ctx, "blocklist", id); err != nil {
+		return nil, nil, err
+	}
+	return allow, block, nil
+}
+
+func (s *Store) listValues(ctx context.Context, table string, id int64) ([]string, error) {
+	// The table name is a package constant, never caller input.
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT value FROM `+table+` WHERE device_id = ? OR device_id IS NULL ORDER BY id`, id)
+	if err != nil {
+		return nil, fmt.Errorf("list %s: %w", table, err)
+	}
+	defer rows.Close()
+
+	out := make([]string, 0, 4)
+	seen := make(map[string]struct{}, 4)
+	for rows.Next() {
+		var v string
+		if err := rows.Scan(&v); err != nil {
+			return nil, fmt.Errorf("scan %s: %w", table, err)
+		}
+		v = strings.TrimSpace(v)
+		if v == "" {
+			continue
+		}
+		if _, dup := seen[v]; dup {
+			continue
+		}
+		seen[v] = struct{}{}
+		out = append(out, v)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("list %s: %w", table, err)
+	}
+	return out, nil
+}
+
+// DeviceByConfigTopic finds a device from a topic the device wrote to, for
+// example "restrict/harry_phone_2/status". Only the Control Service reads the
+// topic table, so a topic the service does not know is ignored.
+func (s *Store) DeviceByConfigTopic(ctx context.Context, topic string) (*Device, error) {
+	dev, err := scanDevice(s.db.QueryRowContext(ctx, selectDevice+` WHERE d.topic = ?`, topic))
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, ErrNotFound
+	}
+	if err != nil {
+		return nil, fmt.Errorf("device by topic: %w", err)
+	}
+	return dev, nil
+}
--- a/internal/store/migrations/002_telemetry_unique.sql
+++ b/internal/store/migrations/002_telemetry_unique.sql
@@ -0,0 +1,13 @@
+-- Migration 002 — Phase 3. Telemetry reports arrive over MQTT and repeat.
+--
+-- A device reports a rolling daily total ("Spotify: 600 s today"), not a delta,
+-- so one row per (device, day, app, domain) is the correct shape: a repeat
+-- report replaces the previous total instead of adding to it. Devices may omit
+-- the app or the domain, so both default to '' — SQLite treats NULLs in a
+-- UNIQUE index as distinct, which would defeat the upsert.
+
+UPDATE telemetry SET app = '' WHERE app IS NULL;
+UPDATE telemetry SET domain = '' WHERE domain IS NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_telemetry_daily
+    ON telemetry(device_id, day, app, domain);
--- a/internal/store/reports.go
+++ b/internal/store/reports.go
@@ -0,0 +1,131 @@
+// Reports that arrive from devices over MQTT: presence and telemetry
+// (AGENTS.md §7, §11 Phase 3).
+//
+// The broker is transport only. Every report is written to SQLite here, because
+// SQLite is the source of truth (SYSTEM_MAP.md). A malformed or hostile report
+// must never take the service down: callers validate first, and a failed insert
+// is logged and dropped.
+package store
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// SetPresence records that a device was seen and whether it is online now. It
+// returns whether the online flag changed.
+//
+// Presence comes from the broker's Last Will and Testament: a device publishes
+// a retained "online" status, and the broker publishes its retained will
+// ("online": false) when the connection drops without a clean disconnect.
+//
+// last_seen is refreshed on every report. The audit log only gets an entry when
+// the online flag changes, so a chatty device does not flood the log (FR-15).
+func (s *Store) SetPresence(ctx context.Context, id int64, online bool, at time.Time, actor string) (bool, error) {
+	if at.IsZero() {
+		at = time.Now()
+	}
+	at = at.UTC()
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return false, fmt.Errorf("set presence: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var current int
+	if err := tx.QueryRowContext(ctx, `SELECT online FROM devices WHERE id = ?`, id).Scan(&current); err != nil {
+		return false, fmt.Errorf("set presence: %w", err)
+	}
+
+	if _, err := tx.ExecContext(ctx,
+		`UPDATE devices SET online = ?, last_seen = ?, updated_at = datetime('now') WHERE id = ?`,
+		boolToInt(online), at.Format(time.RFC3339), id); err != nil {
+		return false, fmt.Errorf("set presence: %w", err)
+	}
+
+	changed := (current != 0) != online
+	if changed {
+		action, detail := "device.offline", "last will received"
+		if online {
+			action, detail = "device.online", "status report"
+		}
+		if err := insertAudit(ctx, tx, actor, action, &id, detail); err != nil {
+			return false, err
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		return false, fmt.Errorf("commit presence: %w", err)
+	}
+	return changed, nil
+}
+
+// TelemetryEntry is one app or domain row of a usage report.
+type TelemetryEntry struct {
+	App     string
+	Domain  string
+	Seconds int
+}
+
+// InsertTelemetry upserts a rolling daily usage total per (device, day, app,
+// domain). Reports are repeats, not deltas: a later report for the same key
+// replaces the earlier total (migration 002). It returns the number of rows
+// written.
+func (s *Store) InsertTelemetry(ctx context.Context, id int64, day string, entries []TelemetryEntry) (int, error) {
+	if len(entries) == 0 {
+		return 0, nil
+	}
+
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return 0, fmt.Errorf("insert telemetry: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	n := 0
+	for _, e := range entries {
+		if e.Seconds < 0 {
+			continue
+		}
+		if _, err := tx.ExecContext(ctx, `
+			INSERT INTO telemetry (device_id, app, domain, seconds, day)
+			VALUES (?, ?, ?, ?, ?)
+			ON CONFLICT(device_id, day, app, domain) DO UPDATE SET
+				seconds = excluded.seconds,
+				created_at = datetime('now')`,
+			id, e.App, e.Domain, e.Seconds, day); err != nil {
+			return n, fmt.Errorf("insert telemetry: %w", err)
+		}
+		n++
+	}
+	if err := tx.Commit(); err != nil {
+		return n, fmt.Errorf("commit telemetry: %w", err)
+	}
+	return n, nil
+}
+
+// TelemetryForDay returns one device's usage rows for a day, biggest first.
+func (s *Store) TelemetryForDay(ctx context.Context, id int64, day string) ([]TelemetryEntry, error) {
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT app, domain, seconds FROM telemetry
+		WHERE device_id = ? AND day = ?
+		ORDER BY seconds DESC`, id, day)
+	if err != nil {
+		return nil, fmt.Errorf("telemetry for day: %w", err)
+	}
+	defer rows.Close()
+
+	out := make([]TelemetryEntry, 0, 8)
+	for rows.Next() {
+		var e TelemetryEntry
+		if err := rows.Scan(&e.App, &e.Domain, &e.Seconds); err != nil {
+			return nil, fmt.Errorf("scan telemetry: %w", err)
+		}
+		out = append(out, e)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("telemetry for day: %w", err)
+	}
+	return out, nil
+}
--- a/internal/store/reports_test.go
+++ b/internal/store/reports_test.go
@@ -0,0 +1,187 @@
+package store
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func newTestDevice(t *testing.T) (*Store, *Device) {
+	t.Helper()
+	st := testStore(t)
+	dev, err := st.CreateDevice(t.Context(), "Harry's phone", "Harry", PlatformAndroid, "sam")
+	if err != nil {
+		t.Fatalf("CreateDevice: %v", err)
+	}
+	return st, dev
+}
+
+func TestSetPresenceAuditsOnlyTransitions(t *testing.T) {
+	st, dev := newTestDevice(t)
+	ctx := context.Background()
+
+	// The first "online" flips the flag, so it is audited.
+	changed, err := st.SetPresence(ctx, dev.ID, true, time.Now(), "device")
+	if err != nil {
+		t.Fatalf("SetPresence: %v", err)
+	}
+	if !changed {
+		t.Error("changed = false on the first online report, want true")
+	}
+
+	// A repeat report only refreshes last_seen.
+	changed, err = st.SetPresence(ctx, dev.ID, true, time.Now().Add(time.Minute), "device")
+	if err != nil {
+		t.Fatalf("SetPresence: %v", err)
+	}
+	if changed {
+		t.Error("changed = true on a repeat report, want false")
+	}
+
+	// The will payload flips it back.
+	changed, err = st.SetPresence(ctx, dev.ID, false, time.Now(), "device")
+	if err != nil {
+		t.Fatalf("SetPresence: %v", err)
+	}
+	if !changed {
+		t.Error("changed = false when the device went offline, want true")
+	}
+
+	got, err := st.GetDevice(ctx, dev.ID)
+	if err != nil {
+		t.Fatalf("GetDevice: %v", err)
+	}
+	if got.Online {
+		t.Error("Online = true after the will, want false")
+	}
+	if got.LastSeen == nil {
+		t.Fatal("LastSeen is nil, want the report time")
+	}
+
+	audit, err := st.ListAudit(ctx, 50)
+	if err != nil {
+		t.Fatalf("ListAudit: %v", err)
+	}
+	var online, offline int
+	for _, e := range audit {
+		switch e.Action {
+		case "device.online":
+			online++
+		case "device.offline":
+			offline++
+		}
+	}
+	if online != 1 || offline != 1 {
+		t.Errorf("audit has %d online and %d offline entries, want 1 and 1", online, offline)
+	}
+}
+
+func TestInsertTelemetryReplacesTheDailyTotal(t *testing.T) {
+	st, dev := newTestDevice(t)
+	ctx := context.Background()
+	const day = "2026-10-13"
+
+	n, err := st.InsertTelemetry(ctx, dev.ID, day, []TelemetryEntry{
+		{App: "com.spotify.music", Seconds: 600},
+		{Domain: "youtube.com", Seconds: 120},
+	})
+	if err != nil {
+		t.Fatalf("InsertTelemetry: %v", err)
+	}
+	if n != 2 {
+		t.Errorf("rows = %d, want 2", n)
+	}
+
+	// Devices report rolling totals, so a repeat report replaces the row.
+	if _, err := st.InsertTelemetry(ctx, dev.ID, day, []TelemetryEntry{
+		{App: "com.spotify.music", Seconds: 900},
+	}); err != nil {
+		t.Fatalf("InsertTelemetry repeat: %v", err)
+	}
+
+	rows, err := st.TelemetryForDay(ctx, dev.ID, day)
+	if err != nil {
+		t.Fatalf("TelemetryForDay: %v", err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("rows = %+v, want 2 (a replace, not an insert)", rows)
+	}
+	if rows[0].App != "com.spotify.music" || rows[0].Seconds != 900 {
+		t.Errorf("rows[0] = %+v, want the replaced total 900", rows[0])
+	}
+	if rows[1].Seconds != 120 {
+		t.Errorf("rows[1] = %+v", rows[1])
+	}
+
+	// Another day is a different row.
+	if _, err := st.InsertTelemetry(ctx, dev.ID, "2026-10-14", []TelemetryEntry{
+		{App: "com.spotify.music", Seconds: 60},
+	}); err != nil {
+		t.Fatalf("InsertTelemetry next day: %v", err)
+	}
+	rows, err = st.TelemetryForDay(ctx, dev.ID, day)
+	if err != nil {
+		t.Fatalf("TelemetryForDay: %v", err)
+	}
+	if len(rows) != 2 || rows[0].Seconds != 900 {
+		t.Errorf("rows for %s = %+v, want the day isolated", day, rows)
+	}
+
+	// A negative total is dropped, and an empty report is a no-op.
+	if n, err := st.InsertTelemetry(ctx, dev.ID, day, []TelemetryEntry{{App: "x", Seconds: -5}}); err != nil || n != 0 {
+		t.Errorf("negative telemetry = (%d, %v), want (0, nil)", n, err)
+	}
+	if n, err := st.InsertTelemetry(ctx, dev.ID, day, nil); err != nil || n != 0 {
+		t.Errorf("empty telemetry = (%d, %v), want (0, nil)", n, err)
+	}
+}
+
+func TestDeviceListsAndTopicLookup(t *testing.T) {
+	st, dev := newTestDevice(t)
+	ctx := context.Background()
+
+	// The Phase 6 editors do not exist yet, so seed the tables directly.
+	seed := []struct {
+		table string
+		kind  string
+		value string
+	}{
+		{"allowlist", "app", "com.spotify.music"},
+		{"allowlist", "site", "school.example.edu"},
+		{"blocklist", "app", "com.roblox.client"},
+	}
+	for _, s := range seed {
+		if _, err := st.DB().ExecContext(ctx,
+			`INSERT INTO `+s.table+` (device_id, kind, value) VALUES (?, ?, ?)`,
+			dev.ID, s.kind, s.value); err != nil {
+			t.Fatalf("seed %s: %v", s.table, err)
+		}
+	}
+	// A NULL device_id applies to every device, and duplicates collapse.
+	if _, err := st.DB().ExecContext(ctx,
+		`INSERT INTO allowlist (device_id, kind, value) VALUES (NULL, 'app', 'com.spotify.music')`); err != nil {
+		t.Fatalf("seed global allow: %v", err)
+	}
+
+	allow, block, err := st.DeviceLists(ctx, dev.ID)
+	if err != nil {
+		t.Fatalf("DeviceLists: %v", err)
+	}
+	if len(allow) != 2 || allow[0] != "com.spotify.music" || allow[1] != "school.example.edu" {
+		t.Errorf("allow = %v", allow)
+	}
+	if len(block) != 1 || block[0] != "com.roblox.client" {
+		t.Errorf("block = %v", block)
+	}
+
+	found, err := st.DeviceByConfigTopic(ctx, dev.Topic)
+	if err != nil {
+		t.Fatalf("DeviceByConfigTopic: %v", err)
+	}
+	if found.ID != dev.ID {
+		t.Errorf("DeviceByConfigTopic = %d, want %d", found.ID, dev.ID)
+	}
+	if _, err := st.DeviceByConfigTopic(ctx, "restrict/nope_99/config"); err != ErrNotFound {
+		t.Errorf("unknown topic error = %v, want ErrNotFound", err)
+	}
+}