Phase 6: rules editor + unsubscribe detection (AUD-72)

- FR-11 rules editor: plain-text area at /rules, saved and versioned
  (SaveRules appends a version; restore re-saves an old version), parse
  errors surfaced, one-click re-classify. The reply model now reads the
  rules document too (FR-11).
- FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post
  (migration v6), detect email and web links (internal/unsubscribe), a
  domain-level ignore list, and an explicit record of what was marked. No
  code path contacts an unsubscribe link; acting on one needs Sam's approval.
- Migration v6 is additive; unsubscribe gains url/updated_at.
- Tests: detector, parser, store migration/records/candidates, rules editor,
  unsubscribe handlers, message-detail rendering.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-11 22:24:38 +11:00
parent 7c66e4ad31
commit 00bb0e05c4
24 changed files with 2121 additions and 42 deletions

View File

@@ -159,3 +159,51 @@ func (s *Store) SaveRules(ctx context.Context, text string) (int, error) {
}
return next, nil
}
// RuleVersion is one saved rules revision, for the version list in the editor
// (FR-11).
type RuleVersion struct {
ID int64
Version int
Text string
CreatedAt string
}
// RulesVersions returns saved rules revisions, newest first.
func (s *Store) RulesVersions(ctx context.Context, limit int) ([]RuleVersion, error) {
if limit < 1 {
limit = 20
}
rows, err := s.db.QueryContext(ctx,
`SELECT id, version, text, created_at FROM rules ORDER BY version DESC LIMIT ?`, limit)
if err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
defer rows.Close()
var out []RuleVersion
for rows.Next() {
var v RuleVersion
if err := rows.Scan(&v.ID, &v.Version, &v.Text, &v.CreatedAt); err != nil {
return nil, fmt.Errorf("scan rules version: %w", err)
}
out = append(out, v)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
return out, nil
}
// RuleByVersion returns the text of one saved rules revision. ok is false when
// the version does not exist.
func (s *Store) RuleByVersion(ctx context.Context, version int) (string, bool, error) {
var text string
err := s.db.QueryRowContext(ctx, `SELECT text FROM rules WHERE version = ?`, version).Scan(&text)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, fmt.Errorf("read rules version %d: %w", version, err)
}
return text, true, nil
}

View File

@@ -20,15 +20,19 @@ type Message struct {
// MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply
// uses it for In-Reply-To and References so Gmail keeps the thread.
MessageIDHeader string
FromAddr string
ToAddr string
Subject string
BodyText string
SourceAddress string
DateSent time.Time
ReceivedAt time.Time
IMAPUID int64
IMAPUIDValidity int64
// ListUnsubscribe and ListUnsubscribePost are the raw RFC 2369/8058 headers
// (Phase 6, FR-7). Detection runs over them on demand.
ListUnsubscribe string
ListUnsubscribePost string
FromAddr string
ToAddr string
Subject string
BodyText string
SourceAddress string
DateSent time.Time
ReceivedAt time.Time
IMAPUID int64
IMAPUIDValidity int64
// Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9):
// inbox, delete, saved, or replied.
Folder string
@@ -71,19 +75,22 @@ func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) {
const q = `
INSERT INTO messages (
gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text,
source_address, date_sent, received_at, imap_uid, imap_uidvalidity
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
source_address, list_unsubscribe, list_unsubscribe_post, date_sent, received_at,
imap_uid, imap_uidvalidity
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (gmail_id) DO UPDATE SET
thread_id = excluded.thread_id,
message_id_header = excluded.message_id_header,
from_addr = excluded.from_addr,
to_addr = excluded.to_addr,
subject = excluded.subject,
body_text = excluded.body_text,
source_address = excluded.source_address,
date_sent = excluded.date_sent,
imap_uid = excluded.imap_uid,
imap_uidvalidity = excluded.imap_uidvalidity`
thread_id = excluded.thread_id,
message_id_header = excluded.message_id_header,
from_addr = excluded.from_addr,
to_addr = excluded.to_addr,
subject = excluded.subject,
body_text = excluded.body_text,
source_address = excluded.source_address,
list_unsubscribe = excluded.list_unsubscribe,
list_unsubscribe_post = excluded.list_unsubscribe_post,
date_sent = excluded.date_sent,
imap_uid = excluded.imap_uid,
imap_uidvalidity = excluded.imap_uidvalidity`
receivedAt := m.ReceivedAt
if receivedAt.IsZero() {
@@ -98,6 +105,8 @@ ON CONFLICT (gmail_id) DO UPDATE SET
nullString(m.Subject),
nullString(m.BodyText),
nullString(m.SourceAddress),
nullString(m.ListUnsubscribe),
nullString(m.ListUnsubscribePost),
nullTime(m.DateSent),
receivedAt.UTC().Format(sqlTimeLayout),
m.IMAPUID,
@@ -124,7 +133,8 @@ SELECT m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''),
COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at,
COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0),
COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'),
COALESCE(m.message_id_header, '')
COALESCE(m.message_id_header, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, '')
FROM messages m WHERE m.id = ?`
var (
m Message
@@ -135,6 +145,7 @@ FROM messages m WHERE m.id = ?`
&m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject,
&m.BodyText, &m.SourceAddress, &dateSent, &received,
&m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader,
&m.ListUnsubscribe, &m.ListUnsubscribePost,
)
if err != nil {
return Message{}, fmt.Errorf("load message %d: %w", id, err)

View File

@@ -21,6 +21,10 @@ CREATE TABLE IF NOT EXISTS messages (
-- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and
-- References to it so Gmail files the reply in the same thread.
message_id_header TEXT,
-- Phase 6 (FR-7): the raw RFC 2369/8058 unsubscribe headers, stored so the
-- detector can run without a refetch.
list_unsubscribe TEXT,
list_unsubscribe_post TEXT,
from_addr TEXT,
to_addr TEXT,
subject TEXT,
@@ -94,11 +98,16 @@ CREATE TABLE IF NOT EXISTS actions_log (
CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id);
CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at);
-- FR-7: unsubscribe tracker and ignore list.
-- FR-7: unsubscribe tracker and ignore list. `status` is 'ignored' for a domain
-- on the ignore list, or 'done' for a domain Sam has recorded as unsubscribed.
-- The app never follows a link itself: it only marks and records (AUD-72).
CREATE TABLE IF NOT EXISTS unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
-- url records the link that was detected or marked (Phase 6).
url TEXT NOT NULL DEFAULT '',
updated_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);

View File

@@ -159,6 +159,25 @@ func (s *Store) applyMigrations(ctx context.Context) error {
return fmt.Errorf("record schema version 5: %w", err)
}
}
if version < 6 {
// Phase 6 (FR-7): store the raw List-Unsubscribe headers so detection
// runs without a refetch, and extend the unsubscribe tracker with the
// link it recorded and an update time.
for _, col := range []struct{ table, name, typ string }{
{"messages", "list_unsubscribe", "TEXT"},
{"messages", "list_unsubscribe_post", "TEXT"},
{"unsubscribe", "url", "TEXT NOT NULL DEFAULT ''"},
{"unsubscribe", "updated_at", "TEXT"},
} {
if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil {
return err
}
}
if _, err := s.db.ExecContext(ctx,
`INSERT OR IGNORE INTO schema_migrations (version) VALUES (6)`); err != nil {
return fmt.Errorf("record schema version 6: %w", err)
}
}
return nil
}

View File

@@ -0,0 +1,177 @@
// Unsubscribe tracking for Phase 6 (FR-7).
//
// Two things are persisted here:
//
// - the raw unsubscribe links detected on a message (messages.list_unsubscribe,
// stored by the mail fetch and parsed by internal/unsubscribe); and
// - the domain-level ignore list and the record of what was marked, in the
// `unsubscribe` table.
//
// The app never follows a link. Marking a domain records Sam's decision; the
// actual unsubscribe stays a manual, explicitly approved action (AUD-72).
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
)
// Unsubscribe statuses. `ignored` is the ignore list; `done` records a domain
// Sam has decided to unsubscribe from.
const (
UnsubscribeStatusDone = "done"
UnsubscribeStatusIgnored = "ignored"
)
// ErrUnsubscribeStatus is returned for an unknown unsubscribe status.
var ErrUnsubscribeStatus = errors.New("unsubscribe status must be done or ignored")
// UnsubscribeRecord is one row of the domain-level tracker.
type UnsubscribeRecord struct {
ID int64
Domain string
Status string
URL string
CreatedAt string
UpdatedAt string
}
// ListUnsubscribe returns every domain record, newest first.
func (s *Store) ListUnsubscribe(ctx context.Context) ([]UnsubscribeRecord, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe ORDER BY COALESCE(updated_at, created_at) DESC, id DESC`)
if err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
defer rows.Close()
var out []UnsubscribeRecord
for rows.Next() {
var r UnsubscribeRecord
if err := rows.Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan unsubscribe record: %w", err)
}
out = append(out, r)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
return out, nil
}
// UnsubscribeByDomain loads one domain record. ok is false when the domain is
// not tracked.
func (s *Store) UnsubscribeByDomain(ctx context.Context, domain string) (UnsubscribeRecord, bool, error) {
domain = strings.TrimSpace(domain)
if domain == "" {
return UnsubscribeRecord{}, false, nil
}
var r UnsubscribeRecord
err := s.db.QueryRowContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe WHERE domain = ?`, domain).
Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return UnsubscribeRecord{}, false, nil
}
if err != nil {
return UnsubscribeRecord{}, false, fmt.Errorf("read unsubscribe record %q: %w", domain, err)
}
return r, true, nil
}
// SetUnsubscribeStatus upserts the domain record. An empty url keeps the stored
// one, so an "ignore" does not wipe a link recorded earlier.
func (s *Store) SetUnsubscribeStatus(ctx context.Context, domain, status, url string) error {
domain = strings.TrimSpace(domain)
if domain == "" {
return errors.New("unsubscribe domain must not be empty")
}
if status != UnsubscribeStatusDone && status != UnsubscribeStatusIgnored {
return ErrUnsubscribeStatus
}
const q = `
INSERT INTO unsubscribe (domain, status, url, updated_at)
VALUES (?, ?, ?, datetime('now'))
ON CONFLICT (domain) DO UPDATE SET
status = excluded.status,
url = CASE WHEN excluded.url <> '' THEN excluded.url ELSE unsubscribe.url END,
updated_at = datetime('now')`
if _, err := s.db.ExecContext(ctx, q, domain, status, strings.TrimSpace(url)); err != nil {
return fmt.Errorf("set unsubscribe status for %q: %w", domain, err)
}
return nil
}
// ClearUnsubscribe removes a domain record, taking it off the ignore list.
func (s *Store) ClearUnsubscribe(ctx context.Context, domain string) error {
if _, err := s.db.ExecContext(ctx,
`DELETE FROM unsubscribe WHERE domain = ?`, strings.TrimSpace(domain)); err != nil {
return fmt.Errorf("clear unsubscribe record %q: %w", domain, err)
}
return nil
}
// UnsubscribeCandidate is a message that may carry an unsubscribe link.
type UnsubscribeCandidate struct {
MessageID int64
FromAddr string
Subject string
SourceAddress string
BodyText string
ListUnsubscribe string
ListUnsubscribePost string
Category string
ReceivedAt time.Time
}
// UnsubscribeCandidates returns messages with a List-Unsubscribe header or a
// body that mentions unsubscribing, newest first. The caller runs the detector
// over them, because body parsing lives in internal/unsubscribe.
func (s *Store) UnsubscribeCandidates(ctx context.Context, limit int) ([]UnsubscribeCandidate, error) {
if limit < 1 {
limit = 200
}
const q = `
SELECT m.id, COALESCE(m.from_addr, ''), COALESCE(m.subject, ''),
COALESCE(m.source_address, ''), COALESCE(m.body_text, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, ''),
COALESCE(c.category, ''), m.received_at
FROM messages m
LEFT JOIN classifications c ON c.id = (
SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1
)
WHERE (m.list_unsubscribe IS NOT NULL AND m.list_unsubscribe <> '')
OR instr(lower(COALESCE(m.body_text, '')), 'unsubscribe') > 0
ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC
LIMIT ?`
rows, err := s.db.QueryContext(ctx, q, limit)
if err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
defer rows.Close()
var out []UnsubscribeCandidate
for rows.Next() {
var (
c UnsubscribeCandidate
received string
)
if err := rows.Scan(
&c.MessageID, &c.FromAddr, &c.Subject, &c.SourceAddress,
&c.BodyText, &c.ListUnsubscribe, &c.ListUnsubscribePost,
&c.Category, &received,
); err != nil {
return nil, fmt.Errorf("scan unsubscribe candidate: %w", err)
}
c.ReceivedAt = parseSQLTime(received)
out = append(out, c)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
return out, nil
}

View File

@@ -0,0 +1,193 @@
package store
import (
"context"
"database/sql"
"path/filepath"
"testing"
)
func TestMigrationV6AddsUnsubscribeColumns(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "old.db")
// Simulate a v5 database: messages has no List-Unsubscribe columns and the
// unsubscribe table has no url/updated_at. Opening it must apply migration
// v6.
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open old db: %v", err)
}
stmts := []string{
`CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
gmail_id TEXT NOT NULL UNIQUE,
thread_id TEXT, message_id_header TEXT, from_addr TEXT, to_addr TEXT,
subject TEXT, body_text TEXT, source_address TEXT, date_sent TEXT,
imap_uid INTEGER, imap_uidvalidity INTEGER,
folder TEXT NOT NULL DEFAULT 'inbox',
mailbox TEXT NOT NULL DEFAULT 'INBOX',
received_at TEXT NOT NULL DEFAULT (datetime('now')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`INSERT INTO schema_migrations (version) VALUES (5)`,
}
for _, q := range stmts {
if _, err := db.ExecContext(ctx, q); err != nil {
t.Fatalf("seed old db: %v", err)
}
}
_ = db.Close()
s, err := Open(ctx, path)
if err != nil {
t.Fatalf("Open() on v5 database error = %v", err)
}
defer s.Close()
v, err := s.SchemaVersion(ctx)
if err != nil {
t.Fatalf("SchemaVersion() error = %v", err)
}
if v < 6 {
t.Errorf("SchemaVersion() = %d, want >= 6", v)
}
for _, tc := range []struct{ table, col string }{
{"messages", "list_unsubscribe"},
{"messages", "list_unsubscribe_post"},
{"unsubscribe", "url"},
{"unsubscribe", "updated_at"},
} {
var n int
q := `SELECT COUNT(*) FROM pragma_table_info('` + tc.table + `') WHERE name = ?`
if err := s.DB().QueryRowContext(ctx, q, tc.col).Scan(&n); err != nil {
t.Fatalf("count %s.%s: %v", tc.table, tc.col, err)
}
if n != 1 {
t.Errorf("%s.%s missing after migration v6", tc.table, tc.col)
}
}
}
func TestUnsubscribeRecordUpsert(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t) // reuses the store helper from categories_test.go
if _, ok, err := s.UnsubscribeByDomain(ctx, "brand.example"); err != nil || ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a miss", ok, err)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusIgnored, "https://brand.example/u"); err != nil {
t.Fatalf("SetUnsubscribeStatus(ignore) error = %v", err)
}
rec, ok, err := s.UnsubscribeByDomain(ctx, "brand.example")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec.Status != UnsubscribeStatusIgnored || rec.URL != "https://brand.example/u" {
t.Errorf("record = %+v, want ignored with url", rec)
}
// A later mark with no URL must keep the recorded link.
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusDone, ""); err != nil {
t.Fatalf("SetUnsubscribeStatus(done) error = %v", err)
}
rec, _, _ = s.UnsubscribeByDomain(ctx, "brand.example")
if rec.Status != UnsubscribeStatusDone {
t.Errorf("status = %q, want done", rec.Status)
}
if rec.URL != "https://brand.example/u" {
t.Errorf("url = %q, want the earlier link preserved", rec.URL)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", "bogus", ""); err == nil {
t.Error("SetUnsubscribeStatus(bogus) accepted an unknown status")
}
records, err := s.ListUnsubscribe(ctx)
if err != nil || len(records) != 1 {
t.Fatalf("ListUnsubscribe() = (%v, %v), want one record", records, err)
}
if err := s.ClearUnsubscribe(ctx, "brand.example"); err != nil {
t.Fatalf("ClearUnsubscribe() error = %v", err)
}
if err := s.SetUnsubscribeStatus(ctx, "", UnsubscribeStatusIgnored, ""); err == nil {
t.Error("SetUnsubscribeStatus accepted an empty domain")
}
}
func TestUnsubscribeCandidates(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
seed := func(gmailID, from, body, header string) {
t.Helper()
if _, err := s.UpsertMessage(ctx, &Message{
GmailID: gmailID,
FromAddr: from,
Subject: "subject " + gmailID,
BodyText: body,
ListUnsubscribe: header,
}); err != nil {
t.Fatalf("seed %s: %v", gmailID, err)
}
}
seed("header-msg", "Support <support@brand.example>", "hello", "<https://brand.example/u>")
seed("body-msg", "Shop <deals@shop.example>", "You can unsubscribe at https://shop.example/opt-out now.", "")
seed("plain-msg", "Friend <friend@example.com>", "Just saying hi.", "")
got, err := s.UnsubscribeCandidates(ctx, 50)
if err != nil {
t.Fatalf("UnsubscribeCandidates() error = %v", err)
}
if len(got) != 2 {
t.Fatalf("candidates = %d, want 2 (%+v)", len(got), got)
}
ids := map[string]bool{}
for _, c := range got {
ids[c.Subject] = true
}
if !ids["subject body-msg"] || !ids["subject header-msg"] {
t.Errorf("candidates = %+v, want the header and body messages", ids)
}
}
func TestRulesVersionsAndRestore(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
if _, err := s.SaveRules(ctx, "from contains \"a\" => category=Info"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
if _, err := s.SaveRules(ctx, "from contains \"b\" => category=Action"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
versions, err := s.RulesVersions(ctx, 10)
if err != nil {
t.Fatalf("RulesVersions() error = %v", err)
}
if len(versions) < 3 {
t.Fatalf("RulesVersions() = %d, want >= 3 (seed + 2 saves)", len(versions))
}
if versions[0].Version != 3 {
t.Errorf("newest version = %d, want 3", versions[0].Version)
}
text, ok, err := s.RuleByVersion(ctx, 2)
if err != nil || !ok {
t.Fatalf("RuleByVersion(2) = (_, %v, %v), want a hit", ok, err)
}
if text != "from contains \"a\" => category=Info" {
t.Errorf("RuleByVersion(2) = %q", text)
}
if _, ok, _ := s.RuleByVersion(ctx, 999); ok {
t.Error("RuleByVersion(999) reported a hit")
}
}