Phase 2: Gmail IMAP fetch (IPv4), FR-2 source address, real mail in dashboard (AUD-67)

- internal/mailfetch: stdlib-only read-only IMAP client (LOGIN, SELECT, UID SEARCH, UID FETCH) that always dials tcp4; IPv6 to Gmail is broken from .13 and caused a socket EOF. - MIME parsing to text, RFC2047 header decoding, FR-2 source address: X-Pm-Original-To -> To -> Received 'for <addr>' -> Return-Path, never Delivered-To. The matched header is recorded for audit. - store: Message upsert/list/count + LogAction; schema migration v2 adds imap_uid and imap_uidvalidity. - server: dashboard and /partials/inbox render real mail grouped by category with a recommended action; POST /actions/fetch runs a scan (CSRF-checked); nav counts; 'Check mail' button. - config: GMAIL_USER, GMAIL_APP_PASSWORD, IMAP host/port/lookback/max. - cmd: -fetch runs one scan and exits (n8n/CLI entry point, FR-1). - compose/.env.example expose the Gmail + IMAP settings. - tests: FR-2 precedence, MIME bodies, IMAP literal/parser, store upsert/migration, inbox rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-11 20:58:32 +11:00
parent e846400338
commit 1d88f962f6
23 changed files with 2067 additions and 52 deletions
--- a/internal/store/messages.go
+++ b/internal/store/messages.go
@@ -0,0 +1,188 @@
+// Message persistence for the Email Agent (AGENTS.md §7).
+//
+// Phase 2 stores the metadata, body text, and — critically for FR-2 — the
+// source address the mail was forwarded to (X-Pm-Original-To and its
+// fallbacks). Classification columns arrive with Phase 3.
+package store
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Message is one row of the messages table.
+type Message struct {
+	ID              int64
+	GmailID         string
+	ThreadID        string
+	FromAddr        string
+	ToAddr          string
+	Subject         string
+	BodyText        string
+	SourceAddress   string
+	DateSent        time.Time
+	ReceivedAt      time.Time
+	IMAPUID         int64
+	IMAPUIDValidity int64
+}
+
+// MessageListItem is a message plus its latest classification, when present.
+type MessageListItem struct {
+	Message
+	Category          string
+	Importance        string
+	Action            string
+	Confidence        float64
+	HasClassification bool
+}
+
+// UpsertMessage inserts a message, or refreshes the mutable fields when the
+// same Gmail id is seen again. received_at is preserved on update so the
+// original arrival time is stable.
+func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) {
+	const q = `
+INSERT INTO messages (
+    gmail_id, thread_id, from_addr, to_addr, subject, body_text,
+    source_address, date_sent, received_at, imap_uid, imap_uidvalidity
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT (gmail_id) DO UPDATE SET
+    thread_id         = excluded.thread_id,
+    from_addr         = excluded.from_addr,
+    to_addr           = excluded.to_addr,
+    subject           = excluded.subject,
+    body_text         = excluded.body_text,
+    source_address    = excluded.source_address,
+    date_sent         = excluded.date_sent,
+    imap_uid          = excluded.imap_uid,
+    imap_uidvalidity  = excluded.imap_uidvalidity`
+
+	receivedAt := m.ReceivedAt
+	if receivedAt.IsZero() {
+		receivedAt = time.Now().UTC()
+	}
+	_, err := s.db.ExecContext(ctx, q,
+		m.GmailID,
+		nullString(m.ThreadID),
+		nullString(m.FromAddr),
+		nullString(m.ToAddr),
+		nullString(m.Subject),
+		nullString(m.BodyText),
+		nullString(m.SourceAddress),
+		nullTime(m.DateSent),
+		receivedAt.UTC().Format(sqlTimeLayout),
+		m.IMAPUID,
+		m.IMAPUIDValidity,
+	)
+	if err != nil {
+		return 0, fmt.Errorf("upsert message %q: %w", m.GmailID, err)
+	}
+
+	var id int64
+	if err := s.db.QueryRowContext(ctx,
+		`SELECT id FROM messages WHERE gmail_id = ?`, m.GmailID).Scan(&id); err != nil {
+		return 0, fmt.Errorf("read message id %q: %w", m.GmailID, err)
+	}
+	return id, nil
+}
+
+// ListMessages returns the most recent messages, newest first, joined to their
+// latest classification when one exists.
+func (s *Store) ListMessages(ctx context.Context, limit int) ([]MessageListItem, error) {
+	if limit < 1 {
+		limit = 100
+	}
+	const q = `
+SELECT
+    m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''),
+    COALESCE(m.to_addr, ''), COALESCE(m.subject, ''), COALESCE(m.body_text, ''),
+    COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at,
+    COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0),
+    COALESCE(c.category, ''), COALESCE(c.importance, ''),
+    COALESCE(c.action, ''), COALESCE(c.confidence, 0)
+FROM messages m
+LEFT JOIN classifications c ON c.id = (
+    SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1
+)
+ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC
+LIMIT ?`
+
+	rows, err := s.db.QueryContext(ctx, q, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list messages: %w", err)
+	}
+	defer rows.Close()
+
+	var out []MessageListItem
+	for rows.Next() {
+		var (
+			item     MessageListItem
+			dateSent string
+			received string
+		)
+		if err := rows.Scan(
+			&item.ID, &item.GmailID, &item.ThreadID, &item.FromAddr,
+			&item.ToAddr, &item.Subject, &item.BodyText,
+			&item.SourceAddress, &dateSent, &received,
+			&item.IMAPUID, &item.IMAPUIDValidity,
+			&item.Category, &item.Importance, &item.Action, &item.Confidence,
+		); err != nil {
+			return nil, fmt.Errorf("scan message: %w", err)
+		}
+		item.DateSent = parseSQLTime(dateSent)
+		item.ReceivedAt = parseSQLTime(received)
+		item.HasClassification = item.Category != ""
+		out = append(out, item)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("list messages: %w", err)
+	}
+	return out, nil
+}
+
+// CountMessages returns the total number of stored messages.
+func (s *Store) CountMessages(ctx context.Context) (int, error) {
+	var n int
+	if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM messages`).Scan(&n); err != nil {
+		return 0, fmt.Errorf("count messages: %w", err)
+	}
+	return n, nil
+}
+
+// LogAction appends to the actions_log (FR-13). The table is append-only.
+func (s *Store) LogAction(ctx context.Context, action, actor, detail string) error {
+	const q = `INSERT INTO actions_log (message_id, action, actor, detail) VALUES (NULL, ?, ?, ?)`
+	if _, err := s.db.ExecContext(ctx, q, action, nullString(actor), nullString(detail)); err != nil {
+		return fmt.Errorf("log action %q: %w", action, err)
+	}
+	return nil
+}
+
+const sqlTimeLayout = "2006-01-02 15:04:05"
+
+func nullString(v string) any {
+	if strings.TrimSpace(v) == "" {
+		return nil
+	}
+	return v
+}
+
+func nullTime(t time.Time) any {
+	if t.IsZero() {
+		return nil
+	}
+	return t.UTC().Format(sqlTimeLayout)
+}
+
+func parseSQLTime(v string) time.Time {
+	if v == "" {
+		return time.Time{}
+	}
+	for _, layout := range []string{sqlTimeLayout, time.RFC3339, "2006-01-02T15:04:05Z"} {
+		if t, err := time.Parse(layout, v); err == nil {
+			return t.UTC()
+		}
+	}
+	return time.Time{}
+}
--- a/internal/store/schema.sql
+++ b/internal/store/schema.sql
@@ -24,6 +24,10 @@ CREATE TABLE IF NOT EXISTS messages (
    body_text       TEXT,
    source_address  TEXT,
    date_sent       TEXT,
+    -- IMAP bookkeeping (AGENTS.md §9): the mailbox UID and the UIDVALIDITY it
+    -- belongs to. UIDs are only meaningful while UIDVALIDITY is unchanged.
+    imap_uid            INTEGER,
+    imap_uidvalidity    INTEGER,
    received_at     TEXT    NOT NULL DEFAULT (datetime('now')),
    created_at      TEXT    NOT NULL DEFAULT (datetime('now'))
 );
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -76,11 +76,69 @@ func buildDSN(path string) string {
 	return "file:" + abs + "?" + q.Encode()
 }

-// migrate applies the embedded schema. It is safe to run on every start.
+// migrate applies the embedded schema, then any incremental migrations that
+// add columns to a database created by an earlier release. It is safe to run on
+// every start.
 func (s *Store) migrate(ctx context.Context) error {
 	if _, err := s.db.ExecContext(ctx, schemaSQL); err != nil {
 		return fmt.Errorf("apply schema: %w", err)
 	}
+	return s.applyMigrations(ctx)
+}
+
+// applyMigrations brings an existing database up to the latest schema version.
+// Version 1 is the base schema (schema.sql); version 2 adds IMAP bookkeeping
+// columns (AGENTS.md §9 / Phase 2). Each step is idempotent.
+func (s *Store) applyMigrations(ctx context.Context) error {
+	version, err := s.SchemaVersion(ctx)
+	if err != nil {
+		return fmt.Errorf("read schema version: %w", err)
+	}
+	if version < 2 {
+		for _, col := range []struct{ table, name, typ string }{
+			{"messages", "imap_uid", "INTEGER"},
+			{"messages", "imap_uidvalidity", "INTEGER"},
+		} {
+			if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil {
+				return err
+			}
+		}
+		if _, err := s.db.ExecContext(ctx,
+			`INSERT OR IGNORE INTO schema_migrations (version) VALUES (2)`); err != nil {
+			return fmt.Errorf("record schema version 2: %w", err)
+		}
+	}
+	return nil
+}
+
+// ensureColumn adds a column when it is missing. SQLite has no
+// "ADD COLUMN IF NOT EXISTS", so the current shape is inspected first.
+func (s *Store) ensureColumn(ctx context.Context, table, name, typ string) error {
+	rows, err := s.db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table))
+	if err != nil {
+		return fmt.Errorf("inspect %s: %w", table, err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			cid, notnull, pk int
+			dflt             sql.NullString
+			colName, colType string
+		)
+		if err := rows.Scan(&cid, &colName, &colType, &notnull, &dflt, &pk); err != nil {
+			return fmt.Errorf("scan %s columns: %w", table, err)
+		}
+		if colName == name {
+			return nil
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("inspect %s columns: %w", table, err)
+	}
+	if _, err := s.db.ExecContext(ctx,
+		fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, name, typ)); err != nil {
+		return fmt.Errorf("add %s.%s: %w", table, name, err)
+	}
 	return nil
 }

--- a/internal/store/store_test.go
+++ b/internal/store/store_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"path/filepath"
 	"testing"
+	"time"
 )

 func TestOpenCreatesSchema(t *testing.T) {
@@ -83,3 +84,110 @@ func TestMessagesInsertAndRead(t *testing.T) {
 		t.Errorf("subject = %q, want %q", subject, "Hello")
 	}
 }
+
+func TestUpsertAndListMessages(t *testing.T) {
+	ctx := context.Background()
+	s, err := Open(ctx, filepath.Join(t.TempDir(), "email_agent.db"))
+	if err != nil {
+		t.Fatalf("Open() error = %v", err)
+	}
+	defer s.Close()
+
+	m := &Message{
+		GmailID:         "1234000000000000000",
+		ThreadID:        "999",
+		FromAddr:        "favrolfe@gmail.com",
+		ToAddr:          "Hello@where-woof.com",
+		Subject:         "A forwarded hello",
+		BodyText:        "body text",
+		SourceAddress:   "Hello@where-woof.com",
+		DateSent:        time.Date(2026, 10, 1, 9, 30, 0, 0, time.UTC),
+		IMAPUID:         42,
+		IMAPUIDValidity: 7,
+	}
+	id, err := s.UpsertMessage(ctx, m)
+	if err != nil {
+		t.Fatalf("UpsertMessage() error = %v", err)
+	}
+	if id == 0 {
+		t.Fatal("UpsertMessage() returned id 0")
+	}
+
+	// A second upsert with the same Gmail id must update, not duplicate, and
+	// must not move received_at.
+	m.Subject = "Updated subject"
+	id2, err := s.UpsertMessage(ctx, m)
+	if err != nil {
+		t.Fatalf("second UpsertMessage() error = %v", err)
+	}
+	if id2 != id {
+		t.Errorf("second upsert id = %d, want %d", id2, id)
+	}
+
+	n, err := s.CountMessages(ctx)
+	if err != nil {
+		t.Fatalf("CountMessages() error = %v", err)
+	}
+	if n != 1 {
+		t.Fatalf("CountMessages() = %d, want 1", n)
+	}
+
+	list, err := s.ListMessages(ctx, 10)
+	if err != nil {
+		t.Fatalf("ListMessages() error = %v", err)
+	}
+	if len(list) != 1 {
+		t.Fatalf("ListMessages() len = %d, want 1", len(list))
+	}
+	got := list[0]
+	if got.Subject != "Updated subject" {
+		t.Errorf("Subject = %q, want updated value", got.Subject)
+	}
+	if got.SourceAddress != "Hello@where-woof.com" {
+		t.Errorf("SourceAddress = %q", got.SourceAddress)
+	}
+	if got.IMAPUID != 42 || got.IMAPUIDValidity != 7 {
+		t.Errorf("IMAP bookkeeping = (%d, %d), want (42, 7)", got.IMAPUID, got.IMAPUIDValidity)
+	}
+	if got.HasClassification {
+		t.Error("HasClassification = true with no classifications row")
+	}
+}
+
+func TestMigrationAddsIMAPColumns(t *testing.T) {
+	ctx := context.Background()
+	s, err := Open(ctx, filepath.Join(t.TempDir(), "email_agent.db"))
+	if err != nil {
+		t.Fatalf("Open() error = %v", err)
+	}
+	defer s.Close()
+
+	v, err := s.SchemaVersion(ctx)
+	if err != nil {
+		t.Fatalf("SchemaVersion() error = %v", err)
+	}
+	if v < 2 {
+		t.Errorf("SchemaVersion() = %d, want >= 2", v)
+	}
+
+	for _, col := range []string{"imap_uid", "imap_uidvalidity"} {
+		var found bool
+		rows, err := s.DB().QueryContext(ctx, `SELECT name FROM pragma_table_info('messages')`)
+		if err != nil {
+			t.Fatalf("pragma_table_info: %v", err)
+		}
+		for rows.Next() {
+			var name string
+			if err := rows.Scan(&name); err != nil {
+				t.Fatalf("scan: %v", err)
+			}
+			if name == col {
+				found = true
+			}
+		}
+		_ = rows.Close()
+		if !found {
+			t.Errorf("column %q missing after migration", col)
+		}
+	}
+}