diff --git a/.env.example b/.env.example index 1f13f52..4e38435 100644 --- a/.env.example +++ b/.env.example @@ -21,11 +21,18 @@ EMAIL_AGENT_SESSION_SECRET=change-me # doubles as the password. EMAIL_AGENT_PASSWORD= +# --- Gmail IMAP (Phase 2) --------------------------------------------------- +# App password only (AGENTS.md §9). Values live in the host environment, never here. +GMAIL_USER= +GMAIL_APP_PASSWORD= +EMAIL_AGENT_IMAP_HOST=imap.gmail.com +EMAIL_AGENT_IMAP_PORT=993 +EMAIL_AGENT_IMAP_LOOKBACK_DAYS=14 +EMAIL_AGENT_IMAP_MAX_MESSAGES=100 + # --- Integrations (used from later phases) --------------------------------- JEV_BASE_URL= JEV_API_KEY= OMNIROUTE_BASE_URL= OMNIROUTE_API_KEY= -EMAIL_AGENT_GMAIL_CREDENTIALS_FILE= -EMAIL_AGENT_GMAIL_TOKEN_FILE= HUB_TOKEN= diff --git a/README.md b/README.md index 486ccbf..5566428 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,20 @@ pre-planning material and is not authoritative. ## Status -**Phase 1.2 — Session login + empty dashboard.** The Go skeleton (Phase 1.1) -plus a single-user session login, a copied Family Home Lab `tokens.css`, and an -HTMX dashboard shell (left nav, centre list, right detail pane) with dark mode. -Mail fetch, classification, and the rest come in later phases. +**Phase 2 — Read mail (IMAP).** The Go skeleton (Phase 1.1) plus session login +and the HTMX dashboard shell (Phase 1.2), now fetching real mail from Gmail over +**IMAP with an app password, forced to IPv4** (AGENTS.md §9). Messages are stored +in SQLite and listed in the dashboard, grouped by category with a recommended +action per row. The FR-2 source address is resolved from `X-Pm-Original-To`, with +`To`, the PurelyMail `Received … for` clause, and `Return-Path` as fallbacks; +`Delivered-To` is never used. Classification and triage actions come in Phase 3+. ## Layout ``` -cmd/email-agent/ service entry point +cmd/email-agent/ service entry point (`-fetch` runs one scan and exits) internal/config/ environment-driven configuration +internal/mailfetch/ stdlib-only IMAP client, MIME parsing, FR-2 source address internal/session/ server-side session store + signed cookie values internal/store/ SQLite open + schema (internal/store/schema.sql) internal/server/ HTTP routes, auth middleware, HTMX handlers @@ -47,6 +51,29 @@ Key variables: | `EMAIL_AGENT_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` | | `EMAIL_AGENT_SESSION_SECRET` | _(empty)_ | Session cookie signing key. If unset, an ephemeral key is used (sessions reset on restart) | | `EMAIL_AGENT_PASSWORD` | _(empty)_ | Login password for Sam. If unset, `EMAIL_AGENT_SESSION_SECRET` doubles as the password | +| `GMAIL_USER` | _(empty)_ | Gmail account that receives the forwarded mail | +| `GMAIL_APP_PASSWORD` | _(empty)_ | Gmail app password for IMAP/SMTP (no OAuth) | +| `EMAIL_AGENT_IMAP_HOST` | `imap.gmail.com` | IMAP host | +| `EMAIL_AGENT_IMAP_PORT` | `993` | IMAP port (implicit TLS) | +| `EMAIL_AGENT_IMAP_LOOKBACK_DAYS` | `14` | First-scan window | +| `EMAIL_AGENT_IMAP_MAX_MESSAGES` | `100` | Per-scan message cap | + +## Mail sync (Phase 2) + +`internal/mailfetch` speaks the minimum IMAP needed: `LOGIN`, `SELECT INBOX`, +`UID SEARCH SINCE`, and `UID FETCH` of the full RFC822 source plus Gmail's +`X-GM-MSGID`/`X-GM-THRID`. The dialer uses `tcp4` so Gmail is only ever reached +over IPv4 — IPv6 to Gmail is broken from `.13` and previously caused a socket EOF. + +Trigger a scan either from the dashboard (**Check mail**) or from the CLI: + +```sh +docker exec email-agent email-agent -fetch +``` + +Each scan is bounded by the lookback and max-message settings. Gmail's +`X-GM-MSGID` is the stable message key; the IMAP `UID` and `UIDVALIDITY` are +stored alongside it for later label operations. ## Run locally @@ -76,8 +103,9 @@ Caddy on `.35` proxies `mail.lab.audasmedia.com.au` to this port. | `GET` | `/login` | public | Sign-in form (deep-indigo band) | | `POST` | `/login` | public | CSRF-checked password sign-in, sets the session cookie | | `POST` | `/logout` | session | Destroys the session, returns to `/login` | -| `GET` | `/` | session | Empty dashboard shell | +| `GET` | `/` | session | Dashboard: nav counts, real message list | | `GET` | `/partials/inbox` | session | HTMX fragment for the centre list | +| `POST` | `/actions/fetch` | session + CSRF | Runs one IMAP scan, returns the refreshed list | | `GET` | `/static/…` | public | Embedded `tokens.css`, `app.css`, Inter fonts, `htmx.min.js` | Unauthenticated requests to `/` and `/partials/…` redirect to `/login?next=…`. @@ -90,4 +118,5 @@ SQLite is opened with the pure-Go `modernc.org/sqlite` driver (no CGO). The schema in `internal/store/schema.sql` creates the AGENTS.md §7 tables: `messages`, `classifications`, `rules`, `actions_log`, `unsubscribe`, `drafts`, `summaries`, plus `schema_migrations`. Schema init is idempotent and runs on -every start. +every start. Migration v2 adds `imap_uid` and `imap_uidvalidity` to `messages` +for databases created by Phase 1. diff --git a/cmd/email-agent/main.go b/cmd/email-agent/main.go index f27ba1f..f8b4a6d 100644 --- a/cmd/email-agent/main.go +++ b/cmd/email-agent/main.go @@ -13,6 +13,7 @@ import ( "time" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/config" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/server" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/version" @@ -22,6 +23,7 @@ func main() { var ( showVersion = flag.Bool("version", false, "print version information and exit") healthcheck = flag.Bool("healthcheck", false, "probe the local /health endpoint and exit") + fetchNow = flag.Bool("fetch", false, "run one Gmail IMAP scan and exit") ) flag.Parse() @@ -47,6 +49,14 @@ func main() { return } + if *fetchNow { + if err := runFetch(cfg, logger); err != nil { + fmt.Fprintf(os.Stderr, "fetch failed: %v\n", err) + os.Exit(1) + } + return + } + if err := run(cfg, logger); err != nil { logger.Error("fatal", "error", err) os.Exit(1) @@ -91,6 +101,30 @@ func run(cfg *config.Config, logger *slog.Logger) error { return srv.Run(ctx) } +// runFetch performs a single IMAP scan and exits. It is the entry point n8n +// (FR-1) and manual operations use to trigger a sync without the HTTP UI. +func runFetch(cfg *config.Config, logger *slog.Logger) error { + if !cfg.GmailConfigured() { + return mailfetch.ErrNoCredentials + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + st, err := store.Open(ctx, cfg.DBPath) + if err != nil { + return fmt.Errorf("open store: %w", err) + } + defer func() { _ = st.Close() }() + + sum, err := mailfetch.NewFetcher(cfg, st, logger).Fetch(ctx) + if err != nil { + return err + } + fmt.Printf("fetched=%d stored=%d uidvalidity=%d since=%s\n", + sum.Fetched, sum.Stored, sum.UIDValidity, sum.Since.Format("2006-01-02")) + return nil +} + // probeHealth requests the local health endpoint. It exists so the container // healthcheck does not need an external HTTP client in the image. func probeHealth(port int) error { diff --git a/docker-compose.yml b/docker-compose.yml index 57a0a94..ceeb0a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,12 +31,17 @@ services: # and are never committed. EMAIL_AGENT_SESSION_SECRET: ${EMAIL_AGENT_SESSION_SECRET:-} EMAIL_AGENT_PASSWORD: ${EMAIL_AGENT_PASSWORD:-} + # Gmail IMAP + app password (AGENTS.md §8/§9). The dialer forces IPv4. + GMAIL_USER: ${GMAIL_USER:-} + GMAIL_APP_PASSWORD: ${GMAIL_APP_PASSWORD:-} + EMAIL_AGENT_IMAP_HOST: ${EMAIL_AGENT_IMAP_HOST:-imap.gmail.com} + EMAIL_AGENT_IMAP_PORT: ${EMAIL_AGENT_IMAP_PORT:-993} + EMAIL_AGENT_IMAP_LOOKBACK_DAYS: ${EMAIL_AGENT_IMAP_LOOKBACK_DAYS:-14} + EMAIL_AGENT_IMAP_MAX_MESSAGES: ${EMAIL_AGENT_IMAP_MAX_MESSAGES:-100} JEV_BASE_URL: ${JEV_BASE_URL:-} JEV_API_KEY: ${JEV_API_KEY:-} OMNIROUTE_BASE_URL: ${OMNIROUTE_BASE_URL:-} OMNIROUTE_API_KEY: ${OMNIROUTE_API_KEY:-} - EMAIL_AGENT_GMAIL_CREDENTIALS_FILE: ${EMAIL_AGENT_GMAIL_CREDENTIALS_FILE:-} - EMAIL_AGENT_GMAIL_TOKEN_FILE: ${EMAIL_AGENT_GMAIL_TOKEN_FILE:-} HUB_TOKEN: ${HUB_TOKEN:-} volumes: diff --git a/internal/config/config.go b/internal/config/config.go index a19e371..7948938 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,6 +21,13 @@ const ( DefaultEnv = "development" DefaultAddr = ":" + "8790" DefaultDB = "data/email_agent.db" + + // IMAP defaults. Gmail IMAP is implicit TLS on 993. IPv4 is forced in the + // dialer (AGENTS.md §9): IPv6 to Gmail is broken from .13. + DefaultIMAPHost = "imap.gmail.com" + DefaultIMAPPort = 993 + DefaultIMAPLookbackDays = 14 + DefaultIMAPMaxMessages = 100 ) // Config is the fully resolved runtime configuration. @@ -65,10 +72,20 @@ type Config struct { // OmniRouteAPIKey authenticates to OmniRoute. Secret. OmniRouteAPIKey string - // GmailCredentialsFile is the OAuth client credentials JSON path. - GmailCredentialsFile string - // GmailTokenFile is the path where the OAuth token is cached. - GmailTokenFile string + // GmailUser is the Gmail account that receives the forwarded mail. + // Secret (half of the app-password pair). + GmailUser string + // GmailAppPassword is the Gmail app password used for IMAP and, later, SMTP. + // Secret. AGENTS.md §9: app password only, no OAuth. + GmailAppPassword string + // IMAPHost is the Gmail IMAP host (default imap.gmail.com). + IMAPHost string + // IMAPPort is the Gmail IMAP port (default 993, implicit TLS). + IMAPPort int + // IMAPLookbackDays bounds the first scan window. + IMAPLookbackDays int + // IMAPMaxMessages caps how many messages one scan fetches. + IMAPMaxMessages int // HubToken authenticates the read-only Hub API. Secret. HubToken string @@ -77,20 +94,29 @@ type Config struct { // Load reads configuration from the environment, applying defaults. func Load() (*Config, error) { c := &Config{ - Env: getenvDefault("EMAIL_AGENT_ENV", DefaultEnv), - SessionSecret: os.Getenv("EMAIL_AGENT_SESSION_SECRET"), - Password: os.Getenv("EMAIL_AGENT_PASSWORD"), - LogLevel: strings.ToLower(getenvDefault("EMAIL_AGENT_LOG_LEVEL", "info")), - PublicBaseURL: strings.TrimRight(getenvDefault("EMAIL_AGENT_PUBLIC_BASE_URL", "http://localhost:8790"), "/"), - JEVBaseURL: strings.TrimRight(os.Getenv("JEV_BASE_URL"), "/"), - JEVAPIKey: os.Getenv("JEV_API_KEY"), - OmniRouteBaseURL: strings.TrimRight(os.Getenv("OMNIROUTE_BASE_URL"), "/"), - OmniRouteAPIKey: os.Getenv("OMNIROUTE_API_KEY"), - GmailCredentialsFile: os.Getenv("EMAIL_AGENT_GMAIL_CREDENTIALS_FILE"), - GmailTokenFile: os.Getenv("EMAIL_AGENT_GMAIL_TOKEN_FILE"), - HubToken: os.Getenv("HUB_TOKEN"), + Env: getenvDefault("EMAIL_AGENT_ENV", DefaultEnv), + SessionSecret: os.Getenv("EMAIL_AGENT_SESSION_SECRET"), + Password: os.Getenv("EMAIL_AGENT_PASSWORD"), + LogLevel: strings.ToLower(getenvDefault("EMAIL_AGENT_LOG_LEVEL", "info")), + PublicBaseURL: strings.TrimRight(getenvDefault("EMAIL_AGENT_PUBLIC_BASE_URL", "http://localhost:8790"), "/"), + JEVBaseURL: strings.TrimRight(os.Getenv("JEV_BASE_URL"), "/"), + JEVAPIKey: os.Getenv("JEV_API_KEY"), + OmniRouteBaseURL: strings.TrimRight(os.Getenv("OMNIROUTE_BASE_URL"), "/"), + OmniRouteAPIKey: os.Getenv("OMNIROUTE_API_KEY"), + GmailUser: os.Getenv("GMAIL_USER"), + GmailAppPassword: os.Getenv("GMAIL_APP_PASSWORD"), + HubToken: os.Getenv("HUB_TOKEN"), } + imapPort, err := resolveIMAPPort() + if err != nil { + return nil, err + } + c.IMAPHost = getenvDefault("EMAIL_AGENT_IMAP_HOST", DefaultIMAPHost) + c.IMAPPort = imapPort + c.IMAPLookbackDays = getenvDefaultInt("EMAIL_AGENT_IMAP_LOOKBACK_DAYS", DefaultIMAPLookbackDays) + c.IMAPMaxMessages = getenvDefaultInt("EMAIL_AGENT_IMAP_MAX_MESSAGES", DefaultIMAPMaxMessages) + port, err := resolvePort() if err != nil { return nil, err @@ -105,6 +131,33 @@ func Load() (*Config, error) { return c, nil } +// resolveIMAPPort reads EMAIL_AGENT_IMAP_PORT, falling back to DefaultIMAPPort. +func resolveIMAPPort() (int, error) { + raw := strings.TrimSpace(os.Getenv("EMAIL_AGENT_IMAP_PORT")) + if raw == "" { + return DefaultIMAPPort, nil + } + p, err := strconv.Atoi(raw) + if err != nil { + return 0, fmt.Errorf("EMAIL_AGENT_IMAP_PORT %q is not a number: %w", raw, err) + } + return p, nil +} + +// getenvDefaultInt reads a positive integer setting, falling back to def when +// unset or malformed. +func getenvDefaultInt(key string, def int) int { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return def + } + n, err := strconv.Atoi(raw) + if err != nil || n < 1 { + return def + } + return n +} + // resolvePort determines the listening port from EMAIL_AGENT_PORT or the port // component of EMAIL_AGENT_ADDR, falling back to DefaultPort. func resolvePort() (int, error) { @@ -151,9 +204,21 @@ func (c *Config) validate() error { default: return fmt.Errorf("EMAIL_AGENT_LOG_LEVEL %q must be debug, info, warn, or error", c.LogLevel) } + if strings.TrimSpace(c.IMAPHost) == "" { + return fmt.Errorf("EMAIL_AGENT_IMAP_HOST must not be empty") + } + if c.IMAPPort < 1 || c.IMAPPort > 65535 { + return fmt.Errorf("EMAIL_AGENT_IMAP_PORT %d is out of range", c.IMAPPort) + } return nil } +// GmailConfigured reports whether an IMAP login can be attempted. The service +// still starts without credentials; only mail sync is unavailable. +func (c *Config) GmailConfigured() bool { + return strings.TrimSpace(c.GmailUser) != "" && strings.TrimSpace(c.GmailAppPassword) != "" +} + // IsProduction reports whether the service runs in the production environment. func (c *Config) IsProduction() bool { return c.Env == "production" } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3b11c4e..a3065af 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -7,6 +7,8 @@ func TestLoadDefaults(t *testing.T) { for _, k := range []string{ "EMAIL_AGENT_ENV", "EMAIL_AGENT_ADDR", "EMAIL_AGENT_PORT", "EMAIL_AGENT_DB_PATH", "EMAIL_AGENT_LOG_LEVEL", "EMAIL_AGENT_PUBLIC_BASE_URL", + "GMAIL_USER", "GMAIL_APP_PASSWORD", "EMAIL_AGENT_IMAP_HOST", "EMAIL_AGENT_IMAP_PORT", + "EMAIL_AGENT_IMAP_LOOKBACK_DAYS", "EMAIL_AGENT_IMAP_MAX_MESSAGES", } { t.Setenv(k, "") } @@ -27,6 +29,43 @@ func TestLoadDefaults(t *testing.T) { if cfg.DBPath != DefaultDB { t.Errorf("DBPath = %q, want %q", cfg.DBPath, DefaultDB) } + if cfg.IMAPHost != DefaultIMAPHost { + t.Errorf("IMAPHost = %q, want %q", cfg.IMAPHost, DefaultIMAPHost) + } + if cfg.IMAPPort != DefaultIMAPPort { + t.Errorf("IMAPPort = %d, want %d", cfg.IMAPPort, DefaultIMAPPort) + } + if cfg.IMAPLookbackDays != DefaultIMAPLookbackDays { + t.Errorf("IMAPLookbackDays = %d, want %d", cfg.IMAPLookbackDays, DefaultIMAPLookbackDays) + } + if cfg.IMAPMaxMessages != DefaultIMAPMaxMessages { + t.Errorf("IMAPMaxMessages = %d, want %d", cfg.IMAPMaxMessages, DefaultIMAPMaxMessages) + } + if cfg.GmailConfigured() { + t.Error("GmailConfigured() = true with no credentials") + } +} + +func TestGmailConfigured(t *testing.T) { + t.Setenv("GMAIL_USER", "samuelrolfe@gmail.com") + t.Setenv("GMAIL_APP_PASSWORD", "app-password") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if !cfg.GmailConfigured() { + t.Error("GmailConfigured() = false with credentials set") + } + + t.Setenv("GMAIL_APP_PASSWORD", "") + cfg, err = Load() + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.GmailConfigured() { + t.Error("GmailConfigured() = true with no app password") + } } func TestLoadOverrides(t *testing.T) { diff --git a/internal/mailfetch/fetch.go b/internal/mailfetch/fetch.go new file mode 100644 index 0000000..786fc37 --- /dev/null +++ b/internal/mailfetch/fetch.go @@ -0,0 +1,160 @@ +package mailfetch + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strconv" + "strings" + "sync" + "time" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/config" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" +) + +// ErrNoCredentials means IMAP login cannot be attempted. +var ErrNoCredentials = errors.New("GMAIL_USER and GMAIL_APP_PASSWORD are not configured") + +// ErrAlreadyRunning means a scan is already in progress. +var ErrAlreadyRunning = errors.New("a mail scan is already running") + +// Fetcher performs an IMAP scan and stores the results. +type Fetcher struct { + cfg *config.Config + store *store.Store + logger *slog.Logger + + mu sync.Mutex +} + +// NewFetcher builds a Fetcher. +func NewFetcher(cfg *config.Config, st *store.Store, logger *slog.Logger) *Fetcher { + return &Fetcher{cfg: cfg, store: st, logger: logger} +} + +// Summary describes the outcome of one scan. +type Summary struct { + UIDValidity uint32 + Since time.Time + Fetched int + Stored int + Duration time.Duration +} + +// Fetch performs one bounded IMAP scan of INBOX and upserts the results. +func (f *Fetcher) Fetch(ctx context.Context) (Summary, error) { + if !f.cfg.GmailConfigured() { + return Summary{}, ErrNoCredentials + } + if !f.mu.TryLock() { + return Summary{}, ErrAlreadyRunning + } + defer f.mu.Unlock() + + start := time.Now() + client, err := Dial(ctx, f.cfg.IMAPHost, f.cfg.IMAPPort, DefaultTimeout) + if err != nil { + return Summary{}, err + } + defer func() { _ = client.Logout() }() + + if err := client.Login(f.cfg.GmailUser, f.cfg.GmailAppPassword); err != nil { + return Summary{}, err + } + + uidValidity, err := client.Select("INBOX") + if err != nil { + return Summary{}, err + } + + since := time.Now().UTC().AddDate(0, 0, -f.cfg.IMAPLookbackDays) + uids, err := client.SearchSince(since) + if err != nil { + return Summary{}, err + } + if max := f.cfg.IMAPMaxMessages; max > 0 && len(uids) > max { + // Keep the newest; SEARCH returns ascending UIDs. + uids = uids[len(uids)-max:] + } + + results, err := client.Fetch(uids) + if err != nil { + return Summary{}, err + } + + sum := Summary{ + UIDValidity: uidValidity, + Since: since, + Fetched: len(results), + } + now := time.Now().UTC() + headersSeen := map[string]struct{}{} + for _, r := range results { + parsed, perr := ParseMessage(r.Raw) + if perr != nil { + f.logger.Warn("skip unparsable message", "uid", r.UID, "error", perr) + continue + } + if parsed.SourceHeader != "" { + headersSeen[parsed.SourceHeader] = struct{}{} + } + msg := &store.Message{ + GmailID: gmailID(r, uidValidity), + ThreadID: r.GmailThreadID, + FromAddr: parsed.From, + ToAddr: parsed.To, + Subject: parsed.Subject, + BodyText: parsed.BodyText, + SourceAddress: parsed.SourceAddress, + DateSent: parsed.Date, + ReceivedAt: now, + IMAPUID: int64(r.UID), + IMAPUIDValidity: int64(uidValidity), + } + if _, err := f.store.UpsertMessage(ctx, msg); err != nil { + return sum, err + } + sum.Stored++ + } + sum.Duration = time.Since(start) + + if err := f.store.LogAction(ctx, "fetch", "agent", + fmt.Sprintf("uidvalidity=%d since=%s fetched=%d stored=%d source_headers=%s", + uidValidity, since.Format("2006-01-02"), sum.Fetched, sum.Stored, joinKeys(headersSeen))); err != nil { + f.logger.Warn("record fetch action", "error", err) + } + + f.logger.Info("mail scan complete", + "uidvalidity", uidValidity, + "since", since.Format("2006-01-02"), + "fetched", sum.Fetched, + "stored", sum.Stored, + "duration_ms", sum.Duration.Milliseconds(), + ) + return sum, nil +} + +// gmailID prefers Gmail's stable X-GM-MSGID and falls back to a UID key that is +// scoped to the UIDVALIDITY it was issued under. +func gmailID(r FetchResult, uidValidity uint32) string { + if r.GmailMsgID != "" { + return r.GmailMsgID + } + return "uid:" + strconv.FormatUint(uint64(uidValidity), 10) + ":" + strconv.FormatUint(uint64(r.UID), 10) +} + +// joinKeys returns the sorted keys of a string set, for a log line. +func joinKeys(set map[string]struct{}) string { + if len(set) == 0 { + return "none" + } + parts := make([]string, 0, len(set)) + for k := range set { + parts = append(parts, k) + } + slices.Sort(parts) + return strings.Join(parts, ",") +} diff --git a/internal/mailfetch/fetch_test.go b/internal/mailfetch/fetch_test.go new file mode 100644 index 0000000..0403a78 --- /dev/null +++ b/internal/mailfetch/fetch_test.go @@ -0,0 +1,40 @@ +package mailfetch + +import ( + "context" + "errors" + "log/slog" + "testing" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/config" +) + +func TestGmailIDPrefersXGMMsgID(t *testing.T) { + if got := gmailID(FetchResult{UID: 7, GmailMsgID: "555"}, 42); got != "555" { + t.Errorf("gmailID() = %q, want 555", got) + } +} + +func TestGmailIDFallbackIsScopedToUIDValidity(t *testing.T) { + if got := gmailID(FetchResult{UID: 7}, 42); got != "uid:42:7" { + t.Errorf("gmailID() = %q, want uid:42:7", got) + } +} + +func TestJoinKeys(t *testing.T) { + if got := joinKeys(map[string]struct{}{}); got != "none" { + t.Errorf("joinKeys(empty) = %q, want none", got) + } + got := joinKeys(map[string]struct{}{"To": {}, "X-Pm-Original-To": {}}) + if got != "To,X-Pm-Original-To" { + t.Errorf("joinKeys() = %q, want sorted keys", got) + } +} + +func TestFetchWithoutCredentials(t *testing.T) { + cfg := &config.Config{Env: "test", IMAPHost: config.DefaultIMAPHost, IMAPPort: config.DefaultIMAPPort} + f := NewFetcher(cfg, nil, slog.New(slog.DiscardHandler)) + if _, err := f.Fetch(context.Background()); !errors.Is(err, ErrNoCredentials) { + t.Fatalf("Fetch() error = %v, want ErrNoCredentials", err) + } +} diff --git a/internal/mailfetch/imap.go b/internal/mailfetch/imap.go new file mode 100644 index 0000000..ea14b5b --- /dev/null +++ b/internal/mailfetch/imap.go @@ -0,0 +1,328 @@ +// Package mailfetch reads mail from Gmail over IMAP and turns it into stored +// messages. +// +// The IMAP client is intentionally dependency-free: it implements only the +// subset the Email Agent needs (LOGIN, SELECT, UID SEARCH, UID FETCH) and it +// always dials IPv4. IPv6 to Gmail is broken from .13 and previously caused a +// socket EOF (AGENTS.md §9). +package mailfetch + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "regexp" + "strconv" + "strings" + "time" +) + +// DefaultTimeout bounds a single IMAP read/write. +const DefaultTimeout = 60 * time.Second + +// Client is a minimal read-only IMAP client. +type Client struct { + conn net.Conn + br *bufio.Reader + tagSeq int + timeout time.Duration + preAuth bool +} + +// response is one IMAP response: the assembled text (literals removed) plus any +// literal payloads in order of appearance. +type response struct { + line string + literals [][]byte +} + +// ServerError is a tagged NO/BAD completion from the IMAP server. +type ServerError struct { + Command string + Response string +} + +func (e *ServerError) Error() string { + return fmt.Sprintf("imap %s failed: %s", e.Command, e.Response) +} + +// Dial opens an implicit-TLS IMAP connection to host:port, forcing IPv4. +func Dial(ctx context.Context, host string, port int, timeout time.Duration) (*Client, error) { + if timeout <= 0 { + timeout = DefaultTimeout + } + addr := net.JoinHostPort(host, strconv.Itoa(port)) + dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second} + raw, err := dialer.DialContext(ctx, "tcp4", addr) + if err != nil { + return nil, fmt.Errorf("dial %s over IPv4: %w", addr, err) + } + + tlsConn := tls.Client(raw, &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + }) + hsCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := tlsConn.HandshakeContext(hsCtx); err != nil { + _ = raw.Close() + return nil, fmt.Errorf("tls handshake with %s: %w", addr, err) + } + + c := &Client{conn: tlsConn, br: bufio.NewReader(tlsConn), timeout: timeout} + + greeting, err := c.readResponse() + if err != nil { + _ = tlsConn.Close() + return nil, fmt.Errorf("read IMAP greeting: %w", err) + } + upper := strings.ToUpper(greeting.line) + if strings.HasPrefix(upper, "* PREAUTH") { + c.preAuth = true + return c, nil + } + if !strings.HasPrefix(upper, "* OK") { + _ = tlsConn.Close() + return nil, fmt.Errorf("unexpected IMAP greeting: %s", greeting.line) + } + return c, nil +} + +// Login authenticates with an app password. +func (c *Client) Login(user, pass string) error { + if c.preAuth { + return nil + } + return c.exec("LOGIN "+imapQuote(user)+" "+imapQuote(pass), nil) +} + +// Select opens a mailbox and returns its UIDVALIDITY. +func (c *Client) Select(mailbox string) (uint32, error) { + var uidValidity uint32 + err := c.exec("SELECT "+imapQuote(mailbox), func(r *response) error { + if v, ok := parseUIDValidity(r.line); ok { + uidValidity = v + } + return nil + }) + return uidValidity, err +} + +// SearchSince returns the UIDs of messages with an internal date on or after t. +func (c *Client) SearchSince(t time.Time) ([]uint32, error) { + date := t.UTC().Format("02-Jan-2006") + var uids []uint32 + err := c.exec("UID SEARCH SINCE "+date, func(r *response) error { + if !strings.HasPrefix(strings.ToUpper(r.line), "* SEARCH") { + return nil + } + fields := strings.Fields(r.line) + for _, f := range fields[2:] { + if n, err := strconv.ParseUint(f, 10, 32); err == nil { + uids = append(uids, uint32(n)) + } + } + return nil + }) + return uids, err +} + +// FetchResult is one message returned by Fetch. +type FetchResult struct { + UID uint32 + GmailMsgID string + GmailThreadID string + Raw []byte +} + +// fetchChunk bounds how many UIDs go into a single FETCH command. +const fetchChunk = 20 + +// Fetch retrieves the full RFC822 source for each UID, in chunks. +func (c *Client) Fetch(uids []uint32) ([]FetchResult, error) { + var out []FetchResult + for start := 0; start < len(uids); start += fetchChunk { + end := start + fetchChunk + if end > len(uids) { + end = len(uids) + } + set := uidSet(uids[start:end]) + var batch []FetchResult + err := c.exec("UID FETCH "+set+" (UID X-GM-MSGID X-GM-THRID BODY.PEEK[])", func(r *response) error { + if fr, ok := parseFetch(r); ok { + batch = append(batch, fr) + } + return nil + }) + if err != nil { + return out, err + } + out = append(out, batch...) + } + return out, nil +} + +// Logout sends LOGOUT and closes the connection. +func (c *Client) Logout() error { + err := c.exec("LOGOUT", nil) + _ = c.conn.Close() + return err +} + +// Close closes the connection without a graceful logout. +func (c *Client) Close() error { + if c == nil || c.conn == nil { + return nil + } + return c.conn.Close() +} + +// --- protocol plumbing ------------------------------------------------------ + +func (c *Client) exec(command string, onUntagged func(*response) error) error { + c.tagSeq++ + tag := fmt.Sprintf("A%04d", c.tagSeq) + if err := c.writeLine(tag + " " + command); err != nil { + return err + } + for { + resp, err := c.readResponse() + if err != nil { + return err + } + switch { + case strings.HasPrefix(resp.line, tag+" "): + status := strings.TrimSpace(strings.TrimPrefix(resp.line, tag+" ")) + if strings.HasPrefix(strings.ToUpper(status), "OK") { + return nil + } + return &ServerError{Command: command, Response: status} + case strings.HasPrefix(resp.line, "* "): + if onUntagged != nil { + if err := onUntagged(resp); err != nil { + return err + } + } + case strings.HasPrefix(resp.line, "+ "): + return fmt.Errorf("imap server requested a continuation, which this client does not support: %s", resp.line) + default: + // Ignore unexpected non-tagged lines rather than desynchronising. + } + } +} + +func (c *Client) writeLine(s string) error { + if err := c.conn.SetWriteDeadline(time.Now().Add(c.timeout)); err != nil { + return err + } + if _, err := c.conn.Write([]byte(s + "\r\n")); err != nil { + return fmt.Errorf("imap write: %w", err) + } + return nil +} + +// readResponse reads one logical IMAP response, resolving literals. +func (c *Client) readResponse() (*response, error) { + var sb strings.Builder + var lits [][]byte + for { + line, err := c.readLine() + if err != nil { + return nil, err + } + if n, ok := literalLen(line); ok { + sb.WriteString(line[:strings.LastIndexByte(line, '{')]) + buf := make([]byte, n) + if _, err := io.ReadFull(c.br, buf); err != nil { + return nil, fmt.Errorf("read literal: %w", err) + } + lits = append(lits, buf) + continue + } + sb.WriteString(line) + return &response{line: sb.String(), literals: lits}, nil + } +} + +func (c *Client) readLine() (string, error) { + if err := c.conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil { + return "", err + } + line, err := c.br.ReadString('\n') + if err != nil && !(err == io.EOF && line != "") { + return "", fmt.Errorf("imap read: %w", err) + } + return strings.TrimRight(line, "\r\n"), nil +} + +// literalLen reports the byte count of a trailing `{n}` literal marker. +func literalLen(line string) (int, bool) { + if !strings.HasSuffix(line, "}") { + return 0, false + } + i := strings.LastIndexByte(line, '{') + if i < 0 { + return 0, false + } + n, err := strconv.Atoi(line[i+1 : len(line)-1]) + if err != nil || n < 0 { + return 0, false + } + return n, true +} + +var ( + uidValidityRE = regexp.MustCompile(`(?i)\[UIDVALIDITY (\d+)\]`) + fetchUIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])UID (\d+)`) + fetchMsgIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])X-GM-MSGID (\d+)`) + fetchThrIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])X-GM-THRID (\d+)`) +) + +func parseUIDValidity(line string) (uint32, bool) { + m := uidValidityRE.FindStringSubmatch(line) + if m == nil { + return 0, false + } + n, err := strconv.ParseUint(m[1], 10, 32) + if err != nil { + return 0, false + } + return uint32(n), true +} + +func parseFetch(r *response) (FetchResult, bool) { + if len(r.literals) == 0 || !strings.Contains(strings.ToUpper(r.line), "FETCH") { + return FetchResult{}, false + } + fr := FetchResult{Raw: r.literals[0]} + if m := fetchUIDRE.FindStringSubmatch(r.line); m != nil { + if n, err := strconv.ParseUint(m[1], 10, 32); err == nil { + fr.UID = uint32(n) + } + } + if m := fetchMsgIDRE.FindStringSubmatch(r.line); m != nil { + fr.GmailMsgID = m[1] + } + if m := fetchThrIDRE.FindStringSubmatch(r.line); m != nil { + fr.GmailThreadID = m[1] + } + return fr, true +} + +func uidSet(uids []uint32) string { + parts := make([]string, len(uids)) + for i, u := range uids { + parts[i] = strconv.FormatUint(uint64(u), 10) + } + return strings.Join(parts, ",") +} + +// imapQuote quotes a string as an IMAP quoted-string. +func imapQuote(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `"`, `\"`) + return `"` + s + `"` +} diff --git a/internal/mailfetch/imap_test.go b/internal/mailfetch/imap_test.go new file mode 100644 index 0000000..20933fd --- /dev/null +++ b/internal/mailfetch/imap_test.go @@ -0,0 +1,167 @@ +package mailfetch + +import ( + "bufio" + "bytes" + "errors" + "net" + "strings" + "testing" + "time" +) + +// fakeConn is an in-memory net.Conn for protocol tests. +type fakeConn struct { + r *bytes.Reader + w bytes.Buffer +} + +func newFakeConn(script string) *fakeConn { + return &fakeConn{r: bytes.NewReader([]byte(script))} +} + +func (c *fakeConn) Read(p []byte) (int, error) { return c.r.Read(p) } +func (c *fakeConn) Write(p []byte) (int, error) { return c.w.Write(p) } +func (c *fakeConn) Close() error { return nil } +func (c *fakeConn) LocalAddr() net.Addr { return dummyAddr{} } +func (c *fakeConn) RemoteAddr() net.Addr { return dummyAddr{} } +func (c *fakeConn) SetDeadline(time.Time) error { return nil } +func (c *fakeConn) SetReadDeadline(time.Time) error { + return nil +} +func (c *fakeConn) SetWriteDeadline(time.Time) error { return nil } + +type dummyAddr struct{} + +func (dummyAddr) Network() string { return "fake" } +func (dummyAddr) String() string { return "fake" } + +func newFakeClient(script string) (*Client, *fakeConn) { + fc := newFakeConn(script) + return &Client{conn: fc, br: bufio.NewReader(fc), timeout: time.Second}, fc +} + +func TestReadResponseSingleLiteral(t *testing.T) { + script := "* 1 FETCH (UID 10 BODY[] {5}\r\nHELLO)\r\n" + c, _ := newFakeClient(script) + + resp, err := c.readResponse() + if err != nil { + t.Fatalf("readResponse() error = %v", err) + } + if len(resp.literals) != 1 || string(resp.literals[0]) != "HELLO" { + t.Fatalf("literals = %q, want [HELLO]", resp.literals) + } + if !strings.Contains(resp.line, "UID 10") || !strings.Contains(resp.line, ")") { + t.Errorf("assembled line = %q, want UID and closing paren", resp.line) + } +} + +func TestReadResponseLiteralAtEnd(t *testing.T) { + script := "* 1 FETCH (BODY[] {5}\r\nHELLO\r\n\r\n" + c, _ := newFakeClient(script) + + resp, err := c.readResponse() + if err != nil { + t.Fatalf("readResponse() error = %v", err) + } + if len(resp.literals) != 1 || string(resp.literals[0]) != "HELLO" { + t.Fatalf("literals = %q, want [HELLO]", resp.literals) + } +} + +func TestSearchSince(t *testing.T) { + script := "* SEARCH 10 11 12\r\nA0001 OK SEARCH completed\r\n" + c, fc := newFakeClient(script) + + uids, err := c.SearchSince(time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("SearchSince() error = %v", err) + } + want := []uint32{10, 11, 12} + if len(uids) != len(want) { + t.Fatalf("uids = %v, want %v", uids, want) + } + for i := range want { + if uids[i] != want[i] { + t.Errorf("uids[%d] = %d, want %d", i, uids[i], want[i]) + } + } + if !strings.Contains(fc.w.String(), "UID SEARCH SINCE 02-Jan-2026") { + t.Errorf("command sent = %q, want UID SEARCH SINCE 02-Jan-2026", fc.w.String()) + } +} + +func TestFetchParsesGmailIDs(t *testing.T) { + script := "* 1 FETCH (UID 10 X-GM-MSGID 1234000000000000000 X-GM-THRID 999 BODY[] {5}\r\nHELLO)\r\n" + + "A0001 OK FETCH completed\r\n" + c, _ := newFakeClient(script) + + results, err := c.Fetch([]uint32{10}) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("results = %d, want 1", len(results)) + } + if results[0].UID != 10 { + t.Errorf("UID = %d, want 10", results[0].UID) + } + if results[0].GmailMsgID != "1234000000000000000" { + t.Errorf("GmailMsgID = %q", results[0].GmailMsgID) + } + if results[0].GmailThreadID != "999" { + t.Errorf("GmailThreadID = %q", results[0].GmailThreadID) + } + if string(results[0].Raw) != "HELLO" { + t.Errorf("Raw = %q, want HELLO", results[0].Raw) + } +} + +func TestLoginFailureIsServerError(t *testing.T) { + script := "* OK greeting\r\nA0001 NO [AUTHENTICATIONFAILED] Invalid credentials\r\n" + c, _ := newFakeClient(script) + + err := c.Login("user@example.test", "wrong") + if err == nil { + t.Fatal("Login() error = nil, want failure") + } + var se *ServerError + if !errors.As(err, &se) { + t.Fatalf("Login() error = %T, want *ServerError", err) + } + if !strings.Contains(se.Response, "AUTHENTICATIONFAILED") { + t.Errorf("ServerError.Response = %q", se.Response) + } +} + +func TestLiteralLen(t *testing.T) { + cases := []struct { + in string + n int + ok bool + }{ + {"* 1 FETCH (BODY[] {12}", 12, true}, + {"* 1 FETCH (BODY[] {abc}", 0, false}, + {"* 1 FETCH (BODY[])", 0, false}, + {"* SEARCH 1 2", 0, false}, + } + for _, tc := range cases { + n, ok := literalLen(tc.in) + if n != tc.n || ok != tc.ok { + t.Errorf("literalLen(%q) = (%d, %v), want (%d, %v)", tc.in, n, ok, tc.n, tc.ok) + } + } +} + +func TestImapQuote(t *testing.T) { + if got := imapQuote(`pa"ss\word`); got != `"pa\"ss\\word"` { + t.Errorf("imapQuote() = %q", got) + } +} + +func TestUIDSet(t *testing.T) { + if got := uidSet([]uint32{1, 2, 3}); got != "1,2,3" { + t.Errorf("uidSet() = %q, want 1,2,3", got) + } +} diff --git a/internal/mailfetch/message.go b/internal/mailfetch/message.go new file mode 100644 index 0000000..34a579a --- /dev/null +++ b/internal/mailfetch/message.go @@ -0,0 +1,249 @@ +package mailfetch + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "net/textproto" + "regexp" + "strings" + "time" + "unicode/utf8" +) + +// maxBodyBytes caps how much body text is read from one message. +const maxBodyBytes = 1 << 20 // 1 MiB + +// ParsedMessage is the subset of an RFC822 message the agent stores. +type ParsedMessage struct { + From string + To string + Subject string + Date time.Time + SourceAddress string + // SourceHeader records which header the source address came from, so FR-2 + // decisions are auditable. + SourceHeader string + BodyText string +} + +// ParseMessage parses a raw RFC822 message. +func ParseMessage(raw []byte) (*ParsedMessage, error) { + msg, err := mail.ReadMessage(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("parse RFC822 message: %w", err) + } + h := msg.Header + pm := &ParsedMessage{ + From: decodeHeader(h.Get("From")), + To: decodeHeader(h.Get("To")), + Subject: decodeHeader(h.Get("Subject")), + Date: parseDate(h.Get("Date")), + } + pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h)) + pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body) + return pm, nil +} + +// SourceAddress resolves the address the mail arrived at (FR-2). +// +// Order: +// 1. X-Pm-Original-To (added by PurelyMail) +// 2. To +// 3. the PurelyMail "Received ... for
" clause +// 4. Return-Path +// +// Delivered-To is deliberately never used: Gmail overwrites it. +func SourceAddress(h textproto.MIMEHeader) (address, header string) { + if v := headerAddress(h.Get("X-Pm-Original-To")); v != "" { + return v, "X-Pm-Original-To" + } + if v := headerAddress(h.Get("To")); v != "" { + return v, "To" + } + for _, rv := range h.Values("Received") { + if v := receivedFor(rv); v != "" { + return v, "Received" + } + } + if v := headerAddress(h.Get("Return-Path")); v != "" { + return v, "Return-Path" + } + return "", "" +} + +// headerAddress returns the first address in a header value. +func headerAddress(v string) string { + v = strings.TrimSpace(v) + if v == "" { + return "" + } + if list, err := mail.ParseAddressList(v); err == nil && len(list) > 0 { + return strings.TrimSpace(list[0].Address) + } + if a, err := mail.ParseAddress(v); err == nil { + return strings.TrimSpace(a.Address) + } + v = strings.TrimSpace(strings.TrimPrefix(v, "<")) + if i := strings.IndexAny(v, ">,;"); i >= 0 { + v = v[:i] + } + return strings.Trim(strings.TrimSpace(v), "<>") +} + +var receivedForRE = regexp.MustCompile(`(?i)\bfor\s+?`) + +// receivedFor extracts the address from a "Received: ... for " clause. +func receivedFor(v string) string { + m := receivedForRE.FindStringSubmatch(v) + if m == nil { + return "" + } + return strings.Trim(m[1], "<>") +} + +func decodeHeader(v string) string { + if v == "" { + return "" + } + dec := mime.WordDecoder{} + if s, err := dec.DecodeHeader(v); err == nil { + return s + } + return v +} + +func parseDate(v string) time.Time { + if t, err := mail.ParseDate(v); err == nil { + return t.UTC() + } + for _, layout := range []string{time.RFC1123Z, time.RFC1123, "Mon, 2 Jan 2006 15:04:05 -0700"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// --- body decoding ---------------------------------------------------------- + +func extractBody(h textproto.MIMEHeader, body io.Reader) string { + mediaType, params := parseMediaType(h.Get("Content-Type")) + decoded := decodeTransfer(h.Get("Content-Transfer-Encoding"), body) + switch { + case mediaType == "" || mediaType == "text/plain": + return toUTF8(readAll(decoded)) + case mediaType == "text/html": + return htmlToText(toUTF8(readAll(decoded))) + case strings.HasPrefix(mediaType, "multipart/"): + return extractMultipart(params["boundary"], decoded) + default: + // application/*, image/*, … have no text to show. + return "" + } +} + +func extractMultipart(boundary string, r io.Reader) string { + if boundary == "" { + return "" + } + mr := multipart.NewReader(r, boundary) + var plain, html []string + for { + part, err := mr.NextPart() + if err != nil { + break + } + mediaType, _ := parseMediaType(part.Header.Get("Content-Type")) + sub := extractBody(textproto.MIMEHeader(part.Header), part) + _ = part.Close() + if sub == "" { + continue + } + if mediaType == "text/html" { + html = append(html, sub) + } else { + plain = append(plain, sub) + } + } + if len(plain) > 0 { + return strings.TrimSpace(strings.Join(plain, "\n")) + } + return strings.TrimSpace(strings.Join(html, "\n")) +} + +func parseMediaType(v string) (string, map[string]string) { + if strings.TrimSpace(v) == "" { + return "", nil + } + mt, params, err := mime.ParseMediaType(v) + if err != nil { + head := strings.TrimSpace(strings.SplitN(v, ";", 2)[0]) + return strings.ToLower(head), nil + } + return strings.ToLower(mt), params +} + +func decodeTransfer(encoding string, r io.Reader) io.Reader { + switch strings.ToLower(strings.TrimSpace(encoding)) { + case "base64": + return base64.NewDecoder(base64.StdEncoding, r) + case "quoted-printable": + return quotedprintable.NewReader(r) + default: + return r + } +} + +func readAll(r io.Reader) []byte { + b, err := io.ReadAll(io.LimitReader(r, maxBodyBytes)) + if err != nil { + return b + } + return b +} + +// toUTF8 returns valid UTF-8 text, falling back to a Latin-1 interpretation of +// bytes that are not valid UTF-8 (no charset table is available without adding a +// dependency; forwarded personal mail is overwhelmingly UTF-8 or Latin-1). +func toUTF8(b []byte) string { + if utf8.Valid(b) { + return strings.TrimSpace(string(b)) + } + var sb strings.Builder + for _, c := range b { + sb.WriteRune(rune(c)) + } + return strings.TrimSpace(sb.String()) +} + +var ( + htmlBlockRE = regexp.MustCompile(`(?is)<(script|style)[^>]*>.*?`) + htmlTagRE = regexp.MustCompile(`(?s)<[^>]*>`) + htmlSpaceRE = regexp.MustCompile(`[ \t]+`) + htmlNLRE = regexp.MustCompile(`\n{3,}`) +) + +func htmlToText(s string) string { + if s == "" { + return "" + } + s = htmlBlockRE.ReplaceAllString(s, " ") + s = htmlTagRE.ReplaceAllString(s, " ") + s = strings.NewReplacer( + " ", " ", + "&", "&", + "<", "<", + ">", ">", + """, `"`, + "'", "'", + ).Replace(s) + s = htmlSpaceRE.ReplaceAllString(s, " ") + s = htmlNLRE.ReplaceAllString(s, "\n\n") + return strings.TrimSpace(s) +} diff --git a/internal/mailfetch/message_test.go b/internal/mailfetch/message_test.go new file mode 100644 index 0000000..9b5133f --- /dev/null +++ b/internal/mailfetch/message_test.go @@ -0,0 +1,174 @@ +package mailfetch + +import ( + "net/textproto" + "strings" + "testing" +) + +// realForwardedHeaders mirrors the header set seen on a real forwarded message +// (AGENTS.md §13, message id 619). X-Pm-Original-To must win and Delivered-To +// must never be used. +func TestSourceAddressXPMOriginalToWins(t *testing.T) { + h := textproto.MIMEHeader{} + h.Set("X-Pm-Original-To", "Hello@where-woof.com") + h.Set("To", "Hello@where-woof.com") + h.Set("Return-Path", "") + h.Set("Delivered-To", "samuelrolfe@gmail.com") + + addr, header := SourceAddress(h) + if addr != "Hello@where-woof.com" { + t.Errorf("address = %q, want %q", addr, "Hello@where-woof.com") + } + if header != "X-Pm-Original-To" { + t.Errorf("header = %q, want %q", header, "X-Pm-Original-To") + } +} + +func TestSourceAddressFallsBackToTo(t *testing.T) { + h := textproto.MIMEHeader{} + h.Set("To", "Second Choice ") + h.Set("Delivered-To", "samuelrolfe@gmail.com") + + addr, header := SourceAddress(h) + if addr != "second@example.test" { + t.Errorf("address = %q, want %q", addr, "second@example.test") + } + if header != "To" { + t.Errorf("header = %q, want %q", header, "To") + } +} + +func TestSourceAddressFallsBackToReceivedFor(t *testing.T) { + h := textproto.MIMEHeader{} + h.Add("Received", "by mx.example.test with ESMTPS id abc") + h.Add("Received", "from mail.purelymail.com by mx.example.test for ; Mon, 1 Jan 2026 00:00:00 +0000") + h.Set("Return-Path", "") + + addr, header := SourceAddress(h) + if addr != "relay@where-woof.com" { + t.Errorf("address = %q, want %q", addr, "relay@where-woof.com") + } + if header != "Received" { + t.Errorf("header = %q, want %q", header, "Received") + } +} + +func TestSourceAddressFallsBackToReturnPath(t *testing.T) { + h := textproto.MIMEHeader{} + h.Set("Return-Path", "") + + addr, header := SourceAddress(h) + if addr != "sender@example.test" { + t.Errorf("address = %q, want %q", addr, "sender@example.test") + } + if header != "Return-Path" { + t.Errorf("header = %q, want %q", header, "Return-Path") + } +} + +func TestSourceAddressIgnoresDeliveredTo(t *testing.T) { + h := textproto.MIMEHeader{} + h.Set("Delivered-To", "samuelrolfe@gmail.com") + + addr, header := SourceAddress(h) + if addr != "" || header != "" { + t.Errorf("SourceAddress() = (%q, %q), want empty: Delivered-To must not be used", addr, header) + } +} + +func TestParseMessagePlain(t *testing.T) { + raw := strings.Join([]string{ + "From: favrolfe@gmail.com", + "To: Hello@where-woof.com", + "X-Pm-Original-To: Hello@where-woof.com", + "Subject: =?UTF-8?Q?Caf=C3=A9_meeting?=", + "Date: Mon, 02 Oct 2006 15:04:05 -0700", + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: quoted-printable", + "", + "Hello there=2C this is a test.", + }, "\r\n") + + pm, err := ParseMessage([]byte(raw)) + if err != nil { + t.Fatalf("ParseMessage() error = %v", err) + } + if pm.Subject != "Café meeting" { + t.Errorf("Subject = %q, want %q", pm.Subject, "Café meeting") + } + if pm.SourceAddress != "Hello@where-woof.com" || pm.SourceHeader != "X-Pm-Original-To" { + t.Errorf("source = (%q, %q), want (Hello@where-woof.com, X-Pm-Original-To)", pm.SourceAddress, pm.SourceHeader) + } + if !strings.Contains(pm.BodyText, "Hello there, this is a test.") { + t.Errorf("BodyText = %q, want quoted-printable decoded text", pm.BodyText) + } +} + +func TestParseMessageMultipartPrefersPlain(t *testing.T) { + raw := strings.Join([]string{ + "From: sender@example.test", + "To: Hello@where-woof.com", + "Subject: multipart", + "MIME-Version: 1.0", + `Content-Type: multipart/alternative; boundary="BOUND"`, + "", + "--BOUND", + "Content-Type: text/plain; charset=utf-8", + "", + "plain part wins", + "--BOUND", + "Content-Type: text/html; charset=utf-8", + "", + "

html part loses

", + "--BOUND--", + "", + }, "\r\n") + + pm, err := ParseMessage([]byte(raw)) + if err != nil { + t.Fatalf("ParseMessage() error = %v", err) + } + if !strings.Contains(pm.BodyText, "plain part wins") { + t.Errorf("BodyText = %q, want the text/plain part", pm.BodyText) + } + if strings.Contains(pm.BodyText, "html part") { + t.Errorf("BodyText = %q, must not contain the HTML part", pm.BodyText) + } +} + +func TestParseMessageHTMLOnlyIsStripped(t *testing.T) { + raw := strings.Join([]string{ + "From: sender@example.test", + "Subject: html only", + "Content-Type: text/html; charset=utf-8", + "", + "

Hello

World & friends

", + }, "\r\n") + + pm, err := ParseMessage([]byte(raw)) + if err != nil { + t.Fatalf("ParseMessage() error = %v", err) + } + if !strings.Contains(pm.BodyText, "Hello") || !strings.Contains(pm.BodyText, "World & friends") { + t.Errorf("BodyText = %q, want stripped HTML text", pm.BodyText) + } + if strings.Contains(pm.BodyText, "<") { + t.Errorf("BodyText = %q, want no tags", pm.BodyText) + } +} + +func TestHeaderAddressParsing(t *testing.T) { + cases := []struct{ in, want string }{ + {"Hello@where-woof.com", "Hello@where-woof.com"}, + {"", "Hello@where-woof.com"}, + {"Sam Rolfe ", "sam@example.test"}, + {"a@example.test, b@example.test", "a@example.test"}, + {"", ""}, + } + for _, tc := range cases { + if got := headerAddress(tc.in); got != tc.want { + t.Errorf("headerAddress(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/server/auth.go b/internal/server/auth.go index c0ff3d3..3bcaf7c 100644 --- a/internal/server/auth.go +++ b/internal/server/auth.go @@ -213,6 +213,7 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) { type navItem struct { Name string Color string + Count int } type dashboardData struct { @@ -220,27 +221,40 @@ type dashboardData struct { User string CSRFToken string Version string + Total int Categories []navItem Folders []navItem } -// handleDashboard renders the empty inbox shell. +// handleDashboard renders the inbox shell with real category counts. func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { sess := sessionFrom(r) + + // FR-5: counts come from the stored messages. Phase 2 has no classifier + // yet, so every message lands under Unclassified (AGENTS.md §13.2). + items, err := s.store.ListMessages(r.Context(), 200) + if err != nil { + s.logger.Error("load dashboard counts", "error", err) + items = nil + } + counts := categoryCounts(items) + data := dashboardData{ Greeting: partOfDay(time.Now()), User: sess.User, CSRFToken: s.csrfToken(w, r), Version: version.Version, + Total: len(items), // Placeholders until FR-14 (editable categories) and FR-9 (folders) // land. Colours are the design system's decorative sticker palette. Categories: []navItem{ - {Name: "Action", Color: "var(--primary)"}, - {Name: "Important", Color: "var(--sticker-orange)"}, - {Name: "Info", Color: "var(--sticker-sky)"}, - {Name: "Spam", Color: "var(--sticker-red)"}, - {Name: "Newsletter", Color: "var(--sticker-purple)"}, - {Name: "Auto-reply", Color: "var(--sticker-teal)"}, + {Name: "Action", Color: "var(--primary)", Count: counts["action"]}, + {Name: "Important", Color: "var(--sticker-orange)", Count: counts["important"]}, + {Name: "Info", Color: "var(--sticker-sky)", Count: counts["info"]}, + {Name: "Spam", Color: "var(--sticker-red)", Count: counts["spam"]}, + {Name: "Newsletter", Color: "var(--sticker-purple)", Count: counts["newsletter"]}, + {Name: "Auto-reply", Color: "var(--sticker-teal)", Count: counts["auto-reply"]}, + {Name: "Unclassified", Color: "var(--ink-faint)", Count: counts[unclassifiedCategory]}, }, Folders: []navItem{ {Name: "Replied-to", Color: "var(--sticker-green)"}, @@ -254,15 +268,6 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { } } -// handleInboxPartial is the HTMX fragment for the centre list. Phase 1.2 has no -// mail yet, so it renders the empty state. -func (s *Server) handleInboxPartial(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := s.tmpl.ExecuteTemplate(w, "inbox.html", nil); err != nil { - s.logger.Error("render inbox partial", "error", err) - } -} - // cleanNext only allows same-site, absolute-path redirects. func cleanNext(next string) string { if next == "" || !strings.HasPrefix(next, "/") || strings.HasPrefix(next, "//") { diff --git a/internal/server/inbox.go b/internal/server/inbox.go new file mode 100644 index 0000000..56c1e26 --- /dev/null +++ b/internal/server/inbox.go @@ -0,0 +1,219 @@ +// Inbox rendering and the manual mail scan. +// +// Phase 2 lists real mail from the database. Classification is Phase 3, so +// messages without a classification are shown under "Unclassified" with a +// temporary "Review" recommended action; the grouping and action plumbing is +// already category-aware. +package server + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" +) + +// unclassifiedCategory is the fallback bucket before Phase 3 adds JEV +// categories (AGENTS.md §13.2). +const unclassifiedCategory = "unclassified" + +// categoryOrder is the display order for the built-in categories. +var categoryOrder = []string{"action", "important", "info", "spam", "newsletter", "auto-reply"} + +var categoryDisplay = map[string]string{ + "action": "Action", + "important": "Important", + "info": "Info", + "spam": "Spam", + "newsletter": "Newsletter", + "auto-reply": "Auto-reply", + "unclassified": "Unclassified", +} + +type messageRow struct { + ID int64 + From string + Subject string + SourceAddr string + Action string + When string +} + +type messageGroup struct { + Name string + Messages []messageRow +} + +type inboxData struct { + Groups []messageGroup + Total int + Note string + Error string +} + +// loadInbox reads messages and groups them for rendering. +func (s *Server) loadInbox(ctx context.Context) (inboxData, error) { + items, err := s.store.ListMessages(ctx, 200) + if err != nil { + return inboxData{}, err + } + return inboxData{Groups: groupMessages(items), Total: len(items)}, nil +} + +// groupMessages buckets messages by category. Unknown (custom) categories get +// their own group; messages with no classification fall into "Unclassified". +func groupMessages(items []store.MessageListItem) []messageGroup { + buckets := map[string][]messageRow{} + for _, it := range items { + key := strings.ToLower(strings.TrimSpace(it.Category)) + if key == "" { + key = unclassifiedCategory + } + action := strings.TrimSpace(it.Action) + if action == "" { + action = "Review" + } + buckets[key] = append(buckets[key], messageRow{ + ID: it.ID, + From: orDefault(it.FromAddr, "(unknown sender)"), + Subject: orDefault(it.Subject, "(no subject)"), + SourceAddr: it.SourceAddress, + Action: action, + When: formatWhen(it.DateSent, it.ReceivedAt), + }) + } + + // Order: built-in categories first, then custom ones alphabetically, then + // Unclassified last. + var keys []string + for _, k := range categoryOrder { + if _, ok := buckets[k]; ok { + keys = append(keys, k) + } + } + var custom []string + for k := range buckets { + if k == unclassifiedCategory || contains(categoryOrder, k) { + continue + } + custom = append(custom, k) + } + sort.Strings(custom) + keys = append(keys, custom...) + if _, ok := buckets[unclassifiedCategory]; ok { + keys = append(keys, unclassifiedCategory) + } + + groups := make([]messageGroup, 0, len(keys)) + for _, k := range keys { + groups = append(groups, messageGroup{ + Name: displayCategory(k), + Messages: buckets[k], + }) + } + return groups +} + +func displayCategory(key string) string { + if name, ok := categoryDisplay[key]; ok { + return name + } + if key == "" { + return "Unclassified" + } + return strings.ToUpper(key[:1]) + key[1:] +} + +// categoryCounts returns per-category message counts for the left nav. +func categoryCounts(items []store.MessageListItem) map[string]int { + counts := map[string]int{} + for _, it := range items { + key := strings.ToLower(strings.TrimSpace(it.Category)) + if key == "" { + key = unclassifiedCategory + } + counts[key]++ + } + return counts +} + +func (s *Server) renderInbox(w http.ResponseWriter, r *http.Request, note string) { + data, err := s.loadInbox(r.Context()) + if err != nil { + s.logger.Error("load inbox", "error", err) + http.Error(w, "could not load messages", http.StatusInternalServerError) + return + } + data.Note = note + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := s.tmpl.ExecuteTemplate(w, "inbox.html", data); err != nil { + s.logger.Error("render inbox partial", "error", err) + } +} + +// handleInboxPartial is the HTMX fragment for the centre list. +func (s *Server) handleInboxPartial(w http.ResponseWriter, r *http.Request) { + s.renderInbox(w, r, "") +} + +// handleFetchMail runs one IMAP scan, then returns the refreshed list. It is a +// POST because it changes stored state and must carry a CSRF token. +func (s *Server) handleFetchMail(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) + defer cancel() + + sum, err := s.fetcher.Fetch(ctx) + switch { + case err == nil: + s.renderInbox(w, r, fmt.Sprintf("Scanned %d message(s); %d stored or refreshed.", sum.Fetched, sum.Stored)) + case errors.Is(err, mailfetch.ErrNoCredentials): + s.renderInbox(w, r, "Mail is not connected yet: set GMAIL_USER and GMAIL_APP_PASSWORD.") + case errors.Is(err, mailfetch.ErrAlreadyRunning): + s.renderInbox(w, r, "A mail scan is already running.") + default: + s.logger.Error("mail scan failed", "error", err) + s.renderInbox(w, r, "Mail scan failed: "+err.Error()) + } +} + +func orDefault(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return strings.TrimSpace(v) +} + +func contains(list []string, v string) bool { + for _, x := range list { + if x == v { + return true + } + } + return false +} + +// formatWhen renders a compact, timezone-stable timestamp. +func formatWhen(sent, received time.Time) string { + t := sent + if t.IsZero() { + t = received + } + if t.IsZero() { + return "" + } + if t.Year() == time.Now().Year() { + return t.Format("2 Jan 15:04") + } + return t.Format("2 Jan 2006") +} diff --git a/internal/server/server.go b/internal/server/server.go index 67f7097..25fb6c8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -18,6 +18,7 @@ import ( "time" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/config" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/session" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/version" @@ -37,6 +38,7 @@ type Server struct { http *http.Server tmpl *template.Template sessions *session.Manager + fetcher *mailfetch.Fetcher } // New builds a Server. It does not start listening. @@ -61,6 +63,7 @@ func New(cfg *config.Config, st *store.Store, logger *slog.Logger) (*Server, err logger: logger, tmpl: tmpl, sessions: sessions, + fetcher: mailfetch.NewFetcher(cfg, st, logger), } s.http = &http.Server{ Addr: cfg.Addr, @@ -109,6 +112,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /{$}", s.requireAuth(s.handleDashboard)) mux.HandleFunc("GET /partials/inbox", s.requireAuth(s.handleInboxPartial)) + mux.HandleFunc("POST /actions/fetch", s.requireAuth(s.handleFetchMail)) return s.recoverPanic(s.logRequests(mux)) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 3d3d0c7..f1bae7e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -11,6 +11,7 @@ import ( "regexp" "strings" "testing" + "time" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/config" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/session" @@ -152,13 +153,69 @@ func TestLoginFlowAndDashboard(t *testing.T) { t.Fatalf("dashboard status = %d, want %d", rec.Code, http.StatusOK) } body := rec.Body.String() - for _, want := range []string{"Good ", "Inbox", "Categories", "Folders", "Log out"} { + for _, want := range []string{"Good ", "Inbox", "Categories", "Folders", "Log out", "Check mail", "All mail"} { if !strings.Contains(body, want) { t.Errorf("dashboard body missing %q", want) } } } +func TestInboxShowsStoredMail(t *testing.T) { + srv := newTestServer(t) + + if _, err := srv.store.UpsertMessage(context.Background(), &store.Message{ + GmailID: "1234000000000000000", + FromAddr: "favrolfe@gmail.com", + ToAddr: "Hello@where-woof.com", + Subject: "Real forwarded mail", + BodyText: "hello from phase 2", + SourceAddress: "Hello@where-woof.com", + DateSent: time.Now().UTC(), + }); err != nil { + t.Fatalf("seed message: %v", err) + } + + h := srv.Handler() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/login", nil)) + csrf := csrfFromBody(t, rec.Body.String()) + csrfCookie := findCookie(t, rec.Result().Cookies(), csrfCookieName) + + rec = httptest.NewRecorder() + h.ServeHTTP(rec, loginRequest(csrf, testPassword, csrfCookie)) + sessionCookie := findCookie(t, rec.Result().Cookies(), session.CookieName) + if sessionCookie == nil { + t.Fatal("login did not set the session cookie") + } + + req := httptest.NewRequest(http.MethodGet, "/partials/inbox", nil) + req.AddCookie(sessionCookie) + rec = httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("inbox partial status = %d, want %d", rec.Code, http.StatusOK) + } + body := rec.Body.String() + for _, want := range []string{"Real forwarded mail", "favrolfe@gmail.com", "Hello@where-woof.com", "Unclassified", "Review"} { + if !strings.Contains(body, want) { + t.Errorf("inbox partial missing %q", want) + } + } +} + +func TestFetchRequiresCSRF(t *testing.T) { + srv := newTestServer(t) + + req := httptest.NewRequest(http.MethodPost, "/actions/fetch", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + + // Unauthenticated requests are redirected to login before the CSRF check. + if rec.Code != http.StatusSeeOther { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusSeeOther) + } +} + func TestLoginWrongPassword(t *testing.T) { srv := newTestServer(t) h := srv.Handler() diff --git a/internal/store/messages.go b/internal/store/messages.go new file mode 100644 index 0000000..ab174d7 --- /dev/null +++ b/internal/store/messages.go @@ -0,0 +1,188 @@ +// Message persistence for the Email Agent (AGENTS.md §7). +// +// Phase 2 stores the metadata, body text, and — critically for FR-2 — the +// source address the mail was forwarded to (X-Pm-Original-To and its +// fallbacks). Classification columns arrive with Phase 3. +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Message is one row of the messages table. +type Message struct { + ID int64 + GmailID string + ThreadID string + FromAddr string + ToAddr string + Subject string + BodyText string + SourceAddress string + DateSent time.Time + ReceivedAt time.Time + IMAPUID int64 + IMAPUIDValidity int64 +} + +// MessageListItem is a message plus its latest classification, when present. +type MessageListItem struct { + Message + Category string + Importance string + Action string + Confidence float64 + HasClassification bool +} + +// UpsertMessage inserts a message, or refreshes the mutable fields when the +// same Gmail id is seen again. received_at is preserved on update so the +// original arrival time is stable. +func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) { + const q = ` +INSERT INTO messages ( + gmail_id, thread_id, from_addr, to_addr, subject, body_text, + source_address, date_sent, received_at, imap_uid, imap_uidvalidity +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (gmail_id) DO UPDATE SET + thread_id = excluded.thread_id, + from_addr = excluded.from_addr, + to_addr = excluded.to_addr, + subject = excluded.subject, + body_text = excluded.body_text, + source_address = excluded.source_address, + date_sent = excluded.date_sent, + imap_uid = excluded.imap_uid, + imap_uidvalidity = excluded.imap_uidvalidity` + + receivedAt := m.ReceivedAt + if receivedAt.IsZero() { + receivedAt = time.Now().UTC() + } + _, err := s.db.ExecContext(ctx, q, + m.GmailID, + nullString(m.ThreadID), + nullString(m.FromAddr), + nullString(m.ToAddr), + nullString(m.Subject), + nullString(m.BodyText), + nullString(m.SourceAddress), + nullTime(m.DateSent), + receivedAt.UTC().Format(sqlTimeLayout), + m.IMAPUID, + m.IMAPUIDValidity, + ) + if err != nil { + return 0, fmt.Errorf("upsert message %q: %w", m.GmailID, err) + } + + var id int64 + if err := s.db.QueryRowContext(ctx, + `SELECT id FROM messages WHERE gmail_id = ?`, m.GmailID).Scan(&id); err != nil { + return 0, fmt.Errorf("read message id %q: %w", m.GmailID, err) + } + return id, nil +} + +// ListMessages returns the most recent messages, newest first, joined to their +// latest classification when one exists. +func (s *Store) ListMessages(ctx context.Context, limit int) ([]MessageListItem, error) { + if limit < 1 { + limit = 100 + } + const q = ` +SELECT + m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''), + COALESCE(m.to_addr, ''), COALESCE(m.subject, ''), COALESCE(m.body_text, ''), + COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at, + COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0), + COALESCE(c.category, ''), COALESCE(c.importance, ''), + COALESCE(c.action, ''), COALESCE(c.confidence, 0) +FROM messages m +LEFT JOIN classifications c ON c.id = ( + SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1 +) +ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC +LIMIT ?` + + rows, err := s.db.QueryContext(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("list messages: %w", err) + } + defer rows.Close() + + var out []MessageListItem + for rows.Next() { + var ( + item MessageListItem + dateSent string + received string + ) + if err := rows.Scan( + &item.ID, &item.GmailID, &item.ThreadID, &item.FromAddr, + &item.ToAddr, &item.Subject, &item.BodyText, + &item.SourceAddress, &dateSent, &received, + &item.IMAPUID, &item.IMAPUIDValidity, + &item.Category, &item.Importance, &item.Action, &item.Confidence, + ); err != nil { + return nil, fmt.Errorf("scan message: %w", err) + } + item.DateSent = parseSQLTime(dateSent) + item.ReceivedAt = parseSQLTime(received) + item.HasClassification = item.Category != "" + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list messages: %w", err) + } + return out, nil +} + +// CountMessages returns the total number of stored messages. +func (s *Store) CountMessages(ctx context.Context) (int, error) { + var n int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM messages`).Scan(&n); err != nil { + return 0, fmt.Errorf("count messages: %w", err) + } + return n, nil +} + +// LogAction appends to the actions_log (FR-13). The table is append-only. +func (s *Store) LogAction(ctx context.Context, action, actor, detail string) error { + const q = `INSERT INTO actions_log (message_id, action, actor, detail) VALUES (NULL, ?, ?, ?)` + if _, err := s.db.ExecContext(ctx, q, action, nullString(actor), nullString(detail)); err != nil { + return fmt.Errorf("log action %q: %w", action, err) + } + return nil +} + +const sqlTimeLayout = "2006-01-02 15:04:05" + +func nullString(v string) any { + if strings.TrimSpace(v) == "" { + return nil + } + return v +} + +func nullTime(t time.Time) any { + if t.IsZero() { + return nil + } + return t.UTC().Format(sqlTimeLayout) +} + +func parseSQLTime(v string) time.Time { + if v == "" { + return time.Time{} + } + for _, layout := range []string{sqlTimeLayout, time.RFC3339, "2006-01-02T15:04:05Z"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC() + } + } + return time.Time{} +} diff --git a/internal/store/schema.sql b/internal/store/schema.sql index d2bba53..5ac4c56 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -24,6 +24,10 @@ CREATE TABLE IF NOT EXISTS messages ( body_text TEXT, source_address TEXT, date_sent TEXT, + -- IMAP bookkeeping (AGENTS.md §9): the mailbox UID and the UIDVALIDITY it + -- belongs to. UIDs are only meaningful while UIDVALIDITY is unchanged. + imap_uid INTEGER, + imap_uidvalidity INTEGER, received_at TEXT NOT NULL DEFAULT (datetime('now')), created_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/internal/store/store.go b/internal/store/store.go index a42a2cd..c923717 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -76,11 +76,69 @@ func buildDSN(path string) string { return "file:" + abs + "?" + q.Encode() } -// migrate applies the embedded schema. It is safe to run on every start. +// migrate applies the embedded schema, then any incremental migrations that +// add columns to a database created by an earlier release. It is safe to run on +// every start. func (s *Store) migrate(ctx context.Context) error { if _, err := s.db.ExecContext(ctx, schemaSQL); err != nil { return fmt.Errorf("apply schema: %w", err) } + return s.applyMigrations(ctx) +} + +// applyMigrations brings an existing database up to the latest schema version. +// Version 1 is the base schema (schema.sql); version 2 adds IMAP bookkeeping +// columns (AGENTS.md §9 / Phase 2). Each step is idempotent. +func (s *Store) applyMigrations(ctx context.Context) error { + version, err := s.SchemaVersion(ctx) + if err != nil { + return fmt.Errorf("read schema version: %w", err) + } + if version < 2 { + for _, col := range []struct{ table, name, typ string }{ + {"messages", "imap_uid", "INTEGER"}, + {"messages", "imap_uidvalidity", "INTEGER"}, + } { + if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil { + return err + } + } + if _, err := s.db.ExecContext(ctx, + `INSERT OR IGNORE INTO schema_migrations (version) VALUES (2)`); err != nil { + return fmt.Errorf("record schema version 2: %w", err) + } + } + return nil +} + +// ensureColumn adds a column when it is missing. SQLite has no +// "ADD COLUMN IF NOT EXISTS", so the current shape is inspected first. +func (s *Store) ensureColumn(ctx context.Context, table, name, typ string) error { + rows, err := s.db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return fmt.Errorf("inspect %s: %w", table, err) + } + defer rows.Close() + for rows.Next() { + var ( + cid, notnull, pk int + dflt sql.NullString + colName, colType string + ) + if err := rows.Scan(&cid, &colName, &colType, ¬null, &dflt, &pk); err != nil { + return fmt.Errorf("scan %s columns: %w", table, err) + } + if colName == name { + return nil + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("inspect %s columns: %w", table, err) + } + if _, err := s.db.ExecContext(ctx, + fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, name, typ)); err != nil { + return fmt.Errorf("add %s.%s: %w", table, name, err) + } return nil } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 2a0948a..d7247b6 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "testing" + "time" ) func TestOpenCreatesSchema(t *testing.T) { @@ -83,3 +84,110 @@ func TestMessagesInsertAndRead(t *testing.T) { t.Errorf("subject = %q, want %q", subject, "Hello") } } + +func TestUpsertAndListMessages(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "email_agent.db")) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + m := &Message{ + GmailID: "1234000000000000000", + ThreadID: "999", + FromAddr: "favrolfe@gmail.com", + ToAddr: "Hello@where-woof.com", + Subject: "A forwarded hello", + BodyText: "body text", + SourceAddress: "Hello@where-woof.com", + DateSent: time.Date(2026, 10, 1, 9, 30, 0, 0, time.UTC), + IMAPUID: 42, + IMAPUIDValidity: 7, + } + id, err := s.UpsertMessage(ctx, m) + if err != nil { + t.Fatalf("UpsertMessage() error = %v", err) + } + if id == 0 { + t.Fatal("UpsertMessage() returned id 0") + } + + // A second upsert with the same Gmail id must update, not duplicate, and + // must not move received_at. + m.Subject = "Updated subject" + id2, err := s.UpsertMessage(ctx, m) + if err != nil { + t.Fatalf("second UpsertMessage() error = %v", err) + } + if id2 != id { + t.Errorf("second upsert id = %d, want %d", id2, id) + } + + n, err := s.CountMessages(ctx) + if err != nil { + t.Fatalf("CountMessages() error = %v", err) + } + if n != 1 { + t.Fatalf("CountMessages() = %d, want 1", n) + } + + list, err := s.ListMessages(ctx, 10) + if err != nil { + t.Fatalf("ListMessages() error = %v", err) + } + if len(list) != 1 { + t.Fatalf("ListMessages() len = %d, want 1", len(list)) + } + got := list[0] + if got.Subject != "Updated subject" { + t.Errorf("Subject = %q, want updated value", got.Subject) + } + if got.SourceAddress != "Hello@where-woof.com" { + t.Errorf("SourceAddress = %q", got.SourceAddress) + } + if got.IMAPUID != 42 || got.IMAPUIDValidity != 7 { + t.Errorf("IMAP bookkeeping = (%d, %d), want (42, 7)", got.IMAPUID, got.IMAPUIDValidity) + } + if got.HasClassification { + t.Error("HasClassification = true with no classifications row") + } +} + +func TestMigrationAddsIMAPColumns(t *testing.T) { + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "email_agent.db")) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion(ctx) + if err != nil { + t.Fatalf("SchemaVersion() error = %v", err) + } + if v < 2 { + t.Errorf("SchemaVersion() = %d, want >= 2", v) + } + + for _, col := range []string{"imap_uid", "imap_uidvalidity"} { + var found bool + rows, err := s.DB().QueryContext(ctx, `SELECT name FROM pragma_table_info('messages')`) + if err != nil { + t.Fatalf("pragma_table_info: %v", err) + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan: %v", err) + } + if name == col { + found = true + } + } + _ = rows.Close() + if !found { + t.Errorf("column %q missing after migration", col) + } + } +} diff --git a/web/static/app.css b/web/static/app.css index d0ca346..5f6d0b9 100644 --- a/web/static/app.css +++ b/web/static/app.css @@ -215,6 +215,17 @@ a:hover { text-decoration: underline; } border-bottom: 1px solid var(--hairline); } .panel-head h2 { font: var(--type-title); color: var(--ink); } +.panel-actions { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; } + +/* scan status line above the message list */ +.scan-note-slot:empty { display: none; } +.scan-note { + padding: var(--space-xs) var(--space-md); + font: var(--type-body-sm); + color: var(--ink-muted); + background: var(--surface-2); + border-bottom: 1px solid var(--hairline); +} .panel-body { padding: var(--space-md); } @@ -261,6 +272,21 @@ a:hover { text-decoration: underline; } /* centre list */ .msg-list { display: flex; flex-direction: column; } +.msg-group { display: flex; flex-direction: column; } +.msg-group-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-sm); + padding: var(--space-xs) var(--space-md); + background: var(--surface-2); + border-bottom: 1px solid var(--hairline); + font: var(--type-eyebrow); + color: var(--ink-muted); + text-transform: uppercase; + letter-spacing: .125px; +} +.msg-group-head .count { font: var(--type-caption); color: var(--ink-faint); text-transform: none; } .msg-row { display: flex; align-items: center; @@ -270,6 +296,15 @@ a:hover { text-decoration: underline; } border-bottom: 1px solid var(--hairline); } .msg-row:last-child { border-bottom: 0; } +.msg-main { min-width: 0; flex: 1 1 auto; display: flex; flex-direction: column; gap: 2px; } +.msg-from { font: var(--type-body-sm); color: var(--ink); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.msg-subject { font: var(--type-body-sm); color: var(--ink-secondary); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.msg-source { font: var(--type-caption); color: var(--ink-faint); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.msg-source .source-addr { color: var(--ink-muted); } +.msg-meta { display: flex; flex-direction: column; align-items: flex-end; gap: 4px; flex: 0 0 auto; } +.msg-when { font: var(--type-caption); color: var(--ink-faint); white-space: nowrap; } +.action-pill { background: var(--primary-soft, var(--surface-2)); color: var(--primary); border-color: var(--primary); white-space: nowrap; } +.nav-item-active { background: var(--surface-2); color: var(--ink); font-weight: 500; } .empty-state { display: flex; diff --git a/web/templates/dashboard.html b/web/templates/dashboard.html index ddda854..c92ca5a 100644 --- a/web/templates/dashboard.html +++ b/web/templates/dashboard.html @@ -46,11 +46,16 @@
Categories
+ + + All mail + {{.Total}} + {{range .Categories}} {{.Name}} - 0 + {{.Count}} {{end}}
@@ -73,10 +78,17 @@

Inbox

- +
+ + +
diff --git a/web/templates/inbox.html b/web/templates/inbox.html index 09b7388..4a5de35 100644 --- a/web/templates/inbox.html +++ b/web/templates/inbox.html @@ -1,5 +1,33 @@ +
+ {{if .Note}}
{{.Note}}
{{end}} +
+ +{{if .Groups}} + {{range .Groups}} +
+
+ {{.Name}} + {{len .Messages}} +
+ {{range .Messages}} +
+
+
{{.From}}
+
{{.Subject}}
+ {{if .SourceAddr}}
arrived at {{.SourceAddr}}
{{end}} +
+
+ {{.Action}} + {{if .When}}{{.When}}{{end}} +
+
+ {{end}} +
+ {{end}} +{{else}}
No messages yet
-

Mail fetch arrives in Phase 2. New messages will appear here, grouped by category, with a recommended action on each row.

- IMAP scan not connected +

Use Check mail to fetch from Gmail. New messages appear here, grouped by category, with a recommended action on each row.

+ No stored messages
+{{end}}