Files
Sam Rolfe 1d88f962f6 Phase 2: Gmail IMAP fetch (IPv4), FR-2 source address, real mail in dashboard (AUD-67)
- internal/mailfetch: stdlib-only read-only IMAP client (LOGIN, SELECT,
  UID SEARCH, UID FETCH) that always dials tcp4; IPv6 to Gmail is broken
  from .13 and caused a socket EOF.
- MIME parsing to text, RFC2047 header decoding, FR-2 source address:
  X-Pm-Original-To -> To -> Received 'for <addr>' -> Return-Path, never
  Delivered-To. The matched header is recorded for audit.
- store: Message upsert/list/count + LogAction; schema migration v2 adds
  imap_uid and imap_uidvalidity.
- server: dashboard and /partials/inbox render real mail grouped by
  category with a recommended action; POST /actions/fetch runs a scan
  (CSRF-checked); nav counts; 'Check mail' button.
- config: GMAIL_USER, GMAIL_APP_PASSWORD, IMAP host/port/lookback/max.
- cmd: -fetch runs one scan and exits (n8n/CLI entry point, FR-1).
- compose/.env.example expose the Gmail + IMAP settings.
- tests: FR-2 precedence, MIME bodies, IMAP literal/parser, store
  upsert/migration, inbox rendering.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-11 20:59:53 +11:00

329 lines
8.5 KiB
Go

// Package mailfetch reads mail from Gmail over IMAP and turns it into stored
// messages.
//
// The IMAP client is intentionally dependency-free: it implements only the
// subset the Email Agent needs (LOGIN, SELECT, UID SEARCH, UID FETCH) and it
// always dials IPv4. IPv6 to Gmail is broken from .13 and previously caused a
// socket EOF (AGENTS.md §9).
package mailfetch
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"io"
"net"
"regexp"
"strconv"
"strings"
"time"
)
// DefaultTimeout bounds a single IMAP read/write.
const DefaultTimeout = 60 * time.Second
// Client is a minimal read-only IMAP client.
type Client struct {
conn net.Conn
br *bufio.Reader
tagSeq int
timeout time.Duration
preAuth bool
}
// response is one IMAP response: the assembled text (literals removed) plus any
// literal payloads in order of appearance.
type response struct {
line string
literals [][]byte
}
// ServerError is a tagged NO/BAD completion from the IMAP server.
type ServerError struct {
Command string
Response string
}
func (e *ServerError) Error() string {
return fmt.Sprintf("imap %s failed: %s", e.Command, e.Response)
}
// Dial opens an implicit-TLS IMAP connection to host:port, forcing IPv4.
func Dial(ctx context.Context, host string, port int, timeout time.Duration) (*Client, error) {
if timeout <= 0 {
timeout = DefaultTimeout
}
addr := net.JoinHostPort(host, strconv.Itoa(port))
dialer := &net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}
raw, err := dialer.DialContext(ctx, "tcp4", addr)
if err != nil {
return nil, fmt.Errorf("dial %s over IPv4: %w", addr, err)
}
tlsConn := tls.Client(raw, &tls.Config{
ServerName: host,
MinVersion: tls.VersionTLS12,
})
hsCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
if err := tlsConn.HandshakeContext(hsCtx); err != nil {
_ = raw.Close()
return nil, fmt.Errorf("tls handshake with %s: %w", addr, err)
}
c := &Client{conn: tlsConn, br: bufio.NewReader(tlsConn), timeout: timeout}
greeting, err := c.readResponse()
if err != nil {
_ = tlsConn.Close()
return nil, fmt.Errorf("read IMAP greeting: %w", err)
}
upper := strings.ToUpper(greeting.line)
if strings.HasPrefix(upper, "* PREAUTH") {
c.preAuth = true
return c, nil
}
if !strings.HasPrefix(upper, "* OK") {
_ = tlsConn.Close()
return nil, fmt.Errorf("unexpected IMAP greeting: %s", greeting.line)
}
return c, nil
}
// Login authenticates with an app password.
func (c *Client) Login(user, pass string) error {
if c.preAuth {
return nil
}
return c.exec("LOGIN "+imapQuote(user)+" "+imapQuote(pass), nil)
}
// Select opens a mailbox and returns its UIDVALIDITY.
func (c *Client) Select(mailbox string) (uint32, error) {
var uidValidity uint32
err := c.exec("SELECT "+imapQuote(mailbox), func(r *response) error {
if v, ok := parseUIDValidity(r.line); ok {
uidValidity = v
}
return nil
})
return uidValidity, err
}
// SearchSince returns the UIDs of messages with an internal date on or after t.
func (c *Client) SearchSince(t time.Time) ([]uint32, error) {
date := t.UTC().Format("02-Jan-2006")
var uids []uint32
err := c.exec("UID SEARCH SINCE "+date, func(r *response) error {
if !strings.HasPrefix(strings.ToUpper(r.line), "* SEARCH") {
return nil
}
fields := strings.Fields(r.line)
for _, f := range fields[2:] {
if n, err := strconv.ParseUint(f, 10, 32); err == nil {
uids = append(uids, uint32(n))
}
}
return nil
})
return uids, err
}
// FetchResult is one message returned by Fetch.
type FetchResult struct {
UID uint32
GmailMsgID string
GmailThreadID string
Raw []byte
}
// fetchChunk bounds how many UIDs go into a single FETCH command.
const fetchChunk = 20
// Fetch retrieves the full RFC822 source for each UID, in chunks.
func (c *Client) Fetch(uids []uint32) ([]FetchResult, error) {
var out []FetchResult
for start := 0; start < len(uids); start += fetchChunk {
end := start + fetchChunk
if end > len(uids) {
end = len(uids)
}
set := uidSet(uids[start:end])
var batch []FetchResult
err := c.exec("UID FETCH "+set+" (UID X-GM-MSGID X-GM-THRID BODY.PEEK[])", func(r *response) error {
if fr, ok := parseFetch(r); ok {
batch = append(batch, fr)
}
return nil
})
if err != nil {
return out, err
}
out = append(out, batch...)
}
return out, nil
}
// Logout sends LOGOUT and closes the connection.
func (c *Client) Logout() error {
err := c.exec("LOGOUT", nil)
_ = c.conn.Close()
return err
}
// Close closes the connection without a graceful logout.
func (c *Client) Close() error {
if c == nil || c.conn == nil {
return nil
}
return c.conn.Close()
}
// --- protocol plumbing ------------------------------------------------------
func (c *Client) exec(command string, onUntagged func(*response) error) error {
c.tagSeq++
tag := fmt.Sprintf("A%04d", c.tagSeq)
if err := c.writeLine(tag + " " + command); err != nil {
return err
}
for {
resp, err := c.readResponse()
if err != nil {
return err
}
switch {
case strings.HasPrefix(resp.line, tag+" "):
status := strings.TrimSpace(strings.TrimPrefix(resp.line, tag+" "))
if strings.HasPrefix(strings.ToUpper(status), "OK") {
return nil
}
return &ServerError{Command: command, Response: status}
case strings.HasPrefix(resp.line, "* "):
if onUntagged != nil {
if err := onUntagged(resp); err != nil {
return err
}
}
case strings.HasPrefix(resp.line, "+ "):
return fmt.Errorf("imap server requested a continuation, which this client does not support: %s", resp.line)
default:
// Ignore unexpected non-tagged lines rather than desynchronising.
}
}
}
func (c *Client) writeLine(s string) error {
if err := c.conn.SetWriteDeadline(time.Now().Add(c.timeout)); err != nil {
return err
}
if _, err := c.conn.Write([]byte(s + "\r\n")); err != nil {
return fmt.Errorf("imap write: %w", err)
}
return nil
}
// readResponse reads one logical IMAP response, resolving literals.
func (c *Client) readResponse() (*response, error) {
var sb strings.Builder
var lits [][]byte
for {
line, err := c.readLine()
if err != nil {
return nil, err
}
if n, ok := literalLen(line); ok {
sb.WriteString(line[:strings.LastIndexByte(line, '{')])
buf := make([]byte, n)
if _, err := io.ReadFull(c.br, buf); err != nil {
return nil, fmt.Errorf("read literal: %w", err)
}
lits = append(lits, buf)
continue
}
sb.WriteString(line)
return &response{line: sb.String(), literals: lits}, nil
}
}
func (c *Client) readLine() (string, error) {
if err := c.conn.SetReadDeadline(time.Now().Add(c.timeout)); err != nil {
return "", err
}
line, err := c.br.ReadString('\n')
if err != nil && !(err == io.EOF && line != "") {
return "", fmt.Errorf("imap read: %w", err)
}
return strings.TrimRight(line, "\r\n"), nil
}
// literalLen reports the byte count of a trailing `{n}` literal marker.
func literalLen(line string) (int, bool) {
if !strings.HasSuffix(line, "}") {
return 0, false
}
i := strings.LastIndexByte(line, '{')
if i < 0 {
return 0, false
}
n, err := strconv.Atoi(line[i+1 : len(line)-1])
if err != nil || n < 0 {
return 0, false
}
return n, true
}
var (
uidValidityRE = regexp.MustCompile(`(?i)\[UIDVALIDITY (\d+)\]`)
fetchUIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])UID (\d+)`)
fetchMsgIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])X-GM-MSGID (\d+)`)
fetchThrIDRE = regexp.MustCompile(`(?i)(?:^|[\s(])X-GM-THRID (\d+)`)
)
func parseUIDValidity(line string) (uint32, bool) {
m := uidValidityRE.FindStringSubmatch(line)
if m == nil {
return 0, false
}
n, err := strconv.ParseUint(m[1], 10, 32)
if err != nil {
return 0, false
}
return uint32(n), true
}
func parseFetch(r *response) (FetchResult, bool) {
if len(r.literals) == 0 || !strings.Contains(strings.ToUpper(r.line), "FETCH") {
return FetchResult{}, false
}
fr := FetchResult{Raw: r.literals[0]}
if m := fetchUIDRE.FindStringSubmatch(r.line); m != nil {
if n, err := strconv.ParseUint(m[1], 10, 32); err == nil {
fr.UID = uint32(n)
}
}
if m := fetchMsgIDRE.FindStringSubmatch(r.line); m != nil {
fr.GmailMsgID = m[1]
}
if m := fetchThrIDRE.FindStringSubmatch(r.line); m != nil {
fr.GmailThreadID = m[1]
}
return fr, true
}
func uidSet(uids []uint32) string {
parts := make([]string, len(uids))
for i, u := range uids {
parts[i] = strconv.FormatUint(uint64(u), 10)
}
return strings.Join(parts, ",")
}
// imapQuote quotes a string as an IMAP quoted-string.
func imapQuote(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
s = strings.ReplaceAll(s, `"`, `\"`)
return `"` + s + `"`
}