Phase 2: Gmail IMAP fetch (IPv4), FR-2 source address, real mail in dashboard (AUD-67)

- internal/mailfetch: stdlib-only read-only IMAP client (LOGIN, SELECT,
  UID SEARCH, UID FETCH) that always dials tcp4; IPv6 to Gmail is broken
  from .13 and caused a socket EOF.
- MIME parsing to text, RFC2047 header decoding, FR-2 source address:
  X-Pm-Original-To -> To -> Received 'for <addr>' -> Return-Path, never
  Delivered-To. The matched header is recorded for audit.
- store: Message upsert/list/count + LogAction; schema migration v2 adds
  imap_uid and imap_uidvalidity.
- server: dashboard and /partials/inbox render real mail grouped by
  category with a recommended action; POST /actions/fetch runs a scan
  (CSRF-checked); nav counts; 'Check mail' button.
- config: GMAIL_USER, GMAIL_APP_PASSWORD, IMAP host/port/lookback/max.
- cmd: -fetch runs one scan and exits (n8n/CLI entry point, FR-1).
- compose/.env.example expose the Gmail + IMAP settings.
- tests: FR-2 precedence, MIME bodies, IMAP literal/parser, store
  upsert/migration, inbox rendering.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-11 20:58:32 +11:00
parent e846400338
commit 1d88f962f6
23 changed files with 2067 additions and 52 deletions

View File

@@ -0,0 +1,174 @@
package mailfetch
import (
"net/textproto"
"strings"
"testing"
)
// realForwardedHeaders mirrors the header set seen on a real forwarded message
// (AGENTS.md §13, message id 619). X-Pm-Original-To must win and Delivered-To
// must never be used.
func TestSourceAddressXPMOriginalToWins(t *testing.T) {
h := textproto.MIMEHeader{}
h.Set("X-Pm-Original-To", "Hello@where-woof.com")
h.Set("To", "Hello@where-woof.com")
h.Set("Return-Path", "<Hello@where-woof.com>")
h.Set("Delivered-To", "samuelrolfe@gmail.com")
addr, header := SourceAddress(h)
if addr != "Hello@where-woof.com" {
t.Errorf("address = %q, want %q", addr, "Hello@where-woof.com")
}
if header != "X-Pm-Original-To" {
t.Errorf("header = %q, want %q", header, "X-Pm-Original-To")
}
}
func TestSourceAddressFallsBackToTo(t *testing.T) {
h := textproto.MIMEHeader{}
h.Set("To", "Second Choice <second@example.test>")
h.Set("Delivered-To", "samuelrolfe@gmail.com")
addr, header := SourceAddress(h)
if addr != "second@example.test" {
t.Errorf("address = %q, want %q", addr, "second@example.test")
}
if header != "To" {
t.Errorf("header = %q, want %q", header, "To")
}
}
func TestSourceAddressFallsBackToReceivedFor(t *testing.T) {
h := textproto.MIMEHeader{}
h.Add("Received", "by mx.example.test with ESMTPS id abc")
h.Add("Received", "from mail.purelymail.com by mx.example.test for <relay@where-woof.com>; Mon, 1 Jan 2026 00:00:00 +0000")
h.Set("Return-Path", "<fallback@example.test>")
addr, header := SourceAddress(h)
if addr != "relay@where-woof.com" {
t.Errorf("address = %q, want %q", addr, "relay@where-woof.com")
}
if header != "Received" {
t.Errorf("header = %q, want %q", header, "Received")
}
}
func TestSourceAddressFallsBackToReturnPath(t *testing.T) {
h := textproto.MIMEHeader{}
h.Set("Return-Path", "<sender@example.test>")
addr, header := SourceAddress(h)
if addr != "sender@example.test" {
t.Errorf("address = %q, want %q", addr, "sender@example.test")
}
if header != "Return-Path" {
t.Errorf("header = %q, want %q", header, "Return-Path")
}
}
func TestSourceAddressIgnoresDeliveredTo(t *testing.T) {
h := textproto.MIMEHeader{}
h.Set("Delivered-To", "samuelrolfe@gmail.com")
addr, header := SourceAddress(h)
if addr != "" || header != "" {
t.Errorf("SourceAddress() = (%q, %q), want empty: Delivered-To must not be used", addr, header)
}
}
func TestParseMessagePlain(t *testing.T) {
raw := strings.Join([]string{
"From: favrolfe@gmail.com",
"To: Hello@where-woof.com",
"X-Pm-Original-To: Hello@where-woof.com",
"Subject: =?UTF-8?Q?Caf=C3=A9_meeting?=",
"Date: Mon, 02 Oct 2006 15:04:05 -0700",
"Content-Type: text/plain; charset=utf-8",
"Content-Transfer-Encoding: quoted-printable",
"",
"Hello there=2C this is a test.",
}, "\r\n")
pm, err := ParseMessage([]byte(raw))
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if pm.Subject != "Café meeting" {
t.Errorf("Subject = %q, want %q", pm.Subject, "Café meeting")
}
if pm.SourceAddress != "Hello@where-woof.com" || pm.SourceHeader != "X-Pm-Original-To" {
t.Errorf("source = (%q, %q), want (Hello@where-woof.com, X-Pm-Original-To)", pm.SourceAddress, pm.SourceHeader)
}
if !strings.Contains(pm.BodyText, "Hello there, this is a test.") {
t.Errorf("BodyText = %q, want quoted-printable decoded text", pm.BodyText)
}
}
func TestParseMessageMultipartPrefersPlain(t *testing.T) {
raw := strings.Join([]string{
"From: sender@example.test",
"To: Hello@where-woof.com",
"Subject: multipart",
"MIME-Version: 1.0",
`Content-Type: multipart/alternative; boundary="BOUND"`,
"",
"--BOUND",
"Content-Type: text/plain; charset=utf-8",
"",
"plain part wins",
"--BOUND",
"Content-Type: text/html; charset=utf-8",
"",
"<p>html part <b>loses</b></p>",
"--BOUND--",
"",
}, "\r\n")
pm, err := ParseMessage([]byte(raw))
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if !strings.Contains(pm.BodyText, "plain part wins") {
t.Errorf("BodyText = %q, want the text/plain part", pm.BodyText)
}
if strings.Contains(pm.BodyText, "html part") {
t.Errorf("BodyText = %q, must not contain the HTML part", pm.BodyText)
}
}
func TestParseMessageHTMLOnlyIsStripped(t *testing.T) {
raw := strings.Join([]string{
"From: sender@example.test",
"Subject: html only",
"Content-Type: text/html; charset=utf-8",
"",
"<html><body><h1>Hello</h1><p>World&nbsp;&amp; friends</p></body></html>",
}, "\r\n")
pm, err := ParseMessage([]byte(raw))
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if !strings.Contains(pm.BodyText, "Hello") || !strings.Contains(pm.BodyText, "World & friends") {
t.Errorf("BodyText = %q, want stripped HTML text", pm.BodyText)
}
if strings.Contains(pm.BodyText, "<") {
t.Errorf("BodyText = %q, want no tags", pm.BodyText)
}
}
func TestHeaderAddressParsing(t *testing.T) {
cases := []struct{ in, want string }{
{"Hello@where-woof.com", "Hello@where-woof.com"},
{"<Hello@where-woof.com>", "Hello@where-woof.com"},
{"Sam Rolfe <sam@example.test>", "sam@example.test"},
{"a@example.test, b@example.test", "a@example.test"},
{"", ""},
}
for _, tc := range cases {
if got := headerAddress(tc.in); got != tc.want {
t.Errorf("headerAddress(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}