Phase 2: Gmail IMAP fetch (IPv4), FR-2 source address, real mail in dashboard (AUD-67)
- internal/mailfetch: stdlib-only read-only IMAP client (LOGIN, SELECT, UID SEARCH, UID FETCH) that always dials tcp4; IPv6 to Gmail is broken from .13 and caused a socket EOF. - MIME parsing to text, RFC2047 header decoding, FR-2 source address: X-Pm-Original-To -> To -> Received 'for <addr>' -> Return-Path, never Delivered-To. The matched header is recorded for audit. - store: Message upsert/list/count + LogAction; schema migration v2 adds imap_uid and imap_uidvalidity. - server: dashboard and /partials/inbox render real mail grouped by category with a recommended action; POST /actions/fetch runs a scan (CSRF-checked); nav counts; 'Check mail' button. - config: GMAIL_USER, GMAIL_APP_PASSWORD, IMAP host/port/lookback/max. - cmd: -fetch runs one scan and exits (n8n/CLI entry point, FR-1). - compose/.env.example expose the Gmail + IMAP settings. - tests: FR-2 precedence, MIME bodies, IMAP literal/parser, store upsert/migration, inbox rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
249
internal/mailfetch/message.go
Normal file
249
internal/mailfetch/message.go
Normal file
@@ -0,0 +1,249 @@
|
||||
package mailfetch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"mime/quotedprintable"
|
||||
"net/mail"
|
||||
"net/textproto"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// maxBodyBytes caps how much body text is read from one message.
|
||||
const maxBodyBytes = 1 << 20 // 1 MiB
|
||||
|
||||
// ParsedMessage is the subset of an RFC822 message the agent stores.
|
||||
type ParsedMessage struct {
|
||||
From string
|
||||
To string
|
||||
Subject string
|
||||
Date time.Time
|
||||
SourceAddress string
|
||||
// SourceHeader records which header the source address came from, so FR-2
|
||||
// decisions are auditable.
|
||||
SourceHeader string
|
||||
BodyText string
|
||||
}
|
||||
|
||||
// ParseMessage parses a raw RFC822 message.
|
||||
func ParseMessage(raw []byte) (*ParsedMessage, error) {
|
||||
msg, err := mail.ReadMessage(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse RFC822 message: %w", err)
|
||||
}
|
||||
h := msg.Header
|
||||
pm := &ParsedMessage{
|
||||
From: decodeHeader(h.Get("From")),
|
||||
To: decodeHeader(h.Get("To")),
|
||||
Subject: decodeHeader(h.Get("Subject")),
|
||||
Date: parseDate(h.Get("Date")),
|
||||
}
|
||||
pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h))
|
||||
pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body)
|
||||
return pm, nil
|
||||
}
|
||||
|
||||
// SourceAddress resolves the address the mail arrived at (FR-2).
|
||||
//
|
||||
// Order:
|
||||
// 1. X-Pm-Original-To (added by PurelyMail)
|
||||
// 2. To
|
||||
// 3. the PurelyMail "Received ... for <address>" clause
|
||||
// 4. Return-Path
|
||||
//
|
||||
// Delivered-To is deliberately never used: Gmail overwrites it.
|
||||
func SourceAddress(h textproto.MIMEHeader) (address, header string) {
|
||||
if v := headerAddress(h.Get("X-Pm-Original-To")); v != "" {
|
||||
return v, "X-Pm-Original-To"
|
||||
}
|
||||
if v := headerAddress(h.Get("To")); v != "" {
|
||||
return v, "To"
|
||||
}
|
||||
for _, rv := range h.Values("Received") {
|
||||
if v := receivedFor(rv); v != "" {
|
||||
return v, "Received"
|
||||
}
|
||||
}
|
||||
if v := headerAddress(h.Get("Return-Path")); v != "" {
|
||||
return v, "Return-Path"
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// headerAddress returns the first address in a header value.
|
||||
func headerAddress(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
if list, err := mail.ParseAddressList(v); err == nil && len(list) > 0 {
|
||||
return strings.TrimSpace(list[0].Address)
|
||||
}
|
||||
if a, err := mail.ParseAddress(v); err == nil {
|
||||
return strings.TrimSpace(a.Address)
|
||||
}
|
||||
v = strings.TrimSpace(strings.TrimPrefix(v, "<"))
|
||||
if i := strings.IndexAny(v, ">,;"); i >= 0 {
|
||||
v = v[:i]
|
||||
}
|
||||
return strings.Trim(strings.TrimSpace(v), "<>")
|
||||
}
|
||||
|
||||
var receivedForRE = regexp.MustCompile(`(?i)\bfor\s+<?([A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+[A-Za-z0-9])>?`)
|
||||
|
||||
// receivedFor extracts the address from a "Received: ... for <addr>" clause.
|
||||
func receivedFor(v string) string {
|
||||
m := receivedForRE.FindStringSubmatch(v)
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m[1], "<>")
|
||||
}
|
||||
|
||||
func decodeHeader(v string) string {
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
dec := mime.WordDecoder{}
|
||||
if s, err := dec.DecodeHeader(v); err == nil {
|
||||
return s
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func parseDate(v string) time.Time {
|
||||
if t, err := mail.ParseDate(v); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
for _, layout := range []string{time.RFC1123Z, time.RFC1123, "Mon, 2 Jan 2006 15:04:05 -0700"} {
|
||||
if t, err := time.Parse(layout, v); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// --- body decoding ----------------------------------------------------------
|
||||
|
||||
func extractBody(h textproto.MIMEHeader, body io.Reader) string {
|
||||
mediaType, params := parseMediaType(h.Get("Content-Type"))
|
||||
decoded := decodeTransfer(h.Get("Content-Transfer-Encoding"), body)
|
||||
switch {
|
||||
case mediaType == "" || mediaType == "text/plain":
|
||||
return toUTF8(readAll(decoded))
|
||||
case mediaType == "text/html":
|
||||
return htmlToText(toUTF8(readAll(decoded)))
|
||||
case strings.HasPrefix(mediaType, "multipart/"):
|
||||
return extractMultipart(params["boundary"], decoded)
|
||||
default:
|
||||
// application/*, image/*, … have no text to show.
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func extractMultipart(boundary string, r io.Reader) string {
|
||||
if boundary == "" {
|
||||
return ""
|
||||
}
|
||||
mr := multipart.NewReader(r, boundary)
|
||||
var plain, html []string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
mediaType, _ := parseMediaType(part.Header.Get("Content-Type"))
|
||||
sub := extractBody(textproto.MIMEHeader(part.Header), part)
|
||||
_ = part.Close()
|
||||
if sub == "" {
|
||||
continue
|
||||
}
|
||||
if mediaType == "text/html" {
|
||||
html = append(html, sub)
|
||||
} else {
|
||||
plain = append(plain, sub)
|
||||
}
|
||||
}
|
||||
if len(plain) > 0 {
|
||||
return strings.TrimSpace(strings.Join(plain, "\n"))
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(html, "\n"))
|
||||
}
|
||||
|
||||
func parseMediaType(v string) (string, map[string]string) {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return "", nil
|
||||
}
|
||||
mt, params, err := mime.ParseMediaType(v)
|
||||
if err != nil {
|
||||
head := strings.TrimSpace(strings.SplitN(v, ";", 2)[0])
|
||||
return strings.ToLower(head), nil
|
||||
}
|
||||
return strings.ToLower(mt), params
|
||||
}
|
||||
|
||||
func decodeTransfer(encoding string, r io.Reader) io.Reader {
|
||||
switch strings.ToLower(strings.TrimSpace(encoding)) {
|
||||
case "base64":
|
||||
return base64.NewDecoder(base64.StdEncoding, r)
|
||||
case "quoted-printable":
|
||||
return quotedprintable.NewReader(r)
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
func readAll(r io.Reader) []byte {
|
||||
b, err := io.ReadAll(io.LimitReader(r, maxBodyBytes))
|
||||
if err != nil {
|
||||
return b
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// toUTF8 returns valid UTF-8 text, falling back to a Latin-1 interpretation of
|
||||
// bytes that are not valid UTF-8 (no charset table is available without adding a
|
||||
// dependency; forwarded personal mail is overwhelmingly UTF-8 or Latin-1).
|
||||
func toUTF8(b []byte) string {
|
||||
if utf8.Valid(b) {
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
var sb strings.Builder
|
||||
for _, c := range b {
|
||||
sb.WriteRune(rune(c))
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
var (
|
||||
htmlBlockRE = regexp.MustCompile(`(?is)<(script|style)[^>]*>.*?</(?:script|style)>`)
|
||||
htmlTagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
htmlSpaceRE = regexp.MustCompile(`[ \t]+`)
|
||||
htmlNLRE = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
func htmlToText(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
s = htmlBlockRE.ReplaceAllString(s, " ")
|
||||
s = htmlTagRE.ReplaceAllString(s, " ")
|
||||
s = strings.NewReplacer(
|
||||
" ", " ",
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
""", `"`,
|
||||
"'", "'",
|
||||
).Replace(s)
|
||||
s = htmlSpaceRE.ReplaceAllString(s, " ")
|
||||
s = htmlNLRE.ReplaceAllString(s, "\n\n")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
Reference in New Issue
Block a user