email_agent/internal/mailfetch/message.go

package mailfetch

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"mime/quotedprintable"
	"net/mail"
	"net/textproto"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

// maxBodyBytes caps how much body text is read from one message.
const maxBodyBytes = 1 << 20 // 1 MiB

// ParsedMessage is the subset of an RFC822 message the agent stores.
type ParsedMessage struct {
	From          string
	To            string
	Subject       string
	Date          time.Time
	SourceAddress string
	// SourceHeader records which header the source address came from, so FR-2
	// decisions are auditable.
	SourceHeader string
	BodyText     string
}

// ParseMessage parses a raw RFC822 message.
func ParseMessage(raw []byte) (*ParsedMessage, error) {
	msg, err := mail.ReadMessage(bytes.NewReader(raw))
	if err != nil {
		return nil, fmt.Errorf("parse RFC822 message: %w", err)
	}
	h := msg.Header
	pm := &ParsedMessage{
		From:    decodeHeader(h.Get("From")),
		To:      decodeHeader(h.Get("To")),
		Subject: decodeHeader(h.Get("Subject")),
		Date:    parseDate(h.Get("Date")),
	}
	pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h))
	pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body)
	return pm, nil
}

// SourceAddress resolves the address the mail arrived at (FR-2).
//
// Order:
//  1. X-Pm-Original-To   (added by PurelyMail)
//  2. To
//  3. the PurelyMail "Received ... for <address>" clause
//  4. Return-Path
//
// Delivered-To is deliberately never used: Gmail overwrites it.
func SourceAddress(h textproto.MIMEHeader) (address, header string) {
	if v := headerAddress(h.Get("X-Pm-Original-To")); v != "" {
		return v, "X-Pm-Original-To"
	}
	if v := headerAddress(h.Get("To")); v != "" {
		return v, "To"
	}
	for _, rv := range h.Values("Received") {
		if v := receivedFor(rv); v != "" {
			return v, "Received"
		}
	}
	if v := headerAddress(h.Get("Return-Path")); v != "" {
		return v, "Return-Path"
	}
	return "", ""
}

// headerAddress returns the first address in a header value.
func headerAddress(v string) string {
	v = strings.TrimSpace(v)
	if v == "" {
		return ""
	}
	if list, err := mail.ParseAddressList(v); err == nil && len(list) > 0 {
		return strings.TrimSpace(list[0].Address)
	}
	if a, err := mail.ParseAddress(v); err == nil {
		return strings.TrimSpace(a.Address)
	}
	v = strings.TrimSpace(strings.TrimPrefix(v, "<"))
	if i := strings.IndexAny(v, ">,;"); i >= 0 {
		v = v[:i]
	}
	return strings.Trim(strings.TrimSpace(v), "<>")
}

var receivedForRE = regexp.MustCompile(`(?i)\bfor\s+<?([A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+[A-Za-z0-9])>?`)

// receivedFor extracts the address from a "Received: ... for <addr>" clause.
func receivedFor(v string) string {
	m := receivedForRE.FindStringSubmatch(v)
	if m == nil {
		return ""
	}
	return strings.Trim(m[1], "<>")
}

func decodeHeader(v string) string {
	if v == "" {
		return ""
	}
	dec := mime.WordDecoder{}
	if s, err := dec.DecodeHeader(v); err == nil {
		return s
	}
	return v
}

func parseDate(v string) time.Time {
	if t, err := mail.ParseDate(v); err == nil {
		return t.UTC()
	}
	for _, layout := range []string{time.RFC1123Z, time.RFC1123, "Mon, 2 Jan 2006 15:04:05 -0700"} {
		if t, err := time.Parse(layout, v); err == nil {
			return t.UTC()
		}
	}
	return time.Time{}
}

// --- body decoding ----------------------------------------------------------

func extractBody(h textproto.MIMEHeader, body io.Reader) string {
	mediaType, params := parseMediaType(h.Get("Content-Type"))
	decoded := decodeTransfer(h.Get("Content-Transfer-Encoding"), body)
	switch {
	case mediaType == "" || mediaType == "text/plain":
		return toUTF8(readAll(decoded))
	case mediaType == "text/html":
		return htmlToText(toUTF8(readAll(decoded)))
	case strings.HasPrefix(mediaType, "multipart/"):
		return extractMultipart(params["boundary"], decoded)
	default:
		// application/*, image/*, … have no text to show.
		return ""
	}
}

func extractMultipart(boundary string, r io.Reader) string {
	if boundary == "" {
		return ""
	}
	mr := multipart.NewReader(r, boundary)
	var plain, html []string
	for {
		part, err := mr.NextPart()
		if err != nil {
			break
		}
		mediaType, _ := parseMediaType(part.Header.Get("Content-Type"))
		sub := extractBody(textproto.MIMEHeader(part.Header), part)
		_ = part.Close()
		if sub == "" {
			continue
		}
		if mediaType == "text/html" {
			html = append(html, sub)
		} else {
			plain = append(plain, sub)
		}
	}
	if len(plain) > 0 {
		return strings.TrimSpace(strings.Join(plain, "\n"))
	}
	return strings.TrimSpace(strings.Join(html, "\n"))
}

func parseMediaType(v string) (string, map[string]string) {
	if strings.TrimSpace(v) == "" {
		return "", nil
	}
	mt, params, err := mime.ParseMediaType(v)
	if err != nil {
		head := strings.TrimSpace(strings.SplitN(v, ";", 2)[0])
		return strings.ToLower(head), nil
	}
	return strings.ToLower(mt), params
}

func decodeTransfer(encoding string, r io.Reader) io.Reader {
	switch strings.ToLower(strings.TrimSpace(encoding)) {
	case "base64":
		return base64.NewDecoder(base64.StdEncoding, r)
	case "quoted-printable":
		return quotedprintable.NewReader(r)
	default:
		return r
	}
}

func readAll(r io.Reader) []byte {
	b, err := io.ReadAll(io.LimitReader(r, maxBodyBytes))
	if err != nil {
		return b
	}
	return b
}

// toUTF8 returns valid UTF-8 text, falling back to a Latin-1 interpretation of
// bytes that are not valid UTF-8 (no charset table is available without adding a
// dependency; forwarded personal mail is overwhelmingly UTF-8 or Latin-1).
func toUTF8(b []byte) string {
	if utf8.Valid(b) {
		return strings.TrimSpace(string(b))
	}
	var sb strings.Builder
	for _, c := range b {
		sb.WriteRune(rune(c))
	}
	return strings.TrimSpace(sb.String())
}

var (
	htmlBlockRE = regexp.MustCompile(`(?is)<(script|style)[^>]*>.*?</(?:script|style)>`)
	htmlTagRE   = regexp.MustCompile(`(?s)<[^>]*>`)
	htmlSpaceRE = regexp.MustCompile(`[ \t]+`)
	htmlNLRE    = regexp.MustCompile(`\n{3,}`)
)

func htmlToText(s string) string {
	if s == "" {
		return ""
	}
	s = htmlBlockRE.ReplaceAllString(s, " ")
	s = htmlTagRE.ReplaceAllString(s, " ")
	s = strings.NewReplacer(
		"&nbsp;", " ",
		"&amp;", "&",
		"&lt;", "<",
		"&gt;", ">",
		"&quot;", `"`,
		"&#39;", "'",
	).Replace(s)
	s = htmlSpaceRE.ReplaceAllString(s, " ")
	s = htmlNLRE.ReplaceAllString(s, "\n\n")
	return strings.TrimSpace(s)
}