Phase 5: reply drafts — template/LLM draft, edit, SMTP send (AUD-71)

Implements Phase 5 of AGENTS.md §11 (FR-8, FR-15): - Message detail pane (GET /partials/message) with body, recommended action, reply identity, template picker, editable draft, and history. - internal/reply: OmniRoute (OpenAI-compatible) chat client and a template-based draft generator. All non-JEV models go through OmniRoute (§10). - internal/mailops: SMTP send over smtp.gmail.com:587 with STARTTLS and the app password, forcing IPv4 (§9); FR-15 reply identity (send as the arrived-at X-Pm-Original-To address, else "Sam Rolfe"); threading via In-Reply-To/References; -check-smtp safe verification; and a [Gmail]/Sent Mail thread check. - Migration v5 stores messages.message_id_header and drafts.sent_at; mailfetch parses Message-ID. - Send is user-triggered only: CSRF + UI confirm + confirm=yes guard, no auto-send, duplicate-send refused. A sent reply files the original under Replied-to/<address> and logs every step (FR-13). - Tests for reply, mailops SMTP/identity/thread check, store drafts and migration v5, and the server draft/edit/send guards. - docs/phase5-replies.md, README, .env.example, docker-compose updated. Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-11 22:05:35 +11:00
parent ae53374ad1
commit 7ca888f0c6
31 changed files with 2556 additions and 42 deletions
--- a/internal/mailops/mailops.go
+++ b/internal/mailops/mailops.go
@@ -31,6 +31,9 @@ const (
 	LabelDelete   = "Delete"
 	LabelSaved    = "Saved"
 	repliedPrefix = "Replied-to/"
+	// LabelSent is Gmail's IMAP name for the Sent mailbox. Phase 5 checks it to
+	// confirm a reply joined the original thread.
+	LabelSent = "[Gmail]/Sent Mail"
 )

 // ErrNotConfigured means Gmail credentials are missing, so no mailbox action can
@@ -46,25 +49,30 @@ type Conn interface {
 	Move(uid uint32, dest string) error
 	StoreFlags(uid uint32, op string, flag string) error
 	SearchGmailID(gmailID string) ([]uint32, error)
+	SearchGmailThreadID(threadID string) ([]uint32, error)
 	Logout() error
 }

 // dialFunc opens an IMAP connection. It is a field so tests can substitute one.
 type dialFunc func(ctx context.Context, host string, port int, timeout time.Duration) (Conn, error)

-// Client performs triage actions against Gmail IMAP.
+// Client performs triage actions against Gmail IMAP and sends replies over
+// Gmail SMTP.
 type Client struct {
 	cfg    *config.Config
 	logger *slog.Logger
 	dial   dialFunc
+	// smtpDial opens the SMTP connection (IPv4 only). It is a field so tests can
+	// substitute one; the production value is dialSMTP4.
+	smtpDial smtpDialFunc
 }

-// New builds a Client using the real Gmail IMAP dialer.
+// New builds a Client using the real Gmail IMAP dialer and IPv4 SMTP dialer.
 func New(cfg *config.Config, logger *slog.Logger) *Client {
 	if logger == nil {
 		logger = slog.New(slog.DiscardHandler)
 	}
-	return &Client{cfg: cfg, logger: logger, dial: defaultDial}
+	return &Client{cfg: cfg, logger: logger, dial: defaultDial, smtpDial: dialSMTP4}
 }

 // defaultDial dials Gmail over IPv4 (mailfetch.Dial forces tcp4).
@@ -143,6 +151,29 @@ func (c *Client) Mark(ctx context.Context, msg store.Message, mark string) error
 	return nil
 }

+// VerifyThreadUpdated reports whether the Gmail thread now contains a message in
+// the Sent mailbox. Phase 5 calls it after a reply is sent so the DoD "the
+// thread updates" can be checked without touching the recipient.
+func (c *Client) VerifyThreadUpdated(ctx context.Context, threadID string) (bool, error) {
+	if strings.TrimSpace(threadID) == "" {
+		return false, fmt.Errorf("message has no Gmail thread id")
+	}
+	conn, err := c.connect(ctx)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = conn.Logout() }()
+
+	if _, err := conn.Select(LabelSent); err != nil {
+		return false, fmt.Errorf("select %q: %w", LabelSent, err)
+	}
+	uids, err := conn.SearchGmailThreadID(threadID)
+	if err != nil {
+		return false, fmt.Errorf("search thread %s in %q: %w", threadID, LabelSent, err)
+	}
+	return len(uids) > 0, nil
+}
+
 // connect dials and logs in. Callers must Logout when done.
 func (c *Client) connect(ctx context.Context) (Conn, error) {
 	if !c.cfg.GmailConfigured() {
--- a/internal/mailops/mailops_test.go
+++ b/internal/mailops/mailops_test.go
@@ -45,6 +45,10 @@ func (f *fakeConn) SearchGmailID(id string) ([]uint32, error) {
 	f.searched = append(f.searched, id)
 	return f.searchUIDs, f.searchErr
 }
+func (f *fakeConn) SearchGmailThreadID(id string) ([]uint32, error) {
+	f.searched = append(f.searched, "thrid:"+id)
+	return f.searchUIDs, f.searchErr
+}
 func (f *fakeConn) Logout() error { f.loggedOut = true; return nil }

 func configured() *config.Config {
@@ -54,6 +58,8 @@ func configured() *config.Config {
 		IMAPPort:         config.DefaultIMAPPort,
 		GmailUser:        "user@example.test",
 		GmailAppPassword: "app-password",
+		SMTPHost:         config.DefaultSMTPHost,
+		SMTPPort:         config.DefaultSMTPPort,
 	}
 }

--- a/internal/mailops/smtp.go
+++ b/internal/mailops/smtp.go
@@ -0,0 +1,264 @@
+// SMTP send path for Phase 5 replies (FR-8, FR-15).
+//
+// Gmail SMTP is reached over IPv4 only, exactly like the IMAP read path
+// (AGENTS.md §9): IPv6 to Gmail is broken from .13. The connection upgrades with
+// STARTTLS on port 587 and authenticates with the same app password as IMAP.
+//
+// Reply identity (FR-15): the From header is the address the mail arrived at
+// (X-Pm-Original-To). When that is unknown, it falls back to Sam Rolfe at the
+// Gmail account. The SMTP envelope sender is always the authenticated Gmail
+// account, which is what Gmail expects; the From header carries the alias.
+package mailops
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/hex"
+	"fmt"
+	"mime"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"strings"
+	"time"
+
+	"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch"
+	"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
+)
+
+// ReplyFromName is the display name used when the arrived-at address is unknown
+// (AGENTS.md §13.1: "Otherwise send as Sam Rolfe").
+const ReplyFromName = "Sam Rolfe"
+
+// OutgoingMessage is one reply to send.
+type OutgoingMessage struct {
+	// To is the recipient (the original sender).
+	To string
+	// From is the From header value, normally "Sam Rolfe <arrived-at@address>".
+	From string
+	// Subject is the reply subject (already prefixed with Re: when appropriate).
+	Subject string
+	// InReplyTo and References carry the original Message-ID so Gmail threads
+	// the reply.
+	InReplyTo  string
+	References string
+	// Body is the plain-text reply body.
+	Body string
+}
+
+// Sender sends a reply. The live implementation is *Client; tests substitute a
+// fake.
+type Sender interface {
+	Send(ctx context.Context, msg OutgoingMessage) error
+}
+
+// SMTPConfig is the connection detail for the Gmail SMTP submission endpoint.
+type SMTPConfig struct {
+	Host     string
+	Port     int
+	User     string
+	Password string
+}
+
+// smtpDialFunc opens a TCP connection. It is a field so tests can assert the
+// IPv4-only network is used.
+type smtpDialFunc func(ctx context.Context, host string, port int, timeout time.Duration) (net.Conn, error)
+
+// dialSMTP4 forces IPv4, matching mailfetch.Dial on the read path.
+func dialSMTP4(ctx context.Context, host string, port int, timeout time.Duration) (net.Conn, error) {
+	d := net.Dialer{Timeout: timeout}
+	return d.DialContext(ctx, "tcp4", net.JoinHostPort(host, fmt.Sprintf("%d", port)))
+}
+
+// ReplyIdentity returns the From header for a reply (FR-15): the address the
+// mail arrived at, otherwise Sam Rolfe at the Gmail account.
+func ReplyIdentity(msg store.Message, gmailUser string) string {
+	addr := strings.TrimSpace(msg.SourceAddress)
+	if addr == "" {
+		addr = strings.TrimSpace(gmailUser)
+	}
+	if addr == "" {
+		return ReplyFromName
+	}
+	return (&mail.Address{Name: ReplyFromName, Address: addr}).String()
+}
+
+// ReplyTo returns the recipient of a reply: the original sender.
+func ReplyTo(msg store.Message) string {
+	addr := strings.TrimSpace(msg.FromAddr)
+	if addr == "" {
+		return ""
+	}
+	if parsed, err := mail.ParseAddress(addr); err == nil {
+		return parsed.Address
+	}
+	return addr
+}
+
+// ReplySubject prefixes the original subject with "Re: " once.
+func ReplySubject(subject string) string {
+	s := strings.TrimSpace(subject)
+	if s == "" {
+		return "Re: (no subject)"
+	}
+	if strings.HasPrefix(strings.ToLower(s), "re:") {
+		return s
+	}
+	return "Re: " + s
+}
+
+// CheckSMTP verifies the submission path without sending mail: it dials Gmail
+// over IPv4, upgrades with STARTTLS, and authenticates with the app password.
+// It stops before MAIL FROM, so no message is sent. It is the safe part of the
+// Phase 5 send-path verification.
+func (c *Client) CheckSMTP(ctx context.Context) error {
+	if !c.cfg.GmailConfigured() {
+		return ErrNotConfigured
+	}
+	host := c.cfg.SMTPHost
+	port := c.cfg.SMTPPort
+
+	conn, err := c.smtpDial(ctx, host, port, mailfetch.DefaultTimeout)
+	if err != nil {
+		return fmt.Errorf("dial smtp %s:%d (ipv4): %w", host, port, err)
+	}
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		_ = conn.Close()
+		return fmt.Errorf("smtp handshake: %w", err)
+	}
+	defer func() { _ = client.Close() }()
+
+	if ok, _ := client.Extension("STARTTLS"); ok {
+		if err := client.StartTLS(&tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}); err != nil {
+			return fmt.Errorf("smtp starttls: %w", err)
+		}
+	}
+	if err := client.Auth(smtp.PlainAuth("", c.cfg.GmailUser, c.cfg.GmailAppPassword, host)); err != nil {
+		return fmt.Errorf("smtp auth: %w", err)
+	}
+	c.logger.Info("smtp check ok", "host", host, "port", port, "user", c.cfg.GmailUser)
+	return client.Quit()
+}
+
+// Send builds and sends one reply over IPv4 + STARTTLS + the app password.
+func (c *Client) Send(ctx context.Context, msg OutgoingMessage) error {
+	if !c.cfg.GmailConfigured() {
+		return ErrNotConfigured
+	}
+	if strings.TrimSpace(msg.To) == "" {
+		return fmt.Errorf("reply has no recipient")
+	}
+	host := c.cfg.SMTPHost
+	port := c.cfg.SMTPPort
+
+	conn, err := c.smtpDial(ctx, host, port, mailfetch.DefaultTimeout)
+	if err != nil {
+		return fmt.Errorf("dial smtp %s:%d (ipv4): %w", host, port, err)
+	}
+	client, err := smtp.NewClient(conn, host)
+	if err != nil {
+		_ = conn.Close()
+		return fmt.Errorf("smtp handshake: %w", err)
+	}
+	defer func() { _ = client.Close() }()
+
+	if ok, _ := client.Extension("STARTTLS"); ok {
+		if err := client.StartTLS(&tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}); err != nil {
+			return fmt.Errorf("smtp starttls: %w", err)
+		}
+	}
+	if err := client.Auth(smtp.PlainAuth("", c.cfg.GmailUser, c.cfg.GmailAppPassword, host)); err != nil {
+		return fmt.Errorf("smtp auth: %w", err)
+	}
+	// The envelope sender is the authenticated account; the From header carries
+	// the arrived-at alias (FR-15).
+	if err := client.Mail(c.cfg.GmailUser); err != nil {
+		return fmt.Errorf("smtp mail from: %w", err)
+	}
+	if err := client.Rcpt(msg.To); err != nil {
+		return fmt.Errorf("smtp rcpt to %s: %w", msg.To, err)
+	}
+	w, err := client.Data()
+	if err != nil {
+		return fmt.Errorf("smtp data: %w", err)
+	}
+	raw := BuildRFC822(msg, c.cfg.GmailUser, time.Now().UTC())
+	if _, err := w.Write(raw); err != nil {
+		_ = w.Close()
+		return fmt.Errorf("write reply: %w", err)
+	}
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("finish reply: %w", err)
+	}
+	if err := client.Quit(); err != nil {
+		return fmt.Errorf("smtp quit: %w", err)
+	}
+	c.logger.Info("reply sent", "to", msg.To, "from", msg.From, "subject", msg.Subject)
+	return nil
+}
+
+// BuildRFC822 renders the reply as an RFC822 message. The Message-ID is
+// generated and the headers are CRLF-terminated for SMTP. It is exported so the
+// wire format can be tested without a live server.
+func BuildRFC822(msg OutgoingMessage, gmailUser string, now time.Time) []byte {
+	var b strings.Builder
+	writeHeader(&b, "From", msg.From)
+	writeHeader(&b, "To", msg.To)
+	writeHeader(&b, "Subject", msg.Subject)
+	writeHeader(&b, "Date", now.Format(time.RFC1123Z))
+	writeHeader(&b, "Message-ID", generateMessageID(now, gmailUser))
+	if v := strings.TrimSpace(msg.InReplyTo); v != "" {
+		writeHeader(&b, "In-Reply-To", ensureAngle(v))
+	}
+	if v := strings.TrimSpace(msg.References); v != "" {
+		writeHeader(&b, "References", ensureAngle(v))
+	}
+	writeHeader(&b, "MIME-Version", "1.0")
+	writeHeader(&b, "Content-Type", "text/plain; charset=UTF-8")
+	writeHeader(&b, "Content-Transfer-Encoding", "8bit")
+	b.WriteString("\r\n")
+	b.WriteString(strings.ReplaceAll(strings.ReplaceAll(msg.Body, "\r\n", "\n"), "\n", "\r\n"))
+	if !strings.HasSuffix(b.String(), "\r\n") {
+		b.WriteString("\r\n")
+	}
+	return []byte(b.String())
+}
+
+// writeHeader encodes a header value when it contains non-ASCII text.
+func writeHeader(b *strings.Builder, key, value string) {
+	value = strings.TrimSpace(value)
+	for _, r := range value {
+		if r > 127 {
+			value = mime.QEncoding.Encode("UTF-8", value)
+			break
+		}
+	}
+	b.WriteString(key)
+	b.WriteString(": ")
+	b.WriteString(value)
+	b.WriteString("\r\n")
+}
+
+// generateMessageID builds a unique Message-ID for an outgoing reply.
+func generateMessageID(now time.Time, user string) string {
+	domain := "email-agent.local"
+	if user != "" {
+		if i := strings.LastIndex(user, "@"); i >= 0 && i < len(user)-1 {
+			domain = user[i+1:]
+		}
+	}
+	buf := make([]byte, 8)
+	_, _ = rand.Read(buf)
+	return fmt.Sprintf("<%d.%s@%s>", now.UnixNano(), hex.EncodeToString(buf), domain)
+}
+
+// ensureAngle wraps a Message-ID in angle brackets when it is not already.
+func ensureAngle(id string) string {
+	id = strings.TrimSpace(id)
+	if strings.HasPrefix(id, "<") && strings.HasSuffix(id, ">") {
+		return id
+	}
+	return "<" + strings.Trim(id, "<>") + ">"
+}
--- a/internal/mailops/smtp_test.go
+++ b/internal/mailops/smtp_test.go
@@ -0,0 +1,140 @@
+package mailops
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.lab.audasmedia.com.au/sam/email_agent/internal/config"
+	"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
+)
+
+func TestReplyIdentityUsesArrivedAtAddress(t *testing.T) {
+	msg := store.Message{SourceAddress: "Hello@where-woof.com"}
+	if got := ReplyIdentity(msg, "samuelrolfe@gmail.com"); got != `"Sam Rolfe" <Hello@where-woof.com>` {
+		t.Errorf("ReplyIdentity() = %q", got)
+	}
+	fallback := ReplyIdentity(store.Message{}, "samuelrolfe@gmail.com")
+	if fallback != `"Sam Rolfe" <samuelrolfe@gmail.com>` {
+		t.Errorf("fallback = %q", fallback)
+	}
+	if got := ReplyIdentity(store.Message{}, ""); got != "Sam Rolfe" {
+		t.Errorf("bare identity = %q", got)
+	}
+}
+
+func TestReplyToParsesSender(t *testing.T) {
+	if got := ReplyTo(store.Message{FromAddr: "Customer Name <cust@example.test>"}); got != "cust@example.test" {
+		t.Errorf("ReplyTo() = %q", got)
+	}
+	if got := ReplyTo(store.Message{FromAddr: "plain@example.test"}); got != "plain@example.test" {
+		t.Errorf("ReplyTo() = %q", got)
+	}
+	if got := ReplyTo(store.Message{}); got != "" {
+		t.Errorf("ReplyTo() empty = %q", got)
+	}
+}
+
+func TestReplySubjectPrefixesOnce(t *testing.T) {
+	if got := ReplySubject("Booking"); got != "Re: Booking" {
+		t.Errorf("ReplySubject() = %q", got)
+	}
+	if got := ReplySubject("RE: Booking"); got != "RE: Booking" {
+		t.Errorf("ReplySubject() should not double-prefix: %q", got)
+	}
+}
+
+func TestBuildRFC822ThreadsReply(t *testing.T) {
+	raw := string(BuildRFC822(OutgoingMessage{
+		To:         "cust@example.test",
+		From:       `"Sam Rolfe" <hello@where-woof.com>`,
+		Subject:    "Re: Booking",
+		InReplyTo:  "<orig-123@mail.example>",
+		References: "orig-123@mail.example",
+		Body:       "Saturday works.\n\nSam",
+	}, "samuelrolfe@gmail.com", time.Unix(0, 0).UTC()))
+
+	for _, want := range []string{
+		"From: \"Sam Rolfe\" <hello@where-woof.com>\r\n",
+		"To: cust@example.test\r\n",
+		"Subject: Re: Booking\r\n",
+		"In-Reply-To: <orig-123@mail.example>\r\n",
+		"References: <orig-123@mail.example>\r\n",
+		"Content-Type: text/plain; charset=UTF-8\r\n",
+		"\r\nSaturday works.\r\n\r\nSam\r\n",
+	} {
+		if !strings.Contains(raw, want) {
+			t.Errorf("message missing %q\n%s", want, raw)
+		}
+	}
+	if strings.Contains(strings.ReplaceAll(raw, "\r\n", ""), "\n") {
+		t.Errorf("message contains a bare LF; SMTP needs CRLF line endings")
+	}
+	if !strings.Contains(raw, "Message-ID: <") {
+		t.Errorf("message has no generated Message-ID:\n%s", raw)
+	}
+}
+
+func TestSendWithoutCredentials(t *testing.T) {
+	c := New(&config.Config{Env: "test"}, nil)
+	out := OutgoingMessage{To: "a@b.test", From: "Sam Rolfe <x@y.test>", Body: "hi"}
+	if err := c.Send(context.Background(), out); err != ErrNotConfigured {
+		t.Fatalf("Send() error = %v, want ErrNotConfigured", err)
+	}
+}
+
+func TestDialSMTP4ConnectsOverIPv4(t *testing.T) {
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	defer ln.Close()
+
+	port := ln.Addr().(*net.TCPAddr).Port
+	// "localhost" may resolve to ::1 first; the tcp4 network must still reach the
+	// IPv4 listener.
+	conn, err := dialSMTP4(context.Background(), "localhost", port, 2*time.Second)
+	if err != nil {
+		t.Fatalf("dialSMTP4() error = %v", err)
+	}
+	_ = conn.Close()
+}
+
+func TestVerifyThreadUpdatedSearchesSentMailbox(t *testing.T) {
+	fc := &fakeConn{uidValidity: 3, searchUIDs: []uint32{11}}
+	c := newFakeClient(fc)
+
+	ok, err := c.VerifyThreadUpdated(context.Background(), "987654321")
+	if err != nil {
+		t.Fatalf("VerifyThreadUpdated() error = %v", err)
+	}
+	if !ok {
+		t.Error("VerifyThreadUpdated() = false, want true when the thread has a sent message")
+	}
+	if fc.selected != LabelSent {
+		t.Errorf("selected = %q, want %q", fc.selected, LabelSent)
+	}
+	if len(fc.searched) != 1 || fc.searched[0] != "thrid:987654321" {
+		t.Errorf("searched = %v, want [thrid:987654321]", fc.searched)
+	}
+}
+
+func TestVerifyThreadUpdatedEmptyThread(t *testing.T) {
+	c := newFakeClient(&fakeConn{searchUIDs: nil})
+	if _, err := c.VerifyThreadUpdated(context.Background(), ""); err == nil {
+		t.Fatal("VerifyThreadUpdated() error = nil, want an error for a missing thread id")
+	}
+}
+
+func TestVerifyThreadUpdatedNoSentMatch(t *testing.T) {
+	c := newFakeClient(&fakeConn{searchUIDs: nil})
+	ok, err := c.VerifyThreadUpdated(context.Background(), "42")
+	if err != nil {
+		t.Fatalf("VerifyThreadUpdated() error = %v", err)
+	}
+	if ok {
+		t.Error("VerifyThreadUpdated() = true, want false when no sent message matches")
+	}
+}