Phase 6: rules editor + unsubscribe detection (AUD-72)

- FR-11 rules editor: plain-text area at /rules, saved and versioned
  (SaveRules appends a version; restore re-saves an old version), parse
  errors surfaced, one-click re-classify. The reply model now reads the
  rules document too (FR-11).
- FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post
  (migration v6), detect email and web links (internal/unsubscribe), a
  domain-level ignore list, and an explicit record of what was marked. No
  code path contacts an unsubscribe link; acting on one needs Sam's approval.
- Migration v6 is additive; unsubscribe gains url/updated_at.
- Tests: detector, parser, store migration/records/candidates, rules editor,
  unsubscribe handlers, message-detail rendering.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-11 22:24:38 +11:00
parent 7c66e4ad31
commit 00bb0e05c4
24 changed files with 2121 additions and 42 deletions

View File

@@ -102,18 +102,20 @@ func (f *Fetcher) Fetch(ctx context.Context) (Summary, error) {
headersSeen[parsed.SourceHeader] = struct{}{}
}
msg := &store.Message{
GmailID: gmailID(r, uidValidity),
ThreadID: r.GmailThreadID,
MessageIDHeader: parsed.MessageID,
FromAddr: parsed.From,
ToAddr: parsed.To,
Subject: parsed.Subject,
BodyText: parsed.BodyText,
SourceAddress: parsed.SourceAddress,
DateSent: parsed.Date,
ReceivedAt: now,
IMAPUID: int64(r.UID),
IMAPUIDValidity: int64(uidValidity),
GmailID: gmailID(r, uidValidity),
ThreadID: r.GmailThreadID,
MessageIDHeader: parsed.MessageID,
FromAddr: parsed.From,
ToAddr: parsed.To,
Subject: parsed.Subject,
BodyText: parsed.BodyText,
SourceAddress: parsed.SourceAddress,
ListUnsubscribe: parsed.ListUnsubscribe,
ListUnsubscribePost: parsed.ListUnsubscribePost,
DateSent: parsed.Date,
ReceivedAt: now,
IMAPUID: int64(r.UID),
IMAPUIDValidity: int64(uidValidity),
}
if _, err := f.store.UpsertMessage(ctx, msg); err != nil {
return sum, err

View File

@@ -27,8 +27,13 @@ type ParsedMessage struct {
Date time.Time
// MessageID is the RFC822 Message-ID header, used for reply threading
// (Phase 5, FR-8).
MessageID string
SourceAddress string
MessageID string
// ListUnsubscribe and ListUnsubscribePost are the RFC 2369/8058 headers that
// name the unsubscribe targets (Phase 6, FR-7). They are stored raw so the
// detector can run later without a refetch.
ListUnsubscribe string
ListUnsubscribePost string
SourceAddress string
// SourceHeader records which header the source address came from, so FR-2
// decisions are auditable.
SourceHeader string
@@ -43,11 +48,13 @@ func ParseMessage(raw []byte) (*ParsedMessage, error) {
}
h := msg.Header
pm := &ParsedMessage{
From: decodeHeader(h.Get("From")),
To: decodeHeader(h.Get("To")),
Subject: decodeHeader(h.Get("Subject")),
Date: parseDate(h.Get("Date")),
MessageID: strings.TrimSpace(h.Get("Message-Id")),
From: decodeHeader(h.Get("From")),
To: decodeHeader(h.Get("To")),
Subject: decodeHeader(h.Get("Subject")),
Date: parseDate(h.Get("Date")),
MessageID: strings.TrimSpace(h.Get("Message-Id")),
ListUnsubscribe: strings.TrimSpace(h.Get("List-Unsubscribe")),
ListUnsubscribePost: strings.TrimSpace(h.Get("List-Unsubscribe-Post")),
}
pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h))
pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body)

View File

@@ -172,3 +172,34 @@ func TestHeaderAddressParsing(t *testing.T) {
}
}
}
func TestParseMessageCapturesUnsubscribeHeaders(t *testing.T) {
raw := "From: Heritage Fruit Trees <support@heritagefruittrees.com.au>\r\n" +
"To: samuelrolfe@gmail.com\r\n" +
"X-Pm-Original-To: hello@where-woof.com\r\n" +
"Subject: New workshops\r\n" +
"List-Unsubscribe: <mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>\r\n" +
"List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" +
"Message-Id: <abc@mail.example>\r\n" +
"Date: Mon, 01 Jan 2026 10:00:00 +0000\r\n" +
"Content-Type: text/plain; charset=utf-8\r\n" +
"\r\n" +
"Hello, you can unsubscribe at https://heritagefruittrees.com.au/u?id=1.\r\n"
pm, err := ParseMessage([]byte(raw))
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if !strings.Contains(pm.ListUnsubscribe, "mailto:unsub@heritagefruittrees.com.au") {
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
}
if !strings.Contains(pm.ListUnsubscribe, "https://heritagefruittrees.com.au/u?id=1") {
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
}
if pm.ListUnsubscribePost != "List-Unsubscribe=One-Click" {
t.Errorf("ListUnsubscribePost = %q", pm.ListUnsubscribePost)
}
if pm.SourceAddress != "hello@where-woof.com" {
t.Errorf("SourceAddress = %q", pm.SourceAddress)
}
}

View File

@@ -18,6 +18,10 @@ const maxPromptBodyChars = 4000
// maxDraftChars bounds the generated body before it is stored.
const maxDraftChars = 8000
// maxRulesChars bounds how much of the user's rules document is sent to the
// model (FR-11: the LLM reads the rules too).
const maxRulesChars = 2000
// Request is one draft request.
type Request struct {
// Message is the mail being replied to.
@@ -31,6 +35,9 @@ type Request struct {
// Instruction is optional extra guidance Sam typed (for example "say the
// invoice is paid"). It is appended to the template instruction.
Instruction string
// Rules is the current rules document from the editor (FR-11). The model
// reads it so reply style and handling follow the same rules as triage.
Rules string
// Signer is the name used for the sign-off (default "Sam").
Signer string
}
@@ -107,6 +114,11 @@ func BuildPrompt(req Request, tmpl Template) (system, user string) {
b.WriteString(extra)
b.WriteString("\n")
}
if rules := strings.TrimSpace(req.Rules); rules != "" {
b.WriteString("USER RULES (follow when they apply):\n")
b.WriteString(truncate(rules, maxRulesChars))
b.WriteString("\n")
}
if cat := strings.TrimSpace(req.Category); cat != "" {
b.WriteString("CLASSIFICATION: category=")
b.WriteString(cat)

View File

@@ -116,3 +116,16 @@ func TestGeneratorDraftEmptyReplyIsAnError(t *testing.T) {
t.Fatal("Draft() error = nil, want an error for an empty reply")
}
}
func TestBuildPromptIncludesRules(t *testing.T) {
req := Request{
Message: store.Message{FromAddr: "customer@example.test", Subject: "hi", BodyText: "?"},
Action: "reply",
Rules: `from contains "example.test" => category=Action`,
}
tmpl, _ := TemplateByID("answer")
_, user := BuildPrompt(req, tmpl)
if !strings.Contains(user, "USER RULES") || !strings.Contains(user, `from contains "example.test"`) {
t.Errorf("user prompt does not include the rules document:\n%s", user)
}
}

View File

@@ -19,6 +19,7 @@ import (
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
)
// ReplyGenerator drafts a reply body. The live implementation is
@@ -63,6 +64,11 @@ type replyData struct {
LLMReady bool
LLMModel string
// Phase 6 (FR-7): detected unsubscribe links for this message, and the
// domain's status on the ignore/recorded list.
UnsubLinks []unsubscribe.Link
UnsubStatus string
Sent bool
ThreadUpdated bool
RepliedFolder string
@@ -109,6 +115,9 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
}
class := s.latestClass(ctx, id)
// FR-11: the reply model reads the current rules document too.
rulesText, _, _ := s.store.CurrentRules(ctx)
body, templateID, err := s.replies.Draft(ctx, reply.Request{
Message: msg,
Category: class.Category,
@@ -116,6 +125,7 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
Action: class.Action,
TemplateID: r.FormValue("template"),
Instruction: r.FormValue("instruction"),
Rules: rulesText,
Signer: "Sam",
})
if err != nil {
@@ -313,6 +323,16 @@ func (s *Server) renderMessage(w http.ResponseWriter, r *http.Request, id int64,
}
data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == ""
// FR-7: detect unsubscribe links and look up the domain's list status.
if res := unsubscribe.Detect(msg.ListUnsubscribe, msg.ListUnsubscribePost, msg.FromAddr, msg.BodyText); len(res.Links) > 0 {
data.UnsubLinks = res.Links
if rec, ok, rerr := s.store.UnsubscribeByDomain(ctx, res.Links[0].Domain); rerr != nil {
s.logger.Error("load unsubscribe status", "message_id", id, "error", rerr)
} else if ok {
data.UnsubStatus = rec.Status
}
}
if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil {
s.logger.Error("load draft", "message_id", id, "error", derr)
} else if ok {

225
internal/server/rules.go Normal file
View File

@@ -0,0 +1,225 @@
// Rules editor for Phase 6 (FR-4, FR-11).
//
// The rules document is plain text (see internal/classify for the syntax). It is
// saved as a new numbered version on every save, so the current text and its
// history are both available. The classifier reads the newest version and
// applies it as an override; the reply model also reads it.
package server
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify"
)
type ruleRevision struct {
Version int
CreatedAt string
Lines int
Preview string
}
type rulesData struct {
Text string
Version int
Revisions []ruleRevision
Errors []classify.ParseError
RuleCount int
Note string
Error string
CSRFToken string
// ClassifierReady is true when JEV is configured, so the page can offer a
// re-classify action that actually runs.
ClassifierReady bool
TotalMessages int
}
// handleRules renders the rules editor with the current text and version list.
func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) {
s.renderRules(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
}
func (s *Server) renderRules(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
ctx := r.Context()
text, version, err := s.store.CurrentRules(ctx)
if err != nil {
s.logger.Error("load rules", "error", err)
http.Error(w, "could not load the rules", http.StatusInternalServerError)
return
}
parsed := classify.ParseRules(text)
revisions, err := s.store.RulesVersions(ctx, 20)
if err != nil {
s.logger.Error("load rules versions", "error", err)
}
revs := make([]ruleRevision, 0, len(revisions))
for _, rv := range revisions {
revs = append(revs, ruleRevision{
Version: rv.Version,
CreatedAt: rv.CreatedAt,
Lines: countRuleLines(rv.Text),
Preview: firstRuleLine(rv.Text),
})
}
total, err := s.store.CountMessages(ctx)
if err != nil {
s.logger.Error("count messages", "error", err)
}
data := rulesData{
Text: text,
Version: version,
Revisions: revs,
Errors: parsed.Errors,
RuleCount: len(parsed.Rules),
Note: note,
Error: errMsg,
CSRFToken: s.csrfToken(w, r),
ClassifierReady: s.classifier.Configured(),
TotalMessages: total,
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(code)
if err := s.tmpl.ExecuteTemplate(w, "rules.html", data); err != nil {
s.logger.Error("render rules", "error", err)
}
}
// handleRulesSave versions the posted text (FR-11).
func (s *Server) handleRulesSave(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectRules(w, r, "", "Could not read the form.")
return
}
text := r.FormValue("text")
version, err := s.store.SaveRules(r.Context(), text)
if err != nil {
s.logger.Error("save rules", "error", err)
s.redirectRules(w, r, "", "Could not save the rules: "+err.Error())
return
}
parsed := classify.ParseRules(text)
s.logAction(r.Context(), "rules_save", "Sam",
"version="+strconv.Itoa(version)+" rules="+strconv.Itoa(len(parsed.Rules))+" parse_errors="+strconv.Itoa(len(parsed.Errors)))
s.redirectRules(w, r, "Saved rules version "+strconv.Itoa(version)+". Re-classify all to apply it to existing mail.", "")
}
// handleRulesRestore re-saves an older version as the newest one, so history is
// append-only (FR-11).
func (s *Server) handleRulesRestore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectRules(w, r, "", "Could not read the form.")
return
}
version, err := strconv.Atoi(strings.TrimSpace(r.FormValue("version")))
if err != nil || version < 1 {
s.redirectRules(w, r, "", "Unknown rules version.")
return
}
text, ok, err := s.store.RuleByVersion(r.Context(), version)
if err != nil {
s.redirectRules(w, r, "", "Could not load that version.")
return
}
if !ok {
s.redirectRules(w, r, "", "That rules version no longer exists.")
return
}
newVersion, err := s.store.SaveRules(r.Context(), text)
if err != nil {
s.logger.Error("restore rules", "from_version", version, "error", err)
s.redirectRules(w, r, "", "Could not restore that version: "+err.Error())
return
}
s.logAction(r.Context(), "rules_restore", "Sam",
"from_version="+strconv.Itoa(version)+" new_version="+strconv.Itoa(newVersion))
s.redirectRules(w, r, "Restored version "+strconv.Itoa(version)+" as version "+strconv.Itoa(newVersion)+".", "")
}
// handleRulesReclassify re-runs the classifier over every stored message and
// returns to the rules editor. It is the rules page's "apply" action: saving a
// new version does not re-classify by itself.
func (s *Server) handleRulesReclassify(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if rc := http.NewResponseController(w); rc != nil {
_ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute))
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
defer cancel()
sum, err := s.classifier.RunAll(ctx)
switch {
case err == nil:
s.logAction(ctx, "reclassify", "Sam",
fmt.Sprintf("considered=%d classified=%d failed=%d rules_version=%d",
sum.Considered, sum.Classified, sum.Failed, sum.RulesVersion))
s.redirectRules(w, r, fmt.Sprintf(
"Re-classified %d of %d message(s); %d failed. The current rules are now applied.",
sum.Classified, sum.Considered, sum.Failed), "")
case errors.Is(err, classify.ErrNotConfigured):
s.redirectRules(w, r, "", "Classification is not connected yet: set JEV_API_KEY.")
default:
s.logger.Error("rules reclassify failed", "error", err)
s.redirectRules(w, r, "", "Classification failed: "+err.Error())
}
}
// redirectRules is POST/redirect/GET so a refresh does not resubmit.
func (s *Server) redirectRules(w http.ResponseWriter, r *http.Request, note, errMsg string) {
q := url.Values{}
if note != "" {
q.Set("note", note)
}
if errMsg != "" {
q.Set("error", errMsg)
}
http.Redirect(w, r, "/rules?"+q.Encode(), http.StatusSeeOther)
}
// countRuleLines counts the non-empty, non-comment lines in a rules document.
func countRuleLines(text string) int {
n := 0
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
n++
}
return n
}
// firstRuleLine returns a short preview of the first real rule, or a note when
// the document has only comments.
func firstRuleLine(text string) string {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if len(line) > 80 {
return line[:80] + "…"
}
return line
}
return "(comments only)"
}

View File

@@ -0,0 +1,152 @@
package server
import (
"context"
"net/http"
"net/url"
"strings"
"testing"
)
func TestRulesPageRenders(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/rules", session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
for _, want := range []string{"Rules editor", "Save new version", "Versions", "Back to inbox", "Unsubscribe"} {
if !strings.Contains(body, want) {
t.Errorf("rules page missing %q", want)
}
}
}
func TestRulesSaveCreatesVersion(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rule := `from contains "heritagefruittrees.com.au" => category=Action, importance=high, action=file`
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
"csrf": {csrf}, "text": {rule},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
text, version, err := srv.store.CurrentRules(context.Background())
if err != nil {
t.Fatalf("CurrentRules() error = %v", err)
}
if text != rule {
t.Errorf("CurrentRules() text = %q, want the saved rule", text)
}
if version != 2 {
t.Errorf("CurrentRules() version = %d, want 2", version)
}
// The editor shows the new version and active rule count.
body := getWithSession(t, srv, "/rules", session).Body.String()
if !strings.Contains(body, "version 2") || !strings.Contains(body, "1 active rule") {
t.Errorf("rules page did not reflect the saved version; body:\n%s", body)
}
}
func TestRulesSaveRequiresCSRF(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
rec := postForm(t, srv, "/rules/save", session, nil, url.Values{"text": {"anything"}})
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403", rec.Code)
}
}
func TestRulesPageShowsParseErrors(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
"csrf": {csrf}, "text": {"this line has no arrow"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save status = %d, want 303", rec.Code)
}
body := getWithSession(t, srv, "/rules", session).Body.String()
if !strings.Contains(body, "could not be parsed") || !strings.Contains(body, "missing =&gt;") {
t.Errorf("rules page did not surface the parse error; body:\n%s", body)
}
}
func TestRulesRestoreCreatesNewVersion(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
first := `from contains "a" => category=Info`
second := `from contains "b" => category=Action`
for _, text := range []string{first, second} {
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{"csrf": {csrf}, "text": {text}})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save %q status = %d, want 303", text, rec.Code)
}
}
// Restore version 2 (the first saved rule) as a new version.
rec := postForm(t, srv, "/rules/restore", session, csrfCookie, url.Values{
"csrf": {csrf}, "version": {"2"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("restore status = %d, want 303", rec.Code)
}
text, version, err := srv.store.CurrentRules(ctx)
if err != nil {
t.Fatalf("CurrentRules() error = %v", err)
}
if text != first {
t.Errorf("restored text = %q, want %q", text, first)
}
if version != 4 {
t.Errorf("restored version = %d, want 4 (append-only)", version)
}
}
func TestDashboardLinksRulesAndUnsubscribe(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
body := getWithSession(t, srv, "/", session).Body.String()
for _, want := range []string{`href="/rules"`, `href="/unsubscribe"`} {
if !strings.Contains(body, want) {
t.Errorf("dashboard missing %q", want)
}
}
}
func TestRulesCountsLines(t *testing.T) {
text := "# comment\n\nfrom contains \"a\" => category=Info\nfrom contains \"b\" => category=Action"
if got := countRuleLines(text); got != 2 {
t.Errorf("countRuleLines() = %d, want 2", got)
}
if got := firstRuleLine(text); got != `from contains "a" => category=Info` {
t.Errorf("firstRuleLine() = %q", got)
}
if got := firstRuleLine("# only a comment"); got != "(comments only)" {
t.Errorf("firstRuleLine(comment only) = %q", got)
}
}
func TestRulesReclassifyWithoutJEVRedirects(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/rules/reclassify", session, csrfCookie, url.Values{"csrf": {csrf}})
if rec.Code != http.StatusSeeOther {
t.Fatalf("status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
if loc := rec.Header().Get("Location"); !strings.Contains(loc, "JEV_API_KEY") {
t.Errorf("Location = %q, want a JEV-not-configured note", loc)
}
}

View File

@@ -149,6 +149,18 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename))
mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove))
// Phase 6 (FR-4, FR-11): rules editor, saved and versioned.
mux.HandleFunc("GET /rules", s.requireAuth(s.handleRules))
mux.HandleFunc("POST /rules/save", s.requireAuth(s.handleRulesSave))
mux.HandleFunc("POST /rules/restore", s.requireAuth(s.handleRulesRestore))
mux.HandleFunc("POST /rules/reclassify", s.requireAuth(s.handleRulesReclassify))
// Phase 6 (FR-7): unsubscribe detection, ignore list, and records.
mux.HandleFunc("GET /unsubscribe", s.requireAuth(s.handleUnsubscribe))
mux.HandleFunc("POST /unsubscribe/ignore", s.requireAuth(s.handleUnsubscribeIgnore))
mux.HandleFunc("POST /unsubscribe/unignore", s.requireAuth(s.handleUnsubscribeUnignore))
mux.HandleFunc("POST /unsubscribe/mark", s.requireAuth(s.handleUnsubscribeMark))
return s.recoverPanic(s.logRequests(mux))
}

View File

@@ -0,0 +1,218 @@
// Unsubscribe detection view for Phase 6 (FR-7).
//
// The page lists messages with a detected unsubscribe link, the domain-level
// ignore list, and the domains Sam has recorded as unsubscribed. Every action
// records state; none of them contacts the link. Acting on a real unsubscribe
// link is a manual step that needs Sam's explicit approval (AUD-72).
package server
import (
"net/http"
"net/url"
"strings"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
)
type unsubscribeItem struct {
MessageID int64
From string
Subject string
SourceAddr string
Domain string
Links []unsubscribe.Link
Status string
Category string
When string
}
type unsubscribeData struct {
Items []unsubscribeItem
Ignored []store.UnsubscribeRecord
Recorded []store.UnsubscribeRecord
Note string
Error string
CSRFToken string
CandidateCount int
}
// handleUnsubscribe renders the detection list and the ignore/recorded lists.
func (s *Server) handleUnsubscribe(w http.ResponseWriter, r *http.Request) {
s.renderUnsubscribe(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
}
func (s *Server) renderUnsubscribe(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
ctx := r.Context()
records, err := s.store.ListUnsubscribe(ctx)
if err != nil {
s.logger.Error("list unsubscribe records", "error", err)
http.Error(w, "could not load the unsubscribe list", http.StatusInternalServerError)
return
}
candidates, err := s.store.UnsubscribeCandidates(ctx, 300)
if err != nil {
s.logger.Error("list unsubscribe candidates", "error", err)
http.Error(w, "could not load messages", http.StatusInternalServerError)
return
}
status := make(map[string]string, len(records))
for _, rec := range records {
status[rec.Domain] = rec.Status
}
var items []unsubscribeItem
for _, c := range candidates {
res := unsubscribe.Detect(c.ListUnsubscribe, c.ListUnsubscribePost, c.FromAddr, c.BodyText)
if len(res.Links) == 0 {
continue
}
domain := res.Links[0].Domain
items = append(items, unsubscribeItem{
MessageID: c.MessageID,
From: orDefault(c.FromAddr, "(unknown sender)"),
Subject: orDefault(c.Subject, "(no subject)"),
SourceAddr: c.SourceAddress,
Domain: domain,
Links: res.Links,
Status: status[domain],
Category: c.Category,
When: formatWhen(c.ReceivedAt, c.ReceivedAt),
})
}
data := unsubscribeData{
Items: items,
CSRFToken: s.csrfToken(w, r),
Note: note,
Error: errMsg,
CandidateCount: len(items),
}
for _, rec := range records {
switch rec.Status {
case store.UnsubscribeStatusIgnored:
data.Ignored = append(data.Ignored, rec)
case store.UnsubscribeStatusDone:
data.Recorded = append(data.Recorded, rec)
}
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(code)
if err := s.tmpl.ExecuteTemplate(w, "unsubscribe.html", data); err != nil {
s.logger.Error("render unsubscribe", "error", err)
}
}
// handleUnsubscribeIgnore adds a domain to the ignore list (FR-7).
func (s *Server) handleUnsubscribeIgnore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusIgnored, r.FormValue("url")); err != nil {
s.logger.Error("ignore unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not update the ignore list.")
return
}
s.logAction(r.Context(), "unsubscribe_ignore", "Sam", "domain="+domain)
s.redirectUnsubscribe(w, r, "Added "+domain+" to the ignore list. It is never recommended for unsubscribe.", "")
}
// handleUnsubscribeUnignore removes a domain from the ignore list.
func (s *Server) handleUnsubscribeUnignore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.ClearUnsubscribe(r.Context(), domain); err != nil {
s.logger.Error("clear unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not update the list.")
return
}
s.logAction(r.Context(), "unsubscribe_unignore", "Sam", "domain="+domain)
s.redirectUnsubscribe(w, r, "Removed "+domain+" from the unsubscribe list.", "")
}
// handleUnsubscribeMark records that a domain should be unsubscribed from. It
// is a record only: the handler makes no network call and the app never follows
// the link (AUD-72). Following a real link needs Sam's explicit approval.
func (s *Server) handleUnsubscribeMark(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
if r.FormValue("confirm") != "yes" {
s.redirectUnsubscribe(w, r, "", "Marking needs confirmation. The app will record it, not follow the link.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusDone, r.FormValue("url")); err != nil {
s.logger.Error("mark unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not record the unsubscribe.")
return
}
s.logAction(r.Context(), "unsubscribe_marked", "Sam", "domain="+domain+" url="+r.FormValue("url"))
s.redirectUnsubscribe(w, r,
"Recorded "+domain+" for unsubscribe. The app did not contact the link; acting on it is a manual step.", "")
}
// redirectUnsubscribe is POST/redirect/GET so a refresh does not resubmit.
func (s *Server) redirectUnsubscribe(w http.ResponseWriter, r *http.Request, note, errMsg string) {
q := url.Values{}
if note != "" {
q.Set("note", note)
}
if errMsg != "" {
q.Set("error", errMsg)
}
http.Redirect(w, r, "/unsubscribe?"+q.Encode(), http.StatusSeeOther)
}
// cleanDomain validates a domain used as the ignore-list key.
func cleanDomain(raw string) (string, bool) {
domain := strings.ToLower(strings.TrimSpace(raw))
domain = strings.Trim(domain, ".")
if domain == "" || len(domain) > 253 {
return "", false
}
for _, r := range domain {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':
default:
return "", false
}
}
if !strings.Contains(domain, ".") {
return "", false
}
return domain, true
}

View File

@@ -0,0 +1,182 @@
package server
import (
"context"
"net/http"
"net/url"
"strings"
"testing"
"time"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
)
func seedUnsubMessage(t *testing.T, srv *Server) int64 {
t.Helper()
id, err := srv.store.UpsertMessage(context.Background(), &store.Message{
GmailID: "unsub-1",
FromAddr: "Heritage Fruit Trees <support@heritagefruittrees.com.au>",
Subject: "New workshops released",
BodyText: "Hello, you have received a newsletter. Unsubscribe at https://heritagefruittrees.com.au/u?id=1.",
SourceAddress: "samuelrolfe@gmail.com",
ListUnsubscribe: "<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>",
DateSent: time.Now().UTC(),
})
if err != nil {
t.Fatalf("seed message: %v", err)
}
return id
}
func TestUnsubscribePageDetectsLinks(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/unsubscribe", session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
for _, want := range []string{
"Detected links", "heritagefruittrees.com.au",
"https://heritagefruittrees.com.au/u?id=1",
"mailto:unsub@heritagefruittrees.com.au",
"Ignore domain", "Mark for unsubscribe", "Ignore list",
} {
if !strings.Contains(body, want) {
t.Errorf("unsubscribe page missing %q", want)
}
}
}
func TestUnsubscribeIgnoreAndUnignore(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("ignore status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec2.Status != store.UnsubscribeStatusIgnored {
t.Errorf("status = %q, want ignored", rec2.Status)
}
body := getWithSession(t, srv, "/unsubscribe", session).Body.String()
if !strings.Contains(body, "ignored") || !strings.Contains(body, "Un-ignore") {
t.Errorf("ignore list did not render the ignored domain")
}
rec = postForm(t, srv, "/unsubscribe/unignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("unignore status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
t.Error("domain still on the list after unignore")
}
}
func TestUnsubscribeMarkRecordsWithoutFollowingLink(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
// Without confirmation the handler refuses and records nothing.
rec := postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
"url": {"https://heritagefruittrees.com.au/u?id=1"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("mark without confirm status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
t.Error("unconfirmed mark recorded a domain")
}
rec = postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
"url": {"https://heritagefruittrees.com.au/u?id=1"}, "confirm": {"yes"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("mark status = %d, want 303", rec.Code)
}
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec2.Status != store.UnsubscribeStatusDone {
t.Errorf("status = %q, want done", rec2.Status)
}
if rec2.URL != "https://heritagefruittrees.com.au/u?id=1" {
t.Errorf("url = %q, want the recorded link", rec2.URL)
}
}
func TestUnsubscribeMutationsRequireCSRF(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
for _, path := range []string{"/unsubscribe/ignore", "/unsubscribe/unignore", "/unsubscribe/mark"} {
rec := postForm(t, srv, path, session, nil, url.Values{"domain": {"x.example"}})
if rec.Code != http.StatusForbidden {
t.Errorf("%s without CSRF status = %d, want 403", path, rec.Code)
}
}
}
func TestUnsubscribeMarkRejectsBadDomain(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"not a domain!"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(context.Background(), "not a domain!"); ok {
t.Error("bad domain was stored")
}
}
func TestMessageDetailShowsUnsubscribeLinks(t *testing.T) {
srv := newTestServer(t)
id := seedUnsubMessage(t, srv)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/partials/message?id="+itoa(id), session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
if !strings.Contains(body, "Unsubscribe") || !strings.Contains(body, "heritagefruittrees.com.au/u?id=1") {
t.Errorf("message detail did not show the unsubscribe link")
}
}
func TestCleanDomain(t *testing.T) {
cases := map[string]bool{
"brand.example": true,
"Sub.Brand.example": true,
"x-y.example": true,
"": false,
"nodot": false,
"has space.example": false,
"bad!char.example": false,
}
for in, want := range cases {
if _, ok := cleanDomain(in); ok != want {
t.Errorf("cleanDomain(%q) ok = %v, want %v", in, ok, want)
}
}
}

View File

@@ -159,3 +159,51 @@ func (s *Store) SaveRules(ctx context.Context, text string) (int, error) {
}
return next, nil
}
// RuleVersion is one saved rules revision, for the version list in the editor
// (FR-11).
type RuleVersion struct {
ID int64
Version int
Text string
CreatedAt string
}
// RulesVersions returns saved rules revisions, newest first.
func (s *Store) RulesVersions(ctx context.Context, limit int) ([]RuleVersion, error) {
if limit < 1 {
limit = 20
}
rows, err := s.db.QueryContext(ctx,
`SELECT id, version, text, created_at FROM rules ORDER BY version DESC LIMIT ?`, limit)
if err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
defer rows.Close()
var out []RuleVersion
for rows.Next() {
var v RuleVersion
if err := rows.Scan(&v.ID, &v.Version, &v.Text, &v.CreatedAt); err != nil {
return nil, fmt.Errorf("scan rules version: %w", err)
}
out = append(out, v)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
return out, nil
}
// RuleByVersion returns the text of one saved rules revision. ok is false when
// the version does not exist.
func (s *Store) RuleByVersion(ctx context.Context, version int) (string, bool, error) {
var text string
err := s.db.QueryRowContext(ctx, `SELECT text FROM rules WHERE version = ?`, version).Scan(&text)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, fmt.Errorf("read rules version %d: %w", version, err)
}
return text, true, nil
}

View File

@@ -20,15 +20,19 @@ type Message struct {
// MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply
// uses it for In-Reply-To and References so Gmail keeps the thread.
MessageIDHeader string
FromAddr string
ToAddr string
Subject string
BodyText string
SourceAddress string
DateSent time.Time
ReceivedAt time.Time
IMAPUID int64
IMAPUIDValidity int64
// ListUnsubscribe and ListUnsubscribePost are the raw RFC 2369/8058 headers
// (Phase 6, FR-7). Detection runs over them on demand.
ListUnsubscribe string
ListUnsubscribePost string
FromAddr string
ToAddr string
Subject string
BodyText string
SourceAddress string
DateSent time.Time
ReceivedAt time.Time
IMAPUID int64
IMAPUIDValidity int64
// Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9):
// inbox, delete, saved, or replied.
Folder string
@@ -71,19 +75,22 @@ func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) {
const q = `
INSERT INTO messages (
gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text,
source_address, date_sent, received_at, imap_uid, imap_uidvalidity
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
source_address, list_unsubscribe, list_unsubscribe_post, date_sent, received_at,
imap_uid, imap_uidvalidity
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (gmail_id) DO UPDATE SET
thread_id = excluded.thread_id,
message_id_header = excluded.message_id_header,
from_addr = excluded.from_addr,
to_addr = excluded.to_addr,
subject = excluded.subject,
body_text = excluded.body_text,
source_address = excluded.source_address,
date_sent = excluded.date_sent,
imap_uid = excluded.imap_uid,
imap_uidvalidity = excluded.imap_uidvalidity`
thread_id = excluded.thread_id,
message_id_header = excluded.message_id_header,
from_addr = excluded.from_addr,
to_addr = excluded.to_addr,
subject = excluded.subject,
body_text = excluded.body_text,
source_address = excluded.source_address,
list_unsubscribe = excluded.list_unsubscribe,
list_unsubscribe_post = excluded.list_unsubscribe_post,
date_sent = excluded.date_sent,
imap_uid = excluded.imap_uid,
imap_uidvalidity = excluded.imap_uidvalidity`
receivedAt := m.ReceivedAt
if receivedAt.IsZero() {
@@ -98,6 +105,8 @@ ON CONFLICT (gmail_id) DO UPDATE SET
nullString(m.Subject),
nullString(m.BodyText),
nullString(m.SourceAddress),
nullString(m.ListUnsubscribe),
nullString(m.ListUnsubscribePost),
nullTime(m.DateSent),
receivedAt.UTC().Format(sqlTimeLayout),
m.IMAPUID,
@@ -124,7 +133,8 @@ SELECT m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''),
COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at,
COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0),
COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'),
COALESCE(m.message_id_header, '')
COALESCE(m.message_id_header, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, '')
FROM messages m WHERE m.id = ?`
var (
m Message
@@ -135,6 +145,7 @@ FROM messages m WHERE m.id = ?`
&m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject,
&m.BodyText, &m.SourceAddress, &dateSent, &received,
&m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader,
&m.ListUnsubscribe, &m.ListUnsubscribePost,
)
if err != nil {
return Message{}, fmt.Errorf("load message %d: %w", id, err)

View File

@@ -21,6 +21,10 @@ CREATE TABLE IF NOT EXISTS messages (
-- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and
-- References to it so Gmail files the reply in the same thread.
message_id_header TEXT,
-- Phase 6 (FR-7): the raw RFC 2369/8058 unsubscribe headers, stored so the
-- detector can run without a refetch.
list_unsubscribe TEXT,
list_unsubscribe_post TEXT,
from_addr TEXT,
to_addr TEXT,
subject TEXT,
@@ -94,11 +98,16 @@ CREATE TABLE IF NOT EXISTS actions_log (
CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id);
CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at);
-- FR-7: unsubscribe tracker and ignore list.
-- FR-7: unsubscribe tracker and ignore list. `status` is 'ignored' for a domain
-- on the ignore list, or 'done' for a domain Sam has recorded as unsubscribed.
-- The app never follows a link itself: it only marks and records (AUD-72).
CREATE TABLE IF NOT EXISTS unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
-- url records the link that was detected or marked (Phase 6).
url TEXT NOT NULL DEFAULT '',
updated_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);

View File

@@ -159,6 +159,25 @@ func (s *Store) applyMigrations(ctx context.Context) error {
return fmt.Errorf("record schema version 5: %w", err)
}
}
if version < 6 {
// Phase 6 (FR-7): store the raw List-Unsubscribe headers so detection
// runs without a refetch, and extend the unsubscribe tracker with the
// link it recorded and an update time.
for _, col := range []struct{ table, name, typ string }{
{"messages", "list_unsubscribe", "TEXT"},
{"messages", "list_unsubscribe_post", "TEXT"},
{"unsubscribe", "url", "TEXT NOT NULL DEFAULT ''"},
{"unsubscribe", "updated_at", "TEXT"},
} {
if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil {
return err
}
}
if _, err := s.db.ExecContext(ctx,
`INSERT OR IGNORE INTO schema_migrations (version) VALUES (6)`); err != nil {
return fmt.Errorf("record schema version 6: %w", err)
}
}
return nil
}

View File

@@ -0,0 +1,177 @@
// Unsubscribe tracking for Phase 6 (FR-7).
//
// Two things are persisted here:
//
// - the raw unsubscribe links detected on a message (messages.list_unsubscribe,
// stored by the mail fetch and parsed by internal/unsubscribe); and
// - the domain-level ignore list and the record of what was marked, in the
// `unsubscribe` table.
//
// The app never follows a link. Marking a domain records Sam's decision; the
// actual unsubscribe stays a manual, explicitly approved action (AUD-72).
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
)
// Unsubscribe statuses. `ignored` is the ignore list; `done` records a domain
// Sam has decided to unsubscribe from.
const (
UnsubscribeStatusDone = "done"
UnsubscribeStatusIgnored = "ignored"
)
// ErrUnsubscribeStatus is returned for an unknown unsubscribe status.
var ErrUnsubscribeStatus = errors.New("unsubscribe status must be done or ignored")
// UnsubscribeRecord is one row of the domain-level tracker.
type UnsubscribeRecord struct {
ID int64
Domain string
Status string
URL string
CreatedAt string
UpdatedAt string
}
// ListUnsubscribe returns every domain record, newest first.
func (s *Store) ListUnsubscribe(ctx context.Context) ([]UnsubscribeRecord, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe ORDER BY COALESCE(updated_at, created_at) DESC, id DESC`)
if err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
defer rows.Close()
var out []UnsubscribeRecord
for rows.Next() {
var r UnsubscribeRecord
if err := rows.Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan unsubscribe record: %w", err)
}
out = append(out, r)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
return out, nil
}
// UnsubscribeByDomain loads one domain record. ok is false when the domain is
// not tracked.
func (s *Store) UnsubscribeByDomain(ctx context.Context, domain string) (UnsubscribeRecord, bool, error) {
domain = strings.TrimSpace(domain)
if domain == "" {
return UnsubscribeRecord{}, false, nil
}
var r UnsubscribeRecord
err := s.db.QueryRowContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe WHERE domain = ?`, domain).
Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return UnsubscribeRecord{}, false, nil
}
if err != nil {
return UnsubscribeRecord{}, false, fmt.Errorf("read unsubscribe record %q: %w", domain, err)
}
return r, true, nil
}
// SetUnsubscribeStatus upserts the domain record. An empty url keeps the stored
// one, so an "ignore" does not wipe a link recorded earlier.
func (s *Store) SetUnsubscribeStatus(ctx context.Context, domain, status, url string) error {
domain = strings.TrimSpace(domain)
if domain == "" {
return errors.New("unsubscribe domain must not be empty")
}
if status != UnsubscribeStatusDone && status != UnsubscribeStatusIgnored {
return ErrUnsubscribeStatus
}
const q = `
INSERT INTO unsubscribe (domain, status, url, updated_at)
VALUES (?, ?, ?, datetime('now'))
ON CONFLICT (domain) DO UPDATE SET
status = excluded.status,
url = CASE WHEN excluded.url <> '' THEN excluded.url ELSE unsubscribe.url END,
updated_at = datetime('now')`
if _, err := s.db.ExecContext(ctx, q, domain, status, strings.TrimSpace(url)); err != nil {
return fmt.Errorf("set unsubscribe status for %q: %w", domain, err)
}
return nil
}
// ClearUnsubscribe removes a domain record, taking it off the ignore list.
func (s *Store) ClearUnsubscribe(ctx context.Context, domain string) error {
if _, err := s.db.ExecContext(ctx,
`DELETE FROM unsubscribe WHERE domain = ?`, strings.TrimSpace(domain)); err != nil {
return fmt.Errorf("clear unsubscribe record %q: %w", domain, err)
}
return nil
}
// UnsubscribeCandidate is a message that may carry an unsubscribe link.
type UnsubscribeCandidate struct {
MessageID int64
FromAddr string
Subject string
SourceAddress string
BodyText string
ListUnsubscribe string
ListUnsubscribePost string
Category string
ReceivedAt time.Time
}
// UnsubscribeCandidates returns messages with a List-Unsubscribe header or a
// body that mentions unsubscribing, newest first. The caller runs the detector
// over them, because body parsing lives in internal/unsubscribe.
func (s *Store) UnsubscribeCandidates(ctx context.Context, limit int) ([]UnsubscribeCandidate, error) {
if limit < 1 {
limit = 200
}
const q = `
SELECT m.id, COALESCE(m.from_addr, ''), COALESCE(m.subject, ''),
COALESCE(m.source_address, ''), COALESCE(m.body_text, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, ''),
COALESCE(c.category, ''), m.received_at
FROM messages m
LEFT JOIN classifications c ON c.id = (
SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1
)
WHERE (m.list_unsubscribe IS NOT NULL AND m.list_unsubscribe <> '')
OR instr(lower(COALESCE(m.body_text, '')), 'unsubscribe') > 0
ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC
LIMIT ?`
rows, err := s.db.QueryContext(ctx, q, limit)
if err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
defer rows.Close()
var out []UnsubscribeCandidate
for rows.Next() {
var (
c UnsubscribeCandidate
received string
)
if err := rows.Scan(
&c.MessageID, &c.FromAddr, &c.Subject, &c.SourceAddress,
&c.BodyText, &c.ListUnsubscribe, &c.ListUnsubscribePost,
&c.Category, &received,
); err != nil {
return nil, fmt.Errorf("scan unsubscribe candidate: %w", err)
}
c.ReceivedAt = parseSQLTime(received)
out = append(out, c)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
return out, nil
}

View File

@@ -0,0 +1,193 @@
package store
import (
"context"
"database/sql"
"path/filepath"
"testing"
)
func TestMigrationV6AddsUnsubscribeColumns(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "old.db")
// Simulate a v5 database: messages has no List-Unsubscribe columns and the
// unsubscribe table has no url/updated_at. Opening it must apply migration
// v6.
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open old db: %v", err)
}
stmts := []string{
`CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
gmail_id TEXT NOT NULL UNIQUE,
thread_id TEXT, message_id_header TEXT, from_addr TEXT, to_addr TEXT,
subject TEXT, body_text TEXT, source_address TEXT, date_sent TEXT,
imap_uid INTEGER, imap_uidvalidity INTEGER,
folder TEXT NOT NULL DEFAULT 'inbox',
mailbox TEXT NOT NULL DEFAULT 'INBOX',
received_at TEXT NOT NULL DEFAULT (datetime('now')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`INSERT INTO schema_migrations (version) VALUES (5)`,
}
for _, q := range stmts {
if _, err := db.ExecContext(ctx, q); err != nil {
t.Fatalf("seed old db: %v", err)
}
}
_ = db.Close()
s, err := Open(ctx, path)
if err != nil {
t.Fatalf("Open() on v5 database error = %v", err)
}
defer s.Close()
v, err := s.SchemaVersion(ctx)
if err != nil {
t.Fatalf("SchemaVersion() error = %v", err)
}
if v < 6 {
t.Errorf("SchemaVersion() = %d, want >= 6", v)
}
for _, tc := range []struct{ table, col string }{
{"messages", "list_unsubscribe"},
{"messages", "list_unsubscribe_post"},
{"unsubscribe", "url"},
{"unsubscribe", "updated_at"},
} {
var n int
q := `SELECT COUNT(*) FROM pragma_table_info('` + tc.table + `') WHERE name = ?`
if err := s.DB().QueryRowContext(ctx, q, tc.col).Scan(&n); err != nil {
t.Fatalf("count %s.%s: %v", tc.table, tc.col, err)
}
if n != 1 {
t.Errorf("%s.%s missing after migration v6", tc.table, tc.col)
}
}
}
func TestUnsubscribeRecordUpsert(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t) // reuses the store helper from categories_test.go
if _, ok, err := s.UnsubscribeByDomain(ctx, "brand.example"); err != nil || ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a miss", ok, err)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusIgnored, "https://brand.example/u"); err != nil {
t.Fatalf("SetUnsubscribeStatus(ignore) error = %v", err)
}
rec, ok, err := s.UnsubscribeByDomain(ctx, "brand.example")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec.Status != UnsubscribeStatusIgnored || rec.URL != "https://brand.example/u" {
t.Errorf("record = %+v, want ignored with url", rec)
}
// A later mark with no URL must keep the recorded link.
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusDone, ""); err != nil {
t.Fatalf("SetUnsubscribeStatus(done) error = %v", err)
}
rec, _, _ = s.UnsubscribeByDomain(ctx, "brand.example")
if rec.Status != UnsubscribeStatusDone {
t.Errorf("status = %q, want done", rec.Status)
}
if rec.URL != "https://brand.example/u" {
t.Errorf("url = %q, want the earlier link preserved", rec.URL)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", "bogus", ""); err == nil {
t.Error("SetUnsubscribeStatus(bogus) accepted an unknown status")
}
records, err := s.ListUnsubscribe(ctx)
if err != nil || len(records) != 1 {
t.Fatalf("ListUnsubscribe() = (%v, %v), want one record", records, err)
}
if err := s.ClearUnsubscribe(ctx, "brand.example"); err != nil {
t.Fatalf("ClearUnsubscribe() error = %v", err)
}
if err := s.SetUnsubscribeStatus(ctx, "", UnsubscribeStatusIgnored, ""); err == nil {
t.Error("SetUnsubscribeStatus accepted an empty domain")
}
}
func TestUnsubscribeCandidates(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
seed := func(gmailID, from, body, header string) {
t.Helper()
if _, err := s.UpsertMessage(ctx, &Message{
GmailID: gmailID,
FromAddr: from,
Subject: "subject " + gmailID,
BodyText: body,
ListUnsubscribe: header,
}); err != nil {
t.Fatalf("seed %s: %v", gmailID, err)
}
}
seed("header-msg", "Support <support@brand.example>", "hello", "<https://brand.example/u>")
seed("body-msg", "Shop <deals@shop.example>", "You can unsubscribe at https://shop.example/opt-out now.", "")
seed("plain-msg", "Friend <friend@example.com>", "Just saying hi.", "")
got, err := s.UnsubscribeCandidates(ctx, 50)
if err != nil {
t.Fatalf("UnsubscribeCandidates() error = %v", err)
}
if len(got) != 2 {
t.Fatalf("candidates = %d, want 2 (%+v)", len(got), got)
}
ids := map[string]bool{}
for _, c := range got {
ids[c.Subject] = true
}
if !ids["subject body-msg"] || !ids["subject header-msg"] {
t.Errorf("candidates = %+v, want the header and body messages", ids)
}
}
func TestRulesVersionsAndRestore(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
if _, err := s.SaveRules(ctx, "from contains \"a\" => category=Info"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
if _, err := s.SaveRules(ctx, "from contains \"b\" => category=Action"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
versions, err := s.RulesVersions(ctx, 10)
if err != nil {
t.Fatalf("RulesVersions() error = %v", err)
}
if len(versions) < 3 {
t.Fatalf("RulesVersions() = %d, want >= 3 (seed + 2 saves)", len(versions))
}
if versions[0].Version != 3 {
t.Errorf("newest version = %d, want 3", versions[0].Version)
}
text, ok, err := s.RuleByVersion(ctx, 2)
if err != nil || !ok {
t.Fatalf("RuleByVersion(2) = (_, %v, %v), want a hit", ok, err)
}
if text != "from contains \"a\" => category=Info" {
t.Errorf("RuleByVersion(2) = %q", text)
}
if _, ok, _ := s.RuleByVersion(ctx, 999); ok {
t.Error("RuleByVersion(999) reported a hit")
}
}

View File

@@ -0,0 +1,240 @@
// Package unsubscribe detects unsubscribe links (FR-7).
//
// The primary source is the List-Unsubscribe header (RFC 2369) together with
// the List-Unsubscribe-Post header (RFC 8058). When neither is present the body
// is scanned for a URL on a line that mentions "unsubscribe".
//
// Detection only. This package never fetches a URL and never sends mail.
// Acting on an unsubscribe link needs Sam's explicit approval (AGENTS.md §10
// and AUD-72); the app only marks and records.
package unsubscribe
import (
"net/url"
"regexp"
"strings"
)
// Link methods.
const (
MethodHTTP = "http"
MethodMailto = "mailto"
)
// Detection sources. They are stored in the action log so a marking is
// auditable.
const (
SourceListUnsubscribe = "list-unsubscribe"
SourceBody = "body"
)
// Link is one detected unsubscribe target.
type Link struct {
// URL is the raw target, for example "https://x.example/u?id=1".
URL string
// Method is MethodHTTP or MethodMailto.
Method string
// Domain is the ignore-list key. It is the sender's brand domain when that
// is a real domain, otherwise the link host.
Domain string
// Host is the host of the link itself, shown for context.
Host string
// Source is SourceListUnsubscribe or SourceBody.
Source string
}
// Result is the detection outcome for one message.
type Result struct {
Links []Link
// Header is the raw List-Unsubscribe value, if any.
Header string
// OneClick reports a RFC 8058 List-Unsubscribe=One-Click header.
OneClick bool
}
// Detect returns the unsubscribe links for a message. from is the raw From
// header; body is the stored plain-text body.
func Detect(listUnsubscribe, listUnsubscribePost, from, body string) Result {
res := Result{
Header: strings.TrimSpace(listUnsubscribe),
OneClick: isOneClick(listUnsubscribePost),
}
sender := brandDomain(from)
seen := map[string]struct{}{}
add := func(raw, source string) {
link, ok := makeLink(raw, source, sender)
if !ok {
return
}
key := link.Method + ":" + strings.ToLower(link.URL)
if _, dup := seen[key]; dup {
return
}
seen[key] = struct{}{}
res.Links = append(res.Links, link)
}
for _, raw := range headerLinks(res.Header) {
add(raw, SourceListUnsubscribe)
}
if len(res.Links) == 0 {
for _, raw := range bodyLinks(body) {
add(raw, SourceBody)
}
}
return res
}
// isOneClick reports a RFC 8058 one-click unsubscribe header. The documented
// value is "List-Unsubscribe=One-Click".
func isOneClick(v string) bool {
return strings.Contains(strings.ToLower(v), "one-click")
}
var angleRE = regexp.MustCompile(`<([^>]+)>`)
// headerLinks extracts the URIs from a List-Unsubscribe value. The RFC form is
// a comma-separated list of angle-bracketed URIs; a bare URI is also accepted.
func headerLinks(v string) []string {
v = strings.TrimSpace(v)
if v == "" {
return nil
}
var out []string
for _, m := range angleRE.FindAllStringSubmatch(v, -1) {
if s := strings.TrimSpace(m[1]); s != "" {
out = append(out, s)
}
}
if len(out) > 0 {
return out
}
for _, part := range strings.Split(v, ",") {
if s := strings.TrimSpace(part); s != "" {
out = append(out, s)
}
}
return out
}
var (
bodyURLRE = regexp.MustCompile(`(?i)(https?://[^\s<>"')\]]+|mailto:[^\s<>"')\]]+)`)
unsubRE = regexp.MustCompile(`(?i)unsubscribe|opt[\s-]?out|manage (?:your )?(?:email )?preferences`)
)
// bodyLinks finds unsubscribe URLs in a plain-text body. Only lines that
// mention unsubscribing are considered, so a random link is not reported.
func bodyLinks(body string) []string {
body = strings.TrimSpace(body)
if body == "" {
return nil
}
var out []string
for _, line := range strings.Split(body, "\n") {
if !unsubRE.MatchString(line) {
continue
}
for _, m := range bodyURLRE.FindAllString(line, -1) {
out = append(out, trimURL(m))
}
}
return out
}
// trimURL removes trailing punctuation that is part of the sentence, not the
// URL.
func trimURL(s string) string {
return strings.TrimRight(strings.TrimSpace(s), ".,;:!?)\"'")
}
// makeLink validates a raw target and fills in method/host/domain.
func makeLink(raw, source, sender string) (Link, bool) {
raw = trimURL(raw)
raw = strings.Trim(raw, "<>")
if raw == "" {
return Link{}, false
}
link := Link{URL: raw, Source: source}
switch {
case strings.HasPrefix(strings.ToLower(raw), "mailto:"):
link.Method = MethodMailto
link.Host = mailtoHost(raw)
case strings.HasPrefix(strings.ToLower(raw), "http://"),
strings.HasPrefix(strings.ToLower(raw), "https://"):
link.Method = MethodHTTP
link.Host = urlHost(raw)
default:
return Link{}, false
}
link.Domain = sender
if link.Domain == "" {
link.Domain = link.Host
}
return link, link.Domain != ""
}
// Domain returns the domain used as the ignore-list key. It is exported for
// callers that need it without parsing a link.
func Domain(from string) string { return brandDomain(from) }
func brandDomain(from string) string {
addr := extractAddress(from)
i := strings.LastIndex(addr, "@")
if i < 0 || i == len(addr)-1 {
return ""
}
dom := strings.ToLower(strings.Trim(strings.TrimSpace(addr[i+1:]), ">."))
if dom == "" || freeMail[dom] {
return ""
}
return dom
}
// extractAddress pulls the address out of a From header such as
// "Support <support@example.com>".
func extractAddress(v string) string {
v = strings.TrimSpace(v)
if i := strings.LastIndex(v, "<"); i >= 0 {
if j := strings.Index(v[i:], ">"); j > 0 {
return strings.TrimSpace(v[i+1 : i+j])
}
}
return strings.Trim(v, "<> \t")
}
// freeMail domains are consumer mailboxes, not brands. A message forwarded
// through one should key its ignore list on the real link host instead.
var freeMail = map[string]bool{
"gmail.com": true,
"googlemail.com": true,
"outlook.com": true,
"hotmail.com": true,
"live.com": true,
"yahoo.com": true,
"icloud.com": true,
"me.com": true,
"aol.com": true,
"proton.me": true,
"protonmail.com": true,
}
func urlHost(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
func mailtoHost(raw string) string {
rest := raw[len("mailto:"):]
if i := strings.IndexAny(rest, "?,"); i >= 0 {
rest = rest[:i]
}
i := strings.LastIndex(rest, "@")
if i < 0 || i == len(rest)-1 {
return ""
}
return strings.ToLower(strings.Trim(rest[i+1:], ">."))
}

View File

@@ -0,0 +1,126 @@
package unsubscribe
import "testing"
func TestDetectListUnsubscribeHeader(t *testing.T) {
res := Detect(
"<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=42>",
"List-Unsubscribe=One-Click",
"Heritage Fruit Trees <support@heritagefruittrees.com.au>",
"Special offer this week.",
)
if !res.OneClick {
t.Error("OneClick = false, want true")
}
if len(res.Links) != 2 {
t.Fatalf("Links len = %d, want 2 (%+v)", len(res.Links), res.Links)
}
mail := res.Links[0]
if mail.Method != MethodMailto {
t.Errorf("mailto method = %q", mail.Method)
}
if mail.Domain != "heritagefruittrees.com.au" {
t.Errorf("mailto domain = %q", mail.Domain)
}
if mail.Source != SourceListUnsubscribe {
t.Errorf("mailto source = %q", mail.Source)
}
http := res.Links[1]
if http.Method != MethodHTTP {
t.Errorf("http method = %q", http.Method)
}
if http.Host != "heritagefruittrees.com.au" {
t.Errorf("http host = %q", http.Host)
}
if http.Domain != "heritagefruittrees.com.au" {
t.Errorf("http domain = %q", http.Domain)
}
}
func TestDetectBareHeaderURL(t *testing.T) {
res := Detect("https://lists.example.com/unsub?u=abc,", "", "News <news@lists.example.com>", "")
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].URL != "https://lists.example.com/unsub?u=abc" {
t.Errorf("URL = %q", res.Links[0].URL)
}
}
func TestDetectBodyFallback(t *testing.T) {
res := Detect("", "", "Shop <deals@shop.example>", "Thanks for subscribing!\nUnsubscribe here: https://shop.example/opt-out?token=1.\nSee you.")
if res.Header != "" {
t.Errorf("Header = %q, want empty", res.Header)
}
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].Source != SourceBody {
t.Errorf("Source = %q, want %q", res.Links[0].Source, SourceBody)
}
if res.Links[0].URL != "https://shop.example/opt-out?token=1" {
t.Errorf("URL = %q (trailing period should be trimmed)", res.Links[0].URL)
}
}
func TestDetectIgnoresUnrelatedBodyLinks(t *testing.T) {
res := Detect("", "", "Friend <friend@example.com>", "Read more at https://example.com/blog")
if len(res.Links) != 0 {
t.Fatalf("Links = %+v, want none", res.Links)
}
}
func TestDetectBodyMailto(t *testing.T) {
res := Detect("", "", "Brand <hello@brand.example>", "To unsubscribe, email mailto:leave@brand.example")
if len(res.Links) != 1 || res.Links[0].Method != MethodMailto {
t.Fatalf("Links = %+v, want one mailto", res.Links)
}
}
func TestFreeMailSenderUsesLinkHost(t *testing.T) {
res := Detect(
"<https://mailer.example.com/u/9>",
"",
"Someone <someone@gmail.com>",
"",
)
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].Domain != "mailer.example.com" {
t.Errorf("domain = %q, want link host", res.Links[0].Domain)
}
}
func TestDetectDeduplicates(t *testing.T) {
res := Detect(
"<https://x.example/u>, <https://x.example/u>",
"",
"Brand <hello@brand.example>",
"",
)
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1 after de-duplication", res.Links)
}
}
func TestDomain(t *testing.T) {
cases := map[string]string{
"Support <support@heritagefruittrees.com.au>": "heritagefruittrees.com.au",
"plain@brand.example": "brand.example",
"Someone <someone@gmail.com>": "",
"": "",
}
for in, want := range cases {
if got := Domain(in); got != want {
t.Errorf("Domain(%q) = %q, want %q", in, got, want)
}
}
}
func TestEmptyInputs(t *testing.T) {
res := Detect("", "", "", "")
if len(res.Links) != 0 || res.OneClick {
t.Fatalf("Detect(empty) = %+v, want zero value", res)
}
}