Phase 6: rules editor + unsubscribe detection (AUD-72)

- FR-11 rules editor: plain-text area at /rules, saved and versioned
  (SaveRules appends a version; restore re-saves an old version), parse
  errors surfaced, one-click re-classify. The reply model now reads the
  rules document too (FR-11).
- FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post
  (migration v6), detect email and web links (internal/unsubscribe), a
  domain-level ignore list, and an explicit record of what was marked. No
  code path contacts an unsubscribe link; acting on one needs Sam's approval.
- Migration v6 is additive; unsubscribe gains url/updated_at.
- Tests: detector, parser, store migration/records/candidates, rules editor,
  unsubscribe handlers, message-detail rendering.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2026-10-11 22:24:38 +11:00
parent 7c66e4ad31
commit 00bb0e05c4
24 changed files with 2121 additions and 42 deletions

View File

@@ -102,18 +102,20 @@ func (f *Fetcher) Fetch(ctx context.Context) (Summary, error) {
headersSeen[parsed.SourceHeader] = struct{}{} headersSeen[parsed.SourceHeader] = struct{}{}
} }
msg := &store.Message{ msg := &store.Message{
GmailID: gmailID(r, uidValidity), GmailID: gmailID(r, uidValidity),
ThreadID: r.GmailThreadID, ThreadID: r.GmailThreadID,
MessageIDHeader: parsed.MessageID, MessageIDHeader: parsed.MessageID,
FromAddr: parsed.From, FromAddr: parsed.From,
ToAddr: parsed.To, ToAddr: parsed.To,
Subject: parsed.Subject, Subject: parsed.Subject,
BodyText: parsed.BodyText, BodyText: parsed.BodyText,
SourceAddress: parsed.SourceAddress, SourceAddress: parsed.SourceAddress,
DateSent: parsed.Date, ListUnsubscribe: parsed.ListUnsubscribe,
ReceivedAt: now, ListUnsubscribePost: parsed.ListUnsubscribePost,
IMAPUID: int64(r.UID), DateSent: parsed.Date,
IMAPUIDValidity: int64(uidValidity), ReceivedAt: now,
IMAPUID: int64(r.UID),
IMAPUIDValidity: int64(uidValidity),
} }
if _, err := f.store.UpsertMessage(ctx, msg); err != nil { if _, err := f.store.UpsertMessage(ctx, msg); err != nil {
return sum, err return sum, err

View File

@@ -27,8 +27,13 @@ type ParsedMessage struct {
Date time.Time Date time.Time
// MessageID is the RFC822 Message-ID header, used for reply threading // MessageID is the RFC822 Message-ID header, used for reply threading
// (Phase 5, FR-8). // (Phase 5, FR-8).
MessageID string MessageID string
SourceAddress string // ListUnsubscribe and ListUnsubscribePost are the RFC 2369/8058 headers that
// name the unsubscribe targets (Phase 6, FR-7). They are stored raw so the
// detector can run later without a refetch.
ListUnsubscribe string
ListUnsubscribePost string
SourceAddress string
// SourceHeader records which header the source address came from, so FR-2 // SourceHeader records which header the source address came from, so FR-2
// decisions are auditable. // decisions are auditable.
SourceHeader string SourceHeader string
@@ -43,11 +48,13 @@ func ParseMessage(raw []byte) (*ParsedMessage, error) {
} }
h := msg.Header h := msg.Header
pm := &ParsedMessage{ pm := &ParsedMessage{
From: decodeHeader(h.Get("From")), From: decodeHeader(h.Get("From")),
To: decodeHeader(h.Get("To")), To: decodeHeader(h.Get("To")),
Subject: decodeHeader(h.Get("Subject")), Subject: decodeHeader(h.Get("Subject")),
Date: parseDate(h.Get("Date")), Date: parseDate(h.Get("Date")),
MessageID: strings.TrimSpace(h.Get("Message-Id")), MessageID: strings.TrimSpace(h.Get("Message-Id")),
ListUnsubscribe: strings.TrimSpace(h.Get("List-Unsubscribe")),
ListUnsubscribePost: strings.TrimSpace(h.Get("List-Unsubscribe-Post")),
} }
pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h)) pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h))
pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body) pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body)

View File

@@ -172,3 +172,34 @@ func TestHeaderAddressParsing(t *testing.T) {
} }
} }
} }
func TestParseMessageCapturesUnsubscribeHeaders(t *testing.T) {
raw := "From: Heritage Fruit Trees <support@heritagefruittrees.com.au>\r\n" +
"To: samuelrolfe@gmail.com\r\n" +
"X-Pm-Original-To: hello@where-woof.com\r\n" +
"Subject: New workshops\r\n" +
"List-Unsubscribe: <mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>\r\n" +
"List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" +
"Message-Id: <abc@mail.example>\r\n" +
"Date: Mon, 01 Jan 2026 10:00:00 +0000\r\n" +
"Content-Type: text/plain; charset=utf-8\r\n" +
"\r\n" +
"Hello, you can unsubscribe at https://heritagefruittrees.com.au/u?id=1.\r\n"
pm, err := ParseMessage([]byte(raw))
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if !strings.Contains(pm.ListUnsubscribe, "mailto:unsub@heritagefruittrees.com.au") {
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
}
if !strings.Contains(pm.ListUnsubscribe, "https://heritagefruittrees.com.au/u?id=1") {
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
}
if pm.ListUnsubscribePost != "List-Unsubscribe=One-Click" {
t.Errorf("ListUnsubscribePost = %q", pm.ListUnsubscribePost)
}
if pm.SourceAddress != "hello@where-woof.com" {
t.Errorf("SourceAddress = %q", pm.SourceAddress)
}
}

View File

@@ -18,6 +18,10 @@ const maxPromptBodyChars = 4000
// maxDraftChars bounds the generated body before it is stored. // maxDraftChars bounds the generated body before it is stored.
const maxDraftChars = 8000 const maxDraftChars = 8000
// maxRulesChars bounds how much of the user's rules document is sent to the
// model (FR-11: the LLM reads the rules too).
const maxRulesChars = 2000
// Request is one draft request. // Request is one draft request.
type Request struct { type Request struct {
// Message is the mail being replied to. // Message is the mail being replied to.
@@ -31,6 +35,9 @@ type Request struct {
// Instruction is optional extra guidance Sam typed (for example "say the // Instruction is optional extra guidance Sam typed (for example "say the
// invoice is paid"). It is appended to the template instruction. // invoice is paid"). It is appended to the template instruction.
Instruction string Instruction string
// Rules is the current rules document from the editor (FR-11). The model
// reads it so reply style and handling follow the same rules as triage.
Rules string
// Signer is the name used for the sign-off (default "Sam"). // Signer is the name used for the sign-off (default "Sam").
Signer string Signer string
} }
@@ -107,6 +114,11 @@ func BuildPrompt(req Request, tmpl Template) (system, user string) {
b.WriteString(extra) b.WriteString(extra)
b.WriteString("\n") b.WriteString("\n")
} }
if rules := strings.TrimSpace(req.Rules); rules != "" {
b.WriteString("USER RULES (follow when they apply):\n")
b.WriteString(truncate(rules, maxRulesChars))
b.WriteString("\n")
}
if cat := strings.TrimSpace(req.Category); cat != "" { if cat := strings.TrimSpace(req.Category); cat != "" {
b.WriteString("CLASSIFICATION: category=") b.WriteString("CLASSIFICATION: category=")
b.WriteString(cat) b.WriteString(cat)

View File

@@ -116,3 +116,16 @@ func TestGeneratorDraftEmptyReplyIsAnError(t *testing.T) {
t.Fatal("Draft() error = nil, want an error for an empty reply") t.Fatal("Draft() error = nil, want an error for an empty reply")
} }
} }
func TestBuildPromptIncludesRules(t *testing.T) {
req := Request{
Message: store.Message{FromAddr: "customer@example.test", Subject: "hi", BodyText: "?"},
Action: "reply",
Rules: `from contains "example.test" => category=Action`,
}
tmpl, _ := TemplateByID("answer")
_, user := BuildPrompt(req, tmpl)
if !strings.Contains(user, "USER RULES") || !strings.Contains(user, `from contains "example.test"`) {
t.Errorf("user prompt does not include the rules document:\n%s", user)
}
}

View File

@@ -19,6 +19,7 @@ import (
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
) )
// ReplyGenerator drafts a reply body. The live implementation is // ReplyGenerator drafts a reply body. The live implementation is
@@ -63,6 +64,11 @@ type replyData struct {
LLMReady bool LLMReady bool
LLMModel string LLMModel string
// Phase 6 (FR-7): detected unsubscribe links for this message, and the
// domain's status on the ignore/recorded list.
UnsubLinks []unsubscribe.Link
UnsubStatus string
Sent bool Sent bool
ThreadUpdated bool ThreadUpdated bool
RepliedFolder string RepliedFolder string
@@ -109,6 +115,9 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
} }
class := s.latestClass(ctx, id) class := s.latestClass(ctx, id)
// FR-11: the reply model reads the current rules document too.
rulesText, _, _ := s.store.CurrentRules(ctx)
body, templateID, err := s.replies.Draft(ctx, reply.Request{ body, templateID, err := s.replies.Draft(ctx, reply.Request{
Message: msg, Message: msg,
Category: class.Category, Category: class.Category,
@@ -116,6 +125,7 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
Action: class.Action, Action: class.Action,
TemplateID: r.FormValue("template"), TemplateID: r.FormValue("template"),
Instruction: r.FormValue("instruction"), Instruction: r.FormValue("instruction"),
Rules: rulesText,
Signer: "Sam", Signer: "Sam",
}) })
if err != nil { if err != nil {
@@ -313,6 +323,16 @@ func (s *Server) renderMessage(w http.ResponseWriter, r *http.Request, id int64,
} }
data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == "" data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == ""
// FR-7: detect unsubscribe links and look up the domain's list status.
if res := unsubscribe.Detect(msg.ListUnsubscribe, msg.ListUnsubscribePost, msg.FromAddr, msg.BodyText); len(res.Links) > 0 {
data.UnsubLinks = res.Links
if rec, ok, rerr := s.store.UnsubscribeByDomain(ctx, res.Links[0].Domain); rerr != nil {
s.logger.Error("load unsubscribe status", "message_id", id, "error", rerr)
} else if ok {
data.UnsubStatus = rec.Status
}
}
if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil { if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil {
s.logger.Error("load draft", "message_id", id, "error", derr) s.logger.Error("load draft", "message_id", id, "error", derr)
} else if ok { } else if ok {

225
internal/server/rules.go Normal file
View File

@@ -0,0 +1,225 @@
// Rules editor for Phase 6 (FR-4, FR-11).
//
// The rules document is plain text (see internal/classify for the syntax). It is
// saved as a new numbered version on every save, so the current text and its
// history are both available. The classifier reads the newest version and
// applies it as an override; the reply model also reads it.
package server
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify"
)
type ruleRevision struct {
Version int
CreatedAt string
Lines int
Preview string
}
type rulesData struct {
Text string
Version int
Revisions []ruleRevision
Errors []classify.ParseError
RuleCount int
Note string
Error string
CSRFToken string
// ClassifierReady is true when JEV is configured, so the page can offer a
// re-classify action that actually runs.
ClassifierReady bool
TotalMessages int
}
// handleRules renders the rules editor with the current text and version list.
func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) {
s.renderRules(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
}
func (s *Server) renderRules(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
ctx := r.Context()
text, version, err := s.store.CurrentRules(ctx)
if err != nil {
s.logger.Error("load rules", "error", err)
http.Error(w, "could not load the rules", http.StatusInternalServerError)
return
}
parsed := classify.ParseRules(text)
revisions, err := s.store.RulesVersions(ctx, 20)
if err != nil {
s.logger.Error("load rules versions", "error", err)
}
revs := make([]ruleRevision, 0, len(revisions))
for _, rv := range revisions {
revs = append(revs, ruleRevision{
Version: rv.Version,
CreatedAt: rv.CreatedAt,
Lines: countRuleLines(rv.Text),
Preview: firstRuleLine(rv.Text),
})
}
total, err := s.store.CountMessages(ctx)
if err != nil {
s.logger.Error("count messages", "error", err)
}
data := rulesData{
Text: text,
Version: version,
Revisions: revs,
Errors: parsed.Errors,
RuleCount: len(parsed.Rules),
Note: note,
Error: errMsg,
CSRFToken: s.csrfToken(w, r),
ClassifierReady: s.classifier.Configured(),
TotalMessages: total,
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(code)
if err := s.tmpl.ExecuteTemplate(w, "rules.html", data); err != nil {
s.logger.Error("render rules", "error", err)
}
}
// handleRulesSave versions the posted text (FR-11).
func (s *Server) handleRulesSave(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectRules(w, r, "", "Could not read the form.")
return
}
text := r.FormValue("text")
version, err := s.store.SaveRules(r.Context(), text)
if err != nil {
s.logger.Error("save rules", "error", err)
s.redirectRules(w, r, "", "Could not save the rules: "+err.Error())
return
}
parsed := classify.ParseRules(text)
s.logAction(r.Context(), "rules_save", "Sam",
"version="+strconv.Itoa(version)+" rules="+strconv.Itoa(len(parsed.Rules))+" parse_errors="+strconv.Itoa(len(parsed.Errors)))
s.redirectRules(w, r, "Saved rules version "+strconv.Itoa(version)+". Re-classify all to apply it to existing mail.", "")
}
// handleRulesRestore re-saves an older version as the newest one, so history is
// append-only (FR-11).
func (s *Server) handleRulesRestore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectRules(w, r, "", "Could not read the form.")
return
}
version, err := strconv.Atoi(strings.TrimSpace(r.FormValue("version")))
if err != nil || version < 1 {
s.redirectRules(w, r, "", "Unknown rules version.")
return
}
text, ok, err := s.store.RuleByVersion(r.Context(), version)
if err != nil {
s.redirectRules(w, r, "", "Could not load that version.")
return
}
if !ok {
s.redirectRules(w, r, "", "That rules version no longer exists.")
return
}
newVersion, err := s.store.SaveRules(r.Context(), text)
if err != nil {
s.logger.Error("restore rules", "from_version", version, "error", err)
s.redirectRules(w, r, "", "Could not restore that version: "+err.Error())
return
}
s.logAction(r.Context(), "rules_restore", "Sam",
"from_version="+strconv.Itoa(version)+" new_version="+strconv.Itoa(newVersion))
s.redirectRules(w, r, "Restored version "+strconv.Itoa(version)+" as version "+strconv.Itoa(newVersion)+".", "")
}
// handleRulesReclassify re-runs the classifier over every stored message and
// returns to the rules editor. It is the rules page's "apply" action: saving a
// new version does not re-classify by itself.
func (s *Server) handleRulesReclassify(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if rc := http.NewResponseController(w); rc != nil {
_ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute))
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
defer cancel()
sum, err := s.classifier.RunAll(ctx)
switch {
case err == nil:
s.logAction(ctx, "reclassify", "Sam",
fmt.Sprintf("considered=%d classified=%d failed=%d rules_version=%d",
sum.Considered, sum.Classified, sum.Failed, sum.RulesVersion))
s.redirectRules(w, r, fmt.Sprintf(
"Re-classified %d of %d message(s); %d failed. The current rules are now applied.",
sum.Classified, sum.Considered, sum.Failed), "")
case errors.Is(err, classify.ErrNotConfigured):
s.redirectRules(w, r, "", "Classification is not connected yet: set JEV_API_KEY.")
default:
s.logger.Error("rules reclassify failed", "error", err)
s.redirectRules(w, r, "", "Classification failed: "+err.Error())
}
}
// redirectRules is POST/redirect/GET so a refresh does not resubmit.
func (s *Server) redirectRules(w http.ResponseWriter, r *http.Request, note, errMsg string) {
q := url.Values{}
if note != "" {
q.Set("note", note)
}
if errMsg != "" {
q.Set("error", errMsg)
}
http.Redirect(w, r, "/rules?"+q.Encode(), http.StatusSeeOther)
}
// countRuleLines counts the non-empty, non-comment lines in a rules document.
func countRuleLines(text string) int {
n := 0
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
n++
}
return n
}
// firstRuleLine returns a short preview of the first real rule, or a note when
// the document has only comments.
func firstRuleLine(text string) string {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if len(line) > 80 {
return line[:80] + "…"
}
return line
}
return "(comments only)"
}

View File

@@ -0,0 +1,152 @@
package server
import (
"context"
"net/http"
"net/url"
"strings"
"testing"
)
func TestRulesPageRenders(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/rules", session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
for _, want := range []string{"Rules editor", "Save new version", "Versions", "Back to inbox", "Unsubscribe"} {
if !strings.Contains(body, want) {
t.Errorf("rules page missing %q", want)
}
}
}
func TestRulesSaveCreatesVersion(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rule := `from contains "heritagefruittrees.com.au" => category=Action, importance=high, action=file`
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
"csrf": {csrf}, "text": {rule},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
text, version, err := srv.store.CurrentRules(context.Background())
if err != nil {
t.Fatalf("CurrentRules() error = %v", err)
}
if text != rule {
t.Errorf("CurrentRules() text = %q, want the saved rule", text)
}
if version != 2 {
t.Errorf("CurrentRules() version = %d, want 2", version)
}
// The editor shows the new version and active rule count.
body := getWithSession(t, srv, "/rules", session).Body.String()
if !strings.Contains(body, "version 2") || !strings.Contains(body, "1 active rule") {
t.Errorf("rules page did not reflect the saved version; body:\n%s", body)
}
}
func TestRulesSaveRequiresCSRF(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
rec := postForm(t, srv, "/rules/save", session, nil, url.Values{"text": {"anything"}})
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403", rec.Code)
}
}
func TestRulesPageShowsParseErrors(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
"csrf": {csrf}, "text": {"this line has no arrow"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save status = %d, want 303", rec.Code)
}
body := getWithSession(t, srv, "/rules", session).Body.String()
if !strings.Contains(body, "could not be parsed") || !strings.Contains(body, "missing =&gt;") {
t.Errorf("rules page did not surface the parse error; body:\n%s", body)
}
}
func TestRulesRestoreCreatesNewVersion(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
first := `from contains "a" => category=Info`
second := `from contains "b" => category=Action`
for _, text := range []string{first, second} {
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{"csrf": {csrf}, "text": {text}})
if rec.Code != http.StatusSeeOther {
t.Fatalf("save %q status = %d, want 303", text, rec.Code)
}
}
// Restore version 2 (the first saved rule) as a new version.
rec := postForm(t, srv, "/rules/restore", session, csrfCookie, url.Values{
"csrf": {csrf}, "version": {"2"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("restore status = %d, want 303", rec.Code)
}
text, version, err := srv.store.CurrentRules(ctx)
if err != nil {
t.Fatalf("CurrentRules() error = %v", err)
}
if text != first {
t.Errorf("restored text = %q, want %q", text, first)
}
if version != 4 {
t.Errorf("restored version = %d, want 4 (append-only)", version)
}
}
func TestDashboardLinksRulesAndUnsubscribe(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
body := getWithSession(t, srv, "/", session).Body.String()
for _, want := range []string{`href="/rules"`, `href="/unsubscribe"`} {
if !strings.Contains(body, want) {
t.Errorf("dashboard missing %q", want)
}
}
}
func TestRulesCountsLines(t *testing.T) {
text := "# comment\n\nfrom contains \"a\" => category=Info\nfrom contains \"b\" => category=Action"
if got := countRuleLines(text); got != 2 {
t.Errorf("countRuleLines() = %d, want 2", got)
}
if got := firstRuleLine(text); got != `from contains "a" => category=Info` {
t.Errorf("firstRuleLine() = %q", got)
}
if got := firstRuleLine("# only a comment"); got != "(comments only)" {
t.Errorf("firstRuleLine(comment only) = %q", got)
}
}
func TestRulesReclassifyWithoutJEVRedirects(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/rules/reclassify", session, csrfCookie, url.Values{"csrf": {csrf}})
if rec.Code != http.StatusSeeOther {
t.Fatalf("status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
if loc := rec.Header().Get("Location"); !strings.Contains(loc, "JEV_API_KEY") {
t.Errorf("Location = %q, want a JEV-not-configured note", loc)
}
}

View File

@@ -149,6 +149,18 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename)) mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename))
mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove)) mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove))
// Phase 6 (FR-4, FR-11): rules editor, saved and versioned.
mux.HandleFunc("GET /rules", s.requireAuth(s.handleRules))
mux.HandleFunc("POST /rules/save", s.requireAuth(s.handleRulesSave))
mux.HandleFunc("POST /rules/restore", s.requireAuth(s.handleRulesRestore))
mux.HandleFunc("POST /rules/reclassify", s.requireAuth(s.handleRulesReclassify))
// Phase 6 (FR-7): unsubscribe detection, ignore list, and records.
mux.HandleFunc("GET /unsubscribe", s.requireAuth(s.handleUnsubscribe))
mux.HandleFunc("POST /unsubscribe/ignore", s.requireAuth(s.handleUnsubscribeIgnore))
mux.HandleFunc("POST /unsubscribe/unignore", s.requireAuth(s.handleUnsubscribeUnignore))
mux.HandleFunc("POST /unsubscribe/mark", s.requireAuth(s.handleUnsubscribeMark))
return s.recoverPanic(s.logRequests(mux)) return s.recoverPanic(s.logRequests(mux))
} }

View File

@@ -0,0 +1,218 @@
// Unsubscribe detection view for Phase 6 (FR-7).
//
// The page lists messages with a detected unsubscribe link, the domain-level
// ignore list, and the domains Sam has recorded as unsubscribed. Every action
// records state; none of them contacts the link. Acting on a real unsubscribe
// link is a manual step that needs Sam's explicit approval (AUD-72).
package server
import (
"net/http"
"net/url"
"strings"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
)
type unsubscribeItem struct {
MessageID int64
From string
Subject string
SourceAddr string
Domain string
Links []unsubscribe.Link
Status string
Category string
When string
}
type unsubscribeData struct {
Items []unsubscribeItem
Ignored []store.UnsubscribeRecord
Recorded []store.UnsubscribeRecord
Note string
Error string
CSRFToken string
CandidateCount int
}
// handleUnsubscribe renders the detection list and the ignore/recorded lists.
func (s *Server) handleUnsubscribe(w http.ResponseWriter, r *http.Request) {
s.renderUnsubscribe(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
}
func (s *Server) renderUnsubscribe(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
ctx := r.Context()
records, err := s.store.ListUnsubscribe(ctx)
if err != nil {
s.logger.Error("list unsubscribe records", "error", err)
http.Error(w, "could not load the unsubscribe list", http.StatusInternalServerError)
return
}
candidates, err := s.store.UnsubscribeCandidates(ctx, 300)
if err != nil {
s.logger.Error("list unsubscribe candidates", "error", err)
http.Error(w, "could not load messages", http.StatusInternalServerError)
return
}
status := make(map[string]string, len(records))
for _, rec := range records {
status[rec.Domain] = rec.Status
}
var items []unsubscribeItem
for _, c := range candidates {
res := unsubscribe.Detect(c.ListUnsubscribe, c.ListUnsubscribePost, c.FromAddr, c.BodyText)
if len(res.Links) == 0 {
continue
}
domain := res.Links[0].Domain
items = append(items, unsubscribeItem{
MessageID: c.MessageID,
From: orDefault(c.FromAddr, "(unknown sender)"),
Subject: orDefault(c.Subject, "(no subject)"),
SourceAddr: c.SourceAddress,
Domain: domain,
Links: res.Links,
Status: status[domain],
Category: c.Category,
When: formatWhen(c.ReceivedAt, c.ReceivedAt),
})
}
data := unsubscribeData{
Items: items,
CSRFToken: s.csrfToken(w, r),
Note: note,
Error: errMsg,
CandidateCount: len(items),
}
for _, rec := range records {
switch rec.Status {
case store.UnsubscribeStatusIgnored:
data.Ignored = append(data.Ignored, rec)
case store.UnsubscribeStatusDone:
data.Recorded = append(data.Recorded, rec)
}
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(code)
if err := s.tmpl.ExecuteTemplate(w, "unsubscribe.html", data); err != nil {
s.logger.Error("render unsubscribe", "error", err)
}
}
// handleUnsubscribeIgnore adds a domain to the ignore list (FR-7).
func (s *Server) handleUnsubscribeIgnore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusIgnored, r.FormValue("url")); err != nil {
s.logger.Error("ignore unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not update the ignore list.")
return
}
s.logAction(r.Context(), "unsubscribe_ignore", "Sam", "domain="+domain)
s.redirectUnsubscribe(w, r, "Added "+domain+" to the ignore list. It is never recommended for unsubscribe.", "")
}
// handleUnsubscribeUnignore removes a domain from the ignore list.
func (s *Server) handleUnsubscribeUnignore(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.ClearUnsubscribe(r.Context(), domain); err != nil {
s.logger.Error("clear unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not update the list.")
return
}
s.logAction(r.Context(), "unsubscribe_unignore", "Sam", "domain="+domain)
s.redirectUnsubscribe(w, r, "Removed "+domain+" from the unsubscribe list.", "")
}
// handleUnsubscribeMark records that a domain should be unsubscribed from. It
// is a record only: the handler makes no network call and the app never follows
// the link (AUD-72). Following a real link needs Sam's explicit approval.
func (s *Server) handleUnsubscribeMark(w http.ResponseWriter, r *http.Request) {
if !s.csrfOK(r) {
http.Error(w, "invalid CSRF token", http.StatusForbidden)
return
}
if err := r.ParseForm(); err != nil {
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
return
}
if r.FormValue("confirm") != "yes" {
s.redirectUnsubscribe(w, r, "", "Marking needs confirmation. The app will record it, not follow the link.")
return
}
domain, ok := cleanDomain(r.FormValue("domain"))
if !ok {
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
return
}
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusDone, r.FormValue("url")); err != nil {
s.logger.Error("mark unsubscribe domain", "domain", domain, "error", err)
s.redirectUnsubscribe(w, r, "", "Could not record the unsubscribe.")
return
}
s.logAction(r.Context(), "unsubscribe_marked", "Sam", "domain="+domain+" url="+r.FormValue("url"))
s.redirectUnsubscribe(w, r,
"Recorded "+domain+" for unsubscribe. The app did not contact the link; acting on it is a manual step.", "")
}
// redirectUnsubscribe is POST/redirect/GET so a refresh does not resubmit.
func (s *Server) redirectUnsubscribe(w http.ResponseWriter, r *http.Request, note, errMsg string) {
q := url.Values{}
if note != "" {
q.Set("note", note)
}
if errMsg != "" {
q.Set("error", errMsg)
}
http.Redirect(w, r, "/unsubscribe?"+q.Encode(), http.StatusSeeOther)
}
// cleanDomain validates a domain used as the ignore-list key.
func cleanDomain(raw string) (string, bool) {
domain := strings.ToLower(strings.TrimSpace(raw))
domain = strings.Trim(domain, ".")
if domain == "" || len(domain) > 253 {
return "", false
}
for _, r := range domain {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':
default:
return "", false
}
}
if !strings.Contains(domain, ".") {
return "", false
}
return domain, true
}

View File

@@ -0,0 +1,182 @@
package server
import (
"context"
"net/http"
"net/url"
"strings"
"testing"
"time"
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
)
func seedUnsubMessage(t *testing.T, srv *Server) int64 {
t.Helper()
id, err := srv.store.UpsertMessage(context.Background(), &store.Message{
GmailID: "unsub-1",
FromAddr: "Heritage Fruit Trees <support@heritagefruittrees.com.au>",
Subject: "New workshops released",
BodyText: "Hello, you have received a newsletter. Unsubscribe at https://heritagefruittrees.com.au/u?id=1.",
SourceAddress: "samuelrolfe@gmail.com",
ListUnsubscribe: "<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>",
DateSent: time.Now().UTC(),
})
if err != nil {
t.Fatalf("seed message: %v", err)
}
return id
}
func TestUnsubscribePageDetectsLinks(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/unsubscribe", session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
for _, want := range []string{
"Detected links", "heritagefruittrees.com.au",
"https://heritagefruittrees.com.au/u?id=1",
"mailto:unsub@heritagefruittrees.com.au",
"Ignore domain", "Mark for unsubscribe", "Ignore list",
} {
if !strings.Contains(body, want) {
t.Errorf("unsubscribe page missing %q", want)
}
}
}
func TestUnsubscribeIgnoreAndUnignore(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("ignore status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
}
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec2.Status != store.UnsubscribeStatusIgnored {
t.Errorf("status = %q, want ignored", rec2.Status)
}
body := getWithSession(t, srv, "/unsubscribe", session).Body.String()
if !strings.Contains(body, "ignored") || !strings.Contains(body, "Un-ignore") {
t.Errorf("ignore list did not render the ignored domain")
}
rec = postForm(t, srv, "/unsubscribe/unignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("unignore status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
t.Error("domain still on the list after unignore")
}
}
func TestUnsubscribeMarkRecordsWithoutFollowingLink(t *testing.T) {
srv := newTestServer(t)
seedUnsubMessage(t, srv)
session, csrfCookie, csrf := login(t, srv)
ctx := context.Background()
// Without confirmation the handler refuses and records nothing.
rec := postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
"url": {"https://heritagefruittrees.com.au/u?id=1"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("mark without confirm status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
t.Error("unconfirmed mark recorded a domain")
}
rec = postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
"url": {"https://heritagefruittrees.com.au/u?id=1"}, "confirm": {"yes"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("mark status = %d, want 303", rec.Code)
}
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec2.Status != store.UnsubscribeStatusDone {
t.Errorf("status = %q, want done", rec2.Status)
}
if rec2.URL != "https://heritagefruittrees.com.au/u?id=1" {
t.Errorf("url = %q, want the recorded link", rec2.URL)
}
}
func TestUnsubscribeMutationsRequireCSRF(t *testing.T) {
srv := newTestServer(t)
session, _, _ := login(t, srv)
for _, path := range []string{"/unsubscribe/ignore", "/unsubscribe/unignore", "/unsubscribe/mark"} {
rec := postForm(t, srv, path, session, nil, url.Values{"domain": {"x.example"}})
if rec.Code != http.StatusForbidden {
t.Errorf("%s without CSRF status = %d, want 403", path, rec.Code)
}
}
}
func TestUnsubscribeMarkRejectsBadDomain(t *testing.T) {
srv := newTestServer(t)
session, csrfCookie, csrf := login(t, srv)
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
"csrf": {csrf}, "domain": {"not a domain!"},
})
if rec.Code != http.StatusSeeOther {
t.Fatalf("status = %d, want 303", rec.Code)
}
if _, ok, _ := srv.store.UnsubscribeByDomain(context.Background(), "not a domain!"); ok {
t.Error("bad domain was stored")
}
}
func TestMessageDetailShowsUnsubscribeLinks(t *testing.T) {
srv := newTestServer(t)
id := seedUnsubMessage(t, srv)
session, _, _ := login(t, srv)
rec := getWithSession(t, srv, "/partials/message?id="+itoa(id), session)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
body := rec.Body.String()
if !strings.Contains(body, "Unsubscribe") || !strings.Contains(body, "heritagefruittrees.com.au/u?id=1") {
t.Errorf("message detail did not show the unsubscribe link")
}
}
func TestCleanDomain(t *testing.T) {
cases := map[string]bool{
"brand.example": true,
"Sub.Brand.example": true,
"x-y.example": true,
"": false,
"nodot": false,
"has space.example": false,
"bad!char.example": false,
}
for in, want := range cases {
if _, ok := cleanDomain(in); ok != want {
t.Errorf("cleanDomain(%q) ok = %v, want %v", in, ok, want)
}
}
}

View File

@@ -159,3 +159,51 @@ func (s *Store) SaveRules(ctx context.Context, text string) (int, error) {
} }
return next, nil return next, nil
} }
// RuleVersion is one saved rules revision, for the version list in the editor
// (FR-11).
type RuleVersion struct {
ID int64
Version int
Text string
CreatedAt string
}
// RulesVersions returns saved rules revisions, newest first.
func (s *Store) RulesVersions(ctx context.Context, limit int) ([]RuleVersion, error) {
if limit < 1 {
limit = 20
}
rows, err := s.db.QueryContext(ctx,
`SELECT id, version, text, created_at FROM rules ORDER BY version DESC LIMIT ?`, limit)
if err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
defer rows.Close()
var out []RuleVersion
for rows.Next() {
var v RuleVersion
if err := rows.Scan(&v.ID, &v.Version, &v.Text, &v.CreatedAt); err != nil {
return nil, fmt.Errorf("scan rules version: %w", err)
}
out = append(out, v)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list rules versions: %w", err)
}
return out, nil
}
// RuleByVersion returns the text of one saved rules revision. ok is false when
// the version does not exist.
func (s *Store) RuleByVersion(ctx context.Context, version int) (string, bool, error) {
var text string
err := s.db.QueryRowContext(ctx, `SELECT text FROM rules WHERE version = ?`, version).Scan(&text)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, fmt.Errorf("read rules version %d: %w", version, err)
}
return text, true, nil
}

View File

@@ -20,15 +20,19 @@ type Message struct {
// MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply // MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply
// uses it for In-Reply-To and References so Gmail keeps the thread. // uses it for In-Reply-To and References so Gmail keeps the thread.
MessageIDHeader string MessageIDHeader string
FromAddr string // ListUnsubscribe and ListUnsubscribePost are the raw RFC 2369/8058 headers
ToAddr string // (Phase 6, FR-7). Detection runs over them on demand.
Subject string ListUnsubscribe string
BodyText string ListUnsubscribePost string
SourceAddress string FromAddr string
DateSent time.Time ToAddr string
ReceivedAt time.Time Subject string
IMAPUID int64 BodyText string
IMAPUIDValidity int64 SourceAddress string
DateSent time.Time
ReceivedAt time.Time
IMAPUID int64
IMAPUIDValidity int64
// Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9): // Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9):
// inbox, delete, saved, or replied. // inbox, delete, saved, or replied.
Folder string Folder string
@@ -71,19 +75,22 @@ func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) {
const q = ` const q = `
INSERT INTO messages ( INSERT INTO messages (
gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text, gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text,
source_address, date_sent, received_at, imap_uid, imap_uidvalidity source_address, list_unsubscribe, list_unsubscribe_post, date_sent, received_at,
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) imap_uid, imap_uidvalidity
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (gmail_id) DO UPDATE SET ON CONFLICT (gmail_id) DO UPDATE SET
thread_id = excluded.thread_id, thread_id = excluded.thread_id,
message_id_header = excluded.message_id_header, message_id_header = excluded.message_id_header,
from_addr = excluded.from_addr, from_addr = excluded.from_addr,
to_addr = excluded.to_addr, to_addr = excluded.to_addr,
subject = excluded.subject, subject = excluded.subject,
body_text = excluded.body_text, body_text = excluded.body_text,
source_address = excluded.source_address, source_address = excluded.source_address,
date_sent = excluded.date_sent, list_unsubscribe = excluded.list_unsubscribe,
imap_uid = excluded.imap_uid, list_unsubscribe_post = excluded.list_unsubscribe_post,
imap_uidvalidity = excluded.imap_uidvalidity` date_sent = excluded.date_sent,
imap_uid = excluded.imap_uid,
imap_uidvalidity = excluded.imap_uidvalidity`
receivedAt := m.ReceivedAt receivedAt := m.ReceivedAt
if receivedAt.IsZero() { if receivedAt.IsZero() {
@@ -98,6 +105,8 @@ ON CONFLICT (gmail_id) DO UPDATE SET
nullString(m.Subject), nullString(m.Subject),
nullString(m.BodyText), nullString(m.BodyText),
nullString(m.SourceAddress), nullString(m.SourceAddress),
nullString(m.ListUnsubscribe),
nullString(m.ListUnsubscribePost),
nullTime(m.DateSent), nullTime(m.DateSent),
receivedAt.UTC().Format(sqlTimeLayout), receivedAt.UTC().Format(sqlTimeLayout),
m.IMAPUID, m.IMAPUID,
@@ -124,7 +133,8 @@ SELECT m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''),
COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at, COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at,
COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0), COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0),
COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'), COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'),
COALESCE(m.message_id_header, '') COALESCE(m.message_id_header, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, '')
FROM messages m WHERE m.id = ?` FROM messages m WHERE m.id = ?`
var ( var (
m Message m Message
@@ -135,6 +145,7 @@ FROM messages m WHERE m.id = ?`
&m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject, &m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject,
&m.BodyText, &m.SourceAddress, &dateSent, &received, &m.BodyText, &m.SourceAddress, &dateSent, &received,
&m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader, &m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader,
&m.ListUnsubscribe, &m.ListUnsubscribePost,
) )
if err != nil { if err != nil {
return Message{}, fmt.Errorf("load message %d: %w", id, err) return Message{}, fmt.Errorf("load message %d: %w", id, err)

View File

@@ -21,6 +21,10 @@ CREATE TABLE IF NOT EXISTS messages (
-- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and -- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and
-- References to it so Gmail files the reply in the same thread. -- References to it so Gmail files the reply in the same thread.
message_id_header TEXT, message_id_header TEXT,
-- Phase 6 (FR-7): the raw RFC 2369/8058 unsubscribe headers, stored so the
-- detector can run without a refetch.
list_unsubscribe TEXT,
list_unsubscribe_post TEXT,
from_addr TEXT, from_addr TEXT,
to_addr TEXT, to_addr TEXT,
subject TEXT, subject TEXT,
@@ -94,11 +98,16 @@ CREATE TABLE IF NOT EXISTS actions_log (
CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id); CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id);
CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at); CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at);
-- FR-7: unsubscribe tracker and ignore list. -- FR-7: unsubscribe tracker and ignore list. `status` is 'ignored' for a domain
-- on the ignore list, or 'done' for a domain Sam has recorded as unsubscribed.
-- The app never follows a link itself: it only marks and records (AUD-72).
CREATE TABLE IF NOT EXISTS unsubscribe ( CREATE TABLE IF NOT EXISTS unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE, domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')), status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
-- url records the link that was detected or marked (Phase 6).
url TEXT NOT NULL DEFAULT '',
updated_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')) created_at TEXT NOT NULL DEFAULT (datetime('now'))
); );

View File

@@ -159,6 +159,25 @@ func (s *Store) applyMigrations(ctx context.Context) error {
return fmt.Errorf("record schema version 5: %w", err) return fmt.Errorf("record schema version 5: %w", err)
} }
} }
if version < 6 {
// Phase 6 (FR-7): store the raw List-Unsubscribe headers so detection
// runs without a refetch, and extend the unsubscribe tracker with the
// link it recorded and an update time.
for _, col := range []struct{ table, name, typ string }{
{"messages", "list_unsubscribe", "TEXT"},
{"messages", "list_unsubscribe_post", "TEXT"},
{"unsubscribe", "url", "TEXT NOT NULL DEFAULT ''"},
{"unsubscribe", "updated_at", "TEXT"},
} {
if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil {
return err
}
}
if _, err := s.db.ExecContext(ctx,
`INSERT OR IGNORE INTO schema_migrations (version) VALUES (6)`); err != nil {
return fmt.Errorf("record schema version 6: %w", err)
}
}
return nil return nil
} }

View File

@@ -0,0 +1,177 @@
// Unsubscribe tracking for Phase 6 (FR-7).
//
// Two things are persisted here:
//
// - the raw unsubscribe links detected on a message (messages.list_unsubscribe,
// stored by the mail fetch and parsed by internal/unsubscribe); and
// - the domain-level ignore list and the record of what was marked, in the
// `unsubscribe` table.
//
// The app never follows a link. Marking a domain records Sam's decision; the
// actual unsubscribe stays a manual, explicitly approved action (AUD-72).
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
)
// Unsubscribe statuses. `ignored` is the ignore list; `done` records a domain
// Sam has decided to unsubscribe from.
const (
UnsubscribeStatusDone = "done"
UnsubscribeStatusIgnored = "ignored"
)
// ErrUnsubscribeStatus is returned for an unknown unsubscribe status.
var ErrUnsubscribeStatus = errors.New("unsubscribe status must be done or ignored")
// UnsubscribeRecord is one row of the domain-level tracker.
type UnsubscribeRecord struct {
ID int64
Domain string
Status string
URL string
CreatedAt string
UpdatedAt string
}
// ListUnsubscribe returns every domain record, newest first.
func (s *Store) ListUnsubscribe(ctx context.Context) ([]UnsubscribeRecord, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe ORDER BY COALESCE(updated_at, created_at) DESC, id DESC`)
if err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
defer rows.Close()
var out []UnsubscribeRecord
for rows.Next() {
var r UnsubscribeRecord
if err := rows.Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan unsubscribe record: %w", err)
}
out = append(out, r)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe records: %w", err)
}
return out, nil
}
// UnsubscribeByDomain loads one domain record. ok is false when the domain is
// not tracked.
func (s *Store) UnsubscribeByDomain(ctx context.Context, domain string) (UnsubscribeRecord, bool, error) {
domain = strings.TrimSpace(domain)
if domain == "" {
return UnsubscribeRecord{}, false, nil
}
var r UnsubscribeRecord
err := s.db.QueryRowContext(ctx, `
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
FROM unsubscribe WHERE domain = ?`, domain).
Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt)
if errors.Is(err, sql.ErrNoRows) {
return UnsubscribeRecord{}, false, nil
}
if err != nil {
return UnsubscribeRecord{}, false, fmt.Errorf("read unsubscribe record %q: %w", domain, err)
}
return r, true, nil
}
// SetUnsubscribeStatus upserts the domain record. An empty url keeps the stored
// one, so an "ignore" does not wipe a link recorded earlier.
func (s *Store) SetUnsubscribeStatus(ctx context.Context, domain, status, url string) error {
domain = strings.TrimSpace(domain)
if domain == "" {
return errors.New("unsubscribe domain must not be empty")
}
if status != UnsubscribeStatusDone && status != UnsubscribeStatusIgnored {
return ErrUnsubscribeStatus
}
const q = `
INSERT INTO unsubscribe (domain, status, url, updated_at)
VALUES (?, ?, ?, datetime('now'))
ON CONFLICT (domain) DO UPDATE SET
status = excluded.status,
url = CASE WHEN excluded.url <> '' THEN excluded.url ELSE unsubscribe.url END,
updated_at = datetime('now')`
if _, err := s.db.ExecContext(ctx, q, domain, status, strings.TrimSpace(url)); err != nil {
return fmt.Errorf("set unsubscribe status for %q: %w", domain, err)
}
return nil
}
// ClearUnsubscribe removes a domain record, taking it off the ignore list.
func (s *Store) ClearUnsubscribe(ctx context.Context, domain string) error {
if _, err := s.db.ExecContext(ctx,
`DELETE FROM unsubscribe WHERE domain = ?`, strings.TrimSpace(domain)); err != nil {
return fmt.Errorf("clear unsubscribe record %q: %w", domain, err)
}
return nil
}
// UnsubscribeCandidate is a message that may carry an unsubscribe link.
type UnsubscribeCandidate struct {
MessageID int64
FromAddr string
Subject string
SourceAddress string
BodyText string
ListUnsubscribe string
ListUnsubscribePost string
Category string
ReceivedAt time.Time
}
// UnsubscribeCandidates returns messages with a List-Unsubscribe header or a
// body that mentions unsubscribing, newest first. The caller runs the detector
// over them, because body parsing lives in internal/unsubscribe.
func (s *Store) UnsubscribeCandidates(ctx context.Context, limit int) ([]UnsubscribeCandidate, error) {
if limit < 1 {
limit = 200
}
const q = `
SELECT m.id, COALESCE(m.from_addr, ''), COALESCE(m.subject, ''),
COALESCE(m.source_address, ''), COALESCE(m.body_text, ''),
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, ''),
COALESCE(c.category, ''), m.received_at
FROM messages m
LEFT JOIN classifications c ON c.id = (
SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1
)
WHERE (m.list_unsubscribe IS NOT NULL AND m.list_unsubscribe <> '')
OR instr(lower(COALESCE(m.body_text, '')), 'unsubscribe') > 0
ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC
LIMIT ?`
rows, err := s.db.QueryContext(ctx, q, limit)
if err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
defer rows.Close()
var out []UnsubscribeCandidate
for rows.Next() {
var (
c UnsubscribeCandidate
received string
)
if err := rows.Scan(
&c.MessageID, &c.FromAddr, &c.Subject, &c.SourceAddress,
&c.BodyText, &c.ListUnsubscribe, &c.ListUnsubscribePost,
&c.Category, &received,
); err != nil {
return nil, fmt.Errorf("scan unsubscribe candidate: %w", err)
}
c.ReceivedAt = parseSQLTime(received)
out = append(out, c)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
}
return out, nil
}

View File

@@ -0,0 +1,193 @@
package store
import (
"context"
"database/sql"
"path/filepath"
"testing"
)
func TestMigrationV6AddsUnsubscribeColumns(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "old.db")
// Simulate a v5 database: messages has no List-Unsubscribe columns and the
// unsubscribe table has no url/updated_at. Opening it must apply migration
// v6.
db, err := sql.Open("sqlite", "file:"+path)
if err != nil {
t.Fatalf("open old db: %v", err)
}
stmts := []string{
`CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
gmail_id TEXT NOT NULL UNIQUE,
thread_id TEXT, message_id_header TEXT, from_addr TEXT, to_addr TEXT,
subject TEXT, body_text TEXT, source_address TEXT, date_sent TEXT,
imap_uid INTEGER, imap_uidvalidity INTEGER,
folder TEXT NOT NULL DEFAULT 'inbox',
mailbox TEXT NOT NULL DEFAULT 'INBOX',
received_at TEXT NOT NULL DEFAULT (datetime('now')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE unsubscribe (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain TEXT NOT NULL UNIQUE,
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`,
`INSERT INTO schema_migrations (version) VALUES (5)`,
}
for _, q := range stmts {
if _, err := db.ExecContext(ctx, q); err != nil {
t.Fatalf("seed old db: %v", err)
}
}
_ = db.Close()
s, err := Open(ctx, path)
if err != nil {
t.Fatalf("Open() on v5 database error = %v", err)
}
defer s.Close()
v, err := s.SchemaVersion(ctx)
if err != nil {
t.Fatalf("SchemaVersion() error = %v", err)
}
if v < 6 {
t.Errorf("SchemaVersion() = %d, want >= 6", v)
}
for _, tc := range []struct{ table, col string }{
{"messages", "list_unsubscribe"},
{"messages", "list_unsubscribe_post"},
{"unsubscribe", "url"},
{"unsubscribe", "updated_at"},
} {
var n int
q := `SELECT COUNT(*) FROM pragma_table_info('` + tc.table + `') WHERE name = ?`
if err := s.DB().QueryRowContext(ctx, q, tc.col).Scan(&n); err != nil {
t.Fatalf("count %s.%s: %v", tc.table, tc.col, err)
}
if n != 1 {
t.Errorf("%s.%s missing after migration v6", tc.table, tc.col)
}
}
}
func TestUnsubscribeRecordUpsert(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t) // reuses the store helper from categories_test.go
if _, ok, err := s.UnsubscribeByDomain(ctx, "brand.example"); err != nil || ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a miss", ok, err)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusIgnored, "https://brand.example/u"); err != nil {
t.Fatalf("SetUnsubscribeStatus(ignore) error = %v", err)
}
rec, ok, err := s.UnsubscribeByDomain(ctx, "brand.example")
if err != nil || !ok {
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
}
if rec.Status != UnsubscribeStatusIgnored || rec.URL != "https://brand.example/u" {
t.Errorf("record = %+v, want ignored with url", rec)
}
// A later mark with no URL must keep the recorded link.
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusDone, ""); err != nil {
t.Fatalf("SetUnsubscribeStatus(done) error = %v", err)
}
rec, _, _ = s.UnsubscribeByDomain(ctx, "brand.example")
if rec.Status != UnsubscribeStatusDone {
t.Errorf("status = %q, want done", rec.Status)
}
if rec.URL != "https://brand.example/u" {
t.Errorf("url = %q, want the earlier link preserved", rec.URL)
}
if err := s.SetUnsubscribeStatus(ctx, "brand.example", "bogus", ""); err == nil {
t.Error("SetUnsubscribeStatus(bogus) accepted an unknown status")
}
records, err := s.ListUnsubscribe(ctx)
if err != nil || len(records) != 1 {
t.Fatalf("ListUnsubscribe() = (%v, %v), want one record", records, err)
}
if err := s.ClearUnsubscribe(ctx, "brand.example"); err != nil {
t.Fatalf("ClearUnsubscribe() error = %v", err)
}
if err := s.SetUnsubscribeStatus(ctx, "", UnsubscribeStatusIgnored, ""); err == nil {
t.Error("SetUnsubscribeStatus accepted an empty domain")
}
}
func TestUnsubscribeCandidates(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
seed := func(gmailID, from, body, header string) {
t.Helper()
if _, err := s.UpsertMessage(ctx, &Message{
GmailID: gmailID,
FromAddr: from,
Subject: "subject " + gmailID,
BodyText: body,
ListUnsubscribe: header,
}); err != nil {
t.Fatalf("seed %s: %v", gmailID, err)
}
}
seed("header-msg", "Support <support@brand.example>", "hello", "<https://brand.example/u>")
seed("body-msg", "Shop <deals@shop.example>", "You can unsubscribe at https://shop.example/opt-out now.", "")
seed("plain-msg", "Friend <friend@example.com>", "Just saying hi.", "")
got, err := s.UnsubscribeCandidates(ctx, 50)
if err != nil {
t.Fatalf("UnsubscribeCandidates() error = %v", err)
}
if len(got) != 2 {
t.Fatalf("candidates = %d, want 2 (%+v)", len(got), got)
}
ids := map[string]bool{}
for _, c := range got {
ids[c.Subject] = true
}
if !ids["subject body-msg"] || !ids["subject header-msg"] {
t.Errorf("candidates = %+v, want the header and body messages", ids)
}
}
func TestRulesVersionsAndRestore(t *testing.T) {
ctx := context.Background()
s := newCategoryStore(t)
if _, err := s.SaveRules(ctx, "from contains \"a\" => category=Info"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
if _, err := s.SaveRules(ctx, "from contains \"b\" => category=Action"); err != nil {
t.Fatalf("SaveRules() error = %v", err)
}
versions, err := s.RulesVersions(ctx, 10)
if err != nil {
t.Fatalf("RulesVersions() error = %v", err)
}
if len(versions) < 3 {
t.Fatalf("RulesVersions() = %d, want >= 3 (seed + 2 saves)", len(versions))
}
if versions[0].Version != 3 {
t.Errorf("newest version = %d, want 3", versions[0].Version)
}
text, ok, err := s.RuleByVersion(ctx, 2)
if err != nil || !ok {
t.Fatalf("RuleByVersion(2) = (_, %v, %v), want a hit", ok, err)
}
if text != "from contains \"a\" => category=Info" {
t.Errorf("RuleByVersion(2) = %q", text)
}
if _, ok, _ := s.RuleByVersion(ctx, 999); ok {
t.Error("RuleByVersion(999) reported a hit")
}
}

View File

@@ -0,0 +1,240 @@
// Package unsubscribe detects unsubscribe links (FR-7).
//
// The primary source is the List-Unsubscribe header (RFC 2369) together with
// the List-Unsubscribe-Post header (RFC 8058). When neither is present the body
// is scanned for a URL on a line that mentions "unsubscribe".
//
// Detection only. This package never fetches a URL and never sends mail.
// Acting on an unsubscribe link needs Sam's explicit approval (AGENTS.md §10
// and AUD-72); the app only marks and records.
package unsubscribe
import (
"net/url"
"regexp"
"strings"
)
// Link methods.
const (
MethodHTTP = "http"
MethodMailto = "mailto"
)
// Detection sources. They are stored in the action log so a marking is
// auditable.
const (
SourceListUnsubscribe = "list-unsubscribe"
SourceBody = "body"
)
// Link is one detected unsubscribe target.
type Link struct {
// URL is the raw target, for example "https://x.example/u?id=1".
URL string
// Method is MethodHTTP or MethodMailto.
Method string
// Domain is the ignore-list key. It is the sender's brand domain when that
// is a real domain, otherwise the link host.
Domain string
// Host is the host of the link itself, shown for context.
Host string
// Source is SourceListUnsubscribe or SourceBody.
Source string
}
// Result is the detection outcome for one message.
type Result struct {
Links []Link
// Header is the raw List-Unsubscribe value, if any.
Header string
// OneClick reports a RFC 8058 List-Unsubscribe=One-Click header.
OneClick bool
}
// Detect returns the unsubscribe links for a message. from is the raw From
// header; body is the stored plain-text body.
func Detect(listUnsubscribe, listUnsubscribePost, from, body string) Result {
res := Result{
Header: strings.TrimSpace(listUnsubscribe),
OneClick: isOneClick(listUnsubscribePost),
}
sender := brandDomain(from)
seen := map[string]struct{}{}
add := func(raw, source string) {
link, ok := makeLink(raw, source, sender)
if !ok {
return
}
key := link.Method + ":" + strings.ToLower(link.URL)
if _, dup := seen[key]; dup {
return
}
seen[key] = struct{}{}
res.Links = append(res.Links, link)
}
for _, raw := range headerLinks(res.Header) {
add(raw, SourceListUnsubscribe)
}
if len(res.Links) == 0 {
for _, raw := range bodyLinks(body) {
add(raw, SourceBody)
}
}
return res
}
// isOneClick reports a RFC 8058 one-click unsubscribe header. The documented
// value is "List-Unsubscribe=One-Click".
func isOneClick(v string) bool {
return strings.Contains(strings.ToLower(v), "one-click")
}
var angleRE = regexp.MustCompile(`<([^>]+)>`)
// headerLinks extracts the URIs from a List-Unsubscribe value. The RFC form is
// a comma-separated list of angle-bracketed URIs; a bare URI is also accepted.
func headerLinks(v string) []string {
v = strings.TrimSpace(v)
if v == "" {
return nil
}
var out []string
for _, m := range angleRE.FindAllStringSubmatch(v, -1) {
if s := strings.TrimSpace(m[1]); s != "" {
out = append(out, s)
}
}
if len(out) > 0 {
return out
}
for _, part := range strings.Split(v, ",") {
if s := strings.TrimSpace(part); s != "" {
out = append(out, s)
}
}
return out
}
var (
bodyURLRE = regexp.MustCompile(`(?i)(https?://[^\s<>"')\]]+|mailto:[^\s<>"')\]]+)`)
unsubRE = regexp.MustCompile(`(?i)unsubscribe|opt[\s-]?out|manage (?:your )?(?:email )?preferences`)
)
// bodyLinks finds unsubscribe URLs in a plain-text body. Only lines that
// mention unsubscribing are considered, so a random link is not reported.
func bodyLinks(body string) []string {
body = strings.TrimSpace(body)
if body == "" {
return nil
}
var out []string
for _, line := range strings.Split(body, "\n") {
if !unsubRE.MatchString(line) {
continue
}
for _, m := range bodyURLRE.FindAllString(line, -1) {
out = append(out, trimURL(m))
}
}
return out
}
// trimURL removes trailing punctuation that is part of the sentence, not the
// URL.
func trimURL(s string) string {
return strings.TrimRight(strings.TrimSpace(s), ".,;:!?)\"'")
}
// makeLink validates a raw target and fills in method/host/domain.
func makeLink(raw, source, sender string) (Link, bool) {
raw = trimURL(raw)
raw = strings.Trim(raw, "<>")
if raw == "" {
return Link{}, false
}
link := Link{URL: raw, Source: source}
switch {
case strings.HasPrefix(strings.ToLower(raw), "mailto:"):
link.Method = MethodMailto
link.Host = mailtoHost(raw)
case strings.HasPrefix(strings.ToLower(raw), "http://"),
strings.HasPrefix(strings.ToLower(raw), "https://"):
link.Method = MethodHTTP
link.Host = urlHost(raw)
default:
return Link{}, false
}
link.Domain = sender
if link.Domain == "" {
link.Domain = link.Host
}
return link, link.Domain != ""
}
// Domain returns the domain used as the ignore-list key. It is exported for
// callers that need it without parsing a link.
func Domain(from string) string { return brandDomain(from) }
func brandDomain(from string) string {
addr := extractAddress(from)
i := strings.LastIndex(addr, "@")
if i < 0 || i == len(addr)-1 {
return ""
}
dom := strings.ToLower(strings.Trim(strings.TrimSpace(addr[i+1:]), ">."))
if dom == "" || freeMail[dom] {
return ""
}
return dom
}
// extractAddress pulls the address out of a From header such as
// "Support <support@example.com>".
func extractAddress(v string) string {
v = strings.TrimSpace(v)
if i := strings.LastIndex(v, "<"); i >= 0 {
if j := strings.Index(v[i:], ">"); j > 0 {
return strings.TrimSpace(v[i+1 : i+j])
}
}
return strings.Trim(v, "<> \t")
}
// freeMail domains are consumer mailboxes, not brands. A message forwarded
// through one should key its ignore list on the real link host instead.
var freeMail = map[string]bool{
"gmail.com": true,
"googlemail.com": true,
"outlook.com": true,
"hotmail.com": true,
"live.com": true,
"yahoo.com": true,
"icloud.com": true,
"me.com": true,
"aol.com": true,
"proton.me": true,
"protonmail.com": true,
}
func urlHost(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return ""
}
return strings.ToLower(u.Hostname())
}
func mailtoHost(raw string) string {
rest := raw[len("mailto:"):]
if i := strings.IndexAny(rest, "?,"); i >= 0 {
rest = rest[:i]
}
i := strings.LastIndex(rest, "@")
if i < 0 || i == len(rest)-1 {
return ""
}
return strings.ToLower(strings.Trim(rest[i+1:], ">."))
}

View File

@@ -0,0 +1,126 @@
package unsubscribe
import "testing"
func TestDetectListUnsubscribeHeader(t *testing.T) {
res := Detect(
"<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=42>",
"List-Unsubscribe=One-Click",
"Heritage Fruit Trees <support@heritagefruittrees.com.au>",
"Special offer this week.",
)
if !res.OneClick {
t.Error("OneClick = false, want true")
}
if len(res.Links) != 2 {
t.Fatalf("Links len = %d, want 2 (%+v)", len(res.Links), res.Links)
}
mail := res.Links[0]
if mail.Method != MethodMailto {
t.Errorf("mailto method = %q", mail.Method)
}
if mail.Domain != "heritagefruittrees.com.au" {
t.Errorf("mailto domain = %q", mail.Domain)
}
if mail.Source != SourceListUnsubscribe {
t.Errorf("mailto source = %q", mail.Source)
}
http := res.Links[1]
if http.Method != MethodHTTP {
t.Errorf("http method = %q", http.Method)
}
if http.Host != "heritagefruittrees.com.au" {
t.Errorf("http host = %q", http.Host)
}
if http.Domain != "heritagefruittrees.com.au" {
t.Errorf("http domain = %q", http.Domain)
}
}
func TestDetectBareHeaderURL(t *testing.T) {
res := Detect("https://lists.example.com/unsub?u=abc,", "", "News <news@lists.example.com>", "")
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].URL != "https://lists.example.com/unsub?u=abc" {
t.Errorf("URL = %q", res.Links[0].URL)
}
}
func TestDetectBodyFallback(t *testing.T) {
res := Detect("", "", "Shop <deals@shop.example>", "Thanks for subscribing!\nUnsubscribe here: https://shop.example/opt-out?token=1.\nSee you.")
if res.Header != "" {
t.Errorf("Header = %q, want empty", res.Header)
}
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].Source != SourceBody {
t.Errorf("Source = %q, want %q", res.Links[0].Source, SourceBody)
}
if res.Links[0].URL != "https://shop.example/opt-out?token=1" {
t.Errorf("URL = %q (trailing period should be trimmed)", res.Links[0].URL)
}
}
func TestDetectIgnoresUnrelatedBodyLinks(t *testing.T) {
res := Detect("", "", "Friend <friend@example.com>", "Read more at https://example.com/blog")
if len(res.Links) != 0 {
t.Fatalf("Links = %+v, want none", res.Links)
}
}
func TestDetectBodyMailto(t *testing.T) {
res := Detect("", "", "Brand <hello@brand.example>", "To unsubscribe, email mailto:leave@brand.example")
if len(res.Links) != 1 || res.Links[0].Method != MethodMailto {
t.Fatalf("Links = %+v, want one mailto", res.Links)
}
}
func TestFreeMailSenderUsesLinkHost(t *testing.T) {
res := Detect(
"<https://mailer.example.com/u/9>",
"",
"Someone <someone@gmail.com>",
"",
)
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1", res.Links)
}
if res.Links[0].Domain != "mailer.example.com" {
t.Errorf("domain = %q, want link host", res.Links[0].Domain)
}
}
func TestDetectDeduplicates(t *testing.T) {
res := Detect(
"<https://x.example/u>, <https://x.example/u>",
"",
"Brand <hello@brand.example>",
"",
)
if len(res.Links) != 1 {
t.Fatalf("Links = %+v, want 1 after de-duplication", res.Links)
}
}
func TestDomain(t *testing.T) {
cases := map[string]string{
"Support <support@heritagefruittrees.com.au>": "heritagefruittrees.com.au",
"plain@brand.example": "brand.example",
"Someone <someone@gmail.com>": "",
"": "",
}
for in, want := range cases {
if got := Domain(in); got != want {
t.Errorf("Domain(%q) = %q, want %q", in, got, want)
}
}
}
func TestEmptyInputs(t *testing.T) {
res := Detect("", "", "", "")
if len(res.Links) != 0 || res.OneClick {
t.Fatalf("Detect(empty) = %+v, want zero value", res)
}
}

View File

@@ -466,3 +466,68 @@ a:hover { text-decoration: underline; }
@media (max-width: 759px) { @media (max-width: 759px) {
.reply-actions .btn { flex: 1 1 auto; } .reply-actions .btn { flex: 1 1 auto; }
} }
/* Phase 6: rules editor (FR-4, FR-11) and unsubscribe detection (FR-7) */
.rules-panel, .unsub-panel { margin-top: var(--space-md); }
.rules-help, .unsub-help { font: var(--type-body-sm); margin-bottom: var(--space-md); max-width: 78ch; }
.rules-help code { font: var(--type-caption); background: var(--surface-2); padding: 1px 4px; border-radius: 4px; }
.rules-errors { margin: var(--space-xs) 0 0; padding-left: var(--space-md); font: var(--type-body-sm); }
.rules-form { display: grid; gap: var(--space-sm); }
.rules-textarea {
width: 100%;
min-height: 340px;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 13px;
line-height: 1.5;
white-space: pre;
overflow-wrap: normal;
overflow-x: auto;
}
.rules-actions, .rules-reclassify { display: flex; align-items: center; gap: var(--space-sm); flex-wrap: wrap; }
.rules-reclassify { margin-top: var(--space-sm); }
.rules-versions { display: flex; flex-direction: column; }
.rules-version {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--space-sm);
padding: var(--space-sm) 0;
border-bottom: 1px solid var(--hairline);
}
.rules-version:last-child { border-bottom: 0; }
.rules-version-main { display: flex; align-items: center; gap: var(--space-sm); min-width: 0; flex-wrap: wrap; }
.rules-version-preview { font: var(--type-caption); color: var(--ink-secondary); overflow-wrap: anywhere; }
.unsub-list { display: flex; flex-direction: column; }
.unsub-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--space-md);
padding: var(--space-sm) 0;
border-bottom: 1px solid var(--hairline);
}
.unsub-row:last-child { border-bottom: 0; }
.unsub-main { display: grid; gap: 2px; min-width: 0; flex: 1 1 auto; }
.unsub-subject { font: var(--type-body-sm); color: var(--ink); overflow-wrap: anywhere; }
.unsub-links { display: grid; gap: 2px; margin-top: var(--space-xs); }
.unsub-link { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; }
.unsub-source { font: var(--type-caption); color: var(--ink-faint); }
.unsub-url { font: var(--type-caption); color: var(--ink-secondary); overflow-wrap: anywhere; }
.unsub-domain { font: var(--type-caption); color: var(--ink-faint); margin-top: 2px; }
.unsub-actions { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; flex: 0 0 auto; }
.unsub-detected {
border: 1px solid var(--hairline);
border-radius: 4px;
padding: var(--space-sm);
margin-bottom: var(--space-md);
display: grid;
gap: var(--space-xs);
}
.unsub-detected h3 { font: var(--type-eyebrow); text-transform: uppercase; letter-spacing: .125px; color: var(--ink-faint); margin: 0; }
.unsub-link-list { list-style: none; margin: 0; padding: 0; display: grid; gap: var(--space-xs); }
.unsub-link-list li { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; font: var(--type-caption); color: var(--ink-secondary); }
@media (max-width: 759px) {
.unsub-row { flex-direction: column; align-items: stretch; }
.rules-version { flex-direction: column; align-items: stretch; }
}

View File

@@ -26,6 +26,8 @@
<header class="nav-bar"> <header class="nav-bar">
<span class="wordmark">Email Agent</span> <span class="wordmark">Email Agent</span>
<div class="nav-actions"> <div class="nav-actions">
<a class="btn btn-utility" href="/rules">Rules</a>
<a class="btn btn-utility" href="/unsubscribe">Unsubscribe</a>
<button class="btn btn-utility theme-toggle" type="button" onclick="eaToggleTheme()" aria-label="Toggle dark mode"> <button class="btn btn-utility theme-toggle" type="button" onclick="eaToggleTheme()" aria-label="Toggle dark mode">
<span class="icon-light" aria-hidden="true">&#9790;</span> <span class="icon-light" aria-hidden="true">&#9790;</span>
<span class="icon-dark" aria-hidden="true">&#9728;</span> <span class="icon-dark" aria-hidden="true">&#9728;</span>

View File

@@ -13,6 +13,18 @@
</div> </div>
</div> </div>
{{if .UnsubLinks}}
<section class="unsub-detected" aria-label="Unsubscribe">
<h3>Unsubscribe</h3>
{{if eq .UnsubStatus "ignored"}}<span class="pill">ignored</span>{{end}}
{{if eq .UnsubStatus "done"}}<span class="pill">recorded</span>{{end}}
<ul class="unsub-link-list">
{{range .UnsubLinks}}<li><span class="pill">{{.Method}}</span> <span class="unsub-url">{{.URL}}</span></li>{{end}}
</ul>
<a class="btn btn-utility btn-small" href="/unsubscribe">Manage unsubscribe</a>
</section>
{{end}}
<details class="detail-original" open> <details class="detail-original" open>
<summary>Original message</summary> <summary>Original message</summary>
<pre class="detail-text">{{.Message.BodyText}}</pre> <pre class="detail-text">{{.Message.BodyText}}</pre>

125
web/templates/rules.html Normal file
View File

@@ -0,0 +1,125 @@
<!doctype html>
<html lang="en" data-theme="light">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Rules · Email Agent</title>
<link rel="stylesheet" href="/static/tokens.css">
<link rel="stylesheet" href="/static/app.css">
<script>
(function () {
try {
var t = localStorage.getItem("ea-theme");
if (!t) t = matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
document.documentElement.setAttribute("data-theme", t);
} catch (e) {}
})();
function eaToggleTheme() {
var el = document.documentElement;
var next = el.getAttribute("data-theme") === "dark" ? "light" : "dark";
el.setAttribute("data-theme", next);
try { localStorage.setItem("ea-theme", next); } catch (e) {}
}
</script>
</head>
<body>
<header class="nav-bar">
<span class="wordmark">Email Agent</span>
<div class="nav-actions">
<a class="btn btn-utility" href="/">Back to inbox</a>
<a class="btn btn-utility" href="/unsubscribe">Unsubscribe</a>
<button class="btn btn-utility theme-toggle" type="button" onclick="eaToggleTheme()" aria-label="Toggle dark mode">
<span class="icon-light" aria-hidden="true">&#9790;</span>
<span class="icon-dark" aria-hidden="true">&#9728;</span>
</button>
<form method="post" action="/logout">
<input type="hidden" name="csrf" value="{{.CSRFToken}}">
<button class="btn btn-utility" type="submit">Log out</button>
</form>
</div>
</header>
<main class="page">
<h1 class="greeting">Rules</h1>
{{if .Note}}<div class="scan-note" role="status">{{.Note}}</div>{{end}}
{{if .Error}}<div class="alert alert-error" role="alert">{{.Error}}</div>{{end}}
<section class="panel rules-panel" aria-label="Rules editor">
<div class="panel-head">
<h2>Rules editor</h2>
<span class="muted">version {{.Version}} · {{.RuleCount}} active rule(s) · {{.TotalMessages}} message(s)</span>
</div>
<div class="panel-body">
<p class="muted rules-help">
One rule per line: <code>&lt;condition&gt; =&gt; &lt;effects&gt;</code>.
Fields are <code>from</code>, <code>to</code>, <code>subject</code>,
<code>source</code>, and <code>body</code>; operators are
<code>contains</code>, <code>equals</code>, <code>starts_with</code>,
and <code>ends_with</code>. The first matching rule wins and overrides
the JEV fields it names. Lines starting with <code>#</code> are comments.
The classifier and the reply model both read this text.
</p>
{{if .Errors}}
<div class="alert alert-error" role="alert">
<strong>{{len .Errors}} line(s) could not be parsed and are being skipped:</strong>
<ul class="rules-errors">
{{range .Errors}}<li>{{.Error}}</li>{{end}}
</ul>
</div>
{{end}}
<form class="rules-form" method="post" action="/rules/save">
<input type="hidden" name="csrf" value="{{.CSRFToken}}">
<label class="field" for="rules-text">
<span>Rules text (plain text)</span>
</label>
<textarea class="text-input rules-textarea" id="rules-text" name="text" rows="18" spellcheck="false">{{.Text}}</textarea>
<div class="rules-actions">
<button class="btn btn-primary" type="submit">Save new version</button>
<span class="muted faint">Saving appends a version; nothing is re-classified until you re-classify.</span>
</div>
</form>
{{if .ClassifierReady}}
<form class="rules-reclassify" method="post" action="/rules/reclassify"
onsubmit="return confirm('Re-classify every stored message with the current rules?');">
<input type="hidden" name="csrf" value="{{.CSRFToken}}">
<button class="btn btn-utility" type="submit">Re-classify all now</button>
</form>
{{else}}
<p class="faint">Re-classify is unavailable until JEV_API_KEY is set.</p>
{{end}}
</div>
</section>
<section class="panel rules-panel" aria-label="Rules versions">
<div class="panel-head"><h2>Versions</h2></div>
<div class="panel-body">
<div class="rules-versions">
{{range .Revisions}}
<div class="rules-version">
<div class="rules-version-main">
<span class="pill">v{{.Version}}</span>
<span class="muted">{{.CreatedAt}}</span>
<span class="muted">{{.Lines}} rule line(s)</span>
<span class="rules-version-preview">{{.Preview}}</span>
</div>
<form method="post" action="/rules/restore"
onsubmit="return confirm('Restore version {{.Version}} as a new version?');">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="version" value="{{.Version}}">
<button class="btn btn-utility btn-small" type="submit">Restore</button>
</form>
</div>
{{else}}
<p class="muted">No versions stored yet.</p>
{{end}}
</div>
</div>
</section>
</main>
</body>
</html>

View File

@@ -0,0 +1,178 @@
<!doctype html>
<html lang="en" data-theme="light">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Unsubscribe · Email Agent</title>
<link rel="stylesheet" href="/static/tokens.css">
<link rel="stylesheet" href="/static/app.css">
<script>
(function () {
try {
var t = localStorage.getItem("ea-theme");
if (!t) t = matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
document.documentElement.setAttribute("data-theme", t);
} catch (e) {}
})();
function eaToggleTheme() {
var el = document.documentElement;
var next = el.getAttribute("data-theme") === "dark" ? "light" : "dark";
el.setAttribute("data-theme", next);
try { localStorage.setItem("ea-theme", next); } catch (e) {}
}
</script>
</head>
<body>
<header class="nav-bar">
<span class="wordmark">Email Agent</span>
<div class="nav-actions">
<a class="btn btn-utility" href="/">Back to inbox</a>
<a class="btn btn-utility" href="/rules">Rules</a>
<button class="btn btn-utility theme-toggle" type="button" onclick="eaToggleTheme()" aria-label="Toggle dark mode">
<span class="icon-light" aria-hidden="true">&#9790;</span>
<span class="icon-dark" aria-hidden="true">&#9728;</span>
</button>
<form method="post" action="/logout">
<input type="hidden" name="csrf" value="{{.CSRFToken}}">
<button class="btn btn-utility" type="submit">Log out</button>
</form>
</div>
</header>
<main class="page">
<h1 class="greeting">Unsubscribe</h1>
{{if .Note}}<div class="scan-note" role="status">{{.Note}}</div>{{end}}
{{if .Error}}<div class="scan-note scan-note-error" role="alert">{{.Error}}</div>{{end}}
<section class="panel unsub-panel" aria-label="Detected unsubscribe links">
<div class="panel-head">
<h2>Detected links</h2>
<span class="muted">{{.CandidateCount}} message(s) with an unsubscribe link</span>
</div>
<div class="panel-body">
<p class="muted unsub-help">
The app only marks and records. It never opens an unsubscribe link:
acting on a real link is a manual step that needs Sam's approval.
Ignoring a domain keeps it out of the recommendation list.
</p>
<div class="unsub-list">
{{range .Items}}
<div class="unsub-row">
<div class="unsub-main">
<div class="unsub-subject">{{.Subject}}</div>
<div class="msg-from">{{.From}}</div>
{{if .SourceAddr}}<div class="msg-source">arrived at <span class="source-addr">{{.SourceAddr}}</span></div>{{end}}
<div class="unsub-links">
{{range .Links}}
<div class="unsub-link">
<span class="pill">{{.Method}}</span>
<span class="unsub-source">{{.Source}}</span>
<span class="unsub-url">{{.URL}}</span>
</div>
{{end}}
</div>
<div class="unsub-domain">domain: <strong>{{.Domain}}</strong>{{if .Category}} · {{.Category}}{{end}}{{if .When}} · {{.When}}{{end}}</div>
</div>
<div class="unsub-actions">
{{if eq .Status "ignored"}}
<span class="pill">ignored</span>
<form method="post" action="/unsubscribe/unignore">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<button class="btn btn-utility btn-small" type="submit">Un-ignore</button>
</form>
{{else if eq .Status "done"}}
<span class="pill">recorded</span>
<form method="post" action="/unsubscribe/unignore">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<button class="btn btn-utility btn-small" type="submit">Clear record</button>
</form>
{{else}}
<form method="post" action="/unsubscribe/ignore">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<input type="hidden" name="url" value="{{(index .Links 0).URL}}">
<button class="btn btn-utility btn-small" type="submit">Ignore domain</button>
</form>
<form method="post" action="/unsubscribe/mark"
onsubmit="return confirm('Record {{.Domain}} for unsubscribe? The app records it and does NOT contact the link.');">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<input type="hidden" name="url" value="{{(index .Links 0).URL}}">
<input type="hidden" name="confirm" value="yes">
<button class="btn btn-utility btn-small" type="submit">Mark for unsubscribe</button>
</form>
{{end}}
</div>
</div>
{{else}}
<div class="empty-state">
<div class="empty-title">No unsubscribe links detected</div>
<p class="empty-copy">Check mail first. Newsletter and bulk senders usually carry a List-Unsubscribe header.</p>
</div>
{{end}}
</div>
</div>
</section>
<section class="panel unsub-panel" aria-label="Ignore list">
<div class="panel-head">
<h2>Ignore list</h2>
<span class="muted">{{len .Ignored}} domain(s)</span>
</div>
<div class="panel-body">
<div class="rules-versions">
{{range .Ignored}}
<div class="rules-version">
<div class="rules-version-main">
<span class="pill">ignored</span>
<span class="unsub-domain"><strong>{{.Domain}}</strong></span>
{{if .URL}}<span class="unsub-url">{{.URL}}</span>{{end}}
</div>
<form method="post" action="/unsubscribe/unignore">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<button class="btn btn-utility btn-small" type="submit">Un-ignore</button>
</form>
</div>
{{else}}
<p class="muted">No domains ignored.</p>
{{end}}
</div>
</div>
</section>
<section class="panel unsub-panel" aria-label="Recorded unsubscribes">
<div class="panel-head">
<h2>Recorded for unsubscribe</h2>
<span class="muted">{{len .Recorded}} domain(s)</span>
</div>
<div class="panel-body">
<p class="muted">These are recorded only. The app has not contacted any link.</p>
<div class="rules-versions">
{{range .Recorded}}
<div class="rules-version">
<div class="rules-version-main">
<span class="pill">recorded</span>
<span class="unsub-domain"><strong>{{.Domain}}</strong></span>
{{if .URL}}<span class="unsub-url">{{.URL}}</span>{{end}}
<span class="muted">{{.UpdatedAt}}</span>
</div>
<form method="post" action="/unsubscribe/unignore">
<input type="hidden" name="csrf" value="{{$.CSRFToken}}">
<input type="hidden" name="domain" value="{{.Domain}}">
<button class="btn btn-utility btn-small" type="submit">Clear record</button>
</form>
</div>
{{else}}
<p class="muted">Nothing recorded yet.</p>
{{end}}
</div>
</div>
</section>
</main>
</body>
</html>