Phase 6: rules editor + unsubscribe detection (AUD-72)
- FR-11 rules editor: plain-text area at /rules, saved and versioned (SaveRules appends a version; restore re-saves an old version), parse errors surfaced, one-click re-classify. The reply model now reads the rules document too (FR-11). - FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post (migration v6), detect email and web links (internal/unsubscribe), a domain-level ignore list, and an explicit record of what was marked. No code path contacts an unsubscribe link; acting on one needs Sam's approval. - Migration v6 is additive; unsubscribe gains url/updated_at. - Tests: detector, parser, store migration/records/candidates, rules editor, unsubscribe handlers, message-detail rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -102,18 +102,20 @@ func (f *Fetcher) Fetch(ctx context.Context) (Summary, error) {
|
||||
headersSeen[parsed.SourceHeader] = struct{}{}
|
||||
}
|
||||
msg := &store.Message{
|
||||
GmailID: gmailID(r, uidValidity),
|
||||
ThreadID: r.GmailThreadID,
|
||||
MessageIDHeader: parsed.MessageID,
|
||||
FromAddr: parsed.From,
|
||||
ToAddr: parsed.To,
|
||||
Subject: parsed.Subject,
|
||||
BodyText: parsed.BodyText,
|
||||
SourceAddress: parsed.SourceAddress,
|
||||
DateSent: parsed.Date,
|
||||
ReceivedAt: now,
|
||||
IMAPUID: int64(r.UID),
|
||||
IMAPUIDValidity: int64(uidValidity),
|
||||
GmailID: gmailID(r, uidValidity),
|
||||
ThreadID: r.GmailThreadID,
|
||||
MessageIDHeader: parsed.MessageID,
|
||||
FromAddr: parsed.From,
|
||||
ToAddr: parsed.To,
|
||||
Subject: parsed.Subject,
|
||||
BodyText: parsed.BodyText,
|
||||
SourceAddress: parsed.SourceAddress,
|
||||
ListUnsubscribe: parsed.ListUnsubscribe,
|
||||
ListUnsubscribePost: parsed.ListUnsubscribePost,
|
||||
DateSent: parsed.Date,
|
||||
ReceivedAt: now,
|
||||
IMAPUID: int64(r.UID),
|
||||
IMAPUIDValidity: int64(uidValidity),
|
||||
}
|
||||
if _, err := f.store.UpsertMessage(ctx, msg); err != nil {
|
||||
return sum, err
|
||||
|
||||
@@ -27,8 +27,13 @@ type ParsedMessage struct {
|
||||
Date time.Time
|
||||
// MessageID is the RFC822 Message-ID header, used for reply threading
|
||||
// (Phase 5, FR-8).
|
||||
MessageID string
|
||||
SourceAddress string
|
||||
MessageID string
|
||||
// ListUnsubscribe and ListUnsubscribePost are the RFC 2369/8058 headers that
|
||||
// name the unsubscribe targets (Phase 6, FR-7). They are stored raw so the
|
||||
// detector can run later without a refetch.
|
||||
ListUnsubscribe string
|
||||
ListUnsubscribePost string
|
||||
SourceAddress string
|
||||
// SourceHeader records which header the source address came from, so FR-2
|
||||
// decisions are auditable.
|
||||
SourceHeader string
|
||||
@@ -43,11 +48,13 @@ func ParseMessage(raw []byte) (*ParsedMessage, error) {
|
||||
}
|
||||
h := msg.Header
|
||||
pm := &ParsedMessage{
|
||||
From: decodeHeader(h.Get("From")),
|
||||
To: decodeHeader(h.Get("To")),
|
||||
Subject: decodeHeader(h.Get("Subject")),
|
||||
Date: parseDate(h.Get("Date")),
|
||||
MessageID: strings.TrimSpace(h.Get("Message-Id")),
|
||||
From: decodeHeader(h.Get("From")),
|
||||
To: decodeHeader(h.Get("To")),
|
||||
Subject: decodeHeader(h.Get("Subject")),
|
||||
Date: parseDate(h.Get("Date")),
|
||||
MessageID: strings.TrimSpace(h.Get("Message-Id")),
|
||||
ListUnsubscribe: strings.TrimSpace(h.Get("List-Unsubscribe")),
|
||||
ListUnsubscribePost: strings.TrimSpace(h.Get("List-Unsubscribe-Post")),
|
||||
}
|
||||
pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h))
|
||||
pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body)
|
||||
|
||||
@@ -172,3 +172,34 @@ func TestHeaderAddressParsing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMessageCapturesUnsubscribeHeaders(t *testing.T) {
|
||||
raw := "From: Heritage Fruit Trees <support@heritagefruittrees.com.au>\r\n" +
|
||||
"To: samuelrolfe@gmail.com\r\n" +
|
||||
"X-Pm-Original-To: hello@where-woof.com\r\n" +
|
||||
"Subject: New workshops\r\n" +
|
||||
"List-Unsubscribe: <mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>\r\n" +
|
||||
"List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" +
|
||||
"Message-Id: <abc@mail.example>\r\n" +
|
||||
"Date: Mon, 01 Jan 2026 10:00:00 +0000\r\n" +
|
||||
"Content-Type: text/plain; charset=utf-8\r\n" +
|
||||
"\r\n" +
|
||||
"Hello, you can unsubscribe at https://heritagefruittrees.com.au/u?id=1.\r\n"
|
||||
|
||||
pm, err := ParseMessage([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(pm.ListUnsubscribe, "mailto:unsub@heritagefruittrees.com.au") {
|
||||
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
|
||||
}
|
||||
if !strings.Contains(pm.ListUnsubscribe, "https://heritagefruittrees.com.au/u?id=1") {
|
||||
t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe)
|
||||
}
|
||||
if pm.ListUnsubscribePost != "List-Unsubscribe=One-Click" {
|
||||
t.Errorf("ListUnsubscribePost = %q", pm.ListUnsubscribePost)
|
||||
}
|
||||
if pm.SourceAddress != "hello@where-woof.com" {
|
||||
t.Errorf("SourceAddress = %q", pm.SourceAddress)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,10 @@ const maxPromptBodyChars = 4000
|
||||
// maxDraftChars bounds the generated body before it is stored.
|
||||
const maxDraftChars = 8000
|
||||
|
||||
// maxRulesChars bounds how much of the user's rules document is sent to the
|
||||
// model (FR-11: the LLM reads the rules too).
|
||||
const maxRulesChars = 2000
|
||||
|
||||
// Request is one draft request.
|
||||
type Request struct {
|
||||
// Message is the mail being replied to.
|
||||
@@ -31,6 +35,9 @@ type Request struct {
|
||||
// Instruction is optional extra guidance Sam typed (for example "say the
|
||||
// invoice is paid"). It is appended to the template instruction.
|
||||
Instruction string
|
||||
// Rules is the current rules document from the editor (FR-11). The model
|
||||
// reads it so reply style and handling follow the same rules as triage.
|
||||
Rules string
|
||||
// Signer is the name used for the sign-off (default "Sam").
|
||||
Signer string
|
||||
}
|
||||
@@ -107,6 +114,11 @@ func BuildPrompt(req Request, tmpl Template) (system, user string) {
|
||||
b.WriteString(extra)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if rules := strings.TrimSpace(req.Rules); rules != "" {
|
||||
b.WriteString("USER RULES (follow when they apply):\n")
|
||||
b.WriteString(truncate(rules, maxRulesChars))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if cat := strings.TrimSpace(req.Category); cat != "" {
|
||||
b.WriteString("CLASSIFICATION: category=")
|
||||
b.WriteString(cat)
|
||||
|
||||
@@ -116,3 +116,16 @@ func TestGeneratorDraftEmptyReplyIsAnError(t *testing.T) {
|
||||
t.Fatal("Draft() error = nil, want an error for an empty reply")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPromptIncludesRules(t *testing.T) {
|
||||
req := Request{
|
||||
Message: store.Message{FromAddr: "customer@example.test", Subject: "hi", BodyText: "?"},
|
||||
Action: "reply",
|
||||
Rules: `from contains "example.test" => category=Action`,
|
||||
}
|
||||
tmpl, _ := TemplateByID("answer")
|
||||
_, user := BuildPrompt(req, tmpl)
|
||||
if !strings.Contains(user, "USER RULES") || !strings.Contains(user, `from contains "example.test"`) {
|
||||
t.Errorf("user prompt does not include the rules document:\n%s", user)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
|
||||
)
|
||||
|
||||
// ReplyGenerator drafts a reply body. The live implementation is
|
||||
@@ -63,6 +64,11 @@ type replyData struct {
|
||||
LLMReady bool
|
||||
LLMModel string
|
||||
|
||||
// Phase 6 (FR-7): detected unsubscribe links for this message, and the
|
||||
// domain's status on the ignore/recorded list.
|
||||
UnsubLinks []unsubscribe.Link
|
||||
UnsubStatus string
|
||||
|
||||
Sent bool
|
||||
ThreadUpdated bool
|
||||
RepliedFolder string
|
||||
@@ -109,6 +115,9 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
class := s.latestClass(ctx, id)
|
||||
|
||||
// FR-11: the reply model reads the current rules document too.
|
||||
rulesText, _, _ := s.store.CurrentRules(ctx)
|
||||
|
||||
body, templateID, err := s.replies.Draft(ctx, reply.Request{
|
||||
Message: msg,
|
||||
Category: class.Category,
|
||||
@@ -116,6 +125,7 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
|
||||
Action: class.Action,
|
||||
TemplateID: r.FormValue("template"),
|
||||
Instruction: r.FormValue("instruction"),
|
||||
Rules: rulesText,
|
||||
Signer: "Sam",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -313,6 +323,16 @@ func (s *Server) renderMessage(w http.ResponseWriter, r *http.Request, id int64,
|
||||
}
|
||||
data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == ""
|
||||
|
||||
// FR-7: detect unsubscribe links and look up the domain's list status.
|
||||
if res := unsubscribe.Detect(msg.ListUnsubscribe, msg.ListUnsubscribePost, msg.FromAddr, msg.BodyText); len(res.Links) > 0 {
|
||||
data.UnsubLinks = res.Links
|
||||
if rec, ok, rerr := s.store.UnsubscribeByDomain(ctx, res.Links[0].Domain); rerr != nil {
|
||||
s.logger.Error("load unsubscribe status", "message_id", id, "error", rerr)
|
||||
} else if ok {
|
||||
data.UnsubStatus = rec.Status
|
||||
}
|
||||
}
|
||||
|
||||
if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil {
|
||||
s.logger.Error("load draft", "message_id", id, "error", derr)
|
||||
} else if ok {
|
||||
|
||||
225
internal/server/rules.go
Normal file
225
internal/server/rules.go
Normal file
@@ -0,0 +1,225 @@
|
||||
// Rules editor for Phase 6 (FR-4, FR-11).
|
||||
//
|
||||
// The rules document is plain text (see internal/classify for the syntax). It is
|
||||
// saved as a new numbered version on every save, so the current text and its
|
||||
// history are both available. The classifier reads the newest version and
|
||||
// applies it as an override; the reply model also reads it.
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify"
|
||||
)
|
||||
|
||||
type ruleRevision struct {
|
||||
Version int
|
||||
CreatedAt string
|
||||
Lines int
|
||||
Preview string
|
||||
}
|
||||
|
||||
type rulesData struct {
|
||||
Text string
|
||||
Version int
|
||||
Revisions []ruleRevision
|
||||
Errors []classify.ParseError
|
||||
RuleCount int
|
||||
Note string
|
||||
Error string
|
||||
CSRFToken string
|
||||
// ClassifierReady is true when JEV is configured, so the page can offer a
|
||||
// re-classify action that actually runs.
|
||||
ClassifierReady bool
|
||||
TotalMessages int
|
||||
}
|
||||
|
||||
// handleRules renders the rules editor with the current text and version list.
|
||||
func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderRules(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (s *Server) renderRules(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
|
||||
ctx := r.Context()
|
||||
text, version, err := s.store.CurrentRules(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("load rules", "error", err)
|
||||
http.Error(w, "could not load the rules", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
parsed := classify.ParseRules(text)
|
||||
revisions, err := s.store.RulesVersions(ctx, 20)
|
||||
if err != nil {
|
||||
s.logger.Error("load rules versions", "error", err)
|
||||
}
|
||||
|
||||
revs := make([]ruleRevision, 0, len(revisions))
|
||||
for _, rv := range revisions {
|
||||
revs = append(revs, ruleRevision{
|
||||
Version: rv.Version,
|
||||
CreatedAt: rv.CreatedAt,
|
||||
Lines: countRuleLines(rv.Text),
|
||||
Preview: firstRuleLine(rv.Text),
|
||||
})
|
||||
}
|
||||
total, err := s.store.CountMessages(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("count messages", "error", err)
|
||||
}
|
||||
|
||||
data := rulesData{
|
||||
Text: text,
|
||||
Version: version,
|
||||
Revisions: revs,
|
||||
Errors: parsed.Errors,
|
||||
RuleCount: len(parsed.Rules),
|
||||
Note: note,
|
||||
Error: errMsg,
|
||||
CSRFToken: s.csrfToken(w, r),
|
||||
ClassifierReady: s.classifier.Configured(),
|
||||
TotalMessages: total,
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(code)
|
||||
if err := s.tmpl.ExecuteTemplate(w, "rules.html", data); err != nil {
|
||||
s.logger.Error("render rules", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleRulesSave versions the posted text (FR-11).
|
||||
func (s *Server) handleRulesSave(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectRules(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
text := r.FormValue("text")
|
||||
version, err := s.store.SaveRules(r.Context(), text)
|
||||
if err != nil {
|
||||
s.logger.Error("save rules", "error", err)
|
||||
s.redirectRules(w, r, "", "Could not save the rules: "+err.Error())
|
||||
return
|
||||
}
|
||||
parsed := classify.ParseRules(text)
|
||||
s.logAction(r.Context(), "rules_save", "Sam",
|
||||
"version="+strconv.Itoa(version)+" rules="+strconv.Itoa(len(parsed.Rules))+" parse_errors="+strconv.Itoa(len(parsed.Errors)))
|
||||
s.redirectRules(w, r, "Saved rules version "+strconv.Itoa(version)+". Re-classify all to apply it to existing mail.", "")
|
||||
}
|
||||
|
||||
// handleRulesRestore re-saves an older version as the newest one, so history is
|
||||
// append-only (FR-11).
|
||||
func (s *Server) handleRulesRestore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectRules(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
version, err := strconv.Atoi(strings.TrimSpace(r.FormValue("version")))
|
||||
if err != nil || version < 1 {
|
||||
s.redirectRules(w, r, "", "Unknown rules version.")
|
||||
return
|
||||
}
|
||||
text, ok, err := s.store.RuleByVersion(r.Context(), version)
|
||||
if err != nil {
|
||||
s.redirectRules(w, r, "", "Could not load that version.")
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
s.redirectRules(w, r, "", "That rules version no longer exists.")
|
||||
return
|
||||
}
|
||||
newVersion, err := s.store.SaveRules(r.Context(), text)
|
||||
if err != nil {
|
||||
s.logger.Error("restore rules", "from_version", version, "error", err)
|
||||
s.redirectRules(w, r, "", "Could not restore that version: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "rules_restore", "Sam",
|
||||
"from_version="+strconv.Itoa(version)+" new_version="+strconv.Itoa(newVersion))
|
||||
s.redirectRules(w, r, "Restored version "+strconv.Itoa(version)+" as version "+strconv.Itoa(newVersion)+".", "")
|
||||
}
|
||||
|
||||
// handleRulesReclassify re-runs the classifier over every stored message and
|
||||
// returns to the rules editor. It is the rules page's "apply" action: saving a
|
||||
// new version does not re-classify by itself.
|
||||
func (s *Server) handleRulesReclassify(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if rc := http.NewResponseController(w); rc != nil {
|
||||
_ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
sum, err := s.classifier.RunAll(ctx)
|
||||
switch {
|
||||
case err == nil:
|
||||
s.logAction(ctx, "reclassify", "Sam",
|
||||
fmt.Sprintf("considered=%d classified=%d failed=%d rules_version=%d",
|
||||
sum.Considered, sum.Classified, sum.Failed, sum.RulesVersion))
|
||||
s.redirectRules(w, r, fmt.Sprintf(
|
||||
"Re-classified %d of %d message(s); %d failed. The current rules are now applied.",
|
||||
sum.Classified, sum.Considered, sum.Failed), "")
|
||||
case errors.Is(err, classify.ErrNotConfigured):
|
||||
s.redirectRules(w, r, "", "Classification is not connected yet: set JEV_API_KEY.")
|
||||
default:
|
||||
s.logger.Error("rules reclassify failed", "error", err)
|
||||
s.redirectRules(w, r, "", "Classification failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// redirectRules is POST/redirect/GET so a refresh does not resubmit.
|
||||
func (s *Server) redirectRules(w http.ResponseWriter, r *http.Request, note, errMsg string) {
|
||||
q := url.Values{}
|
||||
if note != "" {
|
||||
q.Set("note", note)
|
||||
}
|
||||
if errMsg != "" {
|
||||
q.Set("error", errMsg)
|
||||
}
|
||||
http.Redirect(w, r, "/rules?"+q.Encode(), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// countRuleLines counts the non-empty, non-comment lines in a rules document.
|
||||
func countRuleLines(text string) int {
|
||||
n := 0
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// firstRuleLine returns a short preview of the first real rule, or a note when
|
||||
// the document has only comments.
|
||||
func firstRuleLine(text string) string {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
if len(line) > 80 {
|
||||
return line[:80] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
return "(comments only)"
|
||||
}
|
||||
152
internal/server/rules_test.go
Normal file
152
internal/server/rules_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRulesPageRenders(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/rules", session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{"Rules editor", "Save new version", "Versions", "Back to inbox", "Unsubscribe"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("rules page missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesSaveCreatesVersion(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rule := `from contains "heritagefruittrees.com.au" => category=Action, importance=high, action=file`
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "text": {rule},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
text, version, err := srv.store.CurrentRules(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentRules() error = %v", err)
|
||||
}
|
||||
if text != rule {
|
||||
t.Errorf("CurrentRules() text = %q, want the saved rule", text)
|
||||
}
|
||||
if version != 2 {
|
||||
t.Errorf("CurrentRules() version = %d, want 2", version)
|
||||
}
|
||||
|
||||
// The editor shows the new version and active rule count.
|
||||
body := getWithSession(t, srv, "/rules", session).Body.String()
|
||||
if !strings.Contains(body, "version 2") || !strings.Contains(body, "1 active rule") {
|
||||
t.Errorf("rules page did not reflect the saved version; body:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesSaveRequiresCSRF(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/save", session, nil, url.Values{"text": {"anything"}})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesPageShowsParseErrors(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "text": {"this line has no arrow"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save status = %d, want 303", rec.Code)
|
||||
}
|
||||
body := getWithSession(t, srv, "/rules", session).Body.String()
|
||||
if !strings.Contains(body, "could not be parsed") || !strings.Contains(body, "missing =>") {
|
||||
t.Errorf("rules page did not surface the parse error; body:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesRestoreCreatesNewVersion(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
first := `from contains "a" => category=Info`
|
||||
second := `from contains "b" => category=Action`
|
||||
for _, text := range []string{first, second} {
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{"csrf": {csrf}, "text": {text}})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save %q status = %d, want 303", text, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Restore version 2 (the first saved rule) as a new version.
|
||||
rec := postForm(t, srv, "/rules/restore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "version": {"2"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("restore status = %d, want 303", rec.Code)
|
||||
}
|
||||
text, version, err := srv.store.CurrentRules(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentRules() error = %v", err)
|
||||
}
|
||||
if text != first {
|
||||
t.Errorf("restored text = %q, want %q", text, first)
|
||||
}
|
||||
if version != 4 {
|
||||
t.Errorf("restored version = %d, want 4 (append-only)", version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardLinksRulesAndUnsubscribe(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
body := getWithSession(t, srv, "/", session).Body.String()
|
||||
for _, want := range []string{`href="/rules"`, `href="/unsubscribe"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("dashboard missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesCountsLines(t *testing.T) {
|
||||
text := "# comment\n\nfrom contains \"a\" => category=Info\nfrom contains \"b\" => category=Action"
|
||||
if got := countRuleLines(text); got != 2 {
|
||||
t.Errorf("countRuleLines() = %d, want 2", got)
|
||||
}
|
||||
if got := firstRuleLine(text); got != `from contains "a" => category=Info` {
|
||||
t.Errorf("firstRuleLine() = %q", got)
|
||||
}
|
||||
if got := firstRuleLine("# only a comment"); got != "(comments only)" {
|
||||
t.Errorf("firstRuleLine(comment only) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesReclassifyWithoutJEVRedirects(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/reclassify", session, csrfCookie, url.Values{"csrf": {csrf}})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if loc := rec.Header().Get("Location"); !strings.Contains(loc, "JEV_API_KEY") {
|
||||
t.Errorf("Location = %q, want a JEV-not-configured note", loc)
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,18 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename))
|
||||
mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove))
|
||||
|
||||
// Phase 6 (FR-4, FR-11): rules editor, saved and versioned.
|
||||
mux.HandleFunc("GET /rules", s.requireAuth(s.handleRules))
|
||||
mux.HandleFunc("POST /rules/save", s.requireAuth(s.handleRulesSave))
|
||||
mux.HandleFunc("POST /rules/restore", s.requireAuth(s.handleRulesRestore))
|
||||
mux.HandleFunc("POST /rules/reclassify", s.requireAuth(s.handleRulesReclassify))
|
||||
|
||||
// Phase 6 (FR-7): unsubscribe detection, ignore list, and records.
|
||||
mux.HandleFunc("GET /unsubscribe", s.requireAuth(s.handleUnsubscribe))
|
||||
mux.HandleFunc("POST /unsubscribe/ignore", s.requireAuth(s.handleUnsubscribeIgnore))
|
||||
mux.HandleFunc("POST /unsubscribe/unignore", s.requireAuth(s.handleUnsubscribeUnignore))
|
||||
mux.HandleFunc("POST /unsubscribe/mark", s.requireAuth(s.handleUnsubscribeMark))
|
||||
|
||||
return s.recoverPanic(s.logRequests(mux))
|
||||
}
|
||||
|
||||
|
||||
218
internal/server/unsubscribe.go
Normal file
218
internal/server/unsubscribe.go
Normal file
@@ -0,0 +1,218 @@
|
||||
// Unsubscribe detection view for Phase 6 (FR-7).
|
||||
//
|
||||
// The page lists messages with a detected unsubscribe link, the domain-level
|
||||
// ignore list, and the domains Sam has recorded as unsubscribed. Every action
|
||||
// records state; none of them contacts the link. Acting on a real unsubscribe
|
||||
// link is a manual step that needs Sam's explicit approval (AUD-72).
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
|
||||
)
|
||||
|
||||
type unsubscribeItem struct {
|
||||
MessageID int64
|
||||
From string
|
||||
Subject string
|
||||
SourceAddr string
|
||||
Domain string
|
||||
Links []unsubscribe.Link
|
||||
Status string
|
||||
Category string
|
||||
When string
|
||||
}
|
||||
|
||||
type unsubscribeData struct {
|
||||
Items []unsubscribeItem
|
||||
Ignored []store.UnsubscribeRecord
|
||||
Recorded []store.UnsubscribeRecord
|
||||
Note string
|
||||
Error string
|
||||
CSRFToken string
|
||||
CandidateCount int
|
||||
}
|
||||
|
||||
// handleUnsubscribe renders the detection list and the ignore/recorded lists.
|
||||
func (s *Server) handleUnsubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderUnsubscribe(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (s *Server) renderUnsubscribe(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
|
||||
ctx := r.Context()
|
||||
records, err := s.store.ListUnsubscribe(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("list unsubscribe records", "error", err)
|
||||
http.Error(w, "could not load the unsubscribe list", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
candidates, err := s.store.UnsubscribeCandidates(ctx, 300)
|
||||
if err != nil {
|
||||
s.logger.Error("list unsubscribe candidates", "error", err)
|
||||
http.Error(w, "could not load messages", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
status := make(map[string]string, len(records))
|
||||
for _, rec := range records {
|
||||
status[rec.Domain] = rec.Status
|
||||
}
|
||||
|
||||
var items []unsubscribeItem
|
||||
for _, c := range candidates {
|
||||
res := unsubscribe.Detect(c.ListUnsubscribe, c.ListUnsubscribePost, c.FromAddr, c.BodyText)
|
||||
if len(res.Links) == 0 {
|
||||
continue
|
||||
}
|
||||
domain := res.Links[0].Domain
|
||||
items = append(items, unsubscribeItem{
|
||||
MessageID: c.MessageID,
|
||||
From: orDefault(c.FromAddr, "(unknown sender)"),
|
||||
Subject: orDefault(c.Subject, "(no subject)"),
|
||||
SourceAddr: c.SourceAddress,
|
||||
Domain: domain,
|
||||
Links: res.Links,
|
||||
Status: status[domain],
|
||||
Category: c.Category,
|
||||
When: formatWhen(c.ReceivedAt, c.ReceivedAt),
|
||||
})
|
||||
}
|
||||
|
||||
data := unsubscribeData{
|
||||
Items: items,
|
||||
CSRFToken: s.csrfToken(w, r),
|
||||
Note: note,
|
||||
Error: errMsg,
|
||||
CandidateCount: len(items),
|
||||
}
|
||||
for _, rec := range records {
|
||||
switch rec.Status {
|
||||
case store.UnsubscribeStatusIgnored:
|
||||
data.Ignored = append(data.Ignored, rec)
|
||||
case store.UnsubscribeStatusDone:
|
||||
data.Recorded = append(data.Recorded, rec)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(code)
|
||||
if err := s.tmpl.ExecuteTemplate(w, "unsubscribe.html", data); err != nil {
|
||||
s.logger.Error("render unsubscribe", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleUnsubscribeIgnore adds a domain to the ignore list (FR-7).
|
||||
func (s *Server) handleUnsubscribeIgnore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusIgnored, r.FormValue("url")); err != nil {
|
||||
s.logger.Error("ignore unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not update the ignore list.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_ignore", "Sam", "domain="+domain)
|
||||
s.redirectUnsubscribe(w, r, "Added "+domain+" to the ignore list. It is never recommended for unsubscribe.", "")
|
||||
}
|
||||
|
||||
// handleUnsubscribeUnignore removes a domain from the ignore list.
|
||||
func (s *Server) handleUnsubscribeUnignore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.ClearUnsubscribe(r.Context(), domain); err != nil {
|
||||
s.logger.Error("clear unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not update the list.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_unignore", "Sam", "domain="+domain)
|
||||
s.redirectUnsubscribe(w, r, "Removed "+domain+" from the unsubscribe list.", "")
|
||||
}
|
||||
|
||||
// handleUnsubscribeMark records that a domain should be unsubscribed from. It
|
||||
// is a record only: the handler makes no network call and the app never follows
|
||||
// the link (AUD-72). Following a real link needs Sam's explicit approval.
|
||||
func (s *Server) handleUnsubscribeMark(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
if r.FormValue("confirm") != "yes" {
|
||||
s.redirectUnsubscribe(w, r, "", "Marking needs confirmation. The app will record it, not follow the link.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusDone, r.FormValue("url")); err != nil {
|
||||
s.logger.Error("mark unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not record the unsubscribe.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_marked", "Sam", "domain="+domain+" url="+r.FormValue("url"))
|
||||
s.redirectUnsubscribe(w, r,
|
||||
"Recorded "+domain+" for unsubscribe. The app did not contact the link; acting on it is a manual step.", "")
|
||||
}
|
||||
|
||||
// redirectUnsubscribe is POST/redirect/GET so a refresh does not resubmit.
|
||||
func (s *Server) redirectUnsubscribe(w http.ResponseWriter, r *http.Request, note, errMsg string) {
|
||||
q := url.Values{}
|
||||
if note != "" {
|
||||
q.Set("note", note)
|
||||
}
|
||||
if errMsg != "" {
|
||||
q.Set("error", errMsg)
|
||||
}
|
||||
http.Redirect(w, r, "/unsubscribe?"+q.Encode(), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// cleanDomain validates a domain used as the ignore-list key.
|
||||
func cleanDomain(raw string) (string, bool) {
|
||||
domain := strings.ToLower(strings.TrimSpace(raw))
|
||||
domain = strings.Trim(domain, ".")
|
||||
if domain == "" || len(domain) > 253 {
|
||||
return "", false
|
||||
}
|
||||
for _, r := range domain {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
if !strings.Contains(domain, ".") {
|
||||
return "", false
|
||||
}
|
||||
return domain, true
|
||||
}
|
||||
182
internal/server/unsubscribe_test.go
Normal file
182
internal/server/unsubscribe_test.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
)
|
||||
|
||||
func seedUnsubMessage(t *testing.T, srv *Server) int64 {
|
||||
t.Helper()
|
||||
id, err := srv.store.UpsertMessage(context.Background(), &store.Message{
|
||||
GmailID: "unsub-1",
|
||||
FromAddr: "Heritage Fruit Trees <support@heritagefruittrees.com.au>",
|
||||
Subject: "New workshops released",
|
||||
BodyText: "Hello, you have received a newsletter. Unsubscribe at https://heritagefruittrees.com.au/u?id=1.",
|
||||
SourceAddress: "samuelrolfe@gmail.com",
|
||||
ListUnsubscribe: "<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>",
|
||||
DateSent: time.Now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seed message: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUnsubscribePageDetectsLinks(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/unsubscribe", session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{
|
||||
"Detected links", "heritagefruittrees.com.au",
|
||||
"https://heritagefruittrees.com.au/u?id=1",
|
||||
"mailto:unsub@heritagefruittrees.com.au",
|
||||
"Ignore domain", "Mark for unsubscribe", "Ignore list",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("unsubscribe page missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeIgnoreAndUnignore(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("ignore status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if rec2.Status != store.UnsubscribeStatusIgnored {
|
||||
t.Errorf("status = %q, want ignored", rec2.Status)
|
||||
}
|
||||
body := getWithSession(t, srv, "/unsubscribe", session).Body.String()
|
||||
if !strings.Contains(body, "ignored") || !strings.Contains(body, "Un-ignore") {
|
||||
t.Errorf("ignore list did not render the ignored domain")
|
||||
}
|
||||
|
||||
rec = postForm(t, srv, "/unsubscribe/unignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("unignore status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
|
||||
t.Error("domain still on the list after unignore")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMarkRecordsWithoutFollowingLink(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
// Without confirmation the handler refuses and records nothing.
|
||||
rec := postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
"url": {"https://heritagefruittrees.com.au/u?id=1"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("mark without confirm status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
|
||||
t.Error("unconfirmed mark recorded a domain")
|
||||
}
|
||||
|
||||
rec = postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
"url": {"https://heritagefruittrees.com.au/u?id=1"}, "confirm": {"yes"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("mark status = %d, want 303", rec.Code)
|
||||
}
|
||||
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if rec2.Status != store.UnsubscribeStatusDone {
|
||||
t.Errorf("status = %q, want done", rec2.Status)
|
||||
}
|
||||
if rec2.URL != "https://heritagefruittrees.com.au/u?id=1" {
|
||||
t.Errorf("url = %q, want the recorded link", rec2.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMutationsRequireCSRF(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
for _, path := range []string{"/unsubscribe/ignore", "/unsubscribe/unignore", "/unsubscribe/mark"} {
|
||||
rec := postForm(t, srv, path, session, nil, url.Values{"domain": {"x.example"}})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Errorf("%s without CSRF status = %d, want 403", path, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMarkRejectsBadDomain(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"not a domain!"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(context.Background(), "not a domain!"); ok {
|
||||
t.Error("bad domain was stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageDetailShowsUnsubscribeLinks(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
id := seedUnsubMessage(t, srv)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/partials/message?id="+itoa(id), session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "Unsubscribe") || !strings.Contains(body, "heritagefruittrees.com.au/u?id=1") {
|
||||
t.Errorf("message detail did not show the unsubscribe link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanDomain(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"brand.example": true,
|
||||
"Sub.Brand.example": true,
|
||||
"x-y.example": true,
|
||||
"": false,
|
||||
"nodot": false,
|
||||
"has space.example": false,
|
||||
"bad!char.example": false,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if _, ok := cleanDomain(in); ok != want {
|
||||
t.Errorf("cleanDomain(%q) ok = %v, want %v", in, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -159,3 +159,51 @@ func (s *Store) SaveRules(ctx context.Context, text string) (int, error) {
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
// RuleVersion is one saved rules revision, for the version list in the editor
|
||||
// (FR-11).
|
||||
type RuleVersion struct {
|
||||
ID int64
|
||||
Version int
|
||||
Text string
|
||||
CreatedAt string
|
||||
}
|
||||
|
||||
// RulesVersions returns saved rules revisions, newest first.
|
||||
func (s *Store) RulesVersions(ctx context.Context, limit int) ([]RuleVersion, error) {
|
||||
if limit < 1 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT id, version, text, created_at FROM rules ORDER BY version DESC LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list rules versions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []RuleVersion
|
||||
for rows.Next() {
|
||||
var v RuleVersion
|
||||
if err := rows.Scan(&v.ID, &v.Version, &v.Text, &v.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan rules version: %w", err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("list rules versions: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RuleByVersion returns the text of one saved rules revision. ok is false when
|
||||
// the version does not exist.
|
||||
func (s *Store) RuleByVersion(ctx context.Context, version int) (string, bool, error) {
|
||||
var text string
|
||||
err := s.db.QueryRowContext(ctx, `SELECT text FROM rules WHERE version = ?`, version).Scan(&text)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("read rules version %d: %w", version, err)
|
||||
}
|
||||
return text, true, nil
|
||||
}
|
||||
|
||||
@@ -20,15 +20,19 @@ type Message struct {
|
||||
// MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply
|
||||
// uses it for In-Reply-To and References so Gmail keeps the thread.
|
||||
MessageIDHeader string
|
||||
FromAddr string
|
||||
ToAddr string
|
||||
Subject string
|
||||
BodyText string
|
||||
SourceAddress string
|
||||
DateSent time.Time
|
||||
ReceivedAt time.Time
|
||||
IMAPUID int64
|
||||
IMAPUIDValidity int64
|
||||
// ListUnsubscribe and ListUnsubscribePost are the raw RFC 2369/8058 headers
|
||||
// (Phase 6, FR-7). Detection runs over them on demand.
|
||||
ListUnsubscribe string
|
||||
ListUnsubscribePost string
|
||||
FromAddr string
|
||||
ToAddr string
|
||||
Subject string
|
||||
BodyText string
|
||||
SourceAddress string
|
||||
DateSent time.Time
|
||||
ReceivedAt time.Time
|
||||
IMAPUID int64
|
||||
IMAPUIDValidity int64
|
||||
// Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9):
|
||||
// inbox, delete, saved, or replied.
|
||||
Folder string
|
||||
@@ -71,19 +75,22 @@ func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) {
|
||||
const q = `
|
||||
INSERT INTO messages (
|
||||
gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text,
|
||||
source_address, date_sent, received_at, imap_uid, imap_uidvalidity
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
source_address, list_unsubscribe, list_unsubscribe_post, date_sent, received_at,
|
||||
imap_uid, imap_uidvalidity
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (gmail_id) DO UPDATE SET
|
||||
thread_id = excluded.thread_id,
|
||||
message_id_header = excluded.message_id_header,
|
||||
from_addr = excluded.from_addr,
|
||||
to_addr = excluded.to_addr,
|
||||
subject = excluded.subject,
|
||||
body_text = excluded.body_text,
|
||||
source_address = excluded.source_address,
|
||||
date_sent = excluded.date_sent,
|
||||
imap_uid = excluded.imap_uid,
|
||||
imap_uidvalidity = excluded.imap_uidvalidity`
|
||||
thread_id = excluded.thread_id,
|
||||
message_id_header = excluded.message_id_header,
|
||||
from_addr = excluded.from_addr,
|
||||
to_addr = excluded.to_addr,
|
||||
subject = excluded.subject,
|
||||
body_text = excluded.body_text,
|
||||
source_address = excluded.source_address,
|
||||
list_unsubscribe = excluded.list_unsubscribe,
|
||||
list_unsubscribe_post = excluded.list_unsubscribe_post,
|
||||
date_sent = excluded.date_sent,
|
||||
imap_uid = excluded.imap_uid,
|
||||
imap_uidvalidity = excluded.imap_uidvalidity`
|
||||
|
||||
receivedAt := m.ReceivedAt
|
||||
if receivedAt.IsZero() {
|
||||
@@ -98,6 +105,8 @@ ON CONFLICT (gmail_id) DO UPDATE SET
|
||||
nullString(m.Subject),
|
||||
nullString(m.BodyText),
|
||||
nullString(m.SourceAddress),
|
||||
nullString(m.ListUnsubscribe),
|
||||
nullString(m.ListUnsubscribePost),
|
||||
nullTime(m.DateSent),
|
||||
receivedAt.UTC().Format(sqlTimeLayout),
|
||||
m.IMAPUID,
|
||||
@@ -124,7 +133,8 @@ SELECT m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''),
|
||||
COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at,
|
||||
COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0),
|
||||
COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'),
|
||||
COALESCE(m.message_id_header, '')
|
||||
COALESCE(m.message_id_header, ''),
|
||||
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, '')
|
||||
FROM messages m WHERE m.id = ?`
|
||||
var (
|
||||
m Message
|
||||
@@ -135,6 +145,7 @@ FROM messages m WHERE m.id = ?`
|
||||
&m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject,
|
||||
&m.BodyText, &m.SourceAddress, &dateSent, &received,
|
||||
&m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader,
|
||||
&m.ListUnsubscribe, &m.ListUnsubscribePost,
|
||||
)
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("load message %d: %w", id, err)
|
||||
|
||||
@@ -21,6 +21,10 @@ CREATE TABLE IF NOT EXISTS messages (
|
||||
-- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and
|
||||
-- References to it so Gmail files the reply in the same thread.
|
||||
message_id_header TEXT,
|
||||
-- Phase 6 (FR-7): the raw RFC 2369/8058 unsubscribe headers, stored so the
|
||||
-- detector can run without a refetch.
|
||||
list_unsubscribe TEXT,
|
||||
list_unsubscribe_post TEXT,
|
||||
from_addr TEXT,
|
||||
to_addr TEXT,
|
||||
subject TEXT,
|
||||
@@ -94,11 +98,16 @@ CREATE TABLE IF NOT EXISTS actions_log (
|
||||
CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at);
|
||||
|
||||
-- FR-7: unsubscribe tracker and ignore list.
|
||||
-- FR-7: unsubscribe tracker and ignore list. `status` is 'ignored' for a domain
|
||||
-- on the ignore list, or 'done' for a domain Sam has recorded as unsubscribed.
|
||||
-- The app never follows a link itself: it only marks and records (AUD-72).
|
||||
CREATE TABLE IF NOT EXISTS unsubscribe (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
domain TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
|
||||
-- url records the link that was detected or marked (Phase 6).
|
||||
url TEXT NOT NULL DEFAULT '',
|
||||
updated_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
|
||||
@@ -159,6 +159,25 @@ func (s *Store) applyMigrations(ctx context.Context) error {
|
||||
return fmt.Errorf("record schema version 5: %w", err)
|
||||
}
|
||||
}
|
||||
if version < 6 {
|
||||
// Phase 6 (FR-7): store the raw List-Unsubscribe headers so detection
|
||||
// runs without a refetch, and extend the unsubscribe tracker with the
|
||||
// link it recorded and an update time.
|
||||
for _, col := range []struct{ table, name, typ string }{
|
||||
{"messages", "list_unsubscribe", "TEXT"},
|
||||
{"messages", "list_unsubscribe_post", "TEXT"},
|
||||
{"unsubscribe", "url", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"unsubscribe", "updated_at", "TEXT"},
|
||||
} {
|
||||
if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO schema_migrations (version) VALUES (6)`); err != nil {
|
||||
return fmt.Errorf("record schema version 6: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
177
internal/store/unsubscribe.go
Normal file
177
internal/store/unsubscribe.go
Normal file
@@ -0,0 +1,177 @@
|
||||
// Unsubscribe tracking for Phase 6 (FR-7).
|
||||
//
|
||||
// Two things are persisted here:
|
||||
//
|
||||
// - the raw unsubscribe links detected on a message (messages.list_unsubscribe,
|
||||
// stored by the mail fetch and parsed by internal/unsubscribe); and
|
||||
// - the domain-level ignore list and the record of what was marked, in the
|
||||
// `unsubscribe` table.
|
||||
//
|
||||
// The app never follows a link. Marking a domain records Sam's decision; the
|
||||
// actual unsubscribe stays a manual, explicitly approved action (AUD-72).
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Unsubscribe statuses. `ignored` is the ignore list; `done` records a domain
|
||||
// Sam has decided to unsubscribe from.
|
||||
const (
|
||||
UnsubscribeStatusDone = "done"
|
||||
UnsubscribeStatusIgnored = "ignored"
|
||||
)
|
||||
|
||||
// ErrUnsubscribeStatus is returned for an unknown unsubscribe status.
|
||||
var ErrUnsubscribeStatus = errors.New("unsubscribe status must be done or ignored")
|
||||
|
||||
// UnsubscribeRecord is one row of the domain-level tracker.
|
||||
type UnsubscribeRecord struct {
|
||||
ID int64
|
||||
Domain string
|
||||
Status string
|
||||
URL string
|
||||
CreatedAt string
|
||||
UpdatedAt string
|
||||
}
|
||||
|
||||
// ListUnsubscribe returns every domain record, newest first.
|
||||
func (s *Store) ListUnsubscribe(ctx context.Context) ([]UnsubscribeRecord, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
|
||||
FROM unsubscribe ORDER BY COALESCE(updated_at, created_at) DESC, id DESC`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list unsubscribe records: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []UnsubscribeRecord
|
||||
for rows.Next() {
|
||||
var r UnsubscribeRecord
|
||||
if err := rows.Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("scan unsubscribe record: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("list unsubscribe records: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UnsubscribeByDomain loads one domain record. ok is false when the domain is
|
||||
// not tracked.
|
||||
func (s *Store) UnsubscribeByDomain(ctx context.Context, domain string) (UnsubscribeRecord, bool, error) {
|
||||
domain = strings.TrimSpace(domain)
|
||||
if domain == "" {
|
||||
return UnsubscribeRecord{}, false, nil
|
||||
}
|
||||
var r UnsubscribeRecord
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '')
|
||||
FROM unsubscribe WHERE domain = ?`, domain).
|
||||
Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return UnsubscribeRecord{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return UnsubscribeRecord{}, false, fmt.Errorf("read unsubscribe record %q: %w", domain, err)
|
||||
}
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
// SetUnsubscribeStatus upserts the domain record. An empty url keeps the stored
|
||||
// one, so an "ignore" does not wipe a link recorded earlier.
|
||||
func (s *Store) SetUnsubscribeStatus(ctx context.Context, domain, status, url string) error {
|
||||
domain = strings.TrimSpace(domain)
|
||||
if domain == "" {
|
||||
return errors.New("unsubscribe domain must not be empty")
|
||||
}
|
||||
if status != UnsubscribeStatusDone && status != UnsubscribeStatusIgnored {
|
||||
return ErrUnsubscribeStatus
|
||||
}
|
||||
const q = `
|
||||
INSERT INTO unsubscribe (domain, status, url, updated_at)
|
||||
VALUES (?, ?, ?, datetime('now'))
|
||||
ON CONFLICT (domain) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
url = CASE WHEN excluded.url <> '' THEN excluded.url ELSE unsubscribe.url END,
|
||||
updated_at = datetime('now')`
|
||||
if _, err := s.db.ExecContext(ctx, q, domain, status, strings.TrimSpace(url)); err != nil {
|
||||
return fmt.Errorf("set unsubscribe status for %q: %w", domain, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearUnsubscribe removes a domain record, taking it off the ignore list.
|
||||
func (s *Store) ClearUnsubscribe(ctx context.Context, domain string) error {
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`DELETE FROM unsubscribe WHERE domain = ?`, strings.TrimSpace(domain)); err != nil {
|
||||
return fmt.Errorf("clear unsubscribe record %q: %w", domain, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnsubscribeCandidate is a message that may carry an unsubscribe link.
|
||||
type UnsubscribeCandidate struct {
|
||||
MessageID int64
|
||||
FromAddr string
|
||||
Subject string
|
||||
SourceAddress string
|
||||
BodyText string
|
||||
ListUnsubscribe string
|
||||
ListUnsubscribePost string
|
||||
Category string
|
||||
ReceivedAt time.Time
|
||||
}
|
||||
|
||||
// UnsubscribeCandidates returns messages with a List-Unsubscribe header or a
|
||||
// body that mentions unsubscribing, newest first. The caller runs the detector
|
||||
// over them, because body parsing lives in internal/unsubscribe.
|
||||
func (s *Store) UnsubscribeCandidates(ctx context.Context, limit int) ([]UnsubscribeCandidate, error) {
|
||||
if limit < 1 {
|
||||
limit = 200
|
||||
}
|
||||
const q = `
|
||||
SELECT m.id, COALESCE(m.from_addr, ''), COALESCE(m.subject, ''),
|
||||
COALESCE(m.source_address, ''), COALESCE(m.body_text, ''),
|
||||
COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, ''),
|
||||
COALESCE(c.category, ''), m.received_at
|
||||
FROM messages m
|
||||
LEFT JOIN classifications c ON c.id = (
|
||||
SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1
|
||||
)
|
||||
WHERE (m.list_unsubscribe IS NOT NULL AND m.list_unsubscribe <> '')
|
||||
OR instr(lower(COALESCE(m.body_text, '')), 'unsubscribe') > 0
|
||||
ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC
|
||||
LIMIT ?`
|
||||
rows, err := s.db.QueryContext(ctx, q, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []UnsubscribeCandidate
|
||||
for rows.Next() {
|
||||
var (
|
||||
c UnsubscribeCandidate
|
||||
received string
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&c.MessageID, &c.FromAddr, &c.Subject, &c.SourceAddress,
|
||||
&c.BodyText, &c.ListUnsubscribe, &c.ListUnsubscribePost,
|
||||
&c.Category, &received,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan unsubscribe candidate: %w", err)
|
||||
}
|
||||
c.ReceivedAt = parseSQLTime(received)
|
||||
out = append(out, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("list unsubscribe candidates: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
193
internal/store/unsubscribe_test.go
Normal file
193
internal/store/unsubscribe_test.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMigrationV6AddsUnsubscribeColumns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
|
||||
// Simulate a v5 database: messages has no List-Unsubscribe columns and the
|
||||
// unsubscribe table has no url/updated_at. Opening it must apply migration
|
||||
// v6.
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatalf("open old db: %v", err)
|
||||
}
|
||||
stmts := []string{
|
||||
`CREATE TABLE messages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
gmail_id TEXT NOT NULL UNIQUE,
|
||||
thread_id TEXT, message_id_header TEXT, from_addr TEXT, to_addr TEXT,
|
||||
subject TEXT, body_text TEXT, source_address TEXT, date_sent TEXT,
|
||||
imap_uid INTEGER, imap_uidvalidity INTEGER,
|
||||
folder TEXT NOT NULL DEFAULT 'inbox',
|
||||
mailbox TEXT NOT NULL DEFAULT 'INBOX',
|
||||
received_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
|
||||
`CREATE TABLE unsubscribe (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
domain TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')))`,
|
||||
`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`,
|
||||
`INSERT INTO schema_migrations (version) VALUES (5)`,
|
||||
}
|
||||
for _, q := range stmts {
|
||||
if _, err := db.ExecContext(ctx, q); err != nil {
|
||||
t.Fatalf("seed old db: %v", err)
|
||||
}
|
||||
}
|
||||
_ = db.Close()
|
||||
|
||||
s, err := Open(ctx, path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open() on v5 database error = %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
v, err := s.SchemaVersion(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("SchemaVersion() error = %v", err)
|
||||
}
|
||||
if v < 6 {
|
||||
t.Errorf("SchemaVersion() = %d, want >= 6", v)
|
||||
}
|
||||
for _, tc := range []struct{ table, col string }{
|
||||
{"messages", "list_unsubscribe"},
|
||||
{"messages", "list_unsubscribe_post"},
|
||||
{"unsubscribe", "url"},
|
||||
{"unsubscribe", "updated_at"},
|
||||
} {
|
||||
var n int
|
||||
q := `SELECT COUNT(*) FROM pragma_table_info('` + tc.table + `') WHERE name = ?`
|
||||
if err := s.DB().QueryRowContext(ctx, q, tc.col).Scan(&n); err != nil {
|
||||
t.Fatalf("count %s.%s: %v", tc.table, tc.col, err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("%s.%s missing after migration v6", tc.table, tc.col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeRecordUpsert(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newCategoryStore(t) // reuses the store helper from categories_test.go
|
||||
|
||||
if _, ok, err := s.UnsubscribeByDomain(ctx, "brand.example"); err != nil || ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a miss", ok, err)
|
||||
}
|
||||
|
||||
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusIgnored, "https://brand.example/u"); err != nil {
|
||||
t.Fatalf("SetUnsubscribeStatus(ignore) error = %v", err)
|
||||
}
|
||||
rec, ok, err := s.UnsubscribeByDomain(ctx, "brand.example")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if rec.Status != UnsubscribeStatusIgnored || rec.URL != "https://brand.example/u" {
|
||||
t.Errorf("record = %+v, want ignored with url", rec)
|
||||
}
|
||||
|
||||
// A later mark with no URL must keep the recorded link.
|
||||
if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusDone, ""); err != nil {
|
||||
t.Fatalf("SetUnsubscribeStatus(done) error = %v", err)
|
||||
}
|
||||
rec, _, _ = s.UnsubscribeByDomain(ctx, "brand.example")
|
||||
if rec.Status != UnsubscribeStatusDone {
|
||||
t.Errorf("status = %q, want done", rec.Status)
|
||||
}
|
||||
if rec.URL != "https://brand.example/u" {
|
||||
t.Errorf("url = %q, want the earlier link preserved", rec.URL)
|
||||
}
|
||||
|
||||
if err := s.SetUnsubscribeStatus(ctx, "brand.example", "bogus", ""); err == nil {
|
||||
t.Error("SetUnsubscribeStatus(bogus) accepted an unknown status")
|
||||
}
|
||||
|
||||
records, err := s.ListUnsubscribe(ctx)
|
||||
if err != nil || len(records) != 1 {
|
||||
t.Fatalf("ListUnsubscribe() = (%v, %v), want one record", records, err)
|
||||
}
|
||||
|
||||
if err := s.ClearUnsubscribe(ctx, "brand.example"); err != nil {
|
||||
t.Fatalf("ClearUnsubscribe() error = %v", err)
|
||||
}
|
||||
if err := s.SetUnsubscribeStatus(ctx, "", UnsubscribeStatusIgnored, ""); err == nil {
|
||||
t.Error("SetUnsubscribeStatus accepted an empty domain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newCategoryStore(t)
|
||||
|
||||
seed := func(gmailID, from, body, header string) {
|
||||
t.Helper()
|
||||
if _, err := s.UpsertMessage(ctx, &Message{
|
||||
GmailID: gmailID,
|
||||
FromAddr: from,
|
||||
Subject: "subject " + gmailID,
|
||||
BodyText: body,
|
||||
ListUnsubscribe: header,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", gmailID, err)
|
||||
}
|
||||
}
|
||||
seed("header-msg", "Support <support@brand.example>", "hello", "<https://brand.example/u>")
|
||||
seed("body-msg", "Shop <deals@shop.example>", "You can unsubscribe at https://shop.example/opt-out now.", "")
|
||||
seed("plain-msg", "Friend <friend@example.com>", "Just saying hi.", "")
|
||||
|
||||
got, err := s.UnsubscribeCandidates(ctx, 50)
|
||||
if err != nil {
|
||||
t.Fatalf("UnsubscribeCandidates() error = %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("candidates = %d, want 2 (%+v)", len(got), got)
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for _, c := range got {
|
||||
ids[c.Subject] = true
|
||||
}
|
||||
if !ids["subject body-msg"] || !ids["subject header-msg"] {
|
||||
t.Errorf("candidates = %+v, want the header and body messages", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesVersionsAndRestore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newCategoryStore(t)
|
||||
|
||||
if _, err := s.SaveRules(ctx, "from contains \"a\" => category=Info"); err != nil {
|
||||
t.Fatalf("SaveRules() error = %v", err)
|
||||
}
|
||||
if _, err := s.SaveRules(ctx, "from contains \"b\" => category=Action"); err != nil {
|
||||
t.Fatalf("SaveRules() error = %v", err)
|
||||
}
|
||||
|
||||
versions, err := s.RulesVersions(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RulesVersions() error = %v", err)
|
||||
}
|
||||
if len(versions) < 3 {
|
||||
t.Fatalf("RulesVersions() = %d, want >= 3 (seed + 2 saves)", len(versions))
|
||||
}
|
||||
if versions[0].Version != 3 {
|
||||
t.Errorf("newest version = %d, want 3", versions[0].Version)
|
||||
}
|
||||
|
||||
text, ok, err := s.RuleByVersion(ctx, 2)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("RuleByVersion(2) = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if text != "from contains \"a\" => category=Info" {
|
||||
t.Errorf("RuleByVersion(2) = %q", text)
|
||||
}
|
||||
if _, ok, _ := s.RuleByVersion(ctx, 999); ok {
|
||||
t.Error("RuleByVersion(999) reported a hit")
|
||||
}
|
||||
}
|
||||
240
internal/unsubscribe/unsubscribe.go
Normal file
240
internal/unsubscribe/unsubscribe.go
Normal file
@@ -0,0 +1,240 @@
|
||||
// Package unsubscribe detects unsubscribe links (FR-7).
|
||||
//
|
||||
// The primary source is the List-Unsubscribe header (RFC 2369) together with
|
||||
// the List-Unsubscribe-Post header (RFC 8058). When neither is present the body
|
||||
// is scanned for a URL on a line that mentions "unsubscribe".
|
||||
//
|
||||
// Detection only. This package never fetches a URL and never sends mail.
|
||||
// Acting on an unsubscribe link needs Sam's explicit approval (AGENTS.md §10
|
||||
// and AUD-72); the app only marks and records.
|
||||
package unsubscribe
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Link methods.
|
||||
const (
|
||||
MethodHTTP = "http"
|
||||
MethodMailto = "mailto"
|
||||
)
|
||||
|
||||
// Detection sources. They are stored in the action log so a marking is
|
||||
// auditable.
|
||||
const (
|
||||
SourceListUnsubscribe = "list-unsubscribe"
|
||||
SourceBody = "body"
|
||||
)
|
||||
|
||||
// Link is one detected unsubscribe target.
|
||||
type Link struct {
|
||||
// URL is the raw target, for example "https://x.example/u?id=1".
|
||||
URL string
|
||||
// Method is MethodHTTP or MethodMailto.
|
||||
Method string
|
||||
// Domain is the ignore-list key. It is the sender's brand domain when that
|
||||
// is a real domain, otherwise the link host.
|
||||
Domain string
|
||||
// Host is the host of the link itself, shown for context.
|
||||
Host string
|
||||
// Source is SourceListUnsubscribe or SourceBody.
|
||||
Source string
|
||||
}
|
||||
|
||||
// Result is the detection outcome for one message.
|
||||
type Result struct {
|
||||
Links []Link
|
||||
// Header is the raw List-Unsubscribe value, if any.
|
||||
Header string
|
||||
// OneClick reports a RFC 8058 List-Unsubscribe=One-Click header.
|
||||
OneClick bool
|
||||
}
|
||||
|
||||
// Detect returns the unsubscribe links for a message. from is the raw From
|
||||
// header; body is the stored plain-text body.
|
||||
func Detect(listUnsubscribe, listUnsubscribePost, from, body string) Result {
|
||||
res := Result{
|
||||
Header: strings.TrimSpace(listUnsubscribe),
|
||||
OneClick: isOneClick(listUnsubscribePost),
|
||||
}
|
||||
sender := brandDomain(from)
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
add := func(raw, source string) {
|
||||
link, ok := makeLink(raw, source, sender)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
key := link.Method + ":" + strings.ToLower(link.URL)
|
||||
if _, dup := seen[key]; dup {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
res.Links = append(res.Links, link)
|
||||
}
|
||||
|
||||
for _, raw := range headerLinks(res.Header) {
|
||||
add(raw, SourceListUnsubscribe)
|
||||
}
|
||||
if len(res.Links) == 0 {
|
||||
for _, raw := range bodyLinks(body) {
|
||||
add(raw, SourceBody)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// isOneClick reports a RFC 8058 one-click unsubscribe header. The documented
|
||||
// value is "List-Unsubscribe=One-Click".
|
||||
func isOneClick(v string) bool {
|
||||
return strings.Contains(strings.ToLower(v), "one-click")
|
||||
}
|
||||
|
||||
var angleRE = regexp.MustCompile(`<([^>]+)>`)
|
||||
|
||||
// headerLinks extracts the URIs from a List-Unsubscribe value. The RFC form is
|
||||
// a comma-separated list of angle-bracketed URIs; a bare URI is also accepted.
|
||||
func headerLinks(v string) []string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range angleRE.FindAllStringSubmatch(v, -1) {
|
||||
if s := strings.TrimSpace(m[1]); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
for _, part := range strings.Split(v, ",") {
|
||||
if s := strings.TrimSpace(part); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
bodyURLRE = regexp.MustCompile(`(?i)(https?://[^\s<>"')\]]+|mailto:[^\s<>"')\]]+)`)
|
||||
unsubRE = regexp.MustCompile(`(?i)unsubscribe|opt[\s-]?out|manage (?:your )?(?:email )?preferences`)
|
||||
)
|
||||
|
||||
// bodyLinks finds unsubscribe URLs in a plain-text body. Only lines that
|
||||
// mention unsubscribing are considered, so a random link is not reported.
|
||||
func bodyLinks(body string) []string {
|
||||
body = strings.TrimSpace(body)
|
||||
if body == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if !unsubRE.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
for _, m := range bodyURLRE.FindAllString(line, -1) {
|
||||
out = append(out, trimURL(m))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trimURL removes trailing punctuation that is part of the sentence, not the
|
||||
// URL.
|
||||
func trimURL(s string) string {
|
||||
return strings.TrimRight(strings.TrimSpace(s), ".,;:!?)\"'")
|
||||
}
|
||||
|
||||
// makeLink validates a raw target and fills in method/host/domain.
|
||||
func makeLink(raw, source, sender string) (Link, bool) {
|
||||
raw = trimURL(raw)
|
||||
raw = strings.Trim(raw, "<>")
|
||||
if raw == "" {
|
||||
return Link{}, false
|
||||
}
|
||||
link := Link{URL: raw, Source: source}
|
||||
switch {
|
||||
case strings.HasPrefix(strings.ToLower(raw), "mailto:"):
|
||||
link.Method = MethodMailto
|
||||
link.Host = mailtoHost(raw)
|
||||
case strings.HasPrefix(strings.ToLower(raw), "http://"),
|
||||
strings.HasPrefix(strings.ToLower(raw), "https://"):
|
||||
link.Method = MethodHTTP
|
||||
link.Host = urlHost(raw)
|
||||
default:
|
||||
return Link{}, false
|
||||
}
|
||||
link.Domain = sender
|
||||
if link.Domain == "" {
|
||||
link.Domain = link.Host
|
||||
}
|
||||
return link, link.Domain != ""
|
||||
}
|
||||
|
||||
// Domain returns the domain used as the ignore-list key. It is exported for
|
||||
// callers that need it without parsing a link.
|
||||
func Domain(from string) string { return brandDomain(from) }
|
||||
|
||||
func brandDomain(from string) string {
|
||||
addr := extractAddress(from)
|
||||
i := strings.LastIndex(addr, "@")
|
||||
if i < 0 || i == len(addr)-1 {
|
||||
return ""
|
||||
}
|
||||
dom := strings.ToLower(strings.Trim(strings.TrimSpace(addr[i+1:]), ">."))
|
||||
if dom == "" || freeMail[dom] {
|
||||
return ""
|
||||
}
|
||||
return dom
|
||||
}
|
||||
|
||||
// extractAddress pulls the address out of a From header such as
|
||||
// "Support <support@example.com>".
|
||||
func extractAddress(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if i := strings.LastIndex(v, "<"); i >= 0 {
|
||||
if j := strings.Index(v[i:], ">"); j > 0 {
|
||||
return strings.TrimSpace(v[i+1 : i+j])
|
||||
}
|
||||
}
|
||||
return strings.Trim(v, "<> \t")
|
||||
}
|
||||
|
||||
// freeMail domains are consumer mailboxes, not brands. A message forwarded
|
||||
// through one should key its ignore list on the real link host instead.
|
||||
var freeMail = map[string]bool{
|
||||
"gmail.com": true,
|
||||
"googlemail.com": true,
|
||||
"outlook.com": true,
|
||||
"hotmail.com": true,
|
||||
"live.com": true,
|
||||
"yahoo.com": true,
|
||||
"icloud.com": true,
|
||||
"me.com": true,
|
||||
"aol.com": true,
|
||||
"proton.me": true,
|
||||
"protonmail.com": true,
|
||||
}
|
||||
|
||||
func urlHost(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(u.Hostname())
|
||||
}
|
||||
|
||||
func mailtoHost(raw string) string {
|
||||
rest := raw[len("mailto:"):]
|
||||
if i := strings.IndexAny(rest, "?,"); i >= 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
i := strings.LastIndex(rest, "@")
|
||||
if i < 0 || i == len(rest)-1 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(strings.Trim(rest[i+1:], ">."))
|
||||
}
|
||||
126
internal/unsubscribe/unsubscribe_test.go
Normal file
126
internal/unsubscribe/unsubscribe_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package unsubscribe
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDetectListUnsubscribeHeader(t *testing.T) {
|
||||
res := Detect(
|
||||
"<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=42>",
|
||||
"List-Unsubscribe=One-Click",
|
||||
"Heritage Fruit Trees <support@heritagefruittrees.com.au>",
|
||||
"Special offer this week.",
|
||||
)
|
||||
if !res.OneClick {
|
||||
t.Error("OneClick = false, want true")
|
||||
}
|
||||
if len(res.Links) != 2 {
|
||||
t.Fatalf("Links len = %d, want 2 (%+v)", len(res.Links), res.Links)
|
||||
}
|
||||
mail := res.Links[0]
|
||||
if mail.Method != MethodMailto {
|
||||
t.Errorf("mailto method = %q", mail.Method)
|
||||
}
|
||||
if mail.Domain != "heritagefruittrees.com.au" {
|
||||
t.Errorf("mailto domain = %q", mail.Domain)
|
||||
}
|
||||
if mail.Source != SourceListUnsubscribe {
|
||||
t.Errorf("mailto source = %q", mail.Source)
|
||||
}
|
||||
http := res.Links[1]
|
||||
if http.Method != MethodHTTP {
|
||||
t.Errorf("http method = %q", http.Method)
|
||||
}
|
||||
if http.Host != "heritagefruittrees.com.au" {
|
||||
t.Errorf("http host = %q", http.Host)
|
||||
}
|
||||
if http.Domain != "heritagefruittrees.com.au" {
|
||||
t.Errorf("http domain = %q", http.Domain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBareHeaderURL(t *testing.T) {
|
||||
res := Detect("https://lists.example.com/unsub?u=abc,", "", "News <news@lists.example.com>", "")
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].URL != "https://lists.example.com/unsub?u=abc" {
|
||||
t.Errorf("URL = %q", res.Links[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBodyFallback(t *testing.T) {
|
||||
res := Detect("", "", "Shop <deals@shop.example>", "Thanks for subscribing!\nUnsubscribe here: https://shop.example/opt-out?token=1.\nSee you.")
|
||||
if res.Header != "" {
|
||||
t.Errorf("Header = %q, want empty", res.Header)
|
||||
}
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].Source != SourceBody {
|
||||
t.Errorf("Source = %q, want %q", res.Links[0].Source, SourceBody)
|
||||
}
|
||||
if res.Links[0].URL != "https://shop.example/opt-out?token=1" {
|
||||
t.Errorf("URL = %q (trailing period should be trimmed)", res.Links[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectIgnoresUnrelatedBodyLinks(t *testing.T) {
|
||||
res := Detect("", "", "Friend <friend@example.com>", "Read more at https://example.com/blog")
|
||||
if len(res.Links) != 0 {
|
||||
t.Fatalf("Links = %+v, want none", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBodyMailto(t *testing.T) {
|
||||
res := Detect("", "", "Brand <hello@brand.example>", "To unsubscribe, email mailto:leave@brand.example")
|
||||
if len(res.Links) != 1 || res.Links[0].Method != MethodMailto {
|
||||
t.Fatalf("Links = %+v, want one mailto", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeMailSenderUsesLinkHost(t *testing.T) {
|
||||
res := Detect(
|
||||
"<https://mailer.example.com/u/9>",
|
||||
"",
|
||||
"Someone <someone@gmail.com>",
|
||||
"",
|
||||
)
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].Domain != "mailer.example.com" {
|
||||
t.Errorf("domain = %q, want link host", res.Links[0].Domain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDeduplicates(t *testing.T) {
|
||||
res := Detect(
|
||||
"<https://x.example/u>, <https://x.example/u>",
|
||||
"",
|
||||
"Brand <hello@brand.example>",
|
||||
"",
|
||||
)
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1 after de-duplication", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDomain(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"Support <support@heritagefruittrees.com.au>": "heritagefruittrees.com.au",
|
||||
"plain@brand.example": "brand.example",
|
||||
"Someone <someone@gmail.com>": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := Domain(in); got != want {
|
||||
t.Errorf("Domain(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInputs(t *testing.T) {
|
||||
res := Detect("", "", "", "")
|
||||
if len(res.Links) != 0 || res.OneClick {
|
||||
t.Fatalf("Detect(empty) = %+v, want zero value", res)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user