Phase 6: rules editor + unsubscribe detection (AUD-72)
- FR-11 rules editor: plain-text area at /rules, saved and versioned (SaveRules appends a version; restore re-saves an old version), parse errors surfaced, one-click re-classify. The reply model now reads the rules document too (FR-11). - FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post (migration v6), detect email and web links (internal/unsubscribe), a domain-level ignore list, and an explicit record of what was marked. No code path contacts an unsubscribe link; acting on one needs Sam's approval. - Migration v6 is additive; unsubscribe gains url/updated_at. - Tests: detector, parser, store migration/records/candidates, rules editor, unsubscribe handlers, message-detail rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
240
internal/unsubscribe/unsubscribe.go
Normal file
240
internal/unsubscribe/unsubscribe.go
Normal file
@@ -0,0 +1,240 @@
|
||||
// Package unsubscribe detects unsubscribe links (FR-7).
|
||||
//
|
||||
// The primary source is the List-Unsubscribe header (RFC 2369) together with
|
||||
// the List-Unsubscribe-Post header (RFC 8058). When neither is present the body
|
||||
// is scanned for a URL on a line that mentions "unsubscribe".
|
||||
//
|
||||
// Detection only. This package never fetches a URL and never sends mail.
|
||||
// Acting on an unsubscribe link needs Sam's explicit approval (AGENTS.md §10
|
||||
// and AUD-72); the app only marks and records.
|
||||
package unsubscribe
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Link methods.
|
||||
const (
|
||||
MethodHTTP = "http"
|
||||
MethodMailto = "mailto"
|
||||
)
|
||||
|
||||
// Detection sources. They are stored in the action log so a marking is
|
||||
// auditable.
|
||||
const (
|
||||
SourceListUnsubscribe = "list-unsubscribe"
|
||||
SourceBody = "body"
|
||||
)
|
||||
|
||||
// Link is one detected unsubscribe target.
|
||||
type Link struct {
|
||||
// URL is the raw target, for example "https://x.example/u?id=1".
|
||||
URL string
|
||||
// Method is MethodHTTP or MethodMailto.
|
||||
Method string
|
||||
// Domain is the ignore-list key. It is the sender's brand domain when that
|
||||
// is a real domain, otherwise the link host.
|
||||
Domain string
|
||||
// Host is the host of the link itself, shown for context.
|
||||
Host string
|
||||
// Source is SourceListUnsubscribe or SourceBody.
|
||||
Source string
|
||||
}
|
||||
|
||||
// Result is the detection outcome for one message.
|
||||
type Result struct {
|
||||
Links []Link
|
||||
// Header is the raw List-Unsubscribe value, if any.
|
||||
Header string
|
||||
// OneClick reports a RFC 8058 List-Unsubscribe=One-Click header.
|
||||
OneClick bool
|
||||
}
|
||||
|
||||
// Detect returns the unsubscribe links for a message. from is the raw From
|
||||
// header; body is the stored plain-text body.
|
||||
func Detect(listUnsubscribe, listUnsubscribePost, from, body string) Result {
|
||||
res := Result{
|
||||
Header: strings.TrimSpace(listUnsubscribe),
|
||||
OneClick: isOneClick(listUnsubscribePost),
|
||||
}
|
||||
sender := brandDomain(from)
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
add := func(raw, source string) {
|
||||
link, ok := makeLink(raw, source, sender)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
key := link.Method + ":" + strings.ToLower(link.URL)
|
||||
if _, dup := seen[key]; dup {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
res.Links = append(res.Links, link)
|
||||
}
|
||||
|
||||
for _, raw := range headerLinks(res.Header) {
|
||||
add(raw, SourceListUnsubscribe)
|
||||
}
|
||||
if len(res.Links) == 0 {
|
||||
for _, raw := range bodyLinks(body) {
|
||||
add(raw, SourceBody)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// isOneClick reports a RFC 8058 one-click unsubscribe header. The documented
|
||||
// value is "List-Unsubscribe=One-Click".
|
||||
func isOneClick(v string) bool {
|
||||
return strings.Contains(strings.ToLower(v), "one-click")
|
||||
}
|
||||
|
||||
var angleRE = regexp.MustCompile(`<([^>]+)>`)
|
||||
|
||||
// headerLinks extracts the URIs from a List-Unsubscribe value. The RFC form is
|
||||
// a comma-separated list of angle-bracketed URIs; a bare URI is also accepted.
|
||||
func headerLinks(v string) []string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range angleRE.FindAllStringSubmatch(v, -1) {
|
||||
if s := strings.TrimSpace(m[1]); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
for _, part := range strings.Split(v, ",") {
|
||||
if s := strings.TrimSpace(part); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
bodyURLRE = regexp.MustCompile(`(?i)(https?://[^\s<>"')\]]+|mailto:[^\s<>"')\]]+)`)
|
||||
unsubRE = regexp.MustCompile(`(?i)unsubscribe|opt[\s-]?out|manage (?:your )?(?:email )?preferences`)
|
||||
)
|
||||
|
||||
// bodyLinks finds unsubscribe URLs in a plain-text body. Only lines that
|
||||
// mention unsubscribing are considered, so a random link is not reported.
|
||||
func bodyLinks(body string) []string {
|
||||
body = strings.TrimSpace(body)
|
||||
if body == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if !unsubRE.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
for _, m := range bodyURLRE.FindAllString(line, -1) {
|
||||
out = append(out, trimURL(m))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trimURL removes trailing punctuation that is part of the sentence, not the
|
||||
// URL.
|
||||
func trimURL(s string) string {
|
||||
return strings.TrimRight(strings.TrimSpace(s), ".,;:!?)\"'")
|
||||
}
|
||||
|
||||
// makeLink validates a raw target and fills in method/host/domain.
|
||||
func makeLink(raw, source, sender string) (Link, bool) {
|
||||
raw = trimURL(raw)
|
||||
raw = strings.Trim(raw, "<>")
|
||||
if raw == "" {
|
||||
return Link{}, false
|
||||
}
|
||||
link := Link{URL: raw, Source: source}
|
||||
switch {
|
||||
case strings.HasPrefix(strings.ToLower(raw), "mailto:"):
|
||||
link.Method = MethodMailto
|
||||
link.Host = mailtoHost(raw)
|
||||
case strings.HasPrefix(strings.ToLower(raw), "http://"),
|
||||
strings.HasPrefix(strings.ToLower(raw), "https://"):
|
||||
link.Method = MethodHTTP
|
||||
link.Host = urlHost(raw)
|
||||
default:
|
||||
return Link{}, false
|
||||
}
|
||||
link.Domain = sender
|
||||
if link.Domain == "" {
|
||||
link.Domain = link.Host
|
||||
}
|
||||
return link, link.Domain != ""
|
||||
}
|
||||
|
||||
// Domain returns the domain used as the ignore-list key. It is exported for
|
||||
// callers that need it without parsing a link.
|
||||
func Domain(from string) string { return brandDomain(from) }
|
||||
|
||||
func brandDomain(from string) string {
|
||||
addr := extractAddress(from)
|
||||
i := strings.LastIndex(addr, "@")
|
||||
if i < 0 || i == len(addr)-1 {
|
||||
return ""
|
||||
}
|
||||
dom := strings.ToLower(strings.Trim(strings.TrimSpace(addr[i+1:]), ">."))
|
||||
if dom == "" || freeMail[dom] {
|
||||
return ""
|
||||
}
|
||||
return dom
|
||||
}
|
||||
|
||||
// extractAddress pulls the address out of a From header such as
|
||||
// "Support <support@example.com>".
|
||||
func extractAddress(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if i := strings.LastIndex(v, "<"); i >= 0 {
|
||||
if j := strings.Index(v[i:], ">"); j > 0 {
|
||||
return strings.TrimSpace(v[i+1 : i+j])
|
||||
}
|
||||
}
|
||||
return strings.Trim(v, "<> \t")
|
||||
}
|
||||
|
||||
// freeMail domains are consumer mailboxes, not brands. A message forwarded
|
||||
// through one should key its ignore list on the real link host instead.
|
||||
var freeMail = map[string]bool{
|
||||
"gmail.com": true,
|
||||
"googlemail.com": true,
|
||||
"outlook.com": true,
|
||||
"hotmail.com": true,
|
||||
"live.com": true,
|
||||
"yahoo.com": true,
|
||||
"icloud.com": true,
|
||||
"me.com": true,
|
||||
"aol.com": true,
|
||||
"proton.me": true,
|
||||
"protonmail.com": true,
|
||||
}
|
||||
|
||||
func urlHost(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(u.Hostname())
|
||||
}
|
||||
|
||||
func mailtoHost(raw string) string {
|
||||
rest := raw[len("mailto:"):]
|
||||
if i := strings.IndexAny(rest, "?,"); i >= 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
i := strings.LastIndex(rest, "@")
|
||||
if i < 0 || i == len(rest)-1 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(strings.Trim(rest[i+1:], ">."))
|
||||
}
|
||||
126
internal/unsubscribe/unsubscribe_test.go
Normal file
126
internal/unsubscribe/unsubscribe_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package unsubscribe
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDetectListUnsubscribeHeader(t *testing.T) {
|
||||
res := Detect(
|
||||
"<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=42>",
|
||||
"List-Unsubscribe=One-Click",
|
||||
"Heritage Fruit Trees <support@heritagefruittrees.com.au>",
|
||||
"Special offer this week.",
|
||||
)
|
||||
if !res.OneClick {
|
||||
t.Error("OneClick = false, want true")
|
||||
}
|
||||
if len(res.Links) != 2 {
|
||||
t.Fatalf("Links len = %d, want 2 (%+v)", len(res.Links), res.Links)
|
||||
}
|
||||
mail := res.Links[0]
|
||||
if mail.Method != MethodMailto {
|
||||
t.Errorf("mailto method = %q", mail.Method)
|
||||
}
|
||||
if mail.Domain != "heritagefruittrees.com.au" {
|
||||
t.Errorf("mailto domain = %q", mail.Domain)
|
||||
}
|
||||
if mail.Source != SourceListUnsubscribe {
|
||||
t.Errorf("mailto source = %q", mail.Source)
|
||||
}
|
||||
http := res.Links[1]
|
||||
if http.Method != MethodHTTP {
|
||||
t.Errorf("http method = %q", http.Method)
|
||||
}
|
||||
if http.Host != "heritagefruittrees.com.au" {
|
||||
t.Errorf("http host = %q", http.Host)
|
||||
}
|
||||
if http.Domain != "heritagefruittrees.com.au" {
|
||||
t.Errorf("http domain = %q", http.Domain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBareHeaderURL(t *testing.T) {
|
||||
res := Detect("https://lists.example.com/unsub?u=abc,", "", "News <news@lists.example.com>", "")
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].URL != "https://lists.example.com/unsub?u=abc" {
|
||||
t.Errorf("URL = %q", res.Links[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBodyFallback(t *testing.T) {
|
||||
res := Detect("", "", "Shop <deals@shop.example>", "Thanks for subscribing!\nUnsubscribe here: https://shop.example/opt-out?token=1.\nSee you.")
|
||||
if res.Header != "" {
|
||||
t.Errorf("Header = %q, want empty", res.Header)
|
||||
}
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].Source != SourceBody {
|
||||
t.Errorf("Source = %q, want %q", res.Links[0].Source, SourceBody)
|
||||
}
|
||||
if res.Links[0].URL != "https://shop.example/opt-out?token=1" {
|
||||
t.Errorf("URL = %q (trailing period should be trimmed)", res.Links[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectIgnoresUnrelatedBodyLinks(t *testing.T) {
|
||||
res := Detect("", "", "Friend <friend@example.com>", "Read more at https://example.com/blog")
|
||||
if len(res.Links) != 0 {
|
||||
t.Fatalf("Links = %+v, want none", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectBodyMailto(t *testing.T) {
|
||||
res := Detect("", "", "Brand <hello@brand.example>", "To unsubscribe, email mailto:leave@brand.example")
|
||||
if len(res.Links) != 1 || res.Links[0].Method != MethodMailto {
|
||||
t.Fatalf("Links = %+v, want one mailto", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeMailSenderUsesLinkHost(t *testing.T) {
|
||||
res := Detect(
|
||||
"<https://mailer.example.com/u/9>",
|
||||
"",
|
||||
"Someone <someone@gmail.com>",
|
||||
"",
|
||||
)
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1", res.Links)
|
||||
}
|
||||
if res.Links[0].Domain != "mailer.example.com" {
|
||||
t.Errorf("domain = %q, want link host", res.Links[0].Domain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDeduplicates(t *testing.T) {
|
||||
res := Detect(
|
||||
"<https://x.example/u>, <https://x.example/u>",
|
||||
"",
|
||||
"Brand <hello@brand.example>",
|
||||
"",
|
||||
)
|
||||
if len(res.Links) != 1 {
|
||||
t.Fatalf("Links = %+v, want 1 after de-duplication", res.Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDomain(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"Support <support@heritagefruittrees.com.au>": "heritagefruittrees.com.au",
|
||||
"plain@brand.example": "brand.example",
|
||||
"Someone <someone@gmail.com>": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := Domain(in); got != want {
|
||||
t.Errorf("Domain(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInputs(t *testing.T) {
|
||||
res := Detect("", "", "", "")
|
||||
if len(res.Links) != 0 || res.OneClick {
|
||||
t.Fatalf("Detect(empty) = %+v, want zero value", res)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user