From 00bb0e05c4d263158b72b4b59625cceb61f55f6c Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Sun, 11 Oct 2026 22:24:38 +1100 Subject: [PATCH] Phase 6: rules editor + unsubscribe detection (AUD-72) - FR-11 rules editor: plain-text area at /rules, saved and versioned (SaveRules appends a version; restore re-saves an old version), parse errors surfaced, one-click re-classify. The reply model now reads the rules document too (FR-11). - FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post (migration v6), detect email and web links (internal/unsubscribe), a domain-level ignore list, and an explicit record of what was marked. No code path contacts an unsubscribe link; acting on one needs Sam's approval. - Migration v6 is additive; unsubscribe gains url/updated_at. - Tests: detector, parser, store migration/records/candidates, rules editor, unsubscribe handlers, message-detail rendering. Co-Authored-By: Paperclip --- internal/mailfetch/fetch.go | 26 +-- internal/mailfetch/message.go | 21 +- internal/mailfetch/message_test.go | 31 +++ internal/reply/draft.go | 12 ++ internal/reply/draft_test.go | 13 ++ internal/server/replies.go | 20 ++ internal/server/rules.go | 225 +++++++++++++++++++++ internal/server/rules_test.go | 152 ++++++++++++++ internal/server/server.go | 12 ++ internal/server/unsubscribe.go | 218 ++++++++++++++++++++ internal/server/unsubscribe_test.go | 182 +++++++++++++++++ internal/store/classify.go | 48 +++++ internal/store/messages.go | 55 +++--- internal/store/schema.sql | 11 +- internal/store/store.go | 19 ++ internal/store/unsubscribe.go | 177 +++++++++++++++++ internal/store/unsubscribe_test.go | 193 ++++++++++++++++++ internal/unsubscribe/unsubscribe.go | 240 +++++++++++++++++++++++ internal/unsubscribe/unsubscribe_test.go | 126 ++++++++++++ web/static/app.css | 65 ++++++ web/templates/dashboard.html | 2 + web/templates/message.html | 12 ++ web/templates/rules.html | 125 ++++++++++++ web/templates/unsubscribe.html | 178 +++++++++++++++++ 24 files changed, 2121 insertions(+), 42 deletions(-) create mode 100644 internal/server/rules.go create mode 100644 internal/server/rules_test.go create mode 100644 internal/server/unsubscribe.go create mode 100644 internal/server/unsubscribe_test.go create mode 100644 internal/store/unsubscribe.go create mode 100644 internal/store/unsubscribe_test.go create mode 100644 internal/unsubscribe/unsubscribe.go create mode 100644 internal/unsubscribe/unsubscribe_test.go create mode 100644 web/templates/rules.html create mode 100644 web/templates/unsubscribe.html diff --git a/internal/mailfetch/fetch.go b/internal/mailfetch/fetch.go index f727ced..91dae4f 100644 --- a/internal/mailfetch/fetch.go +++ b/internal/mailfetch/fetch.go @@ -102,18 +102,20 @@ func (f *Fetcher) Fetch(ctx context.Context) (Summary, error) { headersSeen[parsed.SourceHeader] = struct{}{} } msg := &store.Message{ - GmailID: gmailID(r, uidValidity), - ThreadID: r.GmailThreadID, - MessageIDHeader: parsed.MessageID, - FromAddr: parsed.From, - ToAddr: parsed.To, - Subject: parsed.Subject, - BodyText: parsed.BodyText, - SourceAddress: parsed.SourceAddress, - DateSent: parsed.Date, - ReceivedAt: now, - IMAPUID: int64(r.UID), - IMAPUIDValidity: int64(uidValidity), + GmailID: gmailID(r, uidValidity), + ThreadID: r.GmailThreadID, + MessageIDHeader: parsed.MessageID, + FromAddr: parsed.From, + ToAddr: parsed.To, + Subject: parsed.Subject, + BodyText: parsed.BodyText, + SourceAddress: parsed.SourceAddress, + ListUnsubscribe: parsed.ListUnsubscribe, + ListUnsubscribePost: parsed.ListUnsubscribePost, + DateSent: parsed.Date, + ReceivedAt: now, + IMAPUID: int64(r.UID), + IMAPUIDValidity: int64(uidValidity), } if _, err := f.store.UpsertMessage(ctx, msg); err != nil { return sum, err diff --git a/internal/mailfetch/message.go b/internal/mailfetch/message.go index 4f7ca79..06dd57a 100644 --- a/internal/mailfetch/message.go +++ b/internal/mailfetch/message.go @@ -27,8 +27,13 @@ type ParsedMessage struct { Date time.Time // MessageID is the RFC822 Message-ID header, used for reply threading // (Phase 5, FR-8). - MessageID string - SourceAddress string + MessageID string + // ListUnsubscribe and ListUnsubscribePost are the RFC 2369/8058 headers that + // name the unsubscribe targets (Phase 6, FR-7). They are stored raw so the + // detector can run later without a refetch. + ListUnsubscribe string + ListUnsubscribePost string + SourceAddress string // SourceHeader records which header the source address came from, so FR-2 // decisions are auditable. SourceHeader string @@ -43,11 +48,13 @@ func ParseMessage(raw []byte) (*ParsedMessage, error) { } h := msg.Header pm := &ParsedMessage{ - From: decodeHeader(h.Get("From")), - To: decodeHeader(h.Get("To")), - Subject: decodeHeader(h.Get("Subject")), - Date: parseDate(h.Get("Date")), - MessageID: strings.TrimSpace(h.Get("Message-Id")), + From: decodeHeader(h.Get("From")), + To: decodeHeader(h.Get("To")), + Subject: decodeHeader(h.Get("Subject")), + Date: parseDate(h.Get("Date")), + MessageID: strings.TrimSpace(h.Get("Message-Id")), + ListUnsubscribe: strings.TrimSpace(h.Get("List-Unsubscribe")), + ListUnsubscribePost: strings.TrimSpace(h.Get("List-Unsubscribe-Post")), } pm.SourceAddress, pm.SourceHeader = SourceAddress(textproto.MIMEHeader(h)) pm.BodyText = extractBody(textproto.MIMEHeader(h), msg.Body) diff --git a/internal/mailfetch/message_test.go b/internal/mailfetch/message_test.go index 9b5133f..4b6ff2f 100644 --- a/internal/mailfetch/message_test.go +++ b/internal/mailfetch/message_test.go @@ -172,3 +172,34 @@ func TestHeaderAddressParsing(t *testing.T) { } } } + +func TestParseMessageCapturesUnsubscribeHeaders(t *testing.T) { + raw := "From: Heritage Fruit Trees \r\n" + + "To: samuelrolfe@gmail.com\r\n" + + "X-Pm-Original-To: hello@where-woof.com\r\n" + + "Subject: New workshops\r\n" + + "List-Unsubscribe: , \r\n" + + "List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n" + + "Message-Id: \r\n" + + "Date: Mon, 01 Jan 2026 10:00:00 +0000\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "\r\n" + + "Hello, you can unsubscribe at https://heritagefruittrees.com.au/u?id=1.\r\n" + + pm, err := ParseMessage([]byte(raw)) + if err != nil { + t.Fatalf("ParseMessage() error = %v", err) + } + if !strings.Contains(pm.ListUnsubscribe, "mailto:unsub@heritagefruittrees.com.au") { + t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe) + } + if !strings.Contains(pm.ListUnsubscribe, "https://heritagefruittrees.com.au/u?id=1") { + t.Errorf("ListUnsubscribe = %q", pm.ListUnsubscribe) + } + if pm.ListUnsubscribePost != "List-Unsubscribe=One-Click" { + t.Errorf("ListUnsubscribePost = %q", pm.ListUnsubscribePost) + } + if pm.SourceAddress != "hello@where-woof.com" { + t.Errorf("SourceAddress = %q", pm.SourceAddress) + } +} diff --git a/internal/reply/draft.go b/internal/reply/draft.go index ce551d0..9dc04b7 100644 --- a/internal/reply/draft.go +++ b/internal/reply/draft.go @@ -18,6 +18,10 @@ const maxPromptBodyChars = 4000 // maxDraftChars bounds the generated body before it is stored. const maxDraftChars = 8000 +// maxRulesChars bounds how much of the user's rules document is sent to the +// model (FR-11: the LLM reads the rules too). +const maxRulesChars = 2000 + // Request is one draft request. type Request struct { // Message is the mail being replied to. @@ -31,6 +35,9 @@ type Request struct { // Instruction is optional extra guidance Sam typed (for example "say the // invoice is paid"). It is appended to the template instruction. Instruction string + // Rules is the current rules document from the editor (FR-11). The model + // reads it so reply style and handling follow the same rules as triage. + Rules string // Signer is the name used for the sign-off (default "Sam"). Signer string } @@ -107,6 +114,11 @@ func BuildPrompt(req Request, tmpl Template) (system, user string) { b.WriteString(extra) b.WriteString("\n") } + if rules := strings.TrimSpace(req.Rules); rules != "" { + b.WriteString("USER RULES (follow when they apply):\n") + b.WriteString(truncate(rules, maxRulesChars)) + b.WriteString("\n") + } if cat := strings.TrimSpace(req.Category); cat != "" { b.WriteString("CLASSIFICATION: category=") b.WriteString(cat) diff --git a/internal/reply/draft_test.go b/internal/reply/draft_test.go index 3eddd9b..c18693e 100644 --- a/internal/reply/draft_test.go +++ b/internal/reply/draft_test.go @@ -116,3 +116,16 @@ func TestGeneratorDraftEmptyReplyIsAnError(t *testing.T) { t.Fatal("Draft() error = nil, want an error for an empty reply") } } + +func TestBuildPromptIncludesRules(t *testing.T) { + req := Request{ + Message: store.Message{FromAddr: "customer@example.test", Subject: "hi", BodyText: "?"}, + Action: "reply", + Rules: `from contains "example.test" => category=Action`, + } + tmpl, _ := TemplateByID("answer") + _, user := BuildPrompt(req, tmpl) + if !strings.Contains(user, "USER RULES") || !strings.Contains(user, `from contains "example.test"`) { + t.Errorf("user prompt does not include the rules document:\n%s", user) + } +} diff --git a/internal/server/replies.go b/internal/server/replies.go index c66d75f..f99a49b 100644 --- a/internal/server/replies.go +++ b/internal/server/replies.go @@ -19,6 +19,7 @@ import ( "gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply" "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe" ) // ReplyGenerator drafts a reply body. The live implementation is @@ -63,6 +64,11 @@ type replyData struct { LLMReady bool LLMModel string + // Phase 6 (FR-7): detected unsubscribe links for this message, and the + // domain's status on the ignore/recorded list. + UnsubLinks []unsubscribe.Link + UnsubStatus string + Sent bool ThreadUpdated bool RepliedFolder string @@ -109,6 +115,9 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) { } class := s.latestClass(ctx, id) + // FR-11: the reply model reads the current rules document too. + rulesText, _, _ := s.store.CurrentRules(ctx) + body, templateID, err := s.replies.Draft(ctx, reply.Request{ Message: msg, Category: class.Category, @@ -116,6 +125,7 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) { Action: class.Action, TemplateID: r.FormValue("template"), Instruction: r.FormValue("instruction"), + Rules: rulesText, Signer: "Sam", }) if err != nil { @@ -313,6 +323,16 @@ func (s *Server) renderMessage(w http.ResponseWriter, r *http.Request, id int64, } data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == "" + // FR-7: detect unsubscribe links and look up the domain's list status. + if res := unsubscribe.Detect(msg.ListUnsubscribe, msg.ListUnsubscribePost, msg.FromAddr, msg.BodyText); len(res.Links) > 0 { + data.UnsubLinks = res.Links + if rec, ok, rerr := s.store.UnsubscribeByDomain(ctx, res.Links[0].Domain); rerr != nil { + s.logger.Error("load unsubscribe status", "message_id", id, "error", rerr) + } else if ok { + data.UnsubStatus = rec.Status + } + } + if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil { s.logger.Error("load draft", "message_id", id, "error", derr) } else if ok { diff --git a/internal/server/rules.go b/internal/server/rules.go new file mode 100644 index 0000000..40a6d01 --- /dev/null +++ b/internal/server/rules.go @@ -0,0 +1,225 @@ +// Rules editor for Phase 6 (FR-4, FR-11). +// +// The rules document is plain text (see internal/classify for the syntax). It is +// saved as a new numbered version on every save, so the current text and its +// history are both available. The classifier reads the newest version and +// applies it as an override; the reply model also reads it. +package server + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify" +) + +type ruleRevision struct { + Version int + CreatedAt string + Lines int + Preview string +} + +type rulesData struct { + Text string + Version int + Revisions []ruleRevision + Errors []classify.ParseError + RuleCount int + Note string + Error string + CSRFToken string + // ClassifierReady is true when JEV is configured, so the page can offer a + // re-classify action that actually runs. + ClassifierReady bool + TotalMessages int +} + +// handleRules renders the rules editor with the current text and version list. +func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) { + s.renderRules(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error")) +} + +func (s *Server) renderRules(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) { + ctx := r.Context() + text, version, err := s.store.CurrentRules(ctx) + if err != nil { + s.logger.Error("load rules", "error", err) + http.Error(w, "could not load the rules", http.StatusInternalServerError) + return + } + parsed := classify.ParseRules(text) + revisions, err := s.store.RulesVersions(ctx, 20) + if err != nil { + s.logger.Error("load rules versions", "error", err) + } + + revs := make([]ruleRevision, 0, len(revisions)) + for _, rv := range revisions { + revs = append(revs, ruleRevision{ + Version: rv.Version, + CreatedAt: rv.CreatedAt, + Lines: countRuleLines(rv.Text), + Preview: firstRuleLine(rv.Text), + }) + } + total, err := s.store.CountMessages(ctx) + if err != nil { + s.logger.Error("count messages", "error", err) + } + + data := rulesData{ + Text: text, + Version: version, + Revisions: revs, + Errors: parsed.Errors, + RuleCount: len(parsed.Rules), + Note: note, + Error: errMsg, + CSRFToken: s.csrfToken(w, r), + ClassifierReady: s.classifier.Configured(), + TotalMessages: total, + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(code) + if err := s.tmpl.ExecuteTemplate(w, "rules.html", data); err != nil { + s.logger.Error("render rules", "error", err) + } +} + +// handleRulesSave versions the posted text (FR-11). +func (s *Server) handleRulesSave(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + s.redirectRules(w, r, "", "Could not read the form.") + return + } + text := r.FormValue("text") + version, err := s.store.SaveRules(r.Context(), text) + if err != nil { + s.logger.Error("save rules", "error", err) + s.redirectRules(w, r, "", "Could not save the rules: "+err.Error()) + return + } + parsed := classify.ParseRules(text) + s.logAction(r.Context(), "rules_save", "Sam", + "version="+strconv.Itoa(version)+" rules="+strconv.Itoa(len(parsed.Rules))+" parse_errors="+strconv.Itoa(len(parsed.Errors))) + s.redirectRules(w, r, "Saved rules version "+strconv.Itoa(version)+". Re-classify all to apply it to existing mail.", "") +} + +// handleRulesRestore re-saves an older version as the newest one, so history is +// append-only (FR-11). +func (s *Server) handleRulesRestore(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + s.redirectRules(w, r, "", "Could not read the form.") + return + } + version, err := strconv.Atoi(strings.TrimSpace(r.FormValue("version"))) + if err != nil || version < 1 { + s.redirectRules(w, r, "", "Unknown rules version.") + return + } + text, ok, err := s.store.RuleByVersion(r.Context(), version) + if err != nil { + s.redirectRules(w, r, "", "Could not load that version.") + return + } + if !ok { + s.redirectRules(w, r, "", "That rules version no longer exists.") + return + } + newVersion, err := s.store.SaveRules(r.Context(), text) + if err != nil { + s.logger.Error("restore rules", "from_version", version, "error", err) + s.redirectRules(w, r, "", "Could not restore that version: "+err.Error()) + return + } + s.logAction(r.Context(), "rules_restore", "Sam", + "from_version="+strconv.Itoa(version)+" new_version="+strconv.Itoa(newVersion)) + s.redirectRules(w, r, "Restored version "+strconv.Itoa(version)+" as version "+strconv.Itoa(newVersion)+".", "") +} + +// handleRulesReclassify re-runs the classifier over every stored message and +// returns to the rules editor. It is the rules page's "apply" action: saving a +// new version does not re-classify by itself. +func (s *Server) handleRulesReclassify(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if rc := http.NewResponseController(w); rc != nil { + _ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute)) + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute) + defer cancel() + + sum, err := s.classifier.RunAll(ctx) + switch { + case err == nil: + s.logAction(ctx, "reclassify", "Sam", + fmt.Sprintf("considered=%d classified=%d failed=%d rules_version=%d", + sum.Considered, sum.Classified, sum.Failed, sum.RulesVersion)) + s.redirectRules(w, r, fmt.Sprintf( + "Re-classified %d of %d message(s); %d failed. The current rules are now applied.", + sum.Classified, sum.Considered, sum.Failed), "") + case errors.Is(err, classify.ErrNotConfigured): + s.redirectRules(w, r, "", "Classification is not connected yet: set JEV_API_KEY.") + default: + s.logger.Error("rules reclassify failed", "error", err) + s.redirectRules(w, r, "", "Classification failed: "+err.Error()) + } +} + +// redirectRules is POST/redirect/GET so a refresh does not resubmit. +func (s *Server) redirectRules(w http.ResponseWriter, r *http.Request, note, errMsg string) { + q := url.Values{} + if note != "" { + q.Set("note", note) + } + if errMsg != "" { + q.Set("error", errMsg) + } + http.Redirect(w, r, "/rules?"+q.Encode(), http.StatusSeeOther) +} + +// countRuleLines counts the non-empty, non-comment lines in a rules document. +func countRuleLines(text string) int { + n := 0 + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + n++ + } + return n +} + +// firstRuleLine returns a short preview of the first real rule, or a note when +// the document has only comments. +func firstRuleLine(text string) string { + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if len(line) > 80 { + return line[:80] + "…" + } + return line + } + return "(comments only)" +} diff --git a/internal/server/rules_test.go b/internal/server/rules_test.go new file mode 100644 index 0000000..5a6087e --- /dev/null +++ b/internal/server/rules_test.go @@ -0,0 +1,152 @@ +package server + +import ( + "context" + "net/http" + "net/url" + "strings" + "testing" +) + +func TestRulesPageRenders(t *testing.T) { + srv := newTestServer(t) + session, _, _ := login(t, srv) + + rec := getWithSession(t, srv, "/rules", session) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{"Rules editor", "Save new version", "Versions", "Back to inbox", "Unsubscribe"} { + if !strings.Contains(body, want) { + t.Errorf("rules page missing %q", want) + } + } +} + +func TestRulesSaveCreatesVersion(t *testing.T) { + srv := newTestServer(t) + session, csrfCookie, csrf := login(t, srv) + + rule := `from contains "heritagefruittrees.com.au" => category=Action, importance=high, action=file` + rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{ + "csrf": {csrf}, "text": {rule}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("save status = %d, want 303 (body %s)", rec.Code, rec.Body.String()) + } + + text, version, err := srv.store.CurrentRules(context.Background()) + if err != nil { + t.Fatalf("CurrentRules() error = %v", err) + } + if text != rule { + t.Errorf("CurrentRules() text = %q, want the saved rule", text) + } + if version != 2 { + t.Errorf("CurrentRules() version = %d, want 2", version) + } + + // The editor shows the new version and active rule count. + body := getWithSession(t, srv, "/rules", session).Body.String() + if !strings.Contains(body, "version 2") || !strings.Contains(body, "1 active rule") { + t.Errorf("rules page did not reflect the saved version; body:\n%s", body) + } +} + +func TestRulesSaveRequiresCSRF(t *testing.T) { + srv := newTestServer(t) + session, _, _ := login(t, srv) + + rec := postForm(t, srv, "/rules/save", session, nil, url.Values{"text": {"anything"}}) + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", rec.Code) + } +} + +func TestRulesPageShowsParseErrors(t *testing.T) { + srv := newTestServer(t) + session, csrfCookie, csrf := login(t, srv) + + rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{ + "csrf": {csrf}, "text": {"this line has no arrow"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("save status = %d, want 303", rec.Code) + } + body := getWithSession(t, srv, "/rules", session).Body.String() + if !strings.Contains(body, "could not be parsed") || !strings.Contains(body, "missing =>") { + t.Errorf("rules page did not surface the parse error; body:\n%s", body) + } +} + +func TestRulesRestoreCreatesNewVersion(t *testing.T) { + srv := newTestServer(t) + session, csrfCookie, csrf := login(t, srv) + ctx := context.Background() + + first := `from contains "a" => category=Info` + second := `from contains "b" => category=Action` + for _, text := range []string{first, second} { + rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{"csrf": {csrf}, "text": {text}}) + if rec.Code != http.StatusSeeOther { + t.Fatalf("save %q status = %d, want 303", text, rec.Code) + } + } + + // Restore version 2 (the first saved rule) as a new version. + rec := postForm(t, srv, "/rules/restore", session, csrfCookie, url.Values{ + "csrf": {csrf}, "version": {"2"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("restore status = %d, want 303", rec.Code) + } + text, version, err := srv.store.CurrentRules(ctx) + if err != nil { + t.Fatalf("CurrentRules() error = %v", err) + } + if text != first { + t.Errorf("restored text = %q, want %q", text, first) + } + if version != 4 { + t.Errorf("restored version = %d, want 4 (append-only)", version) + } +} + +func TestDashboardLinksRulesAndUnsubscribe(t *testing.T) { + srv := newTestServer(t) + session, _, _ := login(t, srv) + + body := getWithSession(t, srv, "/", session).Body.String() + for _, want := range []string{`href="/rules"`, `href="/unsubscribe"`} { + if !strings.Contains(body, want) { + t.Errorf("dashboard missing %q", want) + } + } +} + +func TestRulesCountsLines(t *testing.T) { + text := "# comment\n\nfrom contains \"a\" => category=Info\nfrom contains \"b\" => category=Action" + if got := countRuleLines(text); got != 2 { + t.Errorf("countRuleLines() = %d, want 2", got) + } + if got := firstRuleLine(text); got != `from contains "a" => category=Info` { + t.Errorf("firstRuleLine() = %q", got) + } + if got := firstRuleLine("# only a comment"); got != "(comments only)" { + t.Errorf("firstRuleLine(comment only) = %q", got) + } +} + +func TestRulesReclassifyWithoutJEVRedirects(t *testing.T) { + srv := newTestServer(t) + session, csrfCookie, csrf := login(t, srv) + + rec := postForm(t, srv, "/rules/reclassify", session, csrfCookie, url.Values{"csrf": {csrf}}) + if rec.Code != http.StatusSeeOther { + t.Fatalf("status = %d, want 303 (body %s)", rec.Code, rec.Body.String()) + } + if loc := rec.Header().Get("Location"); !strings.Contains(loc, "JEV_API_KEY") { + t.Errorf("Location = %q, want a JEV-not-configured note", loc) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 617ac11..a9bb2d3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -149,6 +149,18 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename)) mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove)) + // Phase 6 (FR-4, FR-11): rules editor, saved and versioned. + mux.HandleFunc("GET /rules", s.requireAuth(s.handleRules)) + mux.HandleFunc("POST /rules/save", s.requireAuth(s.handleRulesSave)) + mux.HandleFunc("POST /rules/restore", s.requireAuth(s.handleRulesRestore)) + mux.HandleFunc("POST /rules/reclassify", s.requireAuth(s.handleRulesReclassify)) + + // Phase 6 (FR-7): unsubscribe detection, ignore list, and records. + mux.HandleFunc("GET /unsubscribe", s.requireAuth(s.handleUnsubscribe)) + mux.HandleFunc("POST /unsubscribe/ignore", s.requireAuth(s.handleUnsubscribeIgnore)) + mux.HandleFunc("POST /unsubscribe/unignore", s.requireAuth(s.handleUnsubscribeUnignore)) + mux.HandleFunc("POST /unsubscribe/mark", s.requireAuth(s.handleUnsubscribeMark)) + return s.recoverPanic(s.logRequests(mux)) } diff --git a/internal/server/unsubscribe.go b/internal/server/unsubscribe.go new file mode 100644 index 0000000..28362f9 --- /dev/null +++ b/internal/server/unsubscribe.go @@ -0,0 +1,218 @@ +// Unsubscribe detection view for Phase 6 (FR-7). +// +// The page lists messages with a detected unsubscribe link, the domain-level +// ignore list, and the domains Sam has recorded as unsubscribed. Every action +// records state; none of them contacts the link. Acting on a real unsubscribe +// link is a manual step that needs Sam's explicit approval (AUD-72). +package server + +import ( + "net/http" + "net/url" + "strings" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe" +) + +type unsubscribeItem struct { + MessageID int64 + From string + Subject string + SourceAddr string + Domain string + Links []unsubscribe.Link + Status string + Category string + When string +} + +type unsubscribeData struct { + Items []unsubscribeItem + Ignored []store.UnsubscribeRecord + Recorded []store.UnsubscribeRecord + Note string + Error string + CSRFToken string + CandidateCount int +} + +// handleUnsubscribe renders the detection list and the ignore/recorded lists. +func (s *Server) handleUnsubscribe(w http.ResponseWriter, r *http.Request) { + s.renderUnsubscribe(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error")) +} + +func (s *Server) renderUnsubscribe(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) { + ctx := r.Context() + records, err := s.store.ListUnsubscribe(ctx) + if err != nil { + s.logger.Error("list unsubscribe records", "error", err) + http.Error(w, "could not load the unsubscribe list", http.StatusInternalServerError) + return + } + + candidates, err := s.store.UnsubscribeCandidates(ctx, 300) + if err != nil { + s.logger.Error("list unsubscribe candidates", "error", err) + http.Error(w, "could not load messages", http.StatusInternalServerError) + return + } + + status := make(map[string]string, len(records)) + for _, rec := range records { + status[rec.Domain] = rec.Status + } + + var items []unsubscribeItem + for _, c := range candidates { + res := unsubscribe.Detect(c.ListUnsubscribe, c.ListUnsubscribePost, c.FromAddr, c.BodyText) + if len(res.Links) == 0 { + continue + } + domain := res.Links[0].Domain + items = append(items, unsubscribeItem{ + MessageID: c.MessageID, + From: orDefault(c.FromAddr, "(unknown sender)"), + Subject: orDefault(c.Subject, "(no subject)"), + SourceAddr: c.SourceAddress, + Domain: domain, + Links: res.Links, + Status: status[domain], + Category: c.Category, + When: formatWhen(c.ReceivedAt, c.ReceivedAt), + }) + } + + data := unsubscribeData{ + Items: items, + CSRFToken: s.csrfToken(w, r), + Note: note, + Error: errMsg, + CandidateCount: len(items), + } + for _, rec := range records { + switch rec.Status { + case store.UnsubscribeStatusIgnored: + data.Ignored = append(data.Ignored, rec) + case store.UnsubscribeStatusDone: + data.Recorded = append(data.Recorded, rec) + } + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(code) + if err := s.tmpl.ExecuteTemplate(w, "unsubscribe.html", data); err != nil { + s.logger.Error("render unsubscribe", "error", err) + } +} + +// handleUnsubscribeIgnore adds a domain to the ignore list (FR-7). +func (s *Server) handleUnsubscribeIgnore(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + s.redirectUnsubscribe(w, r, "", "Could not read the form.") + return + } + domain, ok := cleanDomain(r.FormValue("domain")) + if !ok { + s.redirectUnsubscribe(w, r, "", "Unknown domain.") + return + } + if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusIgnored, r.FormValue("url")); err != nil { + s.logger.Error("ignore unsubscribe domain", "domain", domain, "error", err) + s.redirectUnsubscribe(w, r, "", "Could not update the ignore list.") + return + } + s.logAction(r.Context(), "unsubscribe_ignore", "Sam", "domain="+domain) + s.redirectUnsubscribe(w, r, "Added "+domain+" to the ignore list. It is never recommended for unsubscribe.", "") +} + +// handleUnsubscribeUnignore removes a domain from the ignore list. +func (s *Server) handleUnsubscribeUnignore(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + s.redirectUnsubscribe(w, r, "", "Could not read the form.") + return + } + domain, ok := cleanDomain(r.FormValue("domain")) + if !ok { + s.redirectUnsubscribe(w, r, "", "Unknown domain.") + return + } + if err := s.store.ClearUnsubscribe(r.Context(), domain); err != nil { + s.logger.Error("clear unsubscribe domain", "domain", domain, "error", err) + s.redirectUnsubscribe(w, r, "", "Could not update the list.") + return + } + s.logAction(r.Context(), "unsubscribe_unignore", "Sam", "domain="+domain) + s.redirectUnsubscribe(w, r, "Removed "+domain+" from the unsubscribe list.", "") +} + +// handleUnsubscribeMark records that a domain should be unsubscribed from. It +// is a record only: the handler makes no network call and the app never follows +// the link (AUD-72). Following a real link needs Sam's explicit approval. +func (s *Server) handleUnsubscribeMark(w http.ResponseWriter, r *http.Request) { + if !s.csrfOK(r) { + http.Error(w, "invalid CSRF token", http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + s.redirectUnsubscribe(w, r, "", "Could not read the form.") + return + } + if r.FormValue("confirm") != "yes" { + s.redirectUnsubscribe(w, r, "", "Marking needs confirmation. The app will record it, not follow the link.") + return + } + domain, ok := cleanDomain(r.FormValue("domain")) + if !ok { + s.redirectUnsubscribe(w, r, "", "Unknown domain.") + return + } + if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusDone, r.FormValue("url")); err != nil { + s.logger.Error("mark unsubscribe domain", "domain", domain, "error", err) + s.redirectUnsubscribe(w, r, "", "Could not record the unsubscribe.") + return + } + s.logAction(r.Context(), "unsubscribe_marked", "Sam", "domain="+domain+" url="+r.FormValue("url")) + s.redirectUnsubscribe(w, r, + "Recorded "+domain+" for unsubscribe. The app did not contact the link; acting on it is a manual step.", "") +} + +// redirectUnsubscribe is POST/redirect/GET so a refresh does not resubmit. +func (s *Server) redirectUnsubscribe(w http.ResponseWriter, r *http.Request, note, errMsg string) { + q := url.Values{} + if note != "" { + q.Set("note", note) + } + if errMsg != "" { + q.Set("error", errMsg) + } + http.Redirect(w, r, "/unsubscribe?"+q.Encode(), http.StatusSeeOther) +} + +// cleanDomain validates a domain used as the ignore-list key. +func cleanDomain(raw string) (string, bool) { + domain := strings.ToLower(strings.TrimSpace(raw)) + domain = strings.Trim(domain, ".") + if domain == "" || len(domain) > 253 { + return "", false + } + for _, r := range domain { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_': + default: + return "", false + } + } + if !strings.Contains(domain, ".") { + return "", false + } + return domain, true +} diff --git a/internal/server/unsubscribe_test.go b/internal/server/unsubscribe_test.go new file mode 100644 index 0000000..bc4c059 --- /dev/null +++ b/internal/server/unsubscribe_test.go @@ -0,0 +1,182 @@ +package server + +import ( + "context" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "gitea.lab.audasmedia.com.au/sam/email_agent/internal/store" +) + +func seedUnsubMessage(t *testing.T, srv *Server) int64 { + t.Helper() + id, err := srv.store.UpsertMessage(context.Background(), &store.Message{ + GmailID: "unsub-1", + FromAddr: "Heritage Fruit Trees ", + Subject: "New workshops released", + BodyText: "Hello, you have received a newsletter. Unsubscribe at https://heritagefruittrees.com.au/u?id=1.", + SourceAddress: "samuelrolfe@gmail.com", + ListUnsubscribe: ", ", + DateSent: time.Now().UTC(), + }) + if err != nil { + t.Fatalf("seed message: %v", err) + } + return id +} + +func TestUnsubscribePageDetectsLinks(t *testing.T) { + srv := newTestServer(t) + seedUnsubMessage(t, srv) + session, _, _ := login(t, srv) + + rec := getWithSession(t, srv, "/unsubscribe", session) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + "Detected links", "heritagefruittrees.com.au", + "https://heritagefruittrees.com.au/u?id=1", + "mailto:unsub@heritagefruittrees.com.au", + "Ignore domain", "Mark for unsubscribe", "Ignore list", + } { + if !strings.Contains(body, want) { + t.Errorf("unsubscribe page missing %q", want) + } + } +} + +func TestUnsubscribeIgnoreAndUnignore(t *testing.T) { + srv := newTestServer(t) + seedUnsubMessage(t, srv) + session, csrfCookie, csrf := login(t, srv) + ctx := context.Background() + + rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{ + "csrf": {csrf}, "domain": {"heritagefruittrees.com.au"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("ignore status = %d, want 303 (body %s)", rec.Code, rec.Body.String()) + } + rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au") + if err != nil || !ok { + t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err) + } + if rec2.Status != store.UnsubscribeStatusIgnored { + t.Errorf("status = %q, want ignored", rec2.Status) + } + body := getWithSession(t, srv, "/unsubscribe", session).Body.String() + if !strings.Contains(body, "ignored") || !strings.Contains(body, "Un-ignore") { + t.Errorf("ignore list did not render the ignored domain") + } + + rec = postForm(t, srv, "/unsubscribe/unignore", session, csrfCookie, url.Values{ + "csrf": {csrf}, "domain": {"heritagefruittrees.com.au"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("unignore status = %d, want 303", rec.Code) + } + if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok { + t.Error("domain still on the list after unignore") + } +} + +func TestUnsubscribeMarkRecordsWithoutFollowingLink(t *testing.T) { + srv := newTestServer(t) + seedUnsubMessage(t, srv) + session, csrfCookie, csrf := login(t, srv) + ctx := context.Background() + + // Without confirmation the handler refuses and records nothing. + rec := postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{ + "csrf": {csrf}, "domain": {"heritagefruittrees.com.au"}, + "url": {"https://heritagefruittrees.com.au/u?id=1"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("mark without confirm status = %d, want 303", rec.Code) + } + if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok { + t.Error("unconfirmed mark recorded a domain") + } + + rec = postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{ + "csrf": {csrf}, "domain": {"heritagefruittrees.com.au"}, + "url": {"https://heritagefruittrees.com.au/u?id=1"}, "confirm": {"yes"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("mark status = %d, want 303", rec.Code) + } + rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au") + if err != nil || !ok { + t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err) + } + if rec2.Status != store.UnsubscribeStatusDone { + t.Errorf("status = %q, want done", rec2.Status) + } + if rec2.URL != "https://heritagefruittrees.com.au/u?id=1" { + t.Errorf("url = %q, want the recorded link", rec2.URL) + } +} + +func TestUnsubscribeMutationsRequireCSRF(t *testing.T) { + srv := newTestServer(t) + session, _, _ := login(t, srv) + + for _, path := range []string{"/unsubscribe/ignore", "/unsubscribe/unignore", "/unsubscribe/mark"} { + rec := postForm(t, srv, path, session, nil, url.Values{"domain": {"x.example"}}) + if rec.Code != http.StatusForbidden { + t.Errorf("%s without CSRF status = %d, want 403", path, rec.Code) + } + } +} + +func TestUnsubscribeMarkRejectsBadDomain(t *testing.T) { + srv := newTestServer(t) + session, csrfCookie, csrf := login(t, srv) + + rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{ + "csrf": {csrf}, "domain": {"not a domain!"}, + }) + if rec.Code != http.StatusSeeOther { + t.Fatalf("status = %d, want 303", rec.Code) + } + if _, ok, _ := srv.store.UnsubscribeByDomain(context.Background(), "not a domain!"); ok { + t.Error("bad domain was stored") + } +} + +func TestMessageDetailShowsUnsubscribeLinks(t *testing.T) { + srv := newTestServer(t) + id := seedUnsubMessage(t, srv) + session, _, _ := login(t, srv) + + rec := getWithSession(t, srv, "/partials/message?id="+itoa(id), session) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + body := rec.Body.String() + if !strings.Contains(body, "Unsubscribe") || !strings.Contains(body, "heritagefruittrees.com.au/u?id=1") { + t.Errorf("message detail did not show the unsubscribe link") + } +} + +func TestCleanDomain(t *testing.T) { + cases := map[string]bool{ + "brand.example": true, + "Sub.Brand.example": true, + "x-y.example": true, + "": false, + "nodot": false, + "has space.example": false, + "bad!char.example": false, + } + for in, want := range cases { + if _, ok := cleanDomain(in); ok != want { + t.Errorf("cleanDomain(%q) ok = %v, want %v", in, ok, want) + } + } +} diff --git a/internal/store/classify.go b/internal/store/classify.go index 18ffd10..29d9404 100644 --- a/internal/store/classify.go +++ b/internal/store/classify.go @@ -159,3 +159,51 @@ func (s *Store) SaveRules(ctx context.Context, text string) (int, error) { } return next, nil } + +// RuleVersion is one saved rules revision, for the version list in the editor +// (FR-11). +type RuleVersion struct { + ID int64 + Version int + Text string + CreatedAt string +} + +// RulesVersions returns saved rules revisions, newest first. +func (s *Store) RulesVersions(ctx context.Context, limit int) ([]RuleVersion, error) { + if limit < 1 { + limit = 20 + } + rows, err := s.db.QueryContext(ctx, + `SELECT id, version, text, created_at FROM rules ORDER BY version DESC LIMIT ?`, limit) + if err != nil { + return nil, fmt.Errorf("list rules versions: %w", err) + } + defer rows.Close() + var out []RuleVersion + for rows.Next() { + var v RuleVersion + if err := rows.Scan(&v.ID, &v.Version, &v.Text, &v.CreatedAt); err != nil { + return nil, fmt.Errorf("scan rules version: %w", err) + } + out = append(out, v) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list rules versions: %w", err) + } + return out, nil +} + +// RuleByVersion returns the text of one saved rules revision. ok is false when +// the version does not exist. +func (s *Store) RuleByVersion(ctx context.Context, version int) (string, bool, error) { + var text string + err := s.db.QueryRowContext(ctx, `SELECT text FROM rules WHERE version = ?`, version).Scan(&text) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("read rules version %d: %w", version, err) + } + return text, true, nil +} diff --git a/internal/store/messages.go b/internal/store/messages.go index 7426cbf..f290de3 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -20,15 +20,19 @@ type Message struct { // MessageIDHeader is the RFC822 Message-ID header (Phase 5, FR-8). A reply // uses it for In-Reply-To and References so Gmail keeps the thread. MessageIDHeader string - FromAddr string - ToAddr string - Subject string - BodyText string - SourceAddress string - DateSent time.Time - ReceivedAt time.Time - IMAPUID int64 - IMAPUIDValidity int64 + // ListUnsubscribe and ListUnsubscribePost are the raw RFC 2369/8058 headers + // (Phase 6, FR-7). Detection runs over them on demand. + ListUnsubscribe string + ListUnsubscribePost string + FromAddr string + ToAddr string + Subject string + BodyText string + SourceAddress string + DateSent time.Time + ReceivedAt time.Time + IMAPUID int64 + IMAPUIDValidity int64 // Folder is the app folder the message is filed under (Phase 4, FR-6/FR-9): // inbox, delete, saved, or replied. Folder string @@ -71,19 +75,22 @@ func (s *Store) UpsertMessage(ctx context.Context, m *Message) (int64, error) { const q = ` INSERT INTO messages ( gmail_id, thread_id, message_id_header, from_addr, to_addr, subject, body_text, - source_address, date_sent, received_at, imap_uid, imap_uidvalidity -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + source_address, list_unsubscribe, list_unsubscribe_post, date_sent, received_at, + imap_uid, imap_uidvalidity +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (gmail_id) DO UPDATE SET - thread_id = excluded.thread_id, - message_id_header = excluded.message_id_header, - from_addr = excluded.from_addr, - to_addr = excluded.to_addr, - subject = excluded.subject, - body_text = excluded.body_text, - source_address = excluded.source_address, - date_sent = excluded.date_sent, - imap_uid = excluded.imap_uid, - imap_uidvalidity = excluded.imap_uidvalidity` + thread_id = excluded.thread_id, + message_id_header = excluded.message_id_header, + from_addr = excluded.from_addr, + to_addr = excluded.to_addr, + subject = excluded.subject, + body_text = excluded.body_text, + source_address = excluded.source_address, + list_unsubscribe = excluded.list_unsubscribe, + list_unsubscribe_post = excluded.list_unsubscribe_post, + date_sent = excluded.date_sent, + imap_uid = excluded.imap_uid, + imap_uidvalidity = excluded.imap_uidvalidity` receivedAt := m.ReceivedAt if receivedAt.IsZero() { @@ -98,6 +105,8 @@ ON CONFLICT (gmail_id) DO UPDATE SET nullString(m.Subject), nullString(m.BodyText), nullString(m.SourceAddress), + nullString(m.ListUnsubscribe), + nullString(m.ListUnsubscribePost), nullTime(m.DateSent), receivedAt.UTC().Format(sqlTimeLayout), m.IMAPUID, @@ -124,7 +133,8 @@ SELECT m.id, m.gmail_id, COALESCE(m.thread_id, ''), COALESCE(m.from_addr, ''), COALESCE(m.source_address, ''), COALESCE(m.date_sent, ''), m.received_at, COALESCE(m.imap_uid, 0), COALESCE(m.imap_uidvalidity, 0), COALESCE(m.folder, 'inbox'), COALESCE(m.mailbox, 'INBOX'), - COALESCE(m.message_id_header, '') + COALESCE(m.message_id_header, ''), + COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, '') FROM messages m WHERE m.id = ?` var ( m Message @@ -135,6 +145,7 @@ FROM messages m WHERE m.id = ?` &m.ID, &m.GmailID, &m.ThreadID, &m.FromAddr, &m.ToAddr, &m.Subject, &m.BodyText, &m.SourceAddress, &dateSent, &received, &m.IMAPUID, &m.IMAPUIDValidity, &m.Folder, &m.Mailbox, &m.MessageIDHeader, + &m.ListUnsubscribe, &m.ListUnsubscribePost, ) if err != nil { return Message{}, fmt.Errorf("load message %d: %w", id, err) diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 25e1bd4..4cff599 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -21,6 +21,10 @@ CREATE TABLE IF NOT EXISTS messages ( -- Phase 5 (FR-8): the RFC822 Message-ID header. A reply sets In-Reply-To and -- References to it so Gmail files the reply in the same thread. message_id_header TEXT, + -- Phase 6 (FR-7): the raw RFC 2369/8058 unsubscribe headers, stored so the + -- detector can run without a refetch. + list_unsubscribe TEXT, + list_unsubscribe_post TEXT, from_addr TEXT, to_addr TEXT, subject TEXT, @@ -94,11 +98,16 @@ CREATE TABLE IF NOT EXISTS actions_log ( CREATE INDEX IF NOT EXISTS idx_actions_log_message ON actions_log (message_id); CREATE INDEX IF NOT EXISTS idx_actions_log_created ON actions_log (created_at); --- FR-7: unsubscribe tracker and ignore list. +-- FR-7: unsubscribe tracker and ignore list. `status` is 'ignored' for a domain +-- on the ignore list, or 'done' for a domain Sam has recorded as unsubscribed. +-- The app never follows a link itself: it only marks and records (AUD-72). CREATE TABLE IF NOT EXISTS unsubscribe ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT NOT NULL UNIQUE, status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')), + -- url records the link that was detected or marked (Phase 6). + url TEXT NOT NULL DEFAULT '', + updated_at TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/internal/store/store.go b/internal/store/store.go index 88034ae..4b49121 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -159,6 +159,25 @@ func (s *Store) applyMigrations(ctx context.Context) error { return fmt.Errorf("record schema version 5: %w", err) } } + if version < 6 { + // Phase 6 (FR-7): store the raw List-Unsubscribe headers so detection + // runs without a refetch, and extend the unsubscribe tracker with the + // link it recorded and an update time. + for _, col := range []struct{ table, name, typ string }{ + {"messages", "list_unsubscribe", "TEXT"}, + {"messages", "list_unsubscribe_post", "TEXT"}, + {"unsubscribe", "url", "TEXT NOT NULL DEFAULT ''"}, + {"unsubscribe", "updated_at", "TEXT"}, + } { + if err := s.ensureColumn(ctx, col.table, col.name, col.typ); err != nil { + return err + } + } + if _, err := s.db.ExecContext(ctx, + `INSERT OR IGNORE INTO schema_migrations (version) VALUES (6)`); err != nil { + return fmt.Errorf("record schema version 6: %w", err) + } + } return nil } diff --git a/internal/store/unsubscribe.go b/internal/store/unsubscribe.go new file mode 100644 index 0000000..dfda838 --- /dev/null +++ b/internal/store/unsubscribe.go @@ -0,0 +1,177 @@ +// Unsubscribe tracking for Phase 6 (FR-7). +// +// Two things are persisted here: +// +// - the raw unsubscribe links detected on a message (messages.list_unsubscribe, +// stored by the mail fetch and parsed by internal/unsubscribe); and +// - the domain-level ignore list and the record of what was marked, in the +// `unsubscribe` table. +// +// The app never follows a link. Marking a domain records Sam's decision; the +// actual unsubscribe stays a manual, explicitly approved action (AUD-72). +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" +) + +// Unsubscribe statuses. `ignored` is the ignore list; `done` records a domain +// Sam has decided to unsubscribe from. +const ( + UnsubscribeStatusDone = "done" + UnsubscribeStatusIgnored = "ignored" +) + +// ErrUnsubscribeStatus is returned for an unknown unsubscribe status. +var ErrUnsubscribeStatus = errors.New("unsubscribe status must be done or ignored") + +// UnsubscribeRecord is one row of the domain-level tracker. +type UnsubscribeRecord struct { + ID int64 + Domain string + Status string + URL string + CreatedAt string + UpdatedAt string +} + +// ListUnsubscribe returns every domain record, newest first. +func (s *Store) ListUnsubscribe(ctx context.Context) ([]UnsubscribeRecord, error) { + rows, err := s.db.QueryContext(ctx, ` +SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '') +FROM unsubscribe ORDER BY COALESCE(updated_at, created_at) DESC, id DESC`) + if err != nil { + return nil, fmt.Errorf("list unsubscribe records: %w", err) + } + defer rows.Close() + var out []UnsubscribeRecord + for rows.Next() { + var r UnsubscribeRecord + if err := rows.Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, fmt.Errorf("scan unsubscribe record: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list unsubscribe records: %w", err) + } + return out, nil +} + +// UnsubscribeByDomain loads one domain record. ok is false when the domain is +// not tracked. +func (s *Store) UnsubscribeByDomain(ctx context.Context, domain string) (UnsubscribeRecord, bool, error) { + domain = strings.TrimSpace(domain) + if domain == "" { + return UnsubscribeRecord{}, false, nil + } + var r UnsubscribeRecord + err := s.db.QueryRowContext(ctx, ` +SELECT id, domain, status, COALESCE(url, ''), created_at, COALESCE(updated_at, '') +FROM unsubscribe WHERE domain = ?`, domain). + Scan(&r.ID, &r.Domain, &r.Status, &r.URL, &r.CreatedAt, &r.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return UnsubscribeRecord{}, false, nil + } + if err != nil { + return UnsubscribeRecord{}, false, fmt.Errorf("read unsubscribe record %q: %w", domain, err) + } + return r, true, nil +} + +// SetUnsubscribeStatus upserts the domain record. An empty url keeps the stored +// one, so an "ignore" does not wipe a link recorded earlier. +func (s *Store) SetUnsubscribeStatus(ctx context.Context, domain, status, url string) error { + domain = strings.TrimSpace(domain) + if domain == "" { + return errors.New("unsubscribe domain must not be empty") + } + if status != UnsubscribeStatusDone && status != UnsubscribeStatusIgnored { + return ErrUnsubscribeStatus + } + const q = ` +INSERT INTO unsubscribe (domain, status, url, updated_at) +VALUES (?, ?, ?, datetime('now')) +ON CONFLICT (domain) DO UPDATE SET + status = excluded.status, + url = CASE WHEN excluded.url <> '' THEN excluded.url ELSE unsubscribe.url END, + updated_at = datetime('now')` + if _, err := s.db.ExecContext(ctx, q, domain, status, strings.TrimSpace(url)); err != nil { + return fmt.Errorf("set unsubscribe status for %q: %w", domain, err) + } + return nil +} + +// ClearUnsubscribe removes a domain record, taking it off the ignore list. +func (s *Store) ClearUnsubscribe(ctx context.Context, domain string) error { + if _, err := s.db.ExecContext(ctx, + `DELETE FROM unsubscribe WHERE domain = ?`, strings.TrimSpace(domain)); err != nil { + return fmt.Errorf("clear unsubscribe record %q: %w", domain, err) + } + return nil +} + +// UnsubscribeCandidate is a message that may carry an unsubscribe link. +type UnsubscribeCandidate struct { + MessageID int64 + FromAddr string + Subject string + SourceAddress string + BodyText string + ListUnsubscribe string + ListUnsubscribePost string + Category string + ReceivedAt time.Time +} + +// UnsubscribeCandidates returns messages with a List-Unsubscribe header or a +// body that mentions unsubscribing, newest first. The caller runs the detector +// over them, because body parsing lives in internal/unsubscribe. +func (s *Store) UnsubscribeCandidates(ctx context.Context, limit int) ([]UnsubscribeCandidate, error) { + if limit < 1 { + limit = 200 + } + const q = ` +SELECT m.id, COALESCE(m.from_addr, ''), COALESCE(m.subject, ''), + COALESCE(m.source_address, ''), COALESCE(m.body_text, ''), + COALESCE(m.list_unsubscribe, ''), COALESCE(m.list_unsubscribe_post, ''), + COALESCE(c.category, ''), m.received_at +FROM messages m +LEFT JOIN classifications c ON c.id = ( + SELECT id FROM classifications WHERE message_id = m.id ORDER BY id DESC LIMIT 1 +) +WHERE (m.list_unsubscribe IS NOT NULL AND m.list_unsubscribe <> '') + OR instr(lower(COALESCE(m.body_text, '')), 'unsubscribe') > 0 +ORDER BY COALESCE(NULLIF(m.date_sent, ''), m.received_at) DESC, m.id DESC +LIMIT ?` + rows, err := s.db.QueryContext(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("list unsubscribe candidates: %w", err) + } + defer rows.Close() + var out []UnsubscribeCandidate + for rows.Next() { + var ( + c UnsubscribeCandidate + received string + ) + if err := rows.Scan( + &c.MessageID, &c.FromAddr, &c.Subject, &c.SourceAddress, + &c.BodyText, &c.ListUnsubscribe, &c.ListUnsubscribePost, + &c.Category, &received, + ); err != nil { + return nil, fmt.Errorf("scan unsubscribe candidate: %w", err) + } + c.ReceivedAt = parseSQLTime(received) + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list unsubscribe candidates: %w", err) + } + return out, nil +} diff --git a/internal/store/unsubscribe_test.go b/internal/store/unsubscribe_test.go new file mode 100644 index 0000000..3105194 --- /dev/null +++ b/internal/store/unsubscribe_test.go @@ -0,0 +1,193 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +func TestMigrationV6AddsUnsubscribeColumns(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "old.db") + + // Simulate a v5 database: messages has no List-Unsubscribe columns and the + // unsubscribe table has no url/updated_at. Opening it must apply migration + // v6. + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatalf("open old db: %v", err) + } + stmts := []string{ + `CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + gmail_id TEXT NOT NULL UNIQUE, + thread_id TEXT, message_id_header TEXT, from_addr TEXT, to_addr TEXT, + subject TEXT, body_text TEXT, source_address TEXT, date_sent TEXT, + imap_uid INTEGER, imap_uidvalidity INTEGER, + folder TEXT NOT NULL DEFAULT 'inbox', + mailbox TEXT NOT NULL DEFAULT 'INBOX', + received_at TEXT NOT NULL DEFAULT (datetime('now')), + created_at TEXT NOT NULL DEFAULT (datetime('now')))`, + `CREATE TABLE unsubscribe ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'done' CHECK (status IN ('done', 'ignored')), + created_at TEXT NOT NULL DEFAULT (datetime('now')))`, + `CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`, + `INSERT INTO schema_migrations (version) VALUES (5)`, + } + for _, q := range stmts { + if _, err := db.ExecContext(ctx, q); err != nil { + t.Fatalf("seed old db: %v", err) + } + } + _ = db.Close() + + s, err := Open(ctx, path) + if err != nil { + t.Fatalf("Open() on v5 database error = %v", err) + } + defer s.Close() + + v, err := s.SchemaVersion(ctx) + if err != nil { + t.Fatalf("SchemaVersion() error = %v", err) + } + if v < 6 { + t.Errorf("SchemaVersion() = %d, want >= 6", v) + } + for _, tc := range []struct{ table, col string }{ + {"messages", "list_unsubscribe"}, + {"messages", "list_unsubscribe_post"}, + {"unsubscribe", "url"}, + {"unsubscribe", "updated_at"}, + } { + var n int + q := `SELECT COUNT(*) FROM pragma_table_info('` + tc.table + `') WHERE name = ?` + if err := s.DB().QueryRowContext(ctx, q, tc.col).Scan(&n); err != nil { + t.Fatalf("count %s.%s: %v", tc.table, tc.col, err) + } + if n != 1 { + t.Errorf("%s.%s missing after migration v6", tc.table, tc.col) + } + } +} + +func TestUnsubscribeRecordUpsert(t *testing.T) { + ctx := context.Background() + s := newCategoryStore(t) // reuses the store helper from categories_test.go + + if _, ok, err := s.UnsubscribeByDomain(ctx, "brand.example"); err != nil || ok { + t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a miss", ok, err) + } + + if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusIgnored, "https://brand.example/u"); err != nil { + t.Fatalf("SetUnsubscribeStatus(ignore) error = %v", err) + } + rec, ok, err := s.UnsubscribeByDomain(ctx, "brand.example") + if err != nil || !ok { + t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err) + } + if rec.Status != UnsubscribeStatusIgnored || rec.URL != "https://brand.example/u" { + t.Errorf("record = %+v, want ignored with url", rec) + } + + // A later mark with no URL must keep the recorded link. + if err := s.SetUnsubscribeStatus(ctx, "brand.example", UnsubscribeStatusDone, ""); err != nil { + t.Fatalf("SetUnsubscribeStatus(done) error = %v", err) + } + rec, _, _ = s.UnsubscribeByDomain(ctx, "brand.example") + if rec.Status != UnsubscribeStatusDone { + t.Errorf("status = %q, want done", rec.Status) + } + if rec.URL != "https://brand.example/u" { + t.Errorf("url = %q, want the earlier link preserved", rec.URL) + } + + if err := s.SetUnsubscribeStatus(ctx, "brand.example", "bogus", ""); err == nil { + t.Error("SetUnsubscribeStatus(bogus) accepted an unknown status") + } + + records, err := s.ListUnsubscribe(ctx) + if err != nil || len(records) != 1 { + t.Fatalf("ListUnsubscribe() = (%v, %v), want one record", records, err) + } + + if err := s.ClearUnsubscribe(ctx, "brand.example"); err != nil { + t.Fatalf("ClearUnsubscribe() error = %v", err) + } + if err := s.SetUnsubscribeStatus(ctx, "", UnsubscribeStatusIgnored, ""); err == nil { + t.Error("SetUnsubscribeStatus accepted an empty domain") + } +} + +func TestUnsubscribeCandidates(t *testing.T) { + ctx := context.Background() + s := newCategoryStore(t) + + seed := func(gmailID, from, body, header string) { + t.Helper() + if _, err := s.UpsertMessage(ctx, &Message{ + GmailID: gmailID, + FromAddr: from, + Subject: "subject " + gmailID, + BodyText: body, + ListUnsubscribe: header, + }); err != nil { + t.Fatalf("seed %s: %v", gmailID, err) + } + } + seed("header-msg", "Support ", "hello", "") + seed("body-msg", "Shop ", "You can unsubscribe at https://shop.example/opt-out now.", "") + seed("plain-msg", "Friend ", "Just saying hi.", "") + + got, err := s.UnsubscribeCandidates(ctx, 50) + if err != nil { + t.Fatalf("UnsubscribeCandidates() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("candidates = %d, want 2 (%+v)", len(got), got) + } + ids := map[string]bool{} + for _, c := range got { + ids[c.Subject] = true + } + if !ids["subject body-msg"] || !ids["subject header-msg"] { + t.Errorf("candidates = %+v, want the header and body messages", ids) + } +} + +func TestRulesVersionsAndRestore(t *testing.T) { + ctx := context.Background() + s := newCategoryStore(t) + + if _, err := s.SaveRules(ctx, "from contains \"a\" => category=Info"); err != nil { + t.Fatalf("SaveRules() error = %v", err) + } + if _, err := s.SaveRules(ctx, "from contains \"b\" => category=Action"); err != nil { + t.Fatalf("SaveRules() error = %v", err) + } + + versions, err := s.RulesVersions(ctx, 10) + if err != nil { + t.Fatalf("RulesVersions() error = %v", err) + } + if len(versions) < 3 { + t.Fatalf("RulesVersions() = %d, want >= 3 (seed + 2 saves)", len(versions)) + } + if versions[0].Version != 3 { + t.Errorf("newest version = %d, want 3", versions[0].Version) + } + + text, ok, err := s.RuleByVersion(ctx, 2) + if err != nil || !ok { + t.Fatalf("RuleByVersion(2) = (_, %v, %v), want a hit", ok, err) + } + if text != "from contains \"a\" => category=Info" { + t.Errorf("RuleByVersion(2) = %q", text) + } + if _, ok, _ := s.RuleByVersion(ctx, 999); ok { + t.Error("RuleByVersion(999) reported a hit") + } +} diff --git a/internal/unsubscribe/unsubscribe.go b/internal/unsubscribe/unsubscribe.go new file mode 100644 index 0000000..a9bdd44 --- /dev/null +++ b/internal/unsubscribe/unsubscribe.go @@ -0,0 +1,240 @@ +// Package unsubscribe detects unsubscribe links (FR-7). +// +// The primary source is the List-Unsubscribe header (RFC 2369) together with +// the List-Unsubscribe-Post header (RFC 8058). When neither is present the body +// is scanned for a URL on a line that mentions "unsubscribe". +// +// Detection only. This package never fetches a URL and never sends mail. +// Acting on an unsubscribe link needs Sam's explicit approval (AGENTS.md §10 +// and AUD-72); the app only marks and records. +package unsubscribe + +import ( + "net/url" + "regexp" + "strings" +) + +// Link methods. +const ( + MethodHTTP = "http" + MethodMailto = "mailto" +) + +// Detection sources. They are stored in the action log so a marking is +// auditable. +const ( + SourceListUnsubscribe = "list-unsubscribe" + SourceBody = "body" +) + +// Link is one detected unsubscribe target. +type Link struct { + // URL is the raw target, for example "https://x.example/u?id=1". + URL string + // Method is MethodHTTP or MethodMailto. + Method string + // Domain is the ignore-list key. It is the sender's brand domain when that + // is a real domain, otherwise the link host. + Domain string + // Host is the host of the link itself, shown for context. + Host string + // Source is SourceListUnsubscribe or SourceBody. + Source string +} + +// Result is the detection outcome for one message. +type Result struct { + Links []Link + // Header is the raw List-Unsubscribe value, if any. + Header string + // OneClick reports a RFC 8058 List-Unsubscribe=One-Click header. + OneClick bool +} + +// Detect returns the unsubscribe links for a message. from is the raw From +// header; body is the stored plain-text body. +func Detect(listUnsubscribe, listUnsubscribePost, from, body string) Result { + res := Result{ + Header: strings.TrimSpace(listUnsubscribe), + OneClick: isOneClick(listUnsubscribePost), + } + sender := brandDomain(from) + + seen := map[string]struct{}{} + add := func(raw, source string) { + link, ok := makeLink(raw, source, sender) + if !ok { + return + } + key := link.Method + ":" + strings.ToLower(link.URL) + if _, dup := seen[key]; dup { + return + } + seen[key] = struct{}{} + res.Links = append(res.Links, link) + } + + for _, raw := range headerLinks(res.Header) { + add(raw, SourceListUnsubscribe) + } + if len(res.Links) == 0 { + for _, raw := range bodyLinks(body) { + add(raw, SourceBody) + } + } + return res +} + +// isOneClick reports a RFC 8058 one-click unsubscribe header. The documented +// value is "List-Unsubscribe=One-Click". +func isOneClick(v string) bool { + return strings.Contains(strings.ToLower(v), "one-click") +} + +var angleRE = regexp.MustCompile(`<([^>]+)>`) + +// headerLinks extracts the URIs from a List-Unsubscribe value. The RFC form is +// a comma-separated list of angle-bracketed URIs; a bare URI is also accepted. +func headerLinks(v string) []string { + v = strings.TrimSpace(v) + if v == "" { + return nil + } + var out []string + for _, m := range angleRE.FindAllStringSubmatch(v, -1) { + if s := strings.TrimSpace(m[1]); s != "" { + out = append(out, s) + } + } + if len(out) > 0 { + return out + } + for _, part := range strings.Split(v, ",") { + if s := strings.TrimSpace(part); s != "" { + out = append(out, s) + } + } + return out +} + +var ( + bodyURLRE = regexp.MustCompile(`(?i)(https?://[^\s<>"')\]]+|mailto:[^\s<>"')\]]+)`) + unsubRE = regexp.MustCompile(`(?i)unsubscribe|opt[\s-]?out|manage (?:your )?(?:email )?preferences`) +) + +// bodyLinks finds unsubscribe URLs in a plain-text body. Only lines that +// mention unsubscribing are considered, so a random link is not reported. +func bodyLinks(body string) []string { + body = strings.TrimSpace(body) + if body == "" { + return nil + } + var out []string + for _, line := range strings.Split(body, "\n") { + if !unsubRE.MatchString(line) { + continue + } + for _, m := range bodyURLRE.FindAllString(line, -1) { + out = append(out, trimURL(m)) + } + } + return out +} + +// trimURL removes trailing punctuation that is part of the sentence, not the +// URL. +func trimURL(s string) string { + return strings.TrimRight(strings.TrimSpace(s), ".,;:!?)\"'") +} + +// makeLink validates a raw target and fills in method/host/domain. +func makeLink(raw, source, sender string) (Link, bool) { + raw = trimURL(raw) + raw = strings.Trim(raw, "<>") + if raw == "" { + return Link{}, false + } + link := Link{URL: raw, Source: source} + switch { + case strings.HasPrefix(strings.ToLower(raw), "mailto:"): + link.Method = MethodMailto + link.Host = mailtoHost(raw) + case strings.HasPrefix(strings.ToLower(raw), "http://"), + strings.HasPrefix(strings.ToLower(raw), "https://"): + link.Method = MethodHTTP + link.Host = urlHost(raw) + default: + return Link{}, false + } + link.Domain = sender + if link.Domain == "" { + link.Domain = link.Host + } + return link, link.Domain != "" +} + +// Domain returns the domain used as the ignore-list key. It is exported for +// callers that need it without parsing a link. +func Domain(from string) string { return brandDomain(from) } + +func brandDomain(from string) string { + addr := extractAddress(from) + i := strings.LastIndex(addr, "@") + if i < 0 || i == len(addr)-1 { + return "" + } + dom := strings.ToLower(strings.Trim(strings.TrimSpace(addr[i+1:]), ">.")) + if dom == "" || freeMail[dom] { + return "" + } + return dom +} + +// extractAddress pulls the address out of a From header such as +// "Support ". +func extractAddress(v string) string { + v = strings.TrimSpace(v) + if i := strings.LastIndex(v, "<"); i >= 0 { + if j := strings.Index(v[i:], ">"); j > 0 { + return strings.TrimSpace(v[i+1 : i+j]) + } + } + return strings.Trim(v, "<> \t") +} + +// freeMail domains are consumer mailboxes, not brands. A message forwarded +// through one should key its ignore list on the real link host instead. +var freeMail = map[string]bool{ + "gmail.com": true, + "googlemail.com": true, + "outlook.com": true, + "hotmail.com": true, + "live.com": true, + "yahoo.com": true, + "icloud.com": true, + "me.com": true, + "aol.com": true, + "proton.me": true, + "protonmail.com": true, +} + +func urlHost(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + return strings.ToLower(u.Hostname()) +} + +func mailtoHost(raw string) string { + rest := raw[len("mailto:"):] + if i := strings.IndexAny(rest, "?,"); i >= 0 { + rest = rest[:i] + } + i := strings.LastIndex(rest, "@") + if i < 0 || i == len(rest)-1 { + return "" + } + return strings.ToLower(strings.Trim(rest[i+1:], ">.")) +} diff --git a/internal/unsubscribe/unsubscribe_test.go b/internal/unsubscribe/unsubscribe_test.go new file mode 100644 index 0000000..652ca7a --- /dev/null +++ b/internal/unsubscribe/unsubscribe_test.go @@ -0,0 +1,126 @@ +package unsubscribe + +import "testing" + +func TestDetectListUnsubscribeHeader(t *testing.T) { + res := Detect( + ", ", + "List-Unsubscribe=One-Click", + "Heritage Fruit Trees ", + "Special offer this week.", + ) + if !res.OneClick { + t.Error("OneClick = false, want true") + } + if len(res.Links) != 2 { + t.Fatalf("Links len = %d, want 2 (%+v)", len(res.Links), res.Links) + } + mail := res.Links[0] + if mail.Method != MethodMailto { + t.Errorf("mailto method = %q", mail.Method) + } + if mail.Domain != "heritagefruittrees.com.au" { + t.Errorf("mailto domain = %q", mail.Domain) + } + if mail.Source != SourceListUnsubscribe { + t.Errorf("mailto source = %q", mail.Source) + } + http := res.Links[1] + if http.Method != MethodHTTP { + t.Errorf("http method = %q", http.Method) + } + if http.Host != "heritagefruittrees.com.au" { + t.Errorf("http host = %q", http.Host) + } + if http.Domain != "heritagefruittrees.com.au" { + t.Errorf("http domain = %q", http.Domain) + } +} + +func TestDetectBareHeaderURL(t *testing.T) { + res := Detect("https://lists.example.com/unsub?u=abc,", "", "News ", "") + if len(res.Links) != 1 { + t.Fatalf("Links = %+v, want 1", res.Links) + } + if res.Links[0].URL != "https://lists.example.com/unsub?u=abc" { + t.Errorf("URL = %q", res.Links[0].URL) + } +} + +func TestDetectBodyFallback(t *testing.T) { + res := Detect("", "", "Shop ", "Thanks for subscribing!\nUnsubscribe here: https://shop.example/opt-out?token=1.\nSee you.") + if res.Header != "" { + t.Errorf("Header = %q, want empty", res.Header) + } + if len(res.Links) != 1 { + t.Fatalf("Links = %+v, want 1", res.Links) + } + if res.Links[0].Source != SourceBody { + t.Errorf("Source = %q, want %q", res.Links[0].Source, SourceBody) + } + if res.Links[0].URL != "https://shop.example/opt-out?token=1" { + t.Errorf("URL = %q (trailing period should be trimmed)", res.Links[0].URL) + } +} + +func TestDetectIgnoresUnrelatedBodyLinks(t *testing.T) { + res := Detect("", "", "Friend ", "Read more at https://example.com/blog") + if len(res.Links) != 0 { + t.Fatalf("Links = %+v, want none", res.Links) + } +} + +func TestDetectBodyMailto(t *testing.T) { + res := Detect("", "", "Brand ", "To unsubscribe, email mailto:leave@brand.example") + if len(res.Links) != 1 || res.Links[0].Method != MethodMailto { + t.Fatalf("Links = %+v, want one mailto", res.Links) + } +} + +func TestFreeMailSenderUsesLinkHost(t *testing.T) { + res := Detect( + "", + "", + "Someone ", + "", + ) + if len(res.Links) != 1 { + t.Fatalf("Links = %+v, want 1", res.Links) + } + if res.Links[0].Domain != "mailer.example.com" { + t.Errorf("domain = %q, want link host", res.Links[0].Domain) + } +} + +func TestDetectDeduplicates(t *testing.T) { + res := Detect( + ", ", + "", + "Brand ", + "", + ) + if len(res.Links) != 1 { + t.Fatalf("Links = %+v, want 1 after de-duplication", res.Links) + } +} + +func TestDomain(t *testing.T) { + cases := map[string]string{ + "Support ": "heritagefruittrees.com.au", + "plain@brand.example": "brand.example", + "Someone ": "", + "": "", + } + for in, want := range cases { + if got := Domain(in); got != want { + t.Errorf("Domain(%q) = %q, want %q", in, got, want) + } + } +} + +func TestEmptyInputs(t *testing.T) { + res := Detect("", "", "", "") + if len(res.Links) != 0 || res.OneClick { + t.Fatalf("Detect(empty) = %+v, want zero value", res) + } +} diff --git a/web/static/app.css b/web/static/app.css index 4d6162f..23a4f71 100644 --- a/web/static/app.css +++ b/web/static/app.css @@ -466,3 +466,68 @@ a:hover { text-decoration: underline; } @media (max-width: 759px) { .reply-actions .btn { flex: 1 1 auto; } } + +/* Phase 6: rules editor (FR-4, FR-11) and unsubscribe detection (FR-7) */ +.rules-panel, .unsub-panel { margin-top: var(--space-md); } +.rules-help, .unsub-help { font: var(--type-body-sm); margin-bottom: var(--space-md); max-width: 78ch; } +.rules-help code { font: var(--type-caption); background: var(--surface-2); padding: 1px 4px; border-radius: 4px; } +.rules-errors { margin: var(--space-xs) 0 0; padding-left: var(--space-md); font: var(--type-body-sm); } +.rules-form { display: grid; gap: var(--space-sm); } +.rules-textarea { + width: 100%; + min-height: 340px; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + line-height: 1.5; + white-space: pre; + overflow-wrap: normal; + overflow-x: auto; +} +.rules-actions, .rules-reclassify { display: flex; align-items: center; gap: var(--space-sm); flex-wrap: wrap; } +.rules-reclassify { margin-top: var(--space-sm); } +.rules-versions { display: flex; flex-direction: column; } +.rules-version { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-sm); + padding: var(--space-sm) 0; + border-bottom: 1px solid var(--hairline); +} +.rules-version:last-child { border-bottom: 0; } +.rules-version-main { display: flex; align-items: center; gap: var(--space-sm); min-width: 0; flex-wrap: wrap; } +.rules-version-preview { font: var(--type-caption); color: var(--ink-secondary); overflow-wrap: anywhere; } +.unsub-list { display: flex; flex-direction: column; } +.unsub-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-md); + padding: var(--space-sm) 0; + border-bottom: 1px solid var(--hairline); +} +.unsub-row:last-child { border-bottom: 0; } +.unsub-main { display: grid; gap: 2px; min-width: 0; flex: 1 1 auto; } +.unsub-subject { font: var(--type-body-sm); color: var(--ink); overflow-wrap: anywhere; } +.unsub-links { display: grid; gap: 2px; margin-top: var(--space-xs); } +.unsub-link { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; } +.unsub-source { font: var(--type-caption); color: var(--ink-faint); } +.unsub-url { font: var(--type-caption); color: var(--ink-secondary); overflow-wrap: anywhere; } +.unsub-domain { font: var(--type-caption); color: var(--ink-faint); margin-top: 2px; } +.unsub-actions { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; flex: 0 0 auto; } +.unsub-detected { + border: 1px solid var(--hairline); + border-radius: 4px; + padding: var(--space-sm); + margin-bottom: var(--space-md); + display: grid; + gap: var(--space-xs); +} +.unsub-detected h3 { font: var(--type-eyebrow); text-transform: uppercase; letter-spacing: .125px; color: var(--ink-faint); margin: 0; } +.unsub-link-list { list-style: none; margin: 0; padding: 0; display: grid; gap: var(--space-xs); } +.unsub-link-list li { display: flex; align-items: center; gap: var(--space-xs); flex-wrap: wrap; font: var(--type-caption); color: var(--ink-secondary); } +@media (max-width: 759px) { + .unsub-row { flex-direction: column; align-items: stretch; } + .rules-version { flex-direction: column; align-items: stretch; } +} diff --git a/web/templates/dashboard.html b/web/templates/dashboard.html index f8de52a..ecb346b 100644 --- a/web/templates/dashboard.html +++ b/web/templates/dashboard.html @@ -26,6 +26,8 @@
Email Agent
+ Rules + Unsubscribe
+ {{if .UnsubLinks}} +
+

Unsubscribe

+ {{if eq .UnsubStatus "ignored"}}ignored{{end}} + {{if eq .UnsubStatus "done"}}recorded{{end}} +
    + {{range .UnsubLinks}}
  • {{.Method}} {{.URL}}
    • {{end}} +
+ Manage unsubscribe +
+ {{end}} +
Original message 
{{.Message.BodyText}}
diff --git a/web/templates/rules.html b/web/templates/rules.html new file mode 100644 index 0000000..ee09601 --- /dev/null +++ b/web/templates/rules.html @@ -0,0 +1,125 @@ + + + + + + Rules · Email Agent + + + + + +
+ Email Agent +
+ Back to inbox + Unsubscribe + +
+ + +
+
+
+ +
+

Rules

+ + {{if .Note}}
{{.Note}}
{{end}} + {{if .Error}}{{end}} + +
+
+

Rules editor

+ version {{.Version}} · {{.RuleCount}} active rule(s) · {{.TotalMessages}} message(s) +
+ +
+

+ One rule per line: <condition> => <effects>. + Fields are from, to, subject, + source, and body; operators are + contains, equals, starts_with, + and ends_with. The first matching rule wins and overrides + the JEV fields it names. Lines starting with # are comments. + The classifier and the reply model both read this text. +

+ + {{if .Errors}} + + {{end}} + +
+ + + +
+ + Saving appends a version; nothing is re-classified until you re-classify. +
+
+ + {{if .ClassifierReady}} +
+ + +
+ {{else}} +

Re-classify is unavailable until JEV_API_KEY is set.

+ {{end}} +
+
+ +
+

Versions

+
+
+ {{range .Revisions}} +
+
+ v{{.Version}} + {{.CreatedAt}} + {{.Lines}} rule line(s) + {{.Preview}} +
+
+ + + +
+
+ {{else}} +

No versions stored yet.

+ {{end}} +
+
+
+
+ + diff --git a/web/templates/unsubscribe.html b/web/templates/unsubscribe.html new file mode 100644 index 0000000..efe4c01 --- /dev/null +++ b/web/templates/unsubscribe.html @@ -0,0 +1,178 @@ + + + + + + Unsubscribe · Email Agent + + + + + +
+ Email Agent +
+ Back to inbox + Rules + +
+ + +
+
+
+ +
+

Unsubscribe

+ + {{if .Note}}
{{.Note}}
{{end}} + {{if .Error}}{{end}} + +
+
+

Detected links

+ {{.CandidateCount}} message(s) with an unsubscribe link +
+
+

+ The app only marks and records. It never opens an unsubscribe link: + acting on a real link is a manual step that needs Sam's approval. + Ignoring a domain keeps it out of the recommendation list. +

+ +
+ {{range .Items}} +
+
+
{{.Subject}}
+
{{.From}}
+ {{if .SourceAddr}}
arrived at {{.SourceAddr}}
{{end}} +
+ {{range .Links}} +
+ {{.Method}} + {{.Source}} + {{.URL}} +
+ {{end}} +
+
domain: {{.Domain}}{{if .Category}} · {{.Category}}{{end}}{{if .When}} · {{.When}}{{end}}
+
+
+ {{if eq .Status "ignored"}} + ignored +
+ + + +
+ {{else if eq .Status "done"}} + recorded +
+ + + +
+ {{else}} +
+ + + + +
+
+ + + + + +
+ {{end}} +
+
+ {{else}} +
+
No unsubscribe links detected
+

Check mail first. Newsletter and bulk senders usually carry a List-Unsubscribe header.

+
+ {{end}} +
+
+
+ +
+
+

Ignore list

+ {{len .Ignored}} domain(s) +
+
+
+ {{range .Ignored}} +
+
+ ignored + {{.Domain}} + {{if .URL}}{{.URL}}{{end}} +
+
+ + + +
+
+ {{else}} +

No domains ignored.

+ {{end}} +
+
+
+ +
+
+

Recorded for unsubscribe

+ {{len .Recorded}} domain(s) +
+
+

These are recorded only. The app has not contacted any link.

+
+ {{range .Recorded}} +
+
+ recorded + {{.Domain}} + {{if .URL}}{{.URL}}{{end}} + {{.UpdatedAt}} +
+
+ + + +
+
+ {{else}} +

Nothing recorded yet.

+ {{end}} +
+
+
+
+ +