Phase 6: rules editor + unsubscribe detection (AUD-72)
- FR-11 rules editor: plain-text area at /rules, saved and versioned (SaveRules appends a version; restore re-saves an old version), parse errors surfaced, one-click re-classify. The reply model now reads the rules document too (FR-11). - FR-7 unsubscribe: parse and store List-Unsubscribe / List-Unsubscribe-Post (migration v6), detect email and web links (internal/unsubscribe), a domain-level ignore list, and an explicit record of what was marked. No code path contacts an unsubscribe link; acting on one needs Sam's approval. - Migration v6 is additive; unsubscribe gains url/updated_at. - Tests: detector, parser, store migration/records/candidates, rules editor, unsubscribe handlers, message-detail rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailops"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/reply"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
|
||||
)
|
||||
|
||||
// ReplyGenerator drafts a reply body. The live implementation is
|
||||
@@ -63,6 +64,11 @@ type replyData struct {
|
||||
LLMReady bool
|
||||
LLMModel string
|
||||
|
||||
// Phase 6 (FR-7): detected unsubscribe links for this message, and the
|
||||
// domain's status on the ignore/recorded list.
|
||||
UnsubLinks []unsubscribe.Link
|
||||
UnsubStatus string
|
||||
|
||||
Sent bool
|
||||
ThreadUpdated bool
|
||||
RepliedFolder string
|
||||
@@ -109,6 +115,9 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
class := s.latestClass(ctx, id)
|
||||
|
||||
// FR-11: the reply model reads the current rules document too.
|
||||
rulesText, _, _ := s.store.CurrentRules(ctx)
|
||||
|
||||
body, templateID, err := s.replies.Draft(ctx, reply.Request{
|
||||
Message: msg,
|
||||
Category: class.Category,
|
||||
@@ -116,6 +125,7 @@ func (s *Server) handleDraft(w http.ResponseWriter, r *http.Request) {
|
||||
Action: class.Action,
|
||||
TemplateID: r.FormValue("template"),
|
||||
Instruction: r.FormValue("instruction"),
|
||||
Rules: rulesText,
|
||||
Signer: "Sam",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -313,6 +323,16 @@ func (s *Server) renderMessage(w http.ResponseWriter, r *http.Request, id int64,
|
||||
}
|
||||
data.NoThreadHeader = strings.TrimSpace(msg.MessageIDHeader) == ""
|
||||
|
||||
// FR-7: detect unsubscribe links and look up the domain's list status.
|
||||
if res := unsubscribe.Detect(msg.ListUnsubscribe, msg.ListUnsubscribePost, msg.FromAddr, msg.BodyText); len(res.Links) > 0 {
|
||||
data.UnsubLinks = res.Links
|
||||
if rec, ok, rerr := s.store.UnsubscribeByDomain(ctx, res.Links[0].Domain); rerr != nil {
|
||||
s.logger.Error("load unsubscribe status", "message_id", id, "error", rerr)
|
||||
} else if ok {
|
||||
data.UnsubStatus = rec.Status
|
||||
}
|
||||
}
|
||||
|
||||
if d, ok, derr := s.store.LatestDraft(ctx, id); derr != nil {
|
||||
s.logger.Error("load draft", "message_id", id, "error", derr)
|
||||
} else if ok {
|
||||
|
||||
225
internal/server/rules.go
Normal file
225
internal/server/rules.go
Normal file
@@ -0,0 +1,225 @@
|
||||
// Rules editor for Phase 6 (FR-4, FR-11).
|
||||
//
|
||||
// The rules document is plain text (see internal/classify for the syntax). It is
|
||||
// saved as a new numbered version on every save, so the current text and its
|
||||
// history are both available. The classifier reads the newest version and
|
||||
// applies it as an override; the reply model also reads it.
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify"
|
||||
)
|
||||
|
||||
type ruleRevision struct {
|
||||
Version int
|
||||
CreatedAt string
|
||||
Lines int
|
||||
Preview string
|
||||
}
|
||||
|
||||
type rulesData struct {
|
||||
Text string
|
||||
Version int
|
||||
Revisions []ruleRevision
|
||||
Errors []classify.ParseError
|
||||
RuleCount int
|
||||
Note string
|
||||
Error string
|
||||
CSRFToken string
|
||||
// ClassifierReady is true when JEV is configured, so the page can offer a
|
||||
// re-classify action that actually runs.
|
||||
ClassifierReady bool
|
||||
TotalMessages int
|
||||
}
|
||||
|
||||
// handleRules renders the rules editor with the current text and version list.
|
||||
func (s *Server) handleRules(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderRules(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (s *Server) renderRules(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
|
||||
ctx := r.Context()
|
||||
text, version, err := s.store.CurrentRules(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("load rules", "error", err)
|
||||
http.Error(w, "could not load the rules", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
parsed := classify.ParseRules(text)
|
||||
revisions, err := s.store.RulesVersions(ctx, 20)
|
||||
if err != nil {
|
||||
s.logger.Error("load rules versions", "error", err)
|
||||
}
|
||||
|
||||
revs := make([]ruleRevision, 0, len(revisions))
|
||||
for _, rv := range revisions {
|
||||
revs = append(revs, ruleRevision{
|
||||
Version: rv.Version,
|
||||
CreatedAt: rv.CreatedAt,
|
||||
Lines: countRuleLines(rv.Text),
|
||||
Preview: firstRuleLine(rv.Text),
|
||||
})
|
||||
}
|
||||
total, err := s.store.CountMessages(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("count messages", "error", err)
|
||||
}
|
||||
|
||||
data := rulesData{
|
||||
Text: text,
|
||||
Version: version,
|
||||
Revisions: revs,
|
||||
Errors: parsed.Errors,
|
||||
RuleCount: len(parsed.Rules),
|
||||
Note: note,
|
||||
Error: errMsg,
|
||||
CSRFToken: s.csrfToken(w, r),
|
||||
ClassifierReady: s.classifier.Configured(),
|
||||
TotalMessages: total,
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(code)
|
||||
if err := s.tmpl.ExecuteTemplate(w, "rules.html", data); err != nil {
|
||||
s.logger.Error("render rules", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleRulesSave versions the posted text (FR-11).
|
||||
func (s *Server) handleRulesSave(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectRules(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
text := r.FormValue("text")
|
||||
version, err := s.store.SaveRules(r.Context(), text)
|
||||
if err != nil {
|
||||
s.logger.Error("save rules", "error", err)
|
||||
s.redirectRules(w, r, "", "Could not save the rules: "+err.Error())
|
||||
return
|
||||
}
|
||||
parsed := classify.ParseRules(text)
|
||||
s.logAction(r.Context(), "rules_save", "Sam",
|
||||
"version="+strconv.Itoa(version)+" rules="+strconv.Itoa(len(parsed.Rules))+" parse_errors="+strconv.Itoa(len(parsed.Errors)))
|
||||
s.redirectRules(w, r, "Saved rules version "+strconv.Itoa(version)+". Re-classify all to apply it to existing mail.", "")
|
||||
}
|
||||
|
||||
// handleRulesRestore re-saves an older version as the newest one, so history is
|
||||
// append-only (FR-11).
|
||||
func (s *Server) handleRulesRestore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectRules(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
version, err := strconv.Atoi(strings.TrimSpace(r.FormValue("version")))
|
||||
if err != nil || version < 1 {
|
||||
s.redirectRules(w, r, "", "Unknown rules version.")
|
||||
return
|
||||
}
|
||||
text, ok, err := s.store.RuleByVersion(r.Context(), version)
|
||||
if err != nil {
|
||||
s.redirectRules(w, r, "", "Could not load that version.")
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
s.redirectRules(w, r, "", "That rules version no longer exists.")
|
||||
return
|
||||
}
|
||||
newVersion, err := s.store.SaveRules(r.Context(), text)
|
||||
if err != nil {
|
||||
s.logger.Error("restore rules", "from_version", version, "error", err)
|
||||
s.redirectRules(w, r, "", "Could not restore that version: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "rules_restore", "Sam",
|
||||
"from_version="+strconv.Itoa(version)+" new_version="+strconv.Itoa(newVersion))
|
||||
s.redirectRules(w, r, "Restored version "+strconv.Itoa(version)+" as version "+strconv.Itoa(newVersion)+".", "")
|
||||
}
|
||||
|
||||
// handleRulesReclassify re-runs the classifier over every stored message and
|
||||
// returns to the rules editor. It is the rules page's "apply" action: saving a
|
||||
// new version does not re-classify by itself.
|
||||
func (s *Server) handleRulesReclassify(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if rc := http.NewResponseController(w); rc != nil {
|
||||
_ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
sum, err := s.classifier.RunAll(ctx)
|
||||
switch {
|
||||
case err == nil:
|
||||
s.logAction(ctx, "reclassify", "Sam",
|
||||
fmt.Sprintf("considered=%d classified=%d failed=%d rules_version=%d",
|
||||
sum.Considered, sum.Classified, sum.Failed, sum.RulesVersion))
|
||||
s.redirectRules(w, r, fmt.Sprintf(
|
||||
"Re-classified %d of %d message(s); %d failed. The current rules are now applied.",
|
||||
sum.Classified, sum.Considered, sum.Failed), "")
|
||||
case errors.Is(err, classify.ErrNotConfigured):
|
||||
s.redirectRules(w, r, "", "Classification is not connected yet: set JEV_API_KEY.")
|
||||
default:
|
||||
s.logger.Error("rules reclassify failed", "error", err)
|
||||
s.redirectRules(w, r, "", "Classification failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// redirectRules is POST/redirect/GET so a refresh does not resubmit.
|
||||
func (s *Server) redirectRules(w http.ResponseWriter, r *http.Request, note, errMsg string) {
|
||||
q := url.Values{}
|
||||
if note != "" {
|
||||
q.Set("note", note)
|
||||
}
|
||||
if errMsg != "" {
|
||||
q.Set("error", errMsg)
|
||||
}
|
||||
http.Redirect(w, r, "/rules?"+q.Encode(), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// countRuleLines counts the non-empty, non-comment lines in a rules document.
|
||||
func countRuleLines(text string) int {
|
||||
n := 0
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// firstRuleLine returns a short preview of the first real rule, or a note when
|
||||
// the document has only comments.
|
||||
func firstRuleLine(text string) string {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
if len(line) > 80 {
|
||||
return line[:80] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
return "(comments only)"
|
||||
}
|
||||
152
internal/server/rules_test.go
Normal file
152
internal/server/rules_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRulesPageRenders(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/rules", session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{"Rules editor", "Save new version", "Versions", "Back to inbox", "Unsubscribe"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("rules page missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesSaveCreatesVersion(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rule := `from contains "heritagefruittrees.com.au" => category=Action, importance=high, action=file`
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "text": {rule},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
text, version, err := srv.store.CurrentRules(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentRules() error = %v", err)
|
||||
}
|
||||
if text != rule {
|
||||
t.Errorf("CurrentRules() text = %q, want the saved rule", text)
|
||||
}
|
||||
if version != 2 {
|
||||
t.Errorf("CurrentRules() version = %d, want 2", version)
|
||||
}
|
||||
|
||||
// The editor shows the new version and active rule count.
|
||||
body := getWithSession(t, srv, "/rules", session).Body.String()
|
||||
if !strings.Contains(body, "version 2") || !strings.Contains(body, "1 active rule") {
|
||||
t.Errorf("rules page did not reflect the saved version; body:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesSaveRequiresCSRF(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/save", session, nil, url.Values{"text": {"anything"}})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesPageShowsParseErrors(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "text": {"this line has no arrow"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save status = %d, want 303", rec.Code)
|
||||
}
|
||||
body := getWithSession(t, srv, "/rules", session).Body.String()
|
||||
if !strings.Contains(body, "could not be parsed") || !strings.Contains(body, "missing =>") {
|
||||
t.Errorf("rules page did not surface the parse error; body:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesRestoreCreatesNewVersion(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
first := `from contains "a" => category=Info`
|
||||
second := `from contains "b" => category=Action`
|
||||
for _, text := range []string{first, second} {
|
||||
rec := postForm(t, srv, "/rules/save", session, csrfCookie, url.Values{"csrf": {csrf}, "text": {text}})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("save %q status = %d, want 303", text, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Restore version 2 (the first saved rule) as a new version.
|
||||
rec := postForm(t, srv, "/rules/restore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "version": {"2"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("restore status = %d, want 303", rec.Code)
|
||||
}
|
||||
text, version, err := srv.store.CurrentRules(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CurrentRules() error = %v", err)
|
||||
}
|
||||
if text != first {
|
||||
t.Errorf("restored text = %q, want %q", text, first)
|
||||
}
|
||||
if version != 4 {
|
||||
t.Errorf("restored version = %d, want 4 (append-only)", version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardLinksRulesAndUnsubscribe(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
body := getWithSession(t, srv, "/", session).Body.String()
|
||||
for _, want := range []string{`href="/rules"`, `href="/unsubscribe"`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("dashboard missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesCountsLines(t *testing.T) {
|
||||
text := "# comment\n\nfrom contains \"a\" => category=Info\nfrom contains \"b\" => category=Action"
|
||||
if got := countRuleLines(text); got != 2 {
|
||||
t.Errorf("countRuleLines() = %d, want 2", got)
|
||||
}
|
||||
if got := firstRuleLine(text); got != `from contains "a" => category=Info` {
|
||||
t.Errorf("firstRuleLine() = %q", got)
|
||||
}
|
||||
if got := firstRuleLine("# only a comment"); got != "(comments only)" {
|
||||
t.Errorf("firstRuleLine(comment only) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesReclassifyWithoutJEVRedirects(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/rules/reclassify", session, csrfCookie, url.Values{"csrf": {csrf}})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if loc := rec.Header().Get("Location"); !strings.Contains(loc, "JEV_API_KEY") {
|
||||
t.Errorf("Location = %q, want a JEV-not-configured note", loc)
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,18 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /categories/rename", s.requireAuth(s.handleCategoryRename))
|
||||
mux.HandleFunc("POST /categories/remove", s.requireAuth(s.handleCategoryRemove))
|
||||
|
||||
// Phase 6 (FR-4, FR-11): rules editor, saved and versioned.
|
||||
mux.HandleFunc("GET /rules", s.requireAuth(s.handleRules))
|
||||
mux.HandleFunc("POST /rules/save", s.requireAuth(s.handleRulesSave))
|
||||
mux.HandleFunc("POST /rules/restore", s.requireAuth(s.handleRulesRestore))
|
||||
mux.HandleFunc("POST /rules/reclassify", s.requireAuth(s.handleRulesReclassify))
|
||||
|
||||
// Phase 6 (FR-7): unsubscribe detection, ignore list, and records.
|
||||
mux.HandleFunc("GET /unsubscribe", s.requireAuth(s.handleUnsubscribe))
|
||||
mux.HandleFunc("POST /unsubscribe/ignore", s.requireAuth(s.handleUnsubscribeIgnore))
|
||||
mux.HandleFunc("POST /unsubscribe/unignore", s.requireAuth(s.handleUnsubscribeUnignore))
|
||||
mux.HandleFunc("POST /unsubscribe/mark", s.requireAuth(s.handleUnsubscribeMark))
|
||||
|
||||
return s.recoverPanic(s.logRequests(mux))
|
||||
}
|
||||
|
||||
|
||||
218
internal/server/unsubscribe.go
Normal file
218
internal/server/unsubscribe.go
Normal file
@@ -0,0 +1,218 @@
|
||||
// Unsubscribe detection view for Phase 6 (FR-7).
|
||||
//
|
||||
// The page lists messages with a detected unsubscribe link, the domain-level
|
||||
// ignore list, and the domains Sam has recorded as unsubscribed. Every action
|
||||
// records state; none of them contacts the link. Acting on a real unsubscribe
|
||||
// link is a manual step that needs Sam's explicit approval (AUD-72).
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/unsubscribe"
|
||||
)
|
||||
|
||||
type unsubscribeItem struct {
|
||||
MessageID int64
|
||||
From string
|
||||
Subject string
|
||||
SourceAddr string
|
||||
Domain string
|
||||
Links []unsubscribe.Link
|
||||
Status string
|
||||
Category string
|
||||
When string
|
||||
}
|
||||
|
||||
type unsubscribeData struct {
|
||||
Items []unsubscribeItem
|
||||
Ignored []store.UnsubscribeRecord
|
||||
Recorded []store.UnsubscribeRecord
|
||||
Note string
|
||||
Error string
|
||||
CSRFToken string
|
||||
CandidateCount int
|
||||
}
|
||||
|
||||
// handleUnsubscribe renders the detection list and the ignore/recorded lists.
|
||||
func (s *Server) handleUnsubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderUnsubscribe(w, r, http.StatusOK, r.URL.Query().Get("note"), r.URL.Query().Get("error"))
|
||||
}
|
||||
|
||||
func (s *Server) renderUnsubscribe(w http.ResponseWriter, r *http.Request, code int, note, errMsg string) {
|
||||
ctx := r.Context()
|
||||
records, err := s.store.ListUnsubscribe(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("list unsubscribe records", "error", err)
|
||||
http.Error(w, "could not load the unsubscribe list", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
candidates, err := s.store.UnsubscribeCandidates(ctx, 300)
|
||||
if err != nil {
|
||||
s.logger.Error("list unsubscribe candidates", "error", err)
|
||||
http.Error(w, "could not load messages", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
status := make(map[string]string, len(records))
|
||||
for _, rec := range records {
|
||||
status[rec.Domain] = rec.Status
|
||||
}
|
||||
|
||||
var items []unsubscribeItem
|
||||
for _, c := range candidates {
|
||||
res := unsubscribe.Detect(c.ListUnsubscribe, c.ListUnsubscribePost, c.FromAddr, c.BodyText)
|
||||
if len(res.Links) == 0 {
|
||||
continue
|
||||
}
|
||||
domain := res.Links[0].Domain
|
||||
items = append(items, unsubscribeItem{
|
||||
MessageID: c.MessageID,
|
||||
From: orDefault(c.FromAddr, "(unknown sender)"),
|
||||
Subject: orDefault(c.Subject, "(no subject)"),
|
||||
SourceAddr: c.SourceAddress,
|
||||
Domain: domain,
|
||||
Links: res.Links,
|
||||
Status: status[domain],
|
||||
Category: c.Category,
|
||||
When: formatWhen(c.ReceivedAt, c.ReceivedAt),
|
||||
})
|
||||
}
|
||||
|
||||
data := unsubscribeData{
|
||||
Items: items,
|
||||
CSRFToken: s.csrfToken(w, r),
|
||||
Note: note,
|
||||
Error: errMsg,
|
||||
CandidateCount: len(items),
|
||||
}
|
||||
for _, rec := range records {
|
||||
switch rec.Status {
|
||||
case store.UnsubscribeStatusIgnored:
|
||||
data.Ignored = append(data.Ignored, rec)
|
||||
case store.UnsubscribeStatusDone:
|
||||
data.Recorded = append(data.Recorded, rec)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(code)
|
||||
if err := s.tmpl.ExecuteTemplate(w, "unsubscribe.html", data); err != nil {
|
||||
s.logger.Error("render unsubscribe", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleUnsubscribeIgnore adds a domain to the ignore list (FR-7).
|
||||
func (s *Server) handleUnsubscribeIgnore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusIgnored, r.FormValue("url")); err != nil {
|
||||
s.logger.Error("ignore unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not update the ignore list.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_ignore", "Sam", "domain="+domain)
|
||||
s.redirectUnsubscribe(w, r, "Added "+domain+" to the ignore list. It is never recommended for unsubscribe.", "")
|
||||
}
|
||||
|
||||
// handleUnsubscribeUnignore removes a domain from the ignore list.
|
||||
func (s *Server) handleUnsubscribeUnignore(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.ClearUnsubscribe(r.Context(), domain); err != nil {
|
||||
s.logger.Error("clear unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not update the list.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_unignore", "Sam", "domain="+domain)
|
||||
s.redirectUnsubscribe(w, r, "Removed "+domain+" from the unsubscribe list.", "")
|
||||
}
|
||||
|
||||
// handleUnsubscribeMark records that a domain should be unsubscribed from. It
|
||||
// is a record only: the handler makes no network call and the app never follows
|
||||
// the link (AUD-72). Following a real link needs Sam's explicit approval.
|
||||
func (s *Server) handleUnsubscribeMark(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
s.redirectUnsubscribe(w, r, "", "Could not read the form.")
|
||||
return
|
||||
}
|
||||
if r.FormValue("confirm") != "yes" {
|
||||
s.redirectUnsubscribe(w, r, "", "Marking needs confirmation. The app will record it, not follow the link.")
|
||||
return
|
||||
}
|
||||
domain, ok := cleanDomain(r.FormValue("domain"))
|
||||
if !ok {
|
||||
s.redirectUnsubscribe(w, r, "", "Unknown domain.")
|
||||
return
|
||||
}
|
||||
if err := s.store.SetUnsubscribeStatus(r.Context(), domain, store.UnsubscribeStatusDone, r.FormValue("url")); err != nil {
|
||||
s.logger.Error("mark unsubscribe domain", "domain", domain, "error", err)
|
||||
s.redirectUnsubscribe(w, r, "", "Could not record the unsubscribe.")
|
||||
return
|
||||
}
|
||||
s.logAction(r.Context(), "unsubscribe_marked", "Sam", "domain="+domain+" url="+r.FormValue("url"))
|
||||
s.redirectUnsubscribe(w, r,
|
||||
"Recorded "+domain+" for unsubscribe. The app did not contact the link; acting on it is a manual step.", "")
|
||||
}
|
||||
|
||||
// redirectUnsubscribe is POST/redirect/GET so a refresh does not resubmit.
|
||||
func (s *Server) redirectUnsubscribe(w http.ResponseWriter, r *http.Request, note, errMsg string) {
|
||||
q := url.Values{}
|
||||
if note != "" {
|
||||
q.Set("note", note)
|
||||
}
|
||||
if errMsg != "" {
|
||||
q.Set("error", errMsg)
|
||||
}
|
||||
http.Redirect(w, r, "/unsubscribe?"+q.Encode(), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// cleanDomain validates a domain used as the ignore-list key.
|
||||
func cleanDomain(raw string) (string, bool) {
|
||||
domain := strings.ToLower(strings.TrimSpace(raw))
|
||||
domain = strings.Trim(domain, ".")
|
||||
if domain == "" || len(domain) > 253 {
|
||||
return "", false
|
||||
}
|
||||
for _, r := range domain {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '.', r == '-', r == '_':
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
if !strings.Contains(domain, ".") {
|
||||
return "", false
|
||||
}
|
||||
return domain, true
|
||||
}
|
||||
182
internal/server/unsubscribe_test.go
Normal file
182
internal/server/unsubscribe_test.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
)
|
||||
|
||||
func seedUnsubMessage(t *testing.T, srv *Server) int64 {
|
||||
t.Helper()
|
||||
id, err := srv.store.UpsertMessage(context.Background(), &store.Message{
|
||||
GmailID: "unsub-1",
|
||||
FromAddr: "Heritage Fruit Trees <support@heritagefruittrees.com.au>",
|
||||
Subject: "New workshops released",
|
||||
BodyText: "Hello, you have received a newsletter. Unsubscribe at https://heritagefruittrees.com.au/u?id=1.",
|
||||
SourceAddress: "samuelrolfe@gmail.com",
|
||||
ListUnsubscribe: "<mailto:unsub@heritagefruittrees.com.au?subject=unsubscribe>, <https://heritagefruittrees.com.au/u?id=1>",
|
||||
DateSent: time.Now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("seed message: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUnsubscribePageDetectsLinks(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/unsubscribe", session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{
|
||||
"Detected links", "heritagefruittrees.com.au",
|
||||
"https://heritagefruittrees.com.au/u?id=1",
|
||||
"mailto:unsub@heritagefruittrees.com.au",
|
||||
"Ignore domain", "Mark for unsubscribe", "Ignore list",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("unsubscribe page missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeIgnoreAndUnignore(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("ignore status = %d, want 303 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if rec2.Status != store.UnsubscribeStatusIgnored {
|
||||
t.Errorf("status = %q, want ignored", rec2.Status)
|
||||
}
|
||||
body := getWithSession(t, srv, "/unsubscribe", session).Body.String()
|
||||
if !strings.Contains(body, "ignored") || !strings.Contains(body, "Un-ignore") {
|
||||
t.Errorf("ignore list did not render the ignored domain")
|
||||
}
|
||||
|
||||
rec = postForm(t, srv, "/unsubscribe/unignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("unignore status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
|
||||
t.Error("domain still on the list after unignore")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMarkRecordsWithoutFollowingLink(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
seedUnsubMessage(t, srv)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
ctx := context.Background()
|
||||
|
||||
// Without confirmation the handler refuses and records nothing.
|
||||
rec := postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
"url": {"https://heritagefruittrees.com.au/u?id=1"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("mark without confirm status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au"); ok {
|
||||
t.Error("unconfirmed mark recorded a domain")
|
||||
}
|
||||
|
||||
rec = postForm(t, srv, "/unsubscribe/mark", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"heritagefruittrees.com.au"},
|
||||
"url": {"https://heritagefruittrees.com.au/u?id=1"}, "confirm": {"yes"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("mark status = %d, want 303", rec.Code)
|
||||
}
|
||||
rec2, ok, err := srv.store.UnsubscribeByDomain(ctx, "heritagefruittrees.com.au")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UnsubscribeByDomain() = (_, %v, %v), want a hit", ok, err)
|
||||
}
|
||||
if rec2.Status != store.UnsubscribeStatusDone {
|
||||
t.Errorf("status = %q, want done", rec2.Status)
|
||||
}
|
||||
if rec2.URL != "https://heritagefruittrees.com.au/u?id=1" {
|
||||
t.Errorf("url = %q, want the recorded link", rec2.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMutationsRequireCSRF(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
for _, path := range []string{"/unsubscribe/ignore", "/unsubscribe/unignore", "/unsubscribe/mark"} {
|
||||
rec := postForm(t, srv, path, session, nil, url.Values{"domain": {"x.example"}})
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Errorf("%s without CSRF status = %d, want 403", path, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsubscribeMarkRejectsBadDomain(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
session, csrfCookie, csrf := login(t, srv)
|
||||
|
||||
rec := postForm(t, srv, "/unsubscribe/ignore", session, csrfCookie, url.Values{
|
||||
"csrf": {csrf}, "domain": {"not a domain!"},
|
||||
})
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want 303", rec.Code)
|
||||
}
|
||||
if _, ok, _ := srv.store.UnsubscribeByDomain(context.Background(), "not a domain!"); ok {
|
||||
t.Error("bad domain was stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageDetailShowsUnsubscribeLinks(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
id := seedUnsubMessage(t, srv)
|
||||
session, _, _ := login(t, srv)
|
||||
|
||||
rec := getWithSession(t, srv, "/partials/message?id="+itoa(id), session)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "Unsubscribe") || !strings.Contains(body, "heritagefruittrees.com.au/u?id=1") {
|
||||
t.Errorf("message detail did not show the unsubscribe link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanDomain(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"brand.example": true,
|
||||
"Sub.Brand.example": true,
|
||||
"x-y.example": true,
|
||||
"": false,
|
||||
"nodot": false,
|
||||
"has space.example": false,
|
||||
"bad!char.example": false,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if _, ok := cleanDomain(in); ok != want {
|
||||
t.Errorf("cleanDomain(%q) ok = %v, want %v", in, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user