Phase 2: Gmail IMAP fetch (IPv4), FR-2 source address, real mail in dashboard (AUD-67)
- internal/mailfetch: stdlib-only read-only IMAP client (LOGIN, SELECT, UID SEARCH, UID FETCH) that always dials tcp4; IPv6 to Gmail is broken from .13 and caused a socket EOF. - MIME parsing to text, RFC2047 header decoding, FR-2 source address: X-Pm-Original-To -> To -> Received 'for <addr>' -> Return-Path, never Delivered-To. The matched header is recorded for audit. - store: Message upsert/list/count + LogAction; schema migration v2 adds imap_uid and imap_uidvalidity. - server: dashboard and /partials/inbox render real mail grouped by category with a recommended action; POST /actions/fetch runs a scan (CSRF-checked); nav counts; 'Check mail' button. - config: GMAIL_USER, GMAIL_APP_PASSWORD, IMAP host/port/lookback/max. - cmd: -fetch runs one scan and exits (n8n/CLI entry point, FR-1). - compose/.env.example expose the Gmail + IMAP settings. - tests: FR-2 precedence, MIME bodies, IMAP literal/parser, store upsert/migration, inbox rendering. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -213,6 +213,7 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
type navItem struct {
|
||||
Name string
|
||||
Color string
|
||||
Count int
|
||||
}
|
||||
|
||||
type dashboardData struct {
|
||||
@@ -220,27 +221,40 @@ type dashboardData struct {
|
||||
User string
|
||||
CSRFToken string
|
||||
Version string
|
||||
Total int
|
||||
Categories []navItem
|
||||
Folders []navItem
|
||||
}
|
||||
|
||||
// handleDashboard renders the empty inbox shell.
|
||||
// handleDashboard renders the inbox shell with real category counts.
|
||||
func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
sess := sessionFrom(r)
|
||||
|
||||
// FR-5: counts come from the stored messages. Phase 2 has no classifier
|
||||
// yet, so every message lands under Unclassified (AGENTS.md §13.2).
|
||||
items, err := s.store.ListMessages(r.Context(), 200)
|
||||
if err != nil {
|
||||
s.logger.Error("load dashboard counts", "error", err)
|
||||
items = nil
|
||||
}
|
||||
counts := categoryCounts(items)
|
||||
|
||||
data := dashboardData{
|
||||
Greeting: partOfDay(time.Now()),
|
||||
User: sess.User,
|
||||
CSRFToken: s.csrfToken(w, r),
|
||||
Version: version.Version,
|
||||
Total: len(items),
|
||||
// Placeholders until FR-14 (editable categories) and FR-9 (folders)
|
||||
// land. Colours are the design system's decorative sticker palette.
|
||||
Categories: []navItem{
|
||||
{Name: "Action", Color: "var(--primary)"},
|
||||
{Name: "Important", Color: "var(--sticker-orange)"},
|
||||
{Name: "Info", Color: "var(--sticker-sky)"},
|
||||
{Name: "Spam", Color: "var(--sticker-red)"},
|
||||
{Name: "Newsletter", Color: "var(--sticker-purple)"},
|
||||
{Name: "Auto-reply", Color: "var(--sticker-teal)"},
|
||||
{Name: "Action", Color: "var(--primary)", Count: counts["action"]},
|
||||
{Name: "Important", Color: "var(--sticker-orange)", Count: counts["important"]},
|
||||
{Name: "Info", Color: "var(--sticker-sky)", Count: counts["info"]},
|
||||
{Name: "Spam", Color: "var(--sticker-red)", Count: counts["spam"]},
|
||||
{Name: "Newsletter", Color: "var(--sticker-purple)", Count: counts["newsletter"]},
|
||||
{Name: "Auto-reply", Color: "var(--sticker-teal)", Count: counts["auto-reply"]},
|
||||
{Name: "Unclassified", Color: "var(--ink-faint)", Count: counts[unclassifiedCategory]},
|
||||
},
|
||||
Folders: []navItem{
|
||||
{Name: "Replied-to", Color: "var(--sticker-green)"},
|
||||
@@ -254,15 +268,6 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// handleInboxPartial is the HTMX fragment for the centre list. Phase 1.2 has no
|
||||
// mail yet, so it renders the empty state.
|
||||
func (s *Server) handleInboxPartial(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := s.tmpl.ExecuteTemplate(w, "inbox.html", nil); err != nil {
|
||||
s.logger.Error("render inbox partial", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanNext only allows same-site, absolute-path redirects.
|
||||
func cleanNext(next string) string {
|
||||
if next == "" || !strings.HasPrefix(next, "/") || strings.HasPrefix(next, "//") {
|
||||
|
||||
219
internal/server/inbox.go
Normal file
219
internal/server/inbox.go
Normal file
@@ -0,0 +1,219 @@
|
||||
// Inbox rendering and the manual mail scan.
|
||||
//
|
||||
// Phase 2 lists real mail from the database. Classification is Phase 3, so
|
||||
// messages without a classification are shown under "Unclassified" with a
|
||||
// temporary "Review" recommended action; the grouping and action plumbing is
|
||||
// already category-aware.
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
)
|
||||
|
||||
// unclassifiedCategory is the fallback bucket before Phase 3 adds JEV
|
||||
// categories (AGENTS.md §13.2).
|
||||
const unclassifiedCategory = "unclassified"
|
||||
|
||||
// categoryOrder is the display order for the built-in categories.
|
||||
var categoryOrder = []string{"action", "important", "info", "spam", "newsletter", "auto-reply"}
|
||||
|
||||
var categoryDisplay = map[string]string{
|
||||
"action": "Action",
|
||||
"important": "Important",
|
||||
"info": "Info",
|
||||
"spam": "Spam",
|
||||
"newsletter": "Newsletter",
|
||||
"auto-reply": "Auto-reply",
|
||||
"unclassified": "Unclassified",
|
||||
}
|
||||
|
||||
type messageRow struct {
|
||||
ID int64
|
||||
From string
|
||||
Subject string
|
||||
SourceAddr string
|
||||
Action string
|
||||
When string
|
||||
}
|
||||
|
||||
type messageGroup struct {
|
||||
Name string
|
||||
Messages []messageRow
|
||||
}
|
||||
|
||||
type inboxData struct {
|
||||
Groups []messageGroup
|
||||
Total int
|
||||
Note string
|
||||
Error string
|
||||
}
|
||||
|
||||
// loadInbox reads messages and groups them for rendering.
|
||||
func (s *Server) loadInbox(ctx context.Context) (inboxData, error) {
|
||||
items, err := s.store.ListMessages(ctx, 200)
|
||||
if err != nil {
|
||||
return inboxData{}, err
|
||||
}
|
||||
return inboxData{Groups: groupMessages(items), Total: len(items)}, nil
|
||||
}
|
||||
|
||||
// groupMessages buckets messages by category. Unknown (custom) categories get
|
||||
// their own group; messages with no classification fall into "Unclassified".
|
||||
func groupMessages(items []store.MessageListItem) []messageGroup {
|
||||
buckets := map[string][]messageRow{}
|
||||
for _, it := range items {
|
||||
key := strings.ToLower(strings.TrimSpace(it.Category))
|
||||
if key == "" {
|
||||
key = unclassifiedCategory
|
||||
}
|
||||
action := strings.TrimSpace(it.Action)
|
||||
if action == "" {
|
||||
action = "Review"
|
||||
}
|
||||
buckets[key] = append(buckets[key], messageRow{
|
||||
ID: it.ID,
|
||||
From: orDefault(it.FromAddr, "(unknown sender)"),
|
||||
Subject: orDefault(it.Subject, "(no subject)"),
|
||||
SourceAddr: it.SourceAddress,
|
||||
Action: action,
|
||||
When: formatWhen(it.DateSent, it.ReceivedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// Order: built-in categories first, then custom ones alphabetically, then
|
||||
// Unclassified last.
|
||||
var keys []string
|
||||
for _, k := range categoryOrder {
|
||||
if _, ok := buckets[k]; ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
var custom []string
|
||||
for k := range buckets {
|
||||
if k == unclassifiedCategory || contains(categoryOrder, k) {
|
||||
continue
|
||||
}
|
||||
custom = append(custom, k)
|
||||
}
|
||||
sort.Strings(custom)
|
||||
keys = append(keys, custom...)
|
||||
if _, ok := buckets[unclassifiedCategory]; ok {
|
||||
keys = append(keys, unclassifiedCategory)
|
||||
}
|
||||
|
||||
groups := make([]messageGroup, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
groups = append(groups, messageGroup{
|
||||
Name: displayCategory(k),
|
||||
Messages: buckets[k],
|
||||
})
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func displayCategory(key string) string {
|
||||
if name, ok := categoryDisplay[key]; ok {
|
||||
return name
|
||||
}
|
||||
if key == "" {
|
||||
return "Unclassified"
|
||||
}
|
||||
return strings.ToUpper(key[:1]) + key[1:]
|
||||
}
|
||||
|
||||
// categoryCounts returns per-category message counts for the left nav.
|
||||
func categoryCounts(items []store.MessageListItem) map[string]int {
|
||||
counts := map[string]int{}
|
||||
for _, it := range items {
|
||||
key := strings.ToLower(strings.TrimSpace(it.Category))
|
||||
if key == "" {
|
||||
key = unclassifiedCategory
|
||||
}
|
||||
counts[key]++
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
func (s *Server) renderInbox(w http.ResponseWriter, r *http.Request, note string) {
|
||||
data, err := s.loadInbox(r.Context())
|
||||
if err != nil {
|
||||
s.logger.Error("load inbox", "error", err)
|
||||
http.Error(w, "could not load messages", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
data.Note = note
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := s.tmpl.ExecuteTemplate(w, "inbox.html", data); err != nil {
|
||||
s.logger.Error("render inbox partial", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleInboxPartial is the HTMX fragment for the centre list.
|
||||
func (s *Server) handleInboxPartial(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderInbox(w, r, "")
|
||||
}
|
||||
|
||||
// handleFetchMail runs one IMAP scan, then returns the refreshed list. It is a
|
||||
// POST because it changes stored state and must carry a CSRF token.
|
||||
func (s *Server) handleFetchMail(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
sum, err := s.fetcher.Fetch(ctx)
|
||||
switch {
|
||||
case err == nil:
|
||||
s.renderInbox(w, r, fmt.Sprintf("Scanned %d message(s); %d stored or refreshed.", sum.Fetched, sum.Stored))
|
||||
case errors.Is(err, mailfetch.ErrNoCredentials):
|
||||
s.renderInbox(w, r, "Mail is not connected yet: set GMAIL_USER and GMAIL_APP_PASSWORD.")
|
||||
case errors.Is(err, mailfetch.ErrAlreadyRunning):
|
||||
s.renderInbox(w, r, "A mail scan is already running.")
|
||||
default:
|
||||
s.logger.Error("mail scan failed", "error", err)
|
||||
s.renderInbox(w, r, "Mail scan failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func orDefault(v, def string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return def
|
||||
}
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
func contains(list []string, v string) bool {
|
||||
for _, x := range list {
|
||||
if x == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// formatWhen renders a compact, timezone-stable timestamp.
|
||||
func formatWhen(sent, received time.Time) string {
|
||||
t := sent
|
||||
if t.IsZero() {
|
||||
t = received
|
||||
}
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
if t.Year() == time.Now().Year() {
|
||||
return t.Format("2 Jan 15:04")
|
||||
}
|
||||
return t.Format("2 Jan 2006")
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/config"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/session"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/version"
|
||||
@@ -37,6 +38,7 @@ type Server struct {
|
||||
http *http.Server
|
||||
tmpl *template.Template
|
||||
sessions *session.Manager
|
||||
fetcher *mailfetch.Fetcher
|
||||
}
|
||||
|
||||
// New builds a Server. It does not start listening.
|
||||
@@ -61,6 +63,7 @@ func New(cfg *config.Config, st *store.Store, logger *slog.Logger) (*Server, err
|
||||
logger: logger,
|
||||
tmpl: tmpl,
|
||||
sessions: sessions,
|
||||
fetcher: mailfetch.NewFetcher(cfg, st, logger),
|
||||
}
|
||||
s.http = &http.Server{
|
||||
Addr: cfg.Addr,
|
||||
@@ -109,6 +112,7 @@ func (s *Server) Handler() http.Handler {
|
||||
|
||||
mux.HandleFunc("GET /{$}", s.requireAuth(s.handleDashboard))
|
||||
mux.HandleFunc("GET /partials/inbox", s.requireAuth(s.handleInboxPartial))
|
||||
mux.HandleFunc("POST /actions/fetch", s.requireAuth(s.handleFetchMail))
|
||||
|
||||
return s.recoverPanic(s.logRequests(mux))
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/config"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/session"
|
||||
@@ -152,13 +153,69 @@ func TestLoginFlowAndDashboard(t *testing.T) {
|
||||
t.Fatalf("dashboard status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{"Good ", "Inbox", "Categories", "Folders", "Log out"} {
|
||||
for _, want := range []string{"Good ", "Inbox", "Categories", "Folders", "Log out", "Check mail", "All mail"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("dashboard body missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboxShowsStoredMail(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
if _, err := srv.store.UpsertMessage(context.Background(), &store.Message{
|
||||
GmailID: "1234000000000000000",
|
||||
FromAddr: "favrolfe@gmail.com",
|
||||
ToAddr: "Hello@where-woof.com",
|
||||
Subject: "Real forwarded mail",
|
||||
BodyText: "hello from phase 2",
|
||||
SourceAddress: "Hello@where-woof.com",
|
||||
DateSent: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed message: %v", err)
|
||||
}
|
||||
|
||||
h := srv.Handler()
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/login", nil))
|
||||
csrf := csrfFromBody(t, rec.Body.String())
|
||||
csrfCookie := findCookie(t, rec.Result().Cookies(), csrfCookieName)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, loginRequest(csrf, testPassword, csrfCookie))
|
||||
sessionCookie := findCookie(t, rec.Result().Cookies(), session.CookieName)
|
||||
if sessionCookie == nil {
|
||||
t.Fatal("login did not set the session cookie")
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/partials/inbox", nil)
|
||||
req.AddCookie(sessionCookie)
|
||||
rec = httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("inbox partial status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
for _, want := range []string{"Real forwarded mail", "favrolfe@gmail.com", "Hello@where-woof.com", "Unclassified", "Review"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("inbox partial missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRequiresCSRF(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/actions/fetch", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rec, req)
|
||||
|
||||
// Unauthenticated requests are redirected to login before the CSRF check.
|
||||
if rec.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginWrongPassword(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
h := srv.Handler()
|
||||
|
||||
Reference in New Issue
Block a user