Phase 3: JEV classification, rules table, editable categories (AUD-69)
Implements FR-3, FR-4, FR-5 and FR-14. - internal/classify: TypeSafe System One (JEV) HTTP client with the confirmed request/response shape, retry with backoff on 429/529/5xx, a plain-text rule engine (first match wins, overrides only the named fields), and bounded orchestration with a per-run cap. - internal/store: migration v3 adds an editable `categories` table (name = JEV Choice option and classifications key; rename cascades, remove reassigns) and seeds a starter rules version once. Classification persistence, coverage, and pending/all message queries. - internal/server: POST /actions/classify and /actions/reclassify, plus the FR-14 category editor (GET /categories and add/rename/remove). The triage list now groups by the live category list and shows action + confidence. - CLI: -classify and -classify-all for n8n. - Docs: docs/phase3-classify.md with the confirmed JEV shape, a live read-only classification, and the DoD evidence. DoD: all 100 stored messages have a non-null category, importance, and suggested action (0 nulls); JEV picked a newly added/renamed category for 6 real messages over HTTP. One live JEV call recorded in the phase doc. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
@@ -1,9 +1,8 @@
|
||||
// Inbox rendering and the manual mail scan.
|
||||
// Inbox rendering, the manual mail scan, and the classify action.
|
||||
//
|
||||
// Phase 2 lists real mail from the database. Classification is Phase 3, so
|
||||
// messages without a classification are shown under "Unclassified" with a
|
||||
// temporary "Review" recommended action; the grouping and action plumbing is
|
||||
// already category-aware.
|
||||
// Phase 3 groups messages by the live category list (FR-5) and shows the JEV
|
||||
// recommended action plus confidence on each row. Messages without a
|
||||
// classification fall under "Unclassified".
|
||||
package server
|
||||
|
||||
import (
|
||||
@@ -15,26 +14,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/classify"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/mailfetch"
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
)
|
||||
|
||||
// unclassifiedCategory is the fallback bucket before Phase 3 adds JEV
|
||||
// categories (AGENTS.md §13.2).
|
||||
const unclassifiedCategory = "unclassified"
|
||||
|
||||
// categoryOrder is the display order for the built-in categories.
|
||||
var categoryOrder = []string{"action", "important", "info", "spam", "newsletter", "auto-reply"}
|
||||
|
||||
var categoryDisplay = map[string]string{
|
||||
"action": "Action",
|
||||
"important": "Important",
|
||||
"info": "Info",
|
||||
"spam": "Spam",
|
||||
"newsletter": "Newsletter",
|
||||
"auto-reply": "Auto-reply",
|
||||
"unclassified": "Unclassified",
|
||||
}
|
||||
// unclassifiedCategory is the bucket for messages JEV has not classified yet.
|
||||
const unclassifiedCategory = "Unclassified"
|
||||
|
||||
type messageRow struct {
|
||||
ID int64
|
||||
@@ -42,6 +28,8 @@ type messageRow struct {
|
||||
Subject string
|
||||
SourceAddr string
|
||||
Action string
|
||||
Confidence int
|
||||
Classified bool
|
||||
When string
|
||||
}
|
||||
|
||||
@@ -57,21 +45,42 @@ type inboxData struct {
|
||||
Error string
|
||||
}
|
||||
|
||||
// loadInbox reads messages and groups them for rendering.
|
||||
// loadInbox reads messages and groups them by the live category list.
|
||||
func (s *Server) loadInbox(ctx context.Context) (inboxData, error) {
|
||||
items, err := s.store.ListMessages(ctx, 200)
|
||||
if err != nil {
|
||||
return inboxData{}, err
|
||||
}
|
||||
return inboxData{Groups: groupMessages(items), Total: len(items)}, nil
|
||||
categories, err := s.store.ListCategories(ctx)
|
||||
if err != nil {
|
||||
return inboxData{}, err
|
||||
}
|
||||
return inboxData{Groups: groupMessages(items, categories), Total: len(items)}, nil
|
||||
}
|
||||
|
||||
// groupMessages buckets messages by category. Unknown (custom) categories get
|
||||
// their own group; messages with no classification fall into "Unclassified".
|
||||
func groupMessages(items []store.MessageListItem) []messageGroup {
|
||||
// groupMessages buckets messages by category, ordering groups the way the
|
||||
// category list is ordered in the UI. Unknown categories (for example a
|
||||
// classification written before a rename) keep their own group; messages with
|
||||
// no classification fall into "Unclassified" last.
|
||||
func groupMessages(items []store.MessageListItem, categories []store.Category) []messageGroup {
|
||||
canonical := make(map[string]string, len(categories))
|
||||
order := make([]string, 0, len(categories))
|
||||
for _, c := range categories {
|
||||
canonical[strings.ToLower(c.Name)] = c.Name
|
||||
order = append(order, c.Name)
|
||||
}
|
||||
|
||||
buckets := map[string][]messageRow{}
|
||||
for _, it := range items {
|
||||
key := strings.ToLower(strings.TrimSpace(it.Category))
|
||||
raw := strings.TrimSpace(it.Category)
|
||||
key := ""
|
||||
if raw != "" {
|
||||
if name, ok := canonical[strings.ToLower(raw)]; ok {
|
||||
key = name
|
||||
} else {
|
||||
key = raw
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
key = unclassifiedCategory
|
||||
}
|
||||
@@ -85,62 +94,47 @@ func groupMessages(items []store.MessageListItem) []messageGroup {
|
||||
Subject: orDefault(it.Subject, "(no subject)"),
|
||||
SourceAddr: it.SourceAddress,
|
||||
Action: action,
|
||||
Confidence: confidencePercent(it.Confidence),
|
||||
Classified: it.HasClassification,
|
||||
When: formatWhen(it.DateSent, it.ReceivedAt),
|
||||
})
|
||||
}
|
||||
|
||||
// Order: built-in categories first, then custom ones alphabetically, then
|
||||
// Unclassified last.
|
||||
var keys []string
|
||||
for _, k := range categoryOrder {
|
||||
if _, ok := buckets[k]; ok {
|
||||
keys = append(keys, k)
|
||||
for _, name := range order {
|
||||
if _, ok := buckets[name]; ok {
|
||||
keys = append(keys, name)
|
||||
}
|
||||
}
|
||||
var custom []string
|
||||
var unknown []string
|
||||
for k := range buckets {
|
||||
if k == unclassifiedCategory || contains(categoryOrder, k) {
|
||||
if k == unclassifiedCategory || contains(order, k) {
|
||||
continue
|
||||
}
|
||||
custom = append(custom, k)
|
||||
unknown = append(unknown, k)
|
||||
}
|
||||
sort.Strings(custom)
|
||||
keys = append(keys, custom...)
|
||||
sort.Strings(unknown)
|
||||
keys = append(keys, unknown...)
|
||||
if _, ok := buckets[unclassifiedCategory]; ok {
|
||||
keys = append(keys, unclassifiedCategory)
|
||||
}
|
||||
|
||||
groups := make([]messageGroup, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
groups = append(groups, messageGroup{
|
||||
Name: displayCategory(k),
|
||||
Messages: buckets[k],
|
||||
})
|
||||
groups = append(groups, messageGroup{Name: k, Messages: buckets[k]})
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func displayCategory(key string) string {
|
||||
if name, ok := categoryDisplay[key]; ok {
|
||||
return name
|
||||
// confidencePercent turns a 0–1 confidence into a whole percentage.
|
||||
func confidencePercent(v float64) int {
|
||||
if v <= 0 {
|
||||
return 0
|
||||
}
|
||||
if key == "" {
|
||||
return "Unclassified"
|
||||
if v > 1 {
|
||||
v = 1
|
||||
}
|
||||
return strings.ToUpper(key[:1]) + key[1:]
|
||||
}
|
||||
|
||||
// categoryCounts returns per-category message counts for the left nav.
|
||||
func categoryCounts(items []store.MessageListItem) map[string]int {
|
||||
counts := map[string]int{}
|
||||
for _, it := range items {
|
||||
key := strings.ToLower(strings.TrimSpace(it.Category))
|
||||
if key == "" {
|
||||
key = unclassifiedCategory
|
||||
}
|
||||
counts[key]++
|
||||
}
|
||||
return counts
|
||||
return int(v*100 + 0.5)
|
||||
}
|
||||
|
||||
func (s *Server) renderInbox(w http.ResponseWriter, r *http.Request, note string) {
|
||||
@@ -187,6 +181,53 @@ func (s *Server) handleFetchMail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// handleClassify classifies pending messages (FR-3) and returns the refreshed
|
||||
// list. A run can take minutes, so the write deadline is extended for this
|
||||
// handler only.
|
||||
func (s *Server) handleClassify(w http.ResponseWriter, r *http.Request) {
|
||||
s.runClassifier(w, r, false)
|
||||
}
|
||||
|
||||
// handleReclassify re-runs the classifier over every stored message. It is how
|
||||
// a category-list or rules edit is applied to mail that was already classified.
|
||||
func (s *Server) handleReclassify(w http.ResponseWriter, r *http.Request) {
|
||||
s.runClassifier(w, r, true)
|
||||
}
|
||||
|
||||
func (s *Server) runClassifier(w http.ResponseWriter, r *http.Request, all bool) {
|
||||
if !s.csrfOK(r) {
|
||||
http.Error(w, "invalid CSRF token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if rc := http.NewResponseController(w); rc != nil {
|
||||
_ = rc.SetWriteDeadline(time.Now().Add(15 * time.Minute))
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
sum classify.Summary
|
||||
err error
|
||||
)
|
||||
if all {
|
||||
sum, err = s.classifier.RunAll(ctx)
|
||||
} else {
|
||||
sum, err = s.classifier.Run(ctx)
|
||||
}
|
||||
switch {
|
||||
case err == nil:
|
||||
s.renderInbox(w, r, fmt.Sprintf(
|
||||
"Classified %d of %d message(s); %d failed; %d still unclassified.",
|
||||
sum.Classified, sum.Considered, sum.Failed, sum.Remaining))
|
||||
case errors.Is(err, classify.ErrNotConfigured):
|
||||
s.renderInbox(w, r, "Classification is not connected yet: set JEV_API_KEY.")
|
||||
default:
|
||||
s.logger.Error("classification run failed", "error", err)
|
||||
s.renderInbox(w, r, "Classification failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func orDefault(v, def string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return def
|
||||
|
||||
Reference in New Issue
Block a user