Phase 3: JEV classification, rules table, editable categories (AUD-69)
Implements FR-3, FR-4, FR-5 and FR-14. - internal/classify: TypeSafe System One (JEV) HTTP client with the confirmed request/response shape, retry with backoff on 429/529/5xx, a plain-text rule engine (first match wins, overrides only the named fields), and bounded orchestration with a per-run cap. - internal/store: migration v3 adds an editable `categories` table (name = JEV Choice option and classifications key; rename cascades, remove reassigns) and seeds a starter rules version once. Classification persistence, coverage, and pending/all message queries. - internal/server: POST /actions/classify and /actions/reclassify, plus the FR-14 category editor (GET /categories and add/rename/remove). The triage list now groups by the live category list and shows action + confidence. - CLI: -classify and -classify-all for n8n. - Docs: docs/phase3-classify.md with the confirmed JEV shape, a live read-only classification, and the DoD evidence. DoD: all 100 stored messages have a non-null category, importance, and suggested action (0 nulls); JEV picked a newly added/renamed category for 6 real messages over HTTP. One live JEV call recorded in the phase doc. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
288
internal/classify/classify_test.go
Normal file
288
internal/classify/classify_test.go
Normal file
@@ -0,0 +1,288 @@
|
||||
package classify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lab.audasmedia.com.au/sam/email_agent/internal/store"
|
||||
)
|
||||
|
||||
// stubJEV returns a TypeSafe-shaped response with a fixed category/action while
|
||||
// recording how many requests it handled.
|
||||
func stubJEV(t *testing.T, category, importance, action string, calls *int32) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
*calls++
|
||||
var body struct {
|
||||
Questions map[string]any `json:"questions"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if len(body.Questions) != 3 {
|
||||
t.Errorf("JEV stub saw %d questions, want 3", len(body.Questions))
|
||||
}
|
||||
resp := map[string]any{
|
||||
"model": "jev-1.13.0",
|
||||
"answers": map[string]any{
|
||||
"category": map[string]any{"type": "choice", "choice": category, "confidence": 0.8},
|
||||
"importance": map[string]any{"type": "choice", "choice": importance, "confidence": 0.7},
|
||||
"action": map[string]any{"type": "choice", "choice": action, "confidence": 0.6},
|
||||
},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}))
|
||||
}
|
||||
|
||||
func newTestStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.Open(context.Background(), filepath.Join(t.TempDir(), "email_agent.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open() error = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
func seedMessages(t *testing.T, st *store.Store, from string, n int) {
|
||||
t.Helper()
|
||||
for i := 0; i < n; i++ {
|
||||
if _, err := st.UpsertMessage(context.Background(), &store.Message{
|
||||
GmailID: fmt.Sprintf("%s-%d@test", from, i),
|
||||
FromAddr: from,
|
||||
ToAddr: "hello@where-woof.com",
|
||||
Subject: fmt.Sprintf("message %d", i),
|
||||
BodyText: "body",
|
||||
SourceAddress: "hello@where-woof.com",
|
||||
DateSent: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed message: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newTestClassifier(t *testing.T, st *store.Store, baseURL string) *Classifier {
|
||||
t.Helper()
|
||||
client := NewClient(baseURL, "test-key", "jev-latest", slog.New(slog.DiscardHandler))
|
||||
return NewClassifier(client, st, slog.New(slog.DiscardHandler), 2, 50)
|
||||
}
|
||||
|
||||
func TestClassifierClassifiesEveryPendingMessage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "someone@example.com", 3)
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Info", "low", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
sum, err := newTestClassifier(t, st, srv.URL).Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if sum.Considered != 3 || sum.Classified != 3 || sum.Failed != 0 {
|
||||
t.Fatalf("summary = %+v, want 3 considered/classified, 0 failed", sum)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Errorf("JEV calls = %d, want 3", calls)
|
||||
}
|
||||
|
||||
coverage, err := st.ClassificationCoverage(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ClassificationCoverage() error = %v", err)
|
||||
}
|
||||
if coverage.Total != 3 || coverage.Classified != 3 || coverage.Missing() != 0 {
|
||||
t.Fatalf("coverage = %+v, want 3/3 classified", coverage)
|
||||
}
|
||||
|
||||
items, err := st.ListMessages(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessages() error = %v", err)
|
||||
}
|
||||
for _, it := range items {
|
||||
if it.Category == "" || it.Importance == "" || it.Action == "" {
|
||||
t.Errorf("message %d has a null classification: %+v", it.ID, it)
|
||||
}
|
||||
if !it.HasClassification {
|
||||
t.Errorf("message %d HasClassification = false", it.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierRuleOverridesJEV(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "billing@stripe.com", 1)
|
||||
if _, err := st.SaveRules(ctx, `from contains "stripe" => category=Action, importance=high, action=reply`); err != nil {
|
||||
t.Fatalf("SaveRules() error = %v", err)
|
||||
}
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Newsletter", "low", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
sum, err := newTestClassifier(t, st, srv.URL).Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if sum.Classified != 1 {
|
||||
t.Fatalf("summary = %+v", sum)
|
||||
}
|
||||
|
||||
items, err := st.ListMessages(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessages() error = %v", err)
|
||||
}
|
||||
got := items[0]
|
||||
if got.Category != "Action" || got.Importance != "high" || got.Action != "reply" {
|
||||
t.Errorf("rule override not applied: %+v", got)
|
||||
}
|
||||
if got.Confidence != 1 {
|
||||
t.Errorf("Confidence = %v, want 1 for a rule override", got.Confidence)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("JEV calls = %d, want 1 (JEV still consulted)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierIgnoresRuleWithUnknownCategory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "billing@stripe.com", 1)
|
||||
if _, err := st.SaveRules(ctx, `from contains "stripe" => category=NoSuchCategory`); err != nil {
|
||||
t.Fatalf("SaveRules() error = %v", err)
|
||||
}
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Newsletter", "medium", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := newTestClassifier(t, st, srv.URL).Run(ctx); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
items, err := st.ListMessages(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessages() error = %v", err)
|
||||
}
|
||||
if items[0].Category != "Newsletter" {
|
||||
t.Errorf("Category = %q, want the JEV answer Newsletter", items[0].Category)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierFallsBackWhenJEVCategoryUnknown(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "someone@example.com", 1)
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "CategoryThatDoesNotExist", "medium", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := newTestClassifier(t, st, srv.URL).Run(ctx); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
categories, err := st.CategoryNames(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CategoryNames() error = %v", err)
|
||||
}
|
||||
items, err := st.ListMessages(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessages() error = %v", err)
|
||||
}
|
||||
if items[0].Category != categories[0] {
|
||||
t.Errorf("Category = %q, want the first configured category %q", items[0].Category, categories[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierRerunsOnlyPending(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "someone@example.com", 2)
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Info", "low", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
classifier := newTestClassifier(t, st, srv.URL)
|
||||
if _, err := classifier.Run(ctx); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
sum, err := classifier.Run(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("second Run() error = %v", err)
|
||||
}
|
||||
if sum.Considered != 0 {
|
||||
t.Errorf("second run considered %d, want 0 (nothing pending)", sum.Considered)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Errorf("JEV calls = %d, want 2 (only the first run)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierRunAllReclassifiesEverything(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "someone@example.com", 2)
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Info", "low", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
classifier := newTestClassifier(t, st, srv.URL)
|
||||
if _, err := classifier.Run(ctx); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
sum, err := classifier.RunAll(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("RunAll() error = %v", err)
|
||||
}
|
||||
if sum.Considered != 2 || sum.Classified != 2 {
|
||||
t.Fatalf("RunAll() summary = %+v, want 2 considered/classified", sum)
|
||||
}
|
||||
if calls != 4 {
|
||||
t.Errorf("JEV calls = %d, want 4 (2 initial + 2 re-classified)", calls)
|
||||
}
|
||||
|
||||
var n int
|
||||
if err := st.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM classifications`).Scan(&n); err != nil {
|
||||
t.Fatalf("count classifications: %v", err)
|
||||
}
|
||||
if n != 4 {
|
||||
t.Errorf("classification rows = %d, want 4 (history is kept)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierActionLogRecordsRule(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestStore(t)
|
||||
seedMessages(t, st, "billing@stripe.com", 1)
|
||||
if _, err := st.SaveRules(ctx, `from contains "stripe" => category=Action`); err != nil {
|
||||
t.Fatalf("SaveRules() error = %v", err)
|
||||
}
|
||||
|
||||
var calls int32
|
||||
srv := stubJEV(t, "Newsletter", "low", "keep", &calls)
|
||||
defer srv.Close()
|
||||
|
||||
if _, err := newTestClassifier(t, st, srv.URL).Run(ctx); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
var detail string
|
||||
if err := st.DB().QueryRowContext(ctx,
|
||||
`SELECT detail FROM actions_log WHERE action = 'classify' ORDER BY id DESC LIMIT 1`).Scan(&detail); err != nil {
|
||||
t.Fatalf("read classify log: %v", err)
|
||||
}
|
||||
for _, want := range []string{"category=Action", "rule_line=1", "from contains"} {
|
||||
if !strings.Contains(detail, want) {
|
||||
t.Errorf("classify log detail %q missing %q", detail, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user