Phase 4: corpus and retrieval — chunks, Qdrant, FTS5, freshness audit

Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.

- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
  Chunking is block-based, so a boundary cannot fall inside a fenced block, a
  table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
  as a UUID. Re-running updates points; re-chunking deletes the old ones by
  source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
  or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
  written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
  with area/confidence filters and a `terms` column so front-matter tech and
  tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
  optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
  from the environment only) and local-hashing (offline placeholder). The
  provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.

Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.

Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

corpus/tests/test_payloads.py

@@ -0,0 +1,100 @@
+"""Payload rules: every field present, enums enforced, missing field fails the run."""
+
+from __future__ import annotations
+
+import unittest
+
+from corpus import chunker, loader, payloads
+from corpus.config import Config
+from corpus.pipeline import chunk_id
+from corpus.tests import support
+
+
+def first_payload():
+    cfg = Config.from_env(root=support.REPO_ROOT, provider="local-hashing", dim=32)
+    doc = loader.load_documents(cfg)[0]
+    chunk = chunker.chunk_document(doc.body)[0]
+    payload = payloads.build_payload(doc, chunk, cfg)
+    payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
+    return cfg, doc, payload
+
+
+class BuildTest(unittest.TestCase):
+    def test_payload_has_every_required_field(self):
+        cfg, doc, payload = first_payload()
+        payloads.validate_payload(payload)
+        for field in payloads.REQUIRED_FIELDS:
+            self.assertIn(field, payload)
+        self.assertEqual(payload["area"], doc.area)
+        self.assertEqual(payload["confidence"], doc.confidence)
+        self.assertEqual(payload["last_verified"], doc.last_verified)
+        self.assertEqual(payload["embedding_model"], cfg.model)
+        self.assertEqual(payload["embedding_dim"], cfg.dim)
+
+    def test_every_real_document_produces_valid_payloads(self):
+        cfg = Config.from_env(root=support.REPO_ROOT, provider="local-hashing", dim=32)
+        for doc in loader.load_documents(cfg):
+            for chunk in chunker.chunk_document(doc.body):
+                payload = payloads.build_payload(doc, chunk, cfg)
+                payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
+                payloads.validate_payload(payload, where=doc.path)
+
+    def test_source_type_is_doc_by_default(self):
+        _cfg, doc, payload = first_payload()
+        self.assertEqual(payload["source_type"], "doc")
+
+    def test_source_type_from_tags(self):
+        doc = loader.Document(
+            path="content/ai/adr-x.md", source_id="adr-x", area="ai", level=3, title="ADR",
+            claim="c", confidence="delivered", period="2024", tech=[], metrics=[], artefacts=[],
+            public=True, last_verified="2026-01-01", tags=["adr"], body="", digest="0" * 64,
+            url="/ai/a/adr-x/",
+        )
+        self.assertEqual(payloads.source_type_for(doc), "adr")
+
+
+class ValidationTest(unittest.TestCase):
+    def test_missing_field_raises(self):
+        _cfg, _doc, payload = first_payload()
+        for field in ("area", "level", "source_type", "confidence", "period", "artefacts", "last_verified", "text"):
+            broken = dict(payload)
+            del broken[field]
+            with self.assertRaises(payloads.PayloadError, msg=f"missing {field} passed validation"):
+                payloads.validate_payload(broken)
+
+    def test_missing_provenance_field_raises(self):
+        _cfg, _doc, payload = first_payload()
+        for field in payloads.PROVENANCE_FIELDS:
+            broken = dict(payload)
+            del broken[field]
+            with self.assertRaises(payloads.PayloadError, msg=f"missing {field} passed validation"):
+                payloads.validate_payload(broken)
+
+    def test_bad_enums_raise(self):
+        _cfg, _doc, payload = first_payload()
+        for field, value in (
+            ("area", "sales"),
+            ("level", 4),
+            ("source_type", "blog"),
+            ("confidence", "vibes"),
+        ):
+            broken = dict(payload)
+            broken[field] = value
+            with self.assertRaises(payloads.PayloadError, msg=f"{field}={value!r} passed validation"):
+                payloads.validate_payload(broken)
+
+    def test_empty_text_raises(self):
+        _cfg, _doc, payload = first_payload()
+        payload["text"] = "   "
+        with self.assertRaises(payloads.PayloadError):
+            payloads.validate_payload(payload)
+
+    def test_bad_date_raises(self):
+        _cfg, _doc, payload = first_payload()
+        payload["last_verified"] = "last october"
+        with self.assertRaises(payloads.PayloadError):
+            payloads.validate_payload(payload)
+
+
+if __name__ == "__main__":
+    unittest.main()