Sam Rolfe
f9dc574172
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.
- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
Chunking is block-based, so a boundary cannot fall inside a fenced block, a
table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
as a UUID. Re-running updates points; re-chunking deletes the old ones by
source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
with area/confidence filters and a `terms` column so front-matter tech and
tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
from the environment only) and local-hashing (offline placeholder). The
provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.
Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.
Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
111 lines
4.2 KiB
Python
111 lines
4.2 KiB
Python
"""Freshness audit: stale documents, index drift and contract drift."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import unittest
|
|
|
|
from corpus import freshness, pipeline, store as store_module
|
|
from corpus.tests import support
|
|
|
|
|
|
class AuditTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = support.temp_repo()
|
|
self.cfg = support.make_config(self.root)
|
|
self.addCleanup(_cleanup, self.root)
|
|
with support.fake_qdrant(self.cfg):
|
|
pipeline.run(self.cfg, mode="rebuild")
|
|
|
|
def audit(self, today=None, cfg=None):
|
|
with support.fake_qdrant(cfg or self.cfg):
|
|
return freshness.audit(cfg or self.cfg, today=today)
|
|
|
|
def test_clean_soon_after_rebuild(self):
|
|
report = self.audit(today=datetime.date(2026, 10, 5))
|
|
self.assertTrue(report["ok"], report["findings"])
|
|
self.assertEqual(report["corpus"]["documents"], 3)
|
|
|
|
def test_reports_stale_documents(self):
|
|
report = self.audit(today=datetime.date(2027, 10, 5))
|
|
stale = {row["source_path"] for row in report["findings"]["stale"]}
|
|
self.assertEqual(
|
|
stale,
|
|
{
|
|
"content/ai/ai-overview.md",
|
|
"content/edge/l3/edge-zigbee.md",
|
|
"content/platform/l3/platform-containers.md",
|
|
},
|
|
)
|
|
self.assertFalse(report["ok"])
|
|
text = freshness.render_text(report)
|
|
self.assertIn("STALE", text)
|
|
self.assertIn("platform-containers", text)
|
|
|
|
def test_warns_before_a_document_goes_stale(self):
|
|
# platform-containers was verified 2026-08-01. Horizon 180 days, warning
|
|
# 30, so on 2027-01-20 it is 172 days old: due soon, not yet stale.
|
|
report = self.audit(today=datetime.date(2027, 1, 20))
|
|
due = {row["source_path"] for row in report["findings"]["due_soon"]}
|
|
self.assertEqual(due, {"content/platform/l3/platform-containers.md"})
|
|
self.assertEqual(report["findings"]["stale"], [])
|
|
self.assertTrue(report["ok"])
|
|
|
|
def test_reports_index_drift_when_a_source_changes(self):
|
|
support.write_doc(
|
|
self.root,
|
|
"ai/ai-overview.md",
|
|
id="ai-overview", area="ai", level=1, title="Agent Systems",
|
|
claim="Edited but not re-indexed.", confidence="running",
|
|
last_verified="2026-10-05", tags="agents", body="New words for the corpus.",
|
|
)
|
|
report = self.audit(today=datetime.date(2026, 10, 6))
|
|
problems = [row["problem"] for row in report["findings"]["index_drift"]]
|
|
self.assertIn("source changed since it was embedded", problems)
|
|
self.assertFalse(report["ok"])
|
|
|
|
def test_reports_contract_drift_when_the_model_changes(self):
|
|
other = support.make_config(self.root, model="different-model")
|
|
report = self.audit(today=datetime.date(2026, 10, 5), cfg=other)
|
|
self.assertTrue(report["findings"]["contract_drift"])
|
|
|
|
def test_report_is_json_serialisable(self):
|
|
import json
|
|
|
|
report = self.audit(today=datetime.date(2026, 10, 5))
|
|
json.dumps(report)
|
|
|
|
|
|
class AlertTest(unittest.TestCase):
|
|
def test_no_url_means_no_alert(self):
|
|
report = {"ok": True, "findings": {"stale": []}, "corpus": {}, "horizon": {}, "summary": {}}
|
|
self.assertFalse(freshness.alert(report, ""))
|
|
|
|
def test_posts_to_apprise(self):
|
|
report = {
|
|
"generated_at": "2026-10-26T00:00:00Z",
|
|
"today": "2026-10-26",
|
|
"horizon": {"max_age_days": 180, "warn_days": 30},
|
|
"corpus": {
|
|
"documents": 1, "chunks": 1, "sources": 1, "collection": "c",
|
|
"collection_points": 1, "provider": "p", "model": "m", "dim": 8,
|
|
"chunker_version": "v",
|
|
},
|
|
"findings": {"stale": [], "due_soon": [], "index_drift": [], "contract_drift": []},
|
|
"summary": {"stale": 0},
|
|
"ok": True,
|
|
}
|
|
with support.StubServer(lambda method, path, body: (200, "ok")) as server:
|
|
self.assertTrue(freshness.alert(report, server.url))
|
|
self.assertIn("title", server.requests[0]["body"])
|
|
|
|
|
|
def _cleanup(root):
|
|
import shutil
|
|
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|