sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 4: corpus and retrieval — chunks, Qdrant, FTS5, freshness audit

Browse Source 
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.

- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
  Chunking is block-based, so a boundary cannot fall inside a fenced block, a
  table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
  as a UUID. Re-running updates points; re-chunking deletes the old ones by
  source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
  or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
  written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
  with area/confidence filters and a `terms` column so front-matter tech and
  tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
  optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
  from the environment only) and local-hashing (offline placeholder). The
  provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.

Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.

Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Sam Rolfe
2026-10-26 21:04:52 +11:00
parent 24590cc351
commit f9dc574172
28 changed files with 4372 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
corpus/tests/__init__.py Normal file
View File

112
corpus/tests/fake_qdrant.py Normal file
View File

@@ -0,0 +1,112 @@
"""An in-memory stand-in for Qdrant, used by the pipeline tests.
It implements the routes the pipeline actually calls and nothing else, and it is
deliberately dumb: brute-force cosine over a dict. If the pipeline passes
against this, the pipeline's own logic is right; the real server is exercised
separately in `docs/phase-4-notes.md` (rebuild against a live container).
"""
from __future__ import annotations
import math
from corpus.qdrant import Qdrant, QdrantError, QdrantNotFound
class FakeQdrant(Qdrant):
def __init__(self, dim: int):
super().__init__(url="fake://qdrant", timeout=1.0, attempts=1)
self.dim = dim
self.collections = {} # name -> {id: {"vector": [...], "payload": {...}}}
self.calls = []
# -- transport --------------------------------------------------------
def _request(self, method: str, path: str, body=None) -> dict:
self.calls.append((method, path, body))
query = ""
if "?" in path:
path, query = path.split("?", 1)
parts = [part for part in path.split("/") if part]
if not parts:
return {"version": "fake-1.0"}
if parts[0] != "collections":
raise QdrantNotFound(f"no route {path}")
if len(parts) == 2:
name = parts[1]
if method == "PUT":
self.collections.setdefault(name, {})
return {"result": True}
if method == "DELETE":
if name not in self.collections:
raise QdrantNotFound("collection not found")
del self.collections[name]
return {"result": True}
if name not in self.collections:
raise QdrantNotFound("collection not found")
return {
"result": {
"points_count": len(self.collections[name]),
"config": {"params": {"vectors": {"size": self.dim, "distance": "Cosine"}}},
}
}
name = parts[1]
if name not in self.collections:
raise QdrantNotFound("collection not found")
points = self.collections[name]
if parts[2:4] == ["points", "count"]:
return {"result": {"count": len(points)}}
if len(parts) == 3 and parts[2] == "points" and method == "PUT":
for point in (body or {}).get("points", []):
if len(point["vector"]) != self.dim:
raise QdrantError("wrong vector dimension")
points[point["id"]] = {"vector": point["vector"], "payload": point["payload"]}
return {"result": {"status": "completed"}}
if parts[2:4] == ["points", "delete"]:
must = (((body or {}).get("filter") or {}).get("must")) or []
for point_id in list(points):
if _matches(points[point_id]["payload"], must):
del points[point_id]
return {"result": {"status": "completed"}}
if parts[2:4] == ["points", "query"]:
vector = body["query"]
must = (((body or {}).get("filter") or {}).get("must")) or []
scored = []
for point_id, point in points.items():
if not _matches(point["payload"], must):
continue
scored.append(
{
"id": point_id,
"score": _cosine(vector, point["vector"]),
"payload": point["payload"],
}
)
scored.sort(key=lambda row: row["score"], reverse=True)
return {"result": {"points": scored[: int(body.get("limit", 10))]}}
if parts[2:4] == ["points", "scroll"]:
limit = int((body or {}).get("limit", 100))
rows = [
{"id": point_id, "payload": {"source_path": point["payload"].get("source_path")}}
for point_id, point in list(points.items())[:limit]
]
return {"result": {"points": rows, "next_page_offset": None}}
raise QdrantNotFound(f"no route {path}")
def _matches(payload: dict, conditions: list) -> bool:
for condition in conditions:
key = condition.get("key")
expected = (condition.get("match") or {}).get("value")
if payload.get(key) != expected:
return False
return True
def _cosine(left: list, right: list) -> float:
dot = sum(a * b for a, b in zip(left, right))
norm = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))
return dot / norm if norm else 0.0

186
corpus/tests/support.py Normal file
View File

@@ -0,0 +1,186 @@
"""Shared fixtures for the corpus tests: a throwaway content tree and a fake Qdrant."""
from __future__ import annotations
import contextlib
import os
import shutil
import tempfile
from unittest import mock
from corpus.config import Config
from corpus.tests.fake_qdrant import FakeQdrant
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
DOC = """---
id: {id}
area: {area}
level: {level}
title: "{title}"
claim: "{claim}"
confidence: {confidence}
period: "2023–"
tech: [Go, SQLite]
metrics: [{{label: endpoints, value: 8}}]
artefacts: []
public: true
last_verified: {last_verified}
tags: [{tags}]
---
# {title}
{body}
"""
FENCE = """A paragraph before the code block explains the constraint.
```go
package main
func main() {{
println("chunk boundary must never land inside this fence")
}}
```
A paragraph after the code block.
"""
TABLE = """Measured results, one row per endpoint:
| endpoint | latency | loss |
|---|---|---|
| alpha | 12ms | 0.0% |
| beta | 14ms | 0.1% |
| gamma | 9ms | 0.0% |
And a closing paragraph.
"""
PROSE = "\n\n".join(
f"Paragraph {index} describes a delivery constraint in corporate language, "
"with enough words that the chunker has to think about where to break, and "
"never inside a fenced block, a table or a front-matter block."
for index in range(1, 40)
)
DEFAULT_DOCS = {
"ai/ai-overview.md": dict(
id="ai-overview", area="ai", level=1, title="Agent Systems",
claim="Harnesses, memory and routing.", confidence="running",
last_verified="2026-10-01", tags="agents, llm", body=PROSE,
),
"edge/l3/edge-zigbee.md": dict(
id="edge-zigbee", area="edge", level=3, title="Low Power Mesh Sensing",
claim="A constrained mesh that survives power loss.", confidence="delivered",
last_verified="2026-09-15", tags="zigbee, mesh", body=FENCE,
),
"platform/l3/platform-containers.md": dict(
id="platform-containers", area="platform", level=3, title="Container Estate",
claim="Fifty containers with measured resource limits.", confidence="in-progress",
last_verified="2026-08-01", tags="docker", body=TABLE,
),
}
def temp_repo(docs: dict = None) -> str:
"""Create a throwaway repository root with tools/ and content/."""
root = tempfile.mkdtemp(prefix="corpus-test-")
os.makedirs(os.path.join(root, "tools"), exist_ok=True)
shutil.copy(
os.path.join(REPO_ROOT, "tools", "validate_content.py"),
os.path.join(root, "tools", "validate_content.py"),
)
write_docs(root, docs or DEFAULT_DOCS)
return root
def write_docs(root: str, docs: dict) -> None:
for rel, fields in docs.items():
path = os.path.join(root, "content", rel)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as handle:
handle.write(DOC.format(**fields))
def write_doc(root: str, rel: str, **fields) -> None:
path = os.path.join(root, "content", rel)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as handle:
handle.write(DOC.format(**fields))
# A fresh interpreter's import cache is fine; the loader re-reads the file.
def make_config(root: str, **overrides) -> Config:
values = dict(
provider="local-hashing",
model="local-hashing-v1",
dim=32,
qdrant_url="fake://qdrant",
collection="test_corpus",
)
values.update(overrides)
return Config.from_env(root=root, **values)
@contextlib.contextmanager
def fake_qdrant(cfg, fake: FakeQdrant = None):
"""Point every module that builds a Qdrant client at one in-memory fake."""
fake = fake or FakeQdrant(cfg.dim)
with mock.patch("corpus.pipeline.Qdrant", return_value=fake), mock.patch(
"corpus.freshness.Qdrant", return_value=fake
):
yield fake
class StubServer:
"""A throwaway HTTP server on a loopback ephemeral port, for provider tests.
`responder(method, path, body)` returns `(status, payload)`; every request is
recorded with its parsed body so a test can assert the wire shape without a
real provider or a real Qdrant.
"""
def __init__(self, responder):
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
self.requests = []
self._responder = responder
outer = self
class Handler(BaseHTTPRequestHandler):
def _handle(self):
length = int(self.headers.get("Content-Length") or 0)
raw = self.rfile.read(length).decode("utf-8") if length else ""
body = json.loads(raw) if raw.strip() else None
outer.requests.append(
{"method": self.command, "path": self.path, "body": body,
"headers": dict(self.headers)}
)
status, payload = outer._responder(self.command, self.path, body)
data = payload.encode("utf-8") if isinstance(payload, str) else json.dumps(payload).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
do_GET = do_POST = do_PUT = do_DELETE = _handle
def log_message(self, *_args):
pass
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
self.url = f"http://127.0.0.1:{self.server.server_address[1]}"
self._thread = threading.Thread(target=self.server.serve_forever, daemon=True)
def __enter__(self):
self._thread.start()
return self
def __exit__(self, *_exc):
self.server.shutdown()
self.server.server_close()

145
corpus/tests/test_chunker.py Normal file
View File

@@ -0,0 +1,145 @@
"""Chunking rules: size, overlap, determinism, and the hard one — no split inside
a fence, a table or front-matter (docs/phase-4-corpus.md DoD item 3)."""
from __future__ import annotations
import os
import unittest
from corpus import chunker, loader
from corpus.config import Config
from corpus.tests import support
REPO_ROOT = support.REPO_ROOT
def content_documents():
cfg = Config.from_env(root=REPO_ROOT, provider="local-hashing", dim=32)
return loader.load_documents(cfg)
class BlockParsingTest(unittest.TestCase):
def test_fence_is_one_block(self):
blocks = chunker.parse_blocks(support.FENCE)
kinds = [block.kind for block in blocks]
self.assertEqual(kinds, ["para", "fence", "para"])
self.assertIn("chunk boundary must never land", blocks[1].text)
def test_table_is_one_block(self):
blocks = chunker.parse_blocks(support.TABLE)
table = [block for block in blocks if block.kind == "table"]
self.assertEqual(len(table), 1)
self.assertEqual(len(table[0].text.splitlines()), 5) # header, rule, 3 rows
def test_front_matter_in_body_is_refused(self):
with self.assertRaises(chunker.ChunkError):
chunker.parse_blocks("---\nid: x\n---\n\n# Heading\n")
def test_unterminated_fence_runs_to_end(self):
blocks = chunker.parse_blocks("intro\n\n```go\nfunc main() {}\n")
self.assertEqual(blocks[-1].kind, "fence")
self.assertTrue(blocks[-1].text.rstrip().endswith("}"))
def test_headings_start_blocks(self):
blocks = chunker.parse_blocks("# One\n\ntext\n\n## Two\n\nmore\n")
self.assertEqual([block.kind for block in blocks], ["heading", "para", "heading", "para"])
def test_list_items_are_separate_blocks(self):
blocks = chunker.parse_blocks("- one\n- two\n continued\n- three\n")
self.assertEqual([block.kind for block in blocks], ["list", "list", "list"])
self.assertIn("continued", blocks[1].text)
class ChunkingTest(unittest.TestCase):
def test_small_body_is_one_chunk(self):
chunks = chunker.chunk_document("# Title\n\nA short body.\n")
self.assertEqual(len(chunks), 1)
self.assertEqual(chunks[0].index, 0)
self.assertFalse(chunks[0].oversize)
def test_long_body_respects_maximum_and_overlaps(self):
chunks = chunker.chunk_document(support.PROSE)
self.assertGreater(len(chunks), 1)
for chunk in chunks:
self.assertLessEqual(chunk.tokens, chunker.TOKEN_MAX, f"chunk {chunk.index} too large")
# Overlap: consecutive chunks share at least one block.
for previous, current in zip(chunks, chunks[1:]):
self.assertTrue(
set(id(block) for block in previous.blocks) & set(id(block) for block in current.blocks),
f"chunks {previous.index} and {current.index} do not overlap",
)
def test_chunk_indices_are_sequential(self):
chunks = chunker.chunk_document(support.PROSE)
self.assertEqual([chunk.index for chunk in chunks], list(range(len(chunks))))
def test_deterministic(self):
first = chunker.chunk_document(support.PROSE)
second = chunker.chunk_document(support.PROSE)
self.assertEqual([chunk.text for chunk in first], [chunk.text for chunk in second])
def test_code_fence_is_never_split(self):
body = support.PROSE + "\n\n" + support.FENCE + "\n\n" + support.PROSE
chunks = chunker.chunk_document(body)
fences = [block for block in chunker.parse_blocks(body) if block.kind == "fence"]
self.assertTrue(fences)
for fence in fences:
holders = [chunk for chunk in chunks if fence in chunk.blocks]
self.assertTrue(holders, "the fenced block disappeared from every chunk")
for holder in holders:
self.assertIn(fence.text, holder.text, "a chunk holds part of a fenced block")
def test_oversize_single_block_is_flagged_not_split(self):
huge = "```go\n" + "\n".join(f"line{i} := {i}" for i in range(1200)) + "\n```\n"
chunks = chunker.chunk_document(huge)
self.assertEqual(len(chunks), 1)
self.assertTrue(chunks[0].oversize)
self.assertGreater(chunks[0].tokens, chunker.TOKEN_MAX)
class RealContentTest(unittest.TestCase):
"""The same rules, over every document in content/."""
def test_no_boundary_splits_a_protected_region(self):
for doc in content_documents():
regions = chunker.protected_regions(doc.body)
chunks = chunker.chunk_document(doc.body)
chunker.verify_no_split(chunks, regions) # raises on failure
def test_every_chunk_is_whole_blocks(self):
for doc in content_documents():
expected = [block.text for block in chunker.parse_blocks(doc.body)]
seen = []
for chunk in chunker.chunk_document(doc.body):
for block in chunk.blocks:
self.assertIn(block.text, chunk.text)
if not seen or seen[-1] != block.text:
seen.append(block.text)
# Every block appears, in order, exactly as parsed (overlap repeats a
# block at the start of the next chunk, which is expected).
position = 0
for text in expected:
while position < len(seen) and seen[position] != text:
position += 1
self.assertLess(position, len(seen), f"{doc.path}: block was dropped: {text[:40]!r}")
position += 1
def test_chunks_are_within_the_size_band(self):
for doc in content_documents():
for chunk in chunker.chunk_document(doc.body):
if chunk.oversize:
self.assertEqual(len(chunk.blocks), 1)
continue
self.assertLessEqual(chunk.tokens, chunker.TOKEN_MAX, f"{doc.path} chunk {chunk.index}")
def test_content_tree_is_chunked_completely(self):
total = 0
for doc in content_documents():
chunks = chunker.chunk_document(doc.body)
self.assertTrue(chunks, f"{doc.path}: no chunks")
total += len(chunks)
self.assertGreater(total, 0)
if __name__ == "__main__":
unittest.main()

125
corpus/tests/test_embeddings.py Normal file
View File

@@ -0,0 +1,125 @@
"""Providers: determinism offline, and an OpenAI-compatible wire shape through OmniRoute.
There is no direct vendor SDK call anywhere (AGENTS.md §4.5), so the provider
test asserts the HTTP request shape, not a client library's behaviour.
"""
from __future__ import annotations
import math
import unittest
from corpus import embeddings
from corpus.tests import support
class LocalHashingTest(unittest.TestCase):
def test_deterministic_and_unit_length(self):
provider = embeddings.LocalHashingProvider(dim=64)
first = provider.embed(["zigbee mesh latency"])[0]
second = embeddings.LocalHashingProvider(dim=64).embed(["zigbee mesh latency"])[0]
self.assertEqual(first, second)
self.assertEqual(len(first), 64)
self.assertAlmostEqual(math.sqrt(sum(value * value for value in first)), 1.0, places=9)
def test_different_text_differs(self):
provider = embeddings.LocalHashingProvider(dim=64)
a, b = provider.embed(["zigbee", "lora"])
self.assertNotEqual(a, b)
def test_empty_text_still_returns_a_unit_vector(self):
provider = embeddings.LocalHashingProvider(dim=16)
vector = provider.embed([""])[0]
self.assertEqual(len(vector), 16)
self.assertAlmostEqual(math.sqrt(sum(value * value for value in vector)), 1.0, places=9)
def test_stats_count_calls_and_texts(self):
provider = embeddings.LocalHashingProvider(dim=16)
provider.embed(["a", "b", "c"])
provider.embed(["d"])
self.assertEqual(provider.stats(), {"calls": 2, "texts": 4})
provider.reset_stats()
self.assertEqual(provider.stats(), {"calls": 0, "texts": 0})
def test_rejects_a_zero_dimension(self):
with self.assertRaises(embeddings.EmbeddingError):
embeddings.LocalHashingProvider(dim=0)
class OmniRouteTest(unittest.TestCase):
def responder_factory(self, dim=8, status_plan=None):
plan = list(status_plan or [])
state = {"n": 0}
def responder(method, path, body):
state["n"] += 1
if plan:
status = plan.pop(0)
if status != 200:
return status, {"error": "planned failure"}
vectors = [
{"index": index, "embedding": [0.1] * dim}
for index, _text in enumerate(body.get("input") or [])
]
return 200, {"model": body.get("model"), "data": vectors}
return responder
def test_posts_to_v1_embeddings_with_the_configured_model(self):
with support.StubServer(self.responder_factory()) as server:
provider = embeddings.OmniRouteProvider(
api_url=server.url, api_key="test-key", model="text-embedding-3-small", dim=8
)
vectors = provider.embed(["one", "two"])
self.assertEqual(len(vectors), 2)
request = server.requests[0]
self.assertEqual(request["method"], "POST")
self.assertEqual(request["path"], "/v1/embeddings")
self.assertEqual(request["body"]["model"], "text-embedding-3-small")
self.assertEqual(request["body"]["input"], ["one", "two"])
self.assertEqual(request["headers"]["Authorization"], "Bearer test-key")
self.assertEqual(provider.stats(), {"calls": 1, "texts": 2})
def test_batches_requests(self):
with support.StubServer(self.responder_factory()) as server:
provider = embeddings.OmniRouteProvider(
api_url=server.url, api_key="k", model="m", dim=8, batch_size=2
)
provider.embed(["a", "b", "c", "d", "e"])
self.assertEqual(len(server.requests), 3)
self.assertEqual(provider.stats(), {"calls": 3, "texts": 5})
def test_dimension_mismatch_is_refused(self):
with support.StubServer(self.responder_factory(dim=4)) as server:
provider = embeddings.OmniRouteProvider(
api_url=server.url, api_key="k", model="m", dim=8
)
with self.assertRaises(embeddings.EmbeddingError) as caught:
provider.embed(["one"])
self.assertIn("dimension", str(caught.exception))
def test_retries_then_succeeds(self):
with support.StubServer(self.responder_factory(status_plan=[429, 200])) as server:
provider = embeddings.OmniRouteProvider(
api_url=server.url, api_key="k", model="m", dim=8, backoff=0.01
)
vectors = provider.embed(["one"])
self.assertEqual(len(vectors), 1)
self.assertEqual(len(server.requests), 2)
def test_client_error_is_not_retried(self):
with support.StubServer(self.responder_factory(status_plan=[400])) as server:
provider = embeddings.OmniRouteProvider(
api_url=server.url, api_key="k", model="m", dim=8, backoff=0.01
)
with self.assertRaises(embeddings.EmbeddingError):
provider.embed(["one"])
self.assertEqual(len(server.requests), 1)
def test_missing_key_is_refused_before_any_call(self):
with self.assertRaises(embeddings.EmbeddingError):
embeddings.OmniRouteProvider(api_url="http://127.0.0.1:1", api_key="", model="m", dim=8)
if __name__ == "__main__":
unittest.main()

110
corpus/tests/test_freshness.py Normal file
View File

@@ -0,0 +1,110 @@
"""Freshness audit: stale documents, index drift and contract drift."""
from __future__ import annotations
import datetime
import unittest
from corpus import freshness, pipeline, store as store_module
from corpus.tests import support
class AuditTest(unittest.TestCase):
def setUp(self):
self.root = support.temp_repo()
self.cfg = support.make_config(self.root)
self.addCleanup(_cleanup, self.root)
with support.fake_qdrant(self.cfg):
pipeline.run(self.cfg, mode="rebuild")
def audit(self, today=None, cfg=None):
with support.fake_qdrant(cfg or self.cfg):
return freshness.audit(cfg or self.cfg, today=today)
def test_clean_soon_after_rebuild(self):
report = self.audit(today=datetime.date(2026, 10, 5))
self.assertTrue(report["ok"], report["findings"])
self.assertEqual(report["corpus"]["documents"], 3)
def test_reports_stale_documents(self):
report = self.audit(today=datetime.date(2027, 10, 5))
stale = {row["source_path"] for row in report["findings"]["stale"]}
self.assertEqual(
stale,
{
"content/ai/ai-overview.md",
"content/edge/l3/edge-zigbee.md",
"content/platform/l3/platform-containers.md",
},
)
self.assertFalse(report["ok"])
text = freshness.render_text(report)
self.assertIn("STALE", text)
self.assertIn("platform-containers", text)
def test_warns_before_a_document_goes_stale(self):
# platform-containers was verified 2026-08-01. Horizon 180 days, warning
# 30, so on 2027-01-20 it is 172 days old: due soon, not yet stale.
report = self.audit(today=datetime.date(2027, 1, 20))
due = {row["source_path"] for row in report["findings"]["due_soon"]}
self.assertEqual(due, {"content/platform/l3/platform-containers.md"})
self.assertEqual(report["findings"]["stale"], [])
self.assertTrue(report["ok"])
def test_reports_index_drift_when_a_source_changes(self):
support.write_doc(
self.root,
"ai/ai-overview.md",
id="ai-overview", area="ai", level=1, title="Agent Systems",
claim="Edited but not re-indexed.", confidence="running",
last_verified="2026-10-05", tags="agents", body="New words for the corpus.",
)
report = self.audit(today=datetime.date(2026, 10, 6))
problems = [row["problem"] for row in report["findings"]["index_drift"]]
self.assertIn("source changed since it was embedded", problems)
self.assertFalse(report["ok"])
def test_reports_contract_drift_when_the_model_changes(self):
other = support.make_config(self.root, model="different-model")
report = self.audit(today=datetime.date(2026, 10, 5), cfg=other)
self.assertTrue(report["findings"]["contract_drift"])
def test_report_is_json_serialisable(self):
import json
report = self.audit(today=datetime.date(2026, 10, 5))
json.dumps(report)
class AlertTest(unittest.TestCase):
def test_no_url_means_no_alert(self):
report = {"ok": True, "findings": {"stale": []}, "corpus": {}, "horizon": {}, "summary": {}}
self.assertFalse(freshness.alert(report, ""))
def test_posts_to_apprise(self):
report = {
"generated_at": "2026-10-26T00:00:00Z",
"today": "2026-10-26",
"horizon": {"max_age_days": 180, "warn_days": 30},
"corpus": {
"documents": 1, "chunks": 1, "sources": 1, "collection": "c",
"collection_points": 1, "provider": "p", "model": "m", "dim": 8,
"chunker_version": "v",
},
"findings": {"stale": [], "due_soon": [], "index_drift": [], "contract_drift": []},
"summary": {"stale": 0},
"ok": True,
}
with support.StubServer(lambda method, path, body: (200, "ok")) as server:
self.assertTrue(freshness.alert(report, server.url))
self.assertIn("title", server.requests[0]["body"])
def _cleanup(root):
import shutil
shutil.rmtree(root, ignore_errors=True)
if __name__ == "__main__":
unittest.main()

100
corpus/tests/test_payloads.py Normal file
View File

@@ -0,0 +1,100 @@
"""Payload rules: every field present, enums enforced, missing field fails the run."""
from __future__ import annotations
import unittest
from corpus import chunker, loader, payloads
from corpus.config import Config
from corpus.pipeline import chunk_id
from corpus.tests import support
def first_payload():
cfg = Config.from_env(root=support.REPO_ROOT, provider="local-hashing", dim=32)
doc = loader.load_documents(cfg)[0]
chunk = chunker.chunk_document(doc.body)[0]
payload = payloads.build_payload(doc, chunk, cfg)
payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
return cfg, doc, payload
class BuildTest(unittest.TestCase):
def test_payload_has_every_required_field(self):
cfg, doc, payload = first_payload()
payloads.validate_payload(payload)
for field in payloads.REQUIRED_FIELDS:
self.assertIn(field, payload)
self.assertEqual(payload["area"], doc.area)
self.assertEqual(payload["confidence"], doc.confidence)
self.assertEqual(payload["last_verified"], doc.last_verified)
self.assertEqual(payload["embedding_model"], cfg.model)
self.assertEqual(payload["embedding_dim"], cfg.dim)
def test_every_real_document_produces_valid_payloads(self):
cfg = Config.from_env(root=support.REPO_ROOT, provider="local-hashing", dim=32)
for doc in loader.load_documents(cfg):
for chunk in chunker.chunk_document(doc.body):
payload = payloads.build_payload(doc, chunk, cfg)
payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
payloads.validate_payload(payload, where=doc.path)
def test_source_type_is_doc_by_default(self):
_cfg, doc, payload = first_payload()
self.assertEqual(payload["source_type"], "doc")
def test_source_type_from_tags(self):
doc = loader.Document(
path="content/ai/adr-x.md", source_id="adr-x", area="ai", level=3, title="ADR",
claim="c", confidence="delivered", period="2024", tech=[], metrics=[], artefacts=[],
public=True, last_verified="2026-01-01", tags=["adr"], body="", digest="0" * 64,
url="/ai/a/adr-x/",
)
self.assertEqual(payloads.source_type_for(doc), "adr")
class ValidationTest(unittest.TestCase):
def test_missing_field_raises(self):
_cfg, _doc, payload = first_payload()
for field in ("area", "level", "source_type", "confidence", "period", "artefacts", "last_verified", "text"):
broken = dict(payload)
del broken[field]
with self.assertRaises(payloads.PayloadError, msg=f"missing {field} passed validation"):
payloads.validate_payload(broken)
def test_missing_provenance_field_raises(self):
_cfg, _doc, payload = first_payload()
for field in payloads.PROVENANCE_FIELDS:
broken = dict(payload)
del broken[field]
with self.assertRaises(payloads.PayloadError, msg=f"missing {field} passed validation"):
payloads.validate_payload(broken)
def test_bad_enums_raise(self):
_cfg, _doc, payload = first_payload()
for field, value in (
("area", "sales"),
("level", 4),
("source_type", "blog"),
("confidence", "vibes"),
):
broken = dict(payload)
broken[field] = value
with self.assertRaises(payloads.PayloadError, msg=f"{field}={value!r} passed validation"):
payloads.validate_payload(broken)
def test_empty_text_raises(self):
_cfg, _doc, payload = first_payload()
payload["text"] = " "
with self.assertRaises(payloads.PayloadError):
payloads.validate_payload(payload)
def test_bad_date_raises(self):
_cfg, _doc, payload = first_payload()
payload["last_verified"] = "last october"
with self.assertRaises(payloads.PayloadError):
payloads.validate_payload(payload)
if __name__ == "__main__":
unittest.main()

179
corpus/tests/test_pipeline.py Normal file
View File

@@ -0,0 +1,179 @@
"""The pipeline: deterministic ids, no duplicate vectors on re-run, incremental mode,
incomplete payloads never reach the store (docs/phase-4-corpus.md DoD 1-6)."""
from __future__ import annotations
import os
import unittest
from unittest import mock
from corpus import chunker, loader, pipeline, store as store_module
from corpus.tests import support
class RebuildTest(unittest.TestCase):
def setUp(self):
self.root = support.temp_repo()
self.cfg = support.make_config(self.root)
self.addCleanup(_cleanup, self.root)
# One fake for the whole test: a second run must see the first run's points.
self.fake = support.FakeQdrant(self.cfg.dim)
def run_pipeline(self, mode="sync", cfg=None):
cfg = cfg or self.cfg
with support.fake_qdrant(cfg, self.fake):
return pipeline.run(cfg, mode=mode)
def test_rebuild_from_empty_volume(self):
stats = self.run_pipeline("rebuild")
self.assertEqual(stats.documents, 3)
self.assertEqual(stats.added, 3)
self.assertGreater(stats.store_chunks, 0)
self.assertEqual(stats.store_chunks, stats.collection_points)
self.assertEqual(stats.collection_points, len(self.fake.collections[self.cfg.collection]))
self.assertGreater(stats.embed_calls, 0)
def test_running_twice_does_not_duplicate_vectors(self):
first = self.run_pipeline("rebuild")
points_after_first = len(self.fake.collections[self.cfg.collection])
second = self.run_pipeline("sync")
points_after_second = len(self.fake.collections[self.cfg.collection])
self.assertEqual(points_after_first, points_after_second)
self.assertEqual(first.store_chunks, second.store_chunks)
self.assertEqual(second.added, 0)
self.assertEqual(second.updated, 0)
self.assertEqual(second.chunks_written, 0)
def test_incremental_reembeds_only_changed_documents(self):
self.run_pipeline("rebuild")
# Nothing changed: no embedding calls at all.
unchanged = self.run_pipeline("sync")
self.assertEqual(unchanged.embed_calls, 0)
self.assertEqual(unchanged.texts_embedded, 0)
# One document changes: exactly one embedding call, for that document.
support.write_doc(
self.root,
"edge/l3/edge-zigbee.md",
id="edge-zigbee", area="edge", level=3, title="Low Power Mesh Sensing",
claim="A constrained mesh that survives power loss.", confidence="delivered",
last_verified="2026-09-15", tags="zigbee, mesh",
body="The mesh now also reports battery voltage on every node.",
)
changed = self.run_pipeline("sync")
self.assertEqual(changed.added + changed.updated, 1)
self.assertEqual(changed.embed_calls, 1)
expected_chunks = len(
chunker.chunk_document(
loader.load_documents(support.make_config(self.root))[1].body
)
)
self.assertEqual(changed.texts_embedded, expected_chunks)
self.assertEqual(changed.unchanged, 2)
def test_removed_document_is_removed_from_both_indexes(self):
self.run_pipeline("rebuild")
before = self.run_pipeline("sync").store_chunks
os.remove(os.path.join(self.root, "content", "platform", "l3", "platform-containers.md"))
after = self.run_pipeline("sync")
self.assertEqual(after.removed, 1)
self.assertLess(after.store_chunks, before)
self.assertEqual(after.store_chunks, after.collection_points)
with store_module.Store(self.cfg.db_path) as store:
self.assertEqual(store.keyword_search("endpoint latency loss"), [])
def test_ids_are_deterministic(self):
first = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64)
second = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64)
other = pipeline.chunk_id("content/ai/ai-overview.md", 1, "a" * 64)
self.assertEqual(first, second)
self.assertNotEqual(first, other)
self.assertEqual(len(first), 36)
def test_invalid_payload_aborts_before_any_write(self):
real_build = pipeline.payloads.build_payload
def broken(doc, chunk, cfg):
payload = real_build(doc, chunk, cfg)
payload.pop("confidence")
return payload
with mock.patch("corpus.payloads.build_payload", side_effect=broken):
with self.assertRaises(pipeline.PipelineError):
self.run_pipeline("rebuild")
self.assertEqual(self.fake.collections.get(self.cfg.collection, {}), {})
def test_provider_change_is_refused_without_rebuild(self):
self.run_pipeline("rebuild")
other = support.make_config(self.root, model="local-hashing-v2")
with self.assertRaises(pipeline.PipelineError) as caught:
self.run_pipeline("sync", cfg=other)
self.assertIn("governed migration", str(caught.exception))
def test_contract_is_recorded(self):
self.run_pipeline("rebuild")
with store_module.Store(self.cfg.db_path) as store:
contract = store.get_meta("collection")
self.assertEqual(contract["model"], self.cfg.model)
self.assertEqual(contract["dim"], self.cfg.dim)
self.assertEqual(contract["chunker_version"], self.cfg.chunker_version)
self.assertTrue(os.path.exists(self.cfg.meta_path))
class VerifyTest(unittest.TestCase):
def setUp(self):
self.root = support.temp_repo()
self.cfg = support.make_config(self.root)
self.addCleanup(_cleanup, self.root)
def test_clean_after_rebuild(self):
with support.fake_qdrant(self.cfg):
pipeline.run(self.cfg, mode="rebuild")
self.assertEqual(pipeline.verify(self.cfg), [])
def test_detects_source_changed_without_sync(self):
with support.fake_qdrant(self.cfg):
pipeline.run(self.cfg, mode="rebuild")
support.write_doc(
self.root,
"ai/ai-overview.md",
id="ai-overview", area="ai", level=1, title="Agent Systems",
claim="Changed.", confidence="running", last_verified="2026-10-02",
tags="agents", body="A different body entirely.",
)
problems = pipeline.verify(self.cfg)
self.assertTrue(any("changed since it was embedded" in problem for problem in problems), problems)
def test_detects_missing_document(self):
with support.fake_qdrant(self.cfg):
pipeline.run(self.cfg, mode="rebuild")
os.remove(os.path.join(self.root, "content", "ai", "ai-overview.md"))
problems = pipeline.verify(self.cfg)
self.assertTrue(any("no longer in content/" in problem for problem in problems), problems)
class AreaFilterTest(unittest.TestCase):
def setUp(self):
self.root = support.temp_repo()
self.cfg = support.make_config(self.root)
self.addCleanup(_cleanup, self.root)
def test_filter_narrows_the_result_set(self):
with support.fake_qdrant(self.cfg):
pipeline.run(self.cfg, mode="rebuild")
result = pipeline.demo_area_filter(self.cfg, "container estate limits", area="edge")
self.assertTrue(result["unfiltered"])
self.assertTrue(result["filtered"])
self.assertEqual({row["area"] for row in result["filtered"]}, {"edge"})
self.assertGreater(len({row["area"] for row in result["unfiltered"]}), 1)
def _cleanup(root):
import shutil
shutil.rmtree(root, ignore_errors=True)
if __name__ == "__main__":
unittest.main()

104
corpus/tests/test_qdrant_client.py Normal file
View File

@@ -0,0 +1,104 @@
"""Qdrant client: the wire shape the pipeline depends on, and the fallback route."""
from __future__ import annotations
import unittest
from corpus import qdrant as qdrant_module
from corpus.tests import support
class ClientTest(unittest.TestCase):
def test_create_collection_sends_size_and_distance(self):
with support.StubServer(lambda method, path, body: (200, {"result": True})) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
client.create_collection("resume_corpus", 1536)
request = server.requests[0]
self.assertEqual(request["method"], "PUT")
self.assertEqual(request["path"], "/collections/resume_corpus")
body = request["body"]
self.assertEqual(body["vectors"]["size"], 1536)
self.assertEqual(body["vectors"]["distance"], "Cosine")
# A memory-bound host: payloads and the HNSW graph live on disk.
self.assertTrue(body["on_disk_payload"])
self.assertTrue(body["hnsw_config"]["on_disk"])
self.assertFalse(body["vectors"]["on_disk"])
self.assertGreater(body["optimizers_config"]["memmap_threshold"], 0)
def test_upsert_waits_and_batches(self):
with support.StubServer(lambda method, path, body: (200, {"result": {}})) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
points = [{"id": f"id-{index}", "vector": [0.0], "payload": {}} for index in range(5)]
sent = client.upsert("c", points, batch=2)
self.assertEqual(sent, 5)
self.assertEqual(len(server.requests), 3)
self.assertTrue(all("wait=true" in request["path"] for request in server.requests))
self.assertEqual(sum(len(r["body"]["points"]) for r in server.requests), 5)
def test_delete_by_source_sends_a_source_filter(self):
with support.StubServer(lambda method, path, body: (200, {"result": {}})) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
client.delete_by_source("c", "content/edge/l3/edge-zigbee.md")
request = server.requests[0]
self.assertEqual(request["method"], "POST")
self.assertIn("/collections/c/points/delete", request["path"])
self.assertEqual(
request["body"],
{"filter": {"must": [{"key": "source_path", "match": {"value": "content/edge/l3/edge-zigbee.md"}}]}},
)
def test_search_filters_before_searching(self):
responder = lambda method, path, body: (200, {"result": {"points": []}}) # noqa: E731
with support.StubServer(responder) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
client.search("c", [0.1, 0.2], limit=3, area="edge", confidence="delivered")
body = server.requests[0]["body"]
self.assertEqual(server.requests[0]["path"], "/collections/c/points/query")
self.assertEqual(body["limit"], 3)
self.assertEqual(
body["filter"],
{
"must": [
{"key": "area", "match": {"value": "edge"}},
{"key": "confidence", "match": {"value": "delivered"}},
]
},
)
def test_search_falls_back_to_the_legacy_route(self):
def responder(method, path, body):
if path.endswith("/points/query"):
return 404, {"status": "not found"}
return 200, {"result": [{"id": "p1", "score": 0.9, "payload": {"area": "ai"}}]}
with support.StubServer(responder) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
hits = client.search("c", [0.1])
self.assertEqual([hit["id"] for hit in hits], ["p1"])
self.assertEqual([r["path"] for r in server.requests][-1], "/collections/c/points/search")
def test_point_count_is_exact(self):
responder = lambda method, path, body: (200, {"result": {"count": 42}}) # noqa: E731
with support.StubServer(responder) as server:
client = qdrant_module.Qdrant(server.url, attempts=1)
self.assertEqual(client.point_count("c"), 42)
self.assertEqual(server.requests[0]["body"], {"exact": True})
def test_health_is_false_when_unreachable(self):
client = qdrant_module.Qdrant("http://127.0.0.1:1", timeout=0.2, attempts=1)
self.assertFalse(client.health())
def test_upsert_retries_a_server_error(self):
plan = [500, 200]
def responder(method, path, body):
return plan.pop(0), {"result": {}}
with support.StubServer(responder) as server:
client = qdrant_module.Qdrant(server.url, attempts=2)
client.upsert("c", [{"id": "x", "vector": [0.0], "payload": {}}])
self.assertEqual(len(server.requests), 2)
if __name__ == "__main__":
unittest.main()

106
corpus/tests/test_store.py Normal file
View File

@@ -0,0 +1,106 @@
"""FTS5 chunk index, manifest and meta: the keyword half of hybrid retrieval."""
from __future__ import annotations
import unittest
from corpus import chunker, loader, payloads, store as store_module
from corpus.config import Config
from corpus.pipeline import chunk_id
from corpus.tests import support
def build_store(path, cfg, documents):
store = store_module.Store(path)
for doc in documents:
payload_list = []
for chunk in chunker.chunk_document(doc.body):
payload = payloads.build_payload(doc, chunk, cfg)
payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
payload_list.append(payload)
store.write_document(doc, payload_list)
return store
class StoreTest(unittest.TestCase):
def setUp(self):
self.root = support.temp_repo()
self.cfg = support.make_config(self.root)
self.addCleanup(_cleanup, self.root)
self.documents = loader.load_documents(self.cfg)
self.store = build_store(self.cfg.db_path, self.cfg, self.documents)
self.addCleanup(self.store.close)
def test_counts_match_the_documents(self):
self.assertEqual(self.store.source_count(), len(self.documents))
expected = sum(len(chunker.chunk_document(doc.body)) for doc in self.documents)
self.assertEqual(self.store.chunk_count(), expected)
def test_keyword_search_finds_exact_tokens(self):
hits = self.store.keyword_search("zigbee")
self.assertTrue(hits, "no hit for a token that is literally in the document")
self.assertEqual(hits[0]["area"], "edge")
def test_keyword_search_respects_area_filter(self):
all_hits = self.store.keyword_search("latency")
edge_hits = self.store.keyword_search("latency", area="edge")
self.assertTrue(all_hits)
for hit in edge_hits:
self.assertEqual(hit["area"], "edge")
def test_keyword_search_returns_snippets_and_scores(self):
hits = self.store.keyword_search("fence")
self.assertTrue(hits)
self.assertIn("score", hits[0])
self.assertIn("snippet", hits[0])
def test_operator_injection_is_not_honoured(self):
# FTS5 operators must be treated as words, never as query syntax.
hits = self.store.keyword_search('edge OR "NEAR"(a b)')
self.assertIsInstance(hits, list)
def test_manifest_records_the_content_hash(self):
manifest = self.store.manifest()
self.assertEqual(set(manifest), {doc.path for doc in self.documents})
for doc in self.documents:
self.assertEqual(manifest[doc.path]["content_hash"], doc.digest)
self.assertGreater(manifest[doc.path]["chunk_count"], 0)
def test_delete_removes_rows_from_both_tables(self):
target = self.documents[1]
self.store.delete_document(target.path)
self.assertNotIn(target.path, self.store.manifest())
for row in self.store.chunk_payloads():
self.assertNotEqual(row["source_path"], target.path)
def test_meta_roundtrip(self):
self.store.set_meta("collection", {"model": "x", "dim": 8})
self.assertEqual(self.store.get_meta("collection"), {"model": "x", "dim": 8})
self.assertIn("collection", self.store.meta())
def test_payloads_survive_the_roundtrip(self):
rows = self.store.chunk_payloads()
self.assertTrue(rows)
for row in rows:
payloads.validate_payload(row)
class MatchExpressionTest(unittest.TestCase):
def test_terms_become_prefix_queries(self):
self.assertEqual(store_module.match_expression("Zigbee mesh"), '"zigbee"* AND "mesh"*')
def test_empty_query_is_empty(self):
self.assertEqual(store_module.match_expression(" "), "")
def test_punctuation_is_dropped(self):
self.assertEqual(store_module.match_expression("-edge- LoRaWAN, 2023"), '"edge"* AND "lorawan"* AND "2023"*')
def _cleanup(root):
import shutil
shutil.rmtree(root, ignore_errors=True)
if __name__ == "__main__":
unittest.main()