Sam Rolfe
f9dc574172
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.
- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
Chunking is block-based, so a boundary cannot fall inside a fenced block, a
table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
as a UUID. Re-running updates points; re-chunking deletes the old ones by
source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
with area/confidence filters and a `terms` column so front-matter tech and
tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
from the environment only) and local-hashing (offline placeholder). The
provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.
Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.
Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
180 lines
7.5 KiB
Python
180 lines
7.5 KiB
Python
"""The pipeline: deterministic ids, no duplicate vectors on re-run, incremental mode,
|
|
incomplete payloads never reach the store (docs/phase-4-corpus.md DoD 1-6)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
from corpus import chunker, loader, pipeline, store as store_module
|
|
from corpus.tests import support
|
|
|
|
|
|
class RebuildTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = support.temp_repo()
|
|
self.cfg = support.make_config(self.root)
|
|
self.addCleanup(_cleanup, self.root)
|
|
# One fake for the whole test: a second run must see the first run's points.
|
|
self.fake = support.FakeQdrant(self.cfg.dim)
|
|
|
|
def run_pipeline(self, mode="sync", cfg=None):
|
|
cfg = cfg or self.cfg
|
|
with support.fake_qdrant(cfg, self.fake):
|
|
return pipeline.run(cfg, mode=mode)
|
|
|
|
def test_rebuild_from_empty_volume(self):
|
|
stats = self.run_pipeline("rebuild")
|
|
self.assertEqual(stats.documents, 3)
|
|
self.assertEqual(stats.added, 3)
|
|
self.assertGreater(stats.store_chunks, 0)
|
|
self.assertEqual(stats.store_chunks, stats.collection_points)
|
|
self.assertEqual(stats.collection_points, len(self.fake.collections[self.cfg.collection]))
|
|
self.assertGreater(stats.embed_calls, 0)
|
|
|
|
def test_running_twice_does_not_duplicate_vectors(self):
|
|
first = self.run_pipeline("rebuild")
|
|
points_after_first = len(self.fake.collections[self.cfg.collection])
|
|
second = self.run_pipeline("sync")
|
|
points_after_second = len(self.fake.collections[self.cfg.collection])
|
|
self.assertEqual(points_after_first, points_after_second)
|
|
self.assertEqual(first.store_chunks, second.store_chunks)
|
|
self.assertEqual(second.added, 0)
|
|
self.assertEqual(second.updated, 0)
|
|
self.assertEqual(second.chunks_written, 0)
|
|
|
|
def test_incremental_reembeds_only_changed_documents(self):
|
|
self.run_pipeline("rebuild")
|
|
|
|
# Nothing changed: no embedding calls at all.
|
|
unchanged = self.run_pipeline("sync")
|
|
self.assertEqual(unchanged.embed_calls, 0)
|
|
self.assertEqual(unchanged.texts_embedded, 0)
|
|
|
|
# One document changes: exactly one embedding call, for that document.
|
|
support.write_doc(
|
|
self.root,
|
|
"edge/l3/edge-zigbee.md",
|
|
id="edge-zigbee", area="edge", level=3, title="Low Power Mesh Sensing",
|
|
claim="A constrained mesh that survives power loss.", confidence="delivered",
|
|
last_verified="2026-09-15", tags="zigbee, mesh",
|
|
body="The mesh now also reports battery voltage on every node.",
|
|
)
|
|
changed = self.run_pipeline("sync")
|
|
self.assertEqual(changed.added + changed.updated, 1)
|
|
self.assertEqual(changed.embed_calls, 1)
|
|
expected_chunks = len(
|
|
chunker.chunk_document(
|
|
loader.load_documents(support.make_config(self.root))[1].body
|
|
)
|
|
)
|
|
self.assertEqual(changed.texts_embedded, expected_chunks)
|
|
self.assertEqual(changed.unchanged, 2)
|
|
|
|
def test_removed_document_is_removed_from_both_indexes(self):
|
|
self.run_pipeline("rebuild")
|
|
before = self.run_pipeline("sync").store_chunks
|
|
os.remove(os.path.join(self.root, "content", "platform", "l3", "platform-containers.md"))
|
|
after = self.run_pipeline("sync")
|
|
self.assertEqual(after.removed, 1)
|
|
self.assertLess(after.store_chunks, before)
|
|
self.assertEqual(after.store_chunks, after.collection_points)
|
|
with store_module.Store(self.cfg.db_path) as store:
|
|
self.assertEqual(store.keyword_search("endpoint latency loss"), [])
|
|
|
|
def test_ids_are_deterministic(self):
|
|
first = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64)
|
|
second = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64)
|
|
other = pipeline.chunk_id("content/ai/ai-overview.md", 1, "a" * 64)
|
|
self.assertEqual(first, second)
|
|
self.assertNotEqual(first, other)
|
|
self.assertEqual(len(first), 36)
|
|
|
|
def test_invalid_payload_aborts_before_any_write(self):
|
|
real_build = pipeline.payloads.build_payload
|
|
|
|
def broken(doc, chunk, cfg):
|
|
payload = real_build(doc, chunk, cfg)
|
|
payload.pop("confidence")
|
|
return payload
|
|
|
|
with mock.patch("corpus.payloads.build_payload", side_effect=broken):
|
|
with self.assertRaises(pipeline.PipelineError):
|
|
self.run_pipeline("rebuild")
|
|
self.assertEqual(self.fake.collections.get(self.cfg.collection, {}), {})
|
|
|
|
def test_provider_change_is_refused_without_rebuild(self):
|
|
self.run_pipeline("rebuild")
|
|
other = support.make_config(self.root, model="local-hashing-v2")
|
|
with self.assertRaises(pipeline.PipelineError) as caught:
|
|
self.run_pipeline("sync", cfg=other)
|
|
self.assertIn("governed migration", str(caught.exception))
|
|
|
|
def test_contract_is_recorded(self):
|
|
self.run_pipeline("rebuild")
|
|
with store_module.Store(self.cfg.db_path) as store:
|
|
contract = store.get_meta("collection")
|
|
self.assertEqual(contract["model"], self.cfg.model)
|
|
self.assertEqual(contract["dim"], self.cfg.dim)
|
|
self.assertEqual(contract["chunker_version"], self.cfg.chunker_version)
|
|
self.assertTrue(os.path.exists(self.cfg.meta_path))
|
|
|
|
|
|
class VerifyTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = support.temp_repo()
|
|
self.cfg = support.make_config(self.root)
|
|
self.addCleanup(_cleanup, self.root)
|
|
|
|
def test_clean_after_rebuild(self):
|
|
with support.fake_qdrant(self.cfg):
|
|
pipeline.run(self.cfg, mode="rebuild")
|
|
self.assertEqual(pipeline.verify(self.cfg), [])
|
|
|
|
def test_detects_source_changed_without_sync(self):
|
|
with support.fake_qdrant(self.cfg):
|
|
pipeline.run(self.cfg, mode="rebuild")
|
|
support.write_doc(
|
|
self.root,
|
|
"ai/ai-overview.md",
|
|
id="ai-overview", area="ai", level=1, title="Agent Systems",
|
|
claim="Changed.", confidence="running", last_verified="2026-10-02",
|
|
tags="agents", body="A different body entirely.",
|
|
)
|
|
problems = pipeline.verify(self.cfg)
|
|
self.assertTrue(any("changed since it was embedded" in problem for problem in problems), problems)
|
|
|
|
def test_detects_missing_document(self):
|
|
with support.fake_qdrant(self.cfg):
|
|
pipeline.run(self.cfg, mode="rebuild")
|
|
os.remove(os.path.join(self.root, "content", "ai", "ai-overview.md"))
|
|
problems = pipeline.verify(self.cfg)
|
|
self.assertTrue(any("no longer in content/" in problem for problem in problems), problems)
|
|
|
|
|
|
class AreaFilterTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = support.temp_repo()
|
|
self.cfg = support.make_config(self.root)
|
|
self.addCleanup(_cleanup, self.root)
|
|
|
|
def test_filter_narrows_the_result_set(self):
|
|
with support.fake_qdrant(self.cfg):
|
|
pipeline.run(self.cfg, mode="rebuild")
|
|
result = pipeline.demo_area_filter(self.cfg, "container estate limits", area="edge")
|
|
self.assertTrue(result["unfiltered"])
|
|
self.assertTrue(result["filtered"])
|
|
self.assertEqual({row["area"] for row in result["filtered"]}, {"edge"})
|
|
self.assertGreater(len({row["area"] for row in result["unfiltered"]}), 1)
|
|
|
|
|
|
def _cleanup(root):
|
|
import shutil
|
|
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|