"""The pipeline: deterministic ids, no duplicate vectors on re-run, incremental mode, incomplete payloads never reach the store (docs/phase-4-corpus.md DoD 1-6).""" from __future__ import annotations import os import unittest from unittest import mock from corpus import chunker, loader, pipeline, store as store_module from corpus.tests import support class RebuildTest(unittest.TestCase): def setUp(self): self.root = support.temp_repo() self.cfg = support.make_config(self.root) self.addCleanup(_cleanup, self.root) # One fake for the whole test: a second run must see the first run's points. self.fake = support.FakeQdrant(self.cfg.dim) def run_pipeline(self, mode="sync", cfg=None): cfg = cfg or self.cfg with support.fake_qdrant(cfg, self.fake): return pipeline.run(cfg, mode=mode) def test_rebuild_from_empty_volume(self): stats = self.run_pipeline("rebuild") self.assertEqual(stats.documents, 3) self.assertEqual(stats.added, 3) self.assertGreater(stats.store_chunks, 0) self.assertEqual(stats.store_chunks, stats.collection_points) self.assertEqual(stats.collection_points, len(self.fake.collections[self.cfg.collection])) self.assertGreater(stats.embed_calls, 0) def test_running_twice_does_not_duplicate_vectors(self): first = self.run_pipeline("rebuild") points_after_first = len(self.fake.collections[self.cfg.collection]) second = self.run_pipeline("sync") points_after_second = len(self.fake.collections[self.cfg.collection]) self.assertEqual(points_after_first, points_after_second) self.assertEqual(first.store_chunks, second.store_chunks) self.assertEqual(second.added, 0) self.assertEqual(second.updated, 0) self.assertEqual(second.chunks_written, 0) def test_incremental_reembeds_only_changed_documents(self): self.run_pipeline("rebuild") # Nothing changed: no embedding calls at all. unchanged = self.run_pipeline("sync") self.assertEqual(unchanged.embed_calls, 0) self.assertEqual(unchanged.texts_embedded, 0) # One document changes: exactly one embedding call, for that document. support.write_doc( self.root, "edge/l3/edge-zigbee.md", id="edge-zigbee", area="edge", level=3, title="Low Power Mesh Sensing", claim="A constrained mesh that survives power loss.", confidence="delivered", last_verified="2026-09-15", tags="zigbee, mesh", body="The mesh now also reports battery voltage on every node.", ) changed = self.run_pipeline("sync") self.assertEqual(changed.added + changed.updated, 1) self.assertEqual(changed.embed_calls, 1) expected_chunks = len( chunker.chunk_document( loader.load_documents(support.make_config(self.root))[1].body ) ) self.assertEqual(changed.texts_embedded, expected_chunks) self.assertEqual(changed.unchanged, 2) def test_removed_document_is_removed_from_both_indexes(self): self.run_pipeline("rebuild") before = self.run_pipeline("sync").store_chunks os.remove(os.path.join(self.root, "content", "platform", "l3", "platform-containers.md")) after = self.run_pipeline("sync") self.assertEqual(after.removed, 1) self.assertLess(after.store_chunks, before) self.assertEqual(after.store_chunks, after.collection_points) with store_module.Store(self.cfg.db_path) as store: self.assertEqual(store.keyword_search("endpoint latency loss"), []) def test_ids_are_deterministic(self): first = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64) second = pipeline.chunk_id("content/ai/ai-overview.md", 0, "a" * 64) other = pipeline.chunk_id("content/ai/ai-overview.md", 1, "a" * 64) self.assertEqual(first, second) self.assertNotEqual(first, other) self.assertEqual(len(first), 36) def test_invalid_payload_aborts_before_any_write(self): real_build = pipeline.payloads.build_payload def broken(doc, chunk, cfg): payload = real_build(doc, chunk, cfg) payload.pop("confidence") return payload with mock.patch("corpus.payloads.build_payload", side_effect=broken): with self.assertRaises(pipeline.PipelineError): self.run_pipeline("rebuild") self.assertEqual(self.fake.collections.get(self.cfg.collection, {}), {}) def test_provider_change_is_refused_without_rebuild(self): self.run_pipeline("rebuild") other = support.make_config(self.root, model="local-hashing-v2") with self.assertRaises(pipeline.PipelineError) as caught: self.run_pipeline("sync", cfg=other) self.assertIn("governed migration", str(caught.exception)) def test_contract_is_recorded(self): self.run_pipeline("rebuild") with store_module.Store(self.cfg.db_path) as store: contract = store.get_meta("collection") self.assertEqual(contract["model"], self.cfg.model) self.assertEqual(contract["dim"], self.cfg.dim) self.assertEqual(contract["chunker_version"], self.cfg.chunker_version) self.assertTrue(os.path.exists(self.cfg.meta_path)) class VerifyTest(unittest.TestCase): def setUp(self): self.root = support.temp_repo() self.cfg = support.make_config(self.root) self.addCleanup(_cleanup, self.root) def test_clean_after_rebuild(self): with support.fake_qdrant(self.cfg): pipeline.run(self.cfg, mode="rebuild") self.assertEqual(pipeline.verify(self.cfg), []) def test_detects_source_changed_without_sync(self): with support.fake_qdrant(self.cfg): pipeline.run(self.cfg, mode="rebuild") support.write_doc( self.root, "ai/ai-overview.md", id="ai-overview", area="ai", level=1, title="Agent Systems", claim="Changed.", confidence="running", last_verified="2026-10-02", tags="agents", body="A different body entirely.", ) problems = pipeline.verify(self.cfg) self.assertTrue(any("changed since it was embedded" in problem for problem in problems), problems) def test_detects_missing_document(self): with support.fake_qdrant(self.cfg): pipeline.run(self.cfg, mode="rebuild") os.remove(os.path.join(self.root, "content", "ai", "ai-overview.md")) problems = pipeline.verify(self.cfg) self.assertTrue(any("no longer in content/" in problem for problem in problems), problems) class AreaFilterTest(unittest.TestCase): def setUp(self): self.root = support.temp_repo() self.cfg = support.make_config(self.root) self.addCleanup(_cleanup, self.root) def test_filter_narrows_the_result_set(self): with support.fake_qdrant(self.cfg): pipeline.run(self.cfg, mode="rebuild") result = pipeline.demo_area_filter(self.cfg, "container estate limits", area="edge") self.assertTrue(result["unfiltered"]) self.assertTrue(result["filtered"]) self.assertEqual({row["area"] for row in result["filtered"]}, {"edge"}) self.assertGreater(len({row["area"] for row in result["unfiltered"]}), 1) def _cleanup(root): import shutil shutil.rmtree(root, ignore_errors=True) if __name__ == "__main__": unittest.main()