"""FTS5 chunk index, manifest and meta: the keyword half of hybrid retrieval.""" from __future__ import annotations import unittest from corpus import chunker, loader, payloads, store as store_module from corpus.config import Config from corpus.pipeline import chunk_id from corpus.tests import support def build_store(path, cfg, documents): store = store_module.Store(path) for doc in documents: payload_list = [] for chunk in chunker.chunk_document(doc.body): payload = payloads.build_payload(doc, chunk, cfg) payload["id"] = chunk_id(doc.path, chunk.index, doc.digest) payload_list.append(payload) store.write_document(doc, payload_list) return store class StoreTest(unittest.TestCase): def setUp(self): self.root = support.temp_repo() self.cfg = support.make_config(self.root) self.addCleanup(_cleanup, self.root) self.documents = loader.load_documents(self.cfg) self.store = build_store(self.cfg.db_path, self.cfg, self.documents) self.addCleanup(self.store.close) def test_counts_match_the_documents(self): self.assertEqual(self.store.source_count(), len(self.documents)) expected = sum(len(chunker.chunk_document(doc.body)) for doc in self.documents) self.assertEqual(self.store.chunk_count(), expected) def test_keyword_search_finds_exact_tokens(self): hits = self.store.keyword_search("zigbee") self.assertTrue(hits, "no hit for a token that is literally in the document") self.assertEqual(hits[0]["area"], "edge") def test_keyword_search_respects_area_filter(self): all_hits = self.store.keyword_search("latency") edge_hits = self.store.keyword_search("latency", area="edge") self.assertTrue(all_hits) for hit in edge_hits: self.assertEqual(hit["area"], "edge") def test_keyword_search_returns_snippets_and_scores(self): hits = self.store.keyword_search("fence") self.assertTrue(hits) self.assertIn("score", hits[0]) self.assertIn("snippet", hits[0]) def test_operator_injection_is_not_honoured(self): # FTS5 operators must be treated as words, never as query syntax. hits = self.store.keyword_search('edge OR "NEAR"(a b)') self.assertIsInstance(hits, list) def test_manifest_records_the_content_hash(self): manifest = self.store.manifest() self.assertEqual(set(manifest), {doc.path for doc in self.documents}) for doc in self.documents: self.assertEqual(manifest[doc.path]["content_hash"], doc.digest) self.assertGreater(manifest[doc.path]["chunk_count"], 0) def test_delete_removes_rows_from_both_tables(self): target = self.documents[1] self.store.delete_document(target.path) self.assertNotIn(target.path, self.store.manifest()) for row in self.store.chunk_payloads(): self.assertNotEqual(row["source_path"], target.path) def test_meta_roundtrip(self): self.store.set_meta("collection", {"model": "x", "dim": 8}) self.assertEqual(self.store.get_meta("collection"), {"model": "x", "dim": 8}) self.assertIn("collection", self.store.meta()) def test_payloads_survive_the_roundtrip(self): rows = self.store.chunk_payloads() self.assertTrue(rows) for row in rows: payloads.validate_payload(row) class MatchExpressionTest(unittest.TestCase): def test_terms_become_prefix_queries(self): self.assertEqual(store_module.match_expression("Zigbee mesh"), '"zigbee"* AND "mesh"*') def test_empty_query_is_empty(self): self.assertEqual(store_module.match_expression(" "), "") def test_punctuation_is_dropped(self): self.assertEqual(store_module.match_expression("-edge- LoRaWAN, 2023"), '"edge"* AND "lorawan"* AND "2023"*') def _cleanup(root): import shutil shutil.rmtree(root, ignore_errors=True) if __name__ == "__main__": unittest.main()