Sam Rolfe
f9dc574172
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.
- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
Chunking is block-based, so a boundary cannot fall inside a fenced block, a
table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
as a UUID. Re-running updates points; re-chunking deletes the old ones by
source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
with area/confidence filters and a `terms` column so front-matter tech and
tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
from the environment only) and local-hashing (offline placeholder). The
provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.
Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.
Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
"""FTS5 chunk index, manifest and meta: the keyword half of hybrid retrieval."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from corpus import chunker, loader, payloads, store as store_module
|
|
from corpus.config import Config
|
|
from corpus.pipeline import chunk_id
|
|
from corpus.tests import support
|
|
|
|
|
|
def build_store(path, cfg, documents):
|
|
store = store_module.Store(path)
|
|
for doc in documents:
|
|
payload_list = []
|
|
for chunk in chunker.chunk_document(doc.body):
|
|
payload = payloads.build_payload(doc, chunk, cfg)
|
|
payload["id"] = chunk_id(doc.path, chunk.index, doc.digest)
|
|
payload_list.append(payload)
|
|
store.write_document(doc, payload_list)
|
|
return store
|
|
|
|
|
|
class StoreTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.root = support.temp_repo()
|
|
self.cfg = support.make_config(self.root)
|
|
self.addCleanup(_cleanup, self.root)
|
|
self.documents = loader.load_documents(self.cfg)
|
|
self.store = build_store(self.cfg.db_path, self.cfg, self.documents)
|
|
self.addCleanup(self.store.close)
|
|
|
|
def test_counts_match_the_documents(self):
|
|
self.assertEqual(self.store.source_count(), len(self.documents))
|
|
expected = sum(len(chunker.chunk_document(doc.body)) for doc in self.documents)
|
|
self.assertEqual(self.store.chunk_count(), expected)
|
|
|
|
def test_keyword_search_finds_exact_tokens(self):
|
|
hits = self.store.keyword_search("zigbee")
|
|
self.assertTrue(hits, "no hit for a token that is literally in the document")
|
|
self.assertEqual(hits[0]["area"], "edge")
|
|
|
|
def test_keyword_search_respects_area_filter(self):
|
|
all_hits = self.store.keyword_search("latency")
|
|
edge_hits = self.store.keyword_search("latency", area="edge")
|
|
self.assertTrue(all_hits)
|
|
for hit in edge_hits:
|
|
self.assertEqual(hit["area"], "edge")
|
|
|
|
def test_keyword_search_returns_snippets_and_scores(self):
|
|
hits = self.store.keyword_search("fence")
|
|
self.assertTrue(hits)
|
|
self.assertIn("score", hits[0])
|
|
self.assertIn("snippet", hits[0])
|
|
|
|
def test_operator_injection_is_not_honoured(self):
|
|
# FTS5 operators must be treated as words, never as query syntax.
|
|
hits = self.store.keyword_search('edge OR "NEAR"(a b)')
|
|
self.assertIsInstance(hits, list)
|
|
|
|
def test_manifest_records_the_content_hash(self):
|
|
manifest = self.store.manifest()
|
|
self.assertEqual(set(manifest), {doc.path for doc in self.documents})
|
|
for doc in self.documents:
|
|
self.assertEqual(manifest[doc.path]["content_hash"], doc.digest)
|
|
self.assertGreater(manifest[doc.path]["chunk_count"], 0)
|
|
|
|
def test_delete_removes_rows_from_both_tables(self):
|
|
target = self.documents[1]
|
|
self.store.delete_document(target.path)
|
|
self.assertNotIn(target.path, self.store.manifest())
|
|
for row in self.store.chunk_payloads():
|
|
self.assertNotEqual(row["source_path"], target.path)
|
|
|
|
def test_meta_roundtrip(self):
|
|
self.store.set_meta("collection", {"model": "x", "dim": 8})
|
|
self.assertEqual(self.store.get_meta("collection"), {"model": "x", "dim": 8})
|
|
self.assertIn("collection", self.store.meta())
|
|
|
|
def test_payloads_survive_the_roundtrip(self):
|
|
rows = self.store.chunk_payloads()
|
|
self.assertTrue(rows)
|
|
for row in rows:
|
|
payloads.validate_payload(row)
|
|
|
|
|
|
class MatchExpressionTest(unittest.TestCase):
|
|
def test_terms_become_prefix_queries(self):
|
|
self.assertEqual(store_module.match_expression("Zigbee mesh"), '"zigbee"* AND "mesh"*')
|
|
|
|
def test_empty_query_is_empty(self):
|
|
self.assertEqual(store_module.match_expression(" "), "")
|
|
|
|
def test_punctuation_is_dropped(self):
|
|
self.assertEqual(store_module.match_expression("-edge- LoRaWAN, 2023"), '"edge"* AND "lorawan"* AND "2023"*')
|
|
|
|
|
|
def _cleanup(root):
|
|
import shutil
|
|
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|