Sam Rolfe
f9dc574172
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.
- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
Chunking is block-based, so a boundary cannot fall inside a fenced block, a
table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
as a UUID. Re-running updates points; re-chunking deletes the old ones by
source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
with area/confidence filters and a `terms` column so front-matter tech and
tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
from the environment only) and local-hashing (offline placeholder). The
provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.
Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.
Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
"""Qdrant client: the wire shape the pipeline depends on, and the fallback route."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from corpus import qdrant as qdrant_module
|
|
from corpus.tests import support
|
|
|
|
|
|
class ClientTest(unittest.TestCase):
|
|
def test_create_collection_sends_size_and_distance(self):
|
|
with support.StubServer(lambda method, path, body: (200, {"result": True})) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
client.create_collection("resume_corpus", 1536)
|
|
request = server.requests[0]
|
|
self.assertEqual(request["method"], "PUT")
|
|
self.assertEqual(request["path"], "/collections/resume_corpus")
|
|
body = request["body"]
|
|
self.assertEqual(body["vectors"]["size"], 1536)
|
|
self.assertEqual(body["vectors"]["distance"], "Cosine")
|
|
# A memory-bound host: payloads and the HNSW graph live on disk.
|
|
self.assertTrue(body["on_disk_payload"])
|
|
self.assertTrue(body["hnsw_config"]["on_disk"])
|
|
self.assertFalse(body["vectors"]["on_disk"])
|
|
self.assertGreater(body["optimizers_config"]["memmap_threshold"], 0)
|
|
|
|
def test_upsert_waits_and_batches(self):
|
|
with support.StubServer(lambda method, path, body: (200, {"result": {}})) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
points = [{"id": f"id-{index}", "vector": [0.0], "payload": {}} for index in range(5)]
|
|
sent = client.upsert("c", points, batch=2)
|
|
self.assertEqual(sent, 5)
|
|
self.assertEqual(len(server.requests), 3)
|
|
self.assertTrue(all("wait=true" in request["path"] for request in server.requests))
|
|
self.assertEqual(sum(len(r["body"]["points"]) for r in server.requests), 5)
|
|
|
|
def test_delete_by_source_sends_a_source_filter(self):
|
|
with support.StubServer(lambda method, path, body: (200, {"result": {}})) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
client.delete_by_source("c", "content/edge/l3/edge-zigbee.md")
|
|
request = server.requests[0]
|
|
self.assertEqual(request["method"], "POST")
|
|
self.assertIn("/collections/c/points/delete", request["path"])
|
|
self.assertEqual(
|
|
request["body"],
|
|
{"filter": {"must": [{"key": "source_path", "match": {"value": "content/edge/l3/edge-zigbee.md"}}]}},
|
|
)
|
|
|
|
def test_search_filters_before_searching(self):
|
|
responder = lambda method, path, body: (200, {"result": {"points": []}}) # noqa: E731
|
|
with support.StubServer(responder) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
client.search("c", [0.1, 0.2], limit=3, area="edge", confidence="delivered")
|
|
body = server.requests[0]["body"]
|
|
self.assertEqual(server.requests[0]["path"], "/collections/c/points/query")
|
|
self.assertEqual(body["limit"], 3)
|
|
self.assertEqual(
|
|
body["filter"],
|
|
{
|
|
"must": [
|
|
{"key": "area", "match": {"value": "edge"}},
|
|
{"key": "confidence", "match": {"value": "delivered"}},
|
|
]
|
|
},
|
|
)
|
|
|
|
def test_search_falls_back_to_the_legacy_route(self):
|
|
def responder(method, path, body):
|
|
if path.endswith("/points/query"):
|
|
return 404, {"status": "not found"}
|
|
return 200, {"result": [{"id": "p1", "score": 0.9, "payload": {"area": "ai"}}]}
|
|
|
|
with support.StubServer(responder) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
hits = client.search("c", [0.1])
|
|
self.assertEqual([hit["id"] for hit in hits], ["p1"])
|
|
self.assertEqual([r["path"] for r in server.requests][-1], "/collections/c/points/search")
|
|
|
|
def test_point_count_is_exact(self):
|
|
responder = lambda method, path, body: (200, {"result": {"count": 42}}) # noqa: E731
|
|
with support.StubServer(responder) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=1)
|
|
self.assertEqual(client.point_count("c"), 42)
|
|
self.assertEqual(server.requests[0]["body"], {"exact": True})
|
|
|
|
def test_health_is_false_when_unreachable(self):
|
|
client = qdrant_module.Qdrant("http://127.0.0.1:1", timeout=0.2, attempts=1)
|
|
self.assertFalse(client.health())
|
|
|
|
def test_upsert_retries_a_server_error(self):
|
|
plan = [500, 200]
|
|
|
|
def responder(method, path, body):
|
|
return plan.pop(0), {"result": {}}
|
|
|
|
with support.StubServer(responder) as server:
|
|
client = qdrant_module.Qdrant(server.url, attempts=2)
|
|
client.upsert("c", [{"id": "x", "vector": [0.0], "payload": {}}])
|
|
self.assertEqual(len(server.requests), 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|