sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 4: corpus and retrieval — chunks, Qdrant, FTS5, freshness audit

Browse Source 
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.

- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
  Chunking is block-based, so a boundary cannot fall inside a fenced block, a
  table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
  as a UUID. Re-running updates points; re-chunking deletes the old ones by
  source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
  or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
  written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
  with area/confidence filters and a `terms` column so front-matter tech and
  tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
  optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
  from the environment only) and local-hashing (offline placeholder). The
  provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.

Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.

Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Sam Rolfe
2026-10-26 21:04:52 +11:00
parent 24590cc351
commit f9dc574172
28 changed files with 4372 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
corpus/cli.py Normal file
View File

@@ -0,0 +1,257 @@
"""Command line entry point: `python3 -m corpus.cli <command>`.
Commands
sync incremental update of the corpus (the normal command)
rebuild drop the collection and rebuild it from content/ (recovery)
audit freshness audit; reports stale documents, index drift
verify does the index agree with content/ and with Qdrant?
demo-filter the same query with and without an `area` filter
hybrid vectors + FTS5 fused with reciprocal rank fusion
measure Qdrant memory at 500 / 5,000 / 20,000 chunks
info what this build thinks it is configured to do
Exit codes: 0 success, 1 findings or verification problems, 2 configuration or
usage error. Every command accepts `--json` where a machine-readable result is
useful (Prefect reads JSON, a human reads the table).
Configuration is environment-first (`corpus/config.py`); the flags exist so a
one-off run can point at a scratch Qdrant without exporting variables.
"""
from __future__ import annotations
import argparse
import datetime
import json
import os
import sys
from . import config as config_module
from . import freshness, pipeline, store as store_module
from .config import Config, ConfigError
from .qdrant import Qdrant, QdrantError
def main(argv=None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
if args.command is None:
parser.print_help()
return 2
overrides = {
"root": args.root,
"qdrant_url": args.qdrant_url,
"collection": args.collection,
"provider": args.provider,
"model": args.model,
"dim": args.dim,
"db_path": args.db,
"data_dir": args.data,
}
try:
cfg = Config.from_env(**overrides)
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
log = (lambda _msg: None) if args.quiet else (lambda msg: print(msg, file=sys.stderr))
try:
handler = {
"sync": _cmd_sync,
"rebuild": _cmd_rebuild,
"audit": _cmd_audit,
"verify": _cmd_verify,
"demo-filter": _cmd_demo_filter,
"hybrid": _cmd_hybrid,
"measure": _cmd_measure,
"info": _cmd_info,
}[args.command]
return handler(cfg, args, log)
except (pipeline.PipelineError, ConfigError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
except QdrantError as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="python3 -m corpus.cli",
description="Build and audit the Ai Resume retrieval corpus (docs/phase-4-corpus.md).",
)
parser.add_argument("--root", help="repository root (default: parent of corpus/)")
parser.add_argument("--qdrant-url", help="Qdrant REST URL (default http://127.0.0.1:6333)")
parser.add_argument("--collection", help="Qdrant collection name (default resume_corpus)")
parser.add_argument("--provider", choices=sorted(config_module.PROVIDERS), help="embedding provider")
parser.add_argument("--model", help="embedding model name")
parser.add_argument("--dim", type=int, help="embedding dimension")
parser.add_argument("--db", help="SQLite corpus database path")
parser.add_argument("--data", help="corpus data directory")
parser.add_argument("--quiet", action="store_true", help="progress on stderr only becomes silent")
sub = parser.add_subparsers(dest="command")
sync = sub.add_parser("sync", help="incremental update")
sync.add_argument("--json", action="store_true")
rebuild = sub.add_parser("rebuild", help="drop the collection and rebuild from content/")
rebuild.add_argument("--json", action="store_true")
audit = sub.add_parser("audit", help="freshness and drift audit")
audit.add_argument("--json", action="store_true")
audit.add_argument("--today", help="audit as of this date (YYYY-MM-DD), for tests")
audit.add_argument("--fail-on-findings", action="store_true", help="exit 1 when anything is found")
audit.add_argument("--alert", action="store_true", help="POST the report to RESUME_APPRISE_URL")
verify = sub.add_parser("verify", help="check the index against content/ and Qdrant")
verify.add_argument("--json", action="store_true")
demo = sub.add_parser("demo-filter", help="same query with and without an area filter")
demo.add_argument("--query", required=True)
demo.add_argument("--area", help="area to filter on (ai, platform, edge, products, history, about)")
demo.add_argument("--limit", type=int, default=5)
demo.add_argument("--json", action="store_true")
hybrid = sub.add_parser("hybrid", help="vectors + FTS5 fused with RRF")
hybrid.add_argument("--query", required=True)
hybrid.add_argument("--area")
hybrid.add_argument("--limit", type=int, default=5)
hybrid.add_argument("--json", action="store_true")
measure = sub.add_parser("measure", help="Qdrant memory at three corpus sizes")
measure.add_argument("--sizes", default="500,5000,20000")
measure.add_argument("--container", default="", help="Qdrant container name for docker stats")
measure.add_argument("--keep", action="store_true", help="keep the measurement collection")
measure.add_argument("--json", action="store_true")
sub.add_parser("info", help="print the resolved configuration")
return parser
# -- commands --------------------------------------------------------------
def _cmd_sync(cfg, args, log) -> int:
stats = pipeline.run(cfg, mode="sync", log=log)
if args.json:
print(json.dumps(stats.to_dict(), indent=2, sort_keys=True))
else:
print(stats.summary())
return 0
def _cmd_rebuild(cfg, args, log) -> int:
stats = pipeline.run(cfg, mode="rebuild", log=log)
if args.json:
print(json.dumps(stats.to_dict(), indent=2, sort_keys=True))
else:
print(stats.summary())
return 0
def _cmd_audit(cfg, args, log) -> int:
today = datetime.date.fromisoformat(args.today) if args.today else None
report = freshness.audit(cfg, today=today, log=log)
if args.json:
print(json.dumps(report, indent=2, sort_keys=True))
else:
print(freshness.render_text(report))
if args.alert:
try:
sent = freshness.alert(report, cfg.apprise_url)
if not sent:
log("alert: RESUME_APPRISE_URL is not set; nothing sent")
except Exception as exc: # noqa: BLE001 - an alert failure must not hide the audit
print(f"warning: could not send the Apprise alert: {exc}", file=sys.stderr)
if args.fail_on_findings and not report["ok"]:
return 1
return 0
def _cmd_verify(cfg, args, log) -> int:
problems = pipeline.verify(cfg, log=log)
if args.json:
print(json.dumps({"ok": not problems, "problems": problems}, indent=2))
else:
if problems:
print(f"FAIL: {len(problems)} problem(s)")
for problem in problems:
print(f" - {problem}")
else:
print("OK: the index agrees with content/ and Qdrant")
return 1 if problems else 0
def _cmd_demo_filter(cfg, args, log) -> int:
result = pipeline.demo_area_filter(cfg, args.query, area=args.area, limit=args.limit)
if args.json:
print(json.dumps(result, indent=2, sort_keys=True))
return 0
print(f"query: {result['query']!r}")
print("")
print(f"without a filter ({len(result['unfiltered'])} result(s)):")
for row in result["unfiltered"]:
print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}")
print("")
if not args.area:
print("no --area given, so the filtered set is empty; re-run with e.g. --area edge")
return 0
print(f"with area={args.area!r} ({len(result['filtered'])} result(s)):")
for row in result["filtered"]:
print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}")
wrong = [row for row in result["filtered"] if row["area"] != args.area]
if wrong:
print(f"FAIL: {len(wrong)} filtered result(s) are not in area {args.area!r}")
return 1
return 0
def _cmd_hybrid(cfg, args, log) -> int:
rows = pipeline.hybrid_search(cfg, args.query, area=args.area, limit=args.limit)
if args.json:
print(json.dumps(rows, indent=2, sort_keys=True, default=str))
return 0
print(f"hybrid (RRF) query={args.query!r} area={args.area!r}: {len(rows)} result(s)")
for row in rows:
print(f" {row['score']:.4f} {row.get('title')} {row.get('url')} [{row['id'][:8]}]")
return 0
def _cmd_measure(cfg, args, log) -> int:
from . import measure as measure_module
sizes = [int(part) for part in args.sizes.split(",") if part.strip()]
report = measure_module.measure(
cfg, sizes=sizes, container=args.container, keep=args.keep, log=log
)
if args.json:
print(json.dumps(report, indent=2, sort_keys=True))
return 0
print(measure_module.render_text(report))
return 0
def _cmd_info(cfg, args, log) -> int:
print(cfg.describe())
qdrant = Qdrant(cfg.qdrant_url, timeout=cfg.timeout)
print("")
print(f"qdrant: {'answering, version ' + qdrant.version() if qdrant.health() else 'NOT answering'}")
if qdrant.collection_exists(cfg.collection):
print(f"collection {cfg.collection}: {qdrant.point_count(cfg.collection)} point(s)")
else:
print(f"collection {cfg.collection}: does not exist")
if os.path.exists(cfg.db_path):
with store_module.Store(cfg.db_path) as store:
print(f"store: {store.chunk_count()} chunk(s) across {store.source_count()} source(s)")
print(f"contract: {json.dumps(store.get_meta('collection'), sort_keys=True)}")
else:
print(f"store: {cfg.db_path} does not exist")
return 0
if __name__ == "__main__":
sys.exit(main())