"""Command line entry point: `python3 -m corpus.cli `. Commands sync incremental update of the corpus (the normal command) rebuild drop the collection and rebuild it from content/ (recovery) audit freshness audit; reports stale documents, index drift verify does the index agree with content/ and with Qdrant? demo-filter the same query with and without an `area` filter hybrid vectors + FTS5 fused with reciprocal rank fusion measure Qdrant memory at 500 / 5,000 / 20,000 chunks info what this build thinks it is configured to do Exit codes: 0 success, 1 findings or verification problems, 2 configuration or usage error. Every command accepts `--json` where a machine-readable result is useful (Prefect reads JSON, a human reads the table). Configuration is environment-first (`corpus/config.py`); the flags exist so a one-off run can point at a scratch Qdrant without exporting variables. """ from __future__ import annotations import argparse import datetime import json import os import sys from . import config as config_module from . import freshness, guides, pipeline, store as store_module from .config import Config, ConfigError from .qdrant import Qdrant, QdrantError from .sources import graph as graph_module def main(argv=None) -> int: parser = _build_parser() args = parser.parse_args(argv) if args.command is None: parser.print_help() return 2 overrides = { "root": args.root, "qdrant_url": args.qdrant_url, "collection": args.collection, "provider": args.provider, "model": args.model, "dim": args.dim, "db_path": args.db, "data_dir": args.data, "offline": True if args.offline else None, "platforms": tuple(part.strip() for part in args.platforms.split(",") if part.strip()) if args.platforms else None, "graph_path": args.graph_path or None, } try: cfg = Config.from_env(**overrides) except ConfigError as exc: print(f"configuration error: {exc}", file=sys.stderr) return 2 log = (lambda _msg: None) if args.quiet else (lambda msg: print(msg, file=sys.stderr)) try: handler = { "sync": _cmd_sync, "rebuild": _cmd_rebuild, "audit": _cmd_audit, "verify": _cmd_verify, "demo-filter": _cmd_demo_filter, "hybrid": _cmd_hybrid, "measure": _cmd_measure, "info": _cmd_info, "graph": _cmd_graph, "graph-neighbours": _cmd_graph_neighbours, "scan": _cmd_scan, "sources": _cmd_sources, }[args.command] return handler(cfg, args, log) except (pipeline.PipelineError, ConfigError) as exc: print(f"error: {exc}", file=sys.stderr) return 2 except QdrantError as exc: print(f"error: {exc}", file=sys.stderr) return 2 def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python3 -m corpus.cli", description="Build and audit the Ai Resume retrieval corpus (docs/phase-4-corpus.md).", ) parser.add_argument("--root", help="repository root (default: parent of corpus/)") parser.add_argument("--qdrant-url", help="Qdrant REST URL (default http://127.0.0.1:6333)") parser.add_argument("--collection", help="Qdrant collection name (default resume_corpus)") parser.add_argument("--provider", choices=sorted(config_module.PROVIDERS), help="embedding provider") parser.add_argument("--model", help="embedding model name") parser.add_argument("--dim", type=int, help="embedding dimension") parser.add_argument("--db", help="SQLite corpus database path") parser.add_argument("--data", help="corpus data directory") parser.add_argument("--quiet", action="store_true", help="progress on stderr only becomes silent") parser.add_argument("--offline", action="store_true", help="read content/ only; do not harvest the estate") parser.add_argument("--platforms", help="comma-separated platforms to harvest (default: all enabled)") parser.add_argument("--graph-path", help="read graph.json from this path instead of Gitea") sub = parser.add_subparsers(dest="command") sync = sub.add_parser("sync", help="incremental update") sync.add_argument("--json", action="store_true") rebuild = sub.add_parser("rebuild", help="drop the collection and rebuild from content/") rebuild.add_argument("--json", action="store_true") audit = sub.add_parser("audit", help="freshness and drift audit") audit.add_argument("--json", action="store_true") audit.add_argument("--today", help="audit as of this date (YYYY-MM-DD), for tests") audit.add_argument("--fail-on-findings", action="store_true", help="exit 1 when anything is found") audit.add_argument("--alert", action="store_true", help="POST the report to RESUME_APPRISE_URL") verify = sub.add_parser("verify", help="check the index against content/ and Qdrant") verify.add_argument("--json", action="store_true") demo = sub.add_parser("demo-filter", help="same query with and without an area filter") demo.add_argument("--query", required=True) demo.add_argument("--area", help="area to filter on (ai, platform, edge, products, history, about)") demo.add_argument("--limit", type=int, default=5) demo.add_argument("--json", action="store_true") hybrid = sub.add_parser("hybrid", help="vectors + FTS5 fused with RRF") hybrid.add_argument("--query", required=True) hybrid.add_argument("--area") hybrid.add_argument("--limit", type=int, default=5) hybrid.add_argument("--json", action="store_true") measure = sub.add_parser("measure", help="Qdrant memory at three corpus sizes") measure.add_argument("--sizes", default="500,5000,20000") measure.add_argument("--container", default="", help="Qdrant container name for docker stats") measure.add_argument("--keep", action="store_true", help="keep the measurement collection") measure.add_argument("--json", action="store_true") graph = sub.add_parser("graph", help="query graph nodes and edges by keyword or vector") graph.add_argument("--query", required=True) graph.add_argument("--area") graph.add_argument("--limit", type=int, default=10) graph.add_argument("--json", action="store_true") neighbours = sub.add_parser("graph-neighbours", help="edges attached to a graph node id") neighbours.add_argument("--node", required=True) neighbours.add_argument("--limit", type=int, default=50) neighbours.add_argument("--json", action="store_true") scan = sub.add_parser("scan", help="search the finished pool for credential-shaped content") scan.add_argument("--json", action="store_true") sources = sub.add_parser("sources", help="show the platform guides and the last harvest") sources.add_argument("--json", action="store_true") sub.add_parser("info", help="print the resolved configuration") return parser # -- commands -------------------------------------------------------------- def _cmd_sync(cfg, args, log) -> int: stats = pipeline.run(cfg, mode="sync", log=log) if args.json: print(json.dumps(stats.to_dict(), indent=2, sort_keys=True)) else: print(stats.summary()) return 0 def _cmd_rebuild(cfg, args, log) -> int: stats = pipeline.run(cfg, mode="rebuild", log=log) if args.json: print(json.dumps(stats.to_dict(), indent=2, sort_keys=True)) else: print(stats.summary()) return 0 def _cmd_audit(cfg, args, log) -> int: today = datetime.date.fromisoformat(args.today) if args.today else None report = freshness.audit(cfg, today=today, log=log) if args.json: print(json.dumps(report, indent=2, sort_keys=True)) else: print(freshness.render_text(report)) if args.alert: try: sent = freshness.alert(report, cfg.apprise_url) if not sent: log("alert: RESUME_APPRISE_URL is not set; nothing sent") except Exception as exc: # noqa: BLE001 - an alert failure must not hide the audit print(f"warning: could not send the Apprise alert: {exc}", file=sys.stderr) if args.fail_on_findings and not report["ok"]: return 1 return 0 def _cmd_verify(cfg, args, log) -> int: problems = pipeline.verify(cfg, log=log) if args.json: print(json.dumps({"ok": not problems, "problems": problems}, indent=2)) else: if problems: print(f"FAIL: {len(problems)} problem(s)") for problem in problems: print(f" - {problem}") else: print("OK: the index agrees with content/ and Qdrant") return 1 if problems else 0 def _cmd_demo_filter(cfg, args, log) -> int: result = pipeline.demo_area_filter(cfg, args.query, area=args.area, limit=args.limit) if args.json: print(json.dumps(result, indent=2, sort_keys=True)) return 0 print(f"query: {result['query']!r}") print("") print(f"without a filter ({len(result['unfiltered'])} result(s)):") for row in result["unfiltered"]: print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}") print("") if not args.area: print("no --area given, so the filtered set is empty; re-run with e.g. --area edge") return 0 print(f"with area={args.area!r} ({len(result['filtered'])} result(s)):") for row in result["filtered"]: print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}") wrong = [row for row in result["filtered"] if row["area"] != args.area] if wrong: print(f"FAIL: {len(wrong)} filtered result(s) are not in area {args.area!r}") return 1 return 0 def _cmd_hybrid(cfg, args, log) -> int: rows = pipeline.hybrid_search(cfg, args.query, area=args.area, limit=args.limit) if args.json: print(json.dumps(rows, indent=2, sort_keys=True, default=str)) return 0 print(f"hybrid (RRF) query={args.query!r} area={args.area!r}: {len(rows)} result(s)") for row in rows: print(f" {row['score']:.4f} {row.get('title')} {row.get('url')} [{row['id'][:8]}]") return 0 def _cmd_measure(cfg, args, log) -> int: from . import measure as measure_module sizes = [int(part) for part in args.sizes.split(",") if part.strip()] report = measure_module.measure( cfg, sizes=sizes, container=args.container, keep=args.keep, log=log ) if args.json: print(json.dumps(report, indent=2, sort_keys=True)) return 0 print(measure_module.render_text(report)) return 0 def _cmd_graph(cfg, args, log) -> int: with store_module.Store(cfg.db_path) as store: report = graph_module.query(cfg, store, args.query, area=args.area, limit=args.limit) if args.json: print(json.dumps(report, indent=2, sort_keys=True, default=str)) return 0 print(f"graph query {args.query!r} area={args.area!r}") print("") print(f"keyword ({len(report['keyword'])} row(s)):") for row in report["keyword"]: kind = row.get("kind") print(f" [{kind}] {row.get('label')} -> {row.get('source_note')} {row.get('origin_url')}") print("") print(f"vector ({len(report['vector'])} row(s)):") for row in report["vector"]: print(f" {row['score']:.3f} [{row['kind']}] {row['label']} -> {row['source_note']}") if report.get("vector_note"): print(f" note: {report['vector_note']}") return 0 if report["ok"] else 1 def _cmd_graph_neighbours(cfg, args, log) -> int: with store_module.Store(cfg.db_path) as store: node = store.graph_node(args.node) edges = store.graph_neighbours(args.node, limit=args.limit) if args.json: print(json.dumps({"node": node, "edges": edges}, indent=2, sort_keys=True, default=str)) return 0 if not node: print(f"no graph node with id {args.node!r}") return 1 print(f"{node.get('label')} ({node.get('node_type')})") print(f" note: {node.get('source_note')}") print(f" origin: {node.get('origin_url')}") for edge in edges: print( f" {edge.get('source_label')} --{edge.get('relation')}--> {edge.get('target_label')}" f" [{edge.get('source_note')}]" ) return 0 def _cmd_scan(cfg, args, log) -> int: report = pipeline.scan_pool(cfg) if args.json: print(json.dumps(report, indent=2, sort_keys=True)) else: print( f"scanned {report['chunks']} chunk(s): " + ("clean" if report["ok"] else f"{len(report['hits'])} hit(s)") ) for hit in report["hits"]: print(f" {hit}") return 0 if report["ok"] else 1 def _cmd_sources(cfg, args, log) -> int: loaded = guides.load_guides(cfg.root) public = guides.build_public_manifest(loaded) last = {} if os.path.exists(cfg.db_path): with store_module.Store(cfg.db_path) as store: last = store.get_meta("sources") or {} if args.json: print(json.dumps({"guides": public, "last_harvest": last}, indent=2, sort_keys=True)) return 0 for row in public["platforms"]: state = "enabled" if row["enabled"] else "disabled" print(f"{row['name']} ({state}) — {row['what']}") print(f" base: {row['base']} fits: {', '.join(row['fits'])} read: {', '.join(row['read'])}") print(f" include: {json.dumps(row['include'])}") print(f" exclude: {json.dumps(row['exclude'])}") seen = last.get(row["name"]) or {} if seen: print( f" last harvest: {seen.get('documents', 0)} document(s), " f"newest {seen.get('newest') or '—'}" ) print("") return 0 def _cmd_info(cfg, args, log) -> int: print(cfg.describe()) qdrant = Qdrant(cfg.qdrant_url, timeout=cfg.timeout) print("") print(f"qdrant: {'answering, version ' + qdrant.version() if qdrant.health() else 'NOT answering'}") if qdrant.collection_exists(cfg.collection): print(f"collection {cfg.collection}: {qdrant.point_count(cfg.collection)} point(s)") else: print(f"collection {cfg.collection}: does not exist") if os.path.exists(cfg.db_path): with store_module.Store(cfg.db_path) as store: print(f"store: {store.chunk_count()} chunk(s) across {store.source_count()} source(s)") print(f"contract: {json.dumps(store.get_meta('collection'), sort_keys=True)}") print(f"graph: {json.dumps(store.graph_counts(), sort_keys=True)}") sources = store.get_meta("sources") or {} for name in sorted(sources): row = sources[name] print( f"source {name}: {row.get('documents', 0)} document(s), " f"newest {row.get('newest') or '—'}" ) else: print(f"store: {cfg.db_path} does not exist") return 0 if __name__ == "__main__": sys.exit(main())