378 lines
15 KiB
Python
378 lines
15 KiB
Python
"""Command line entry point: `python3 -m corpus.cli <command>`.
|
|
|
|
Commands
|
|
|
|
sync incremental update of the corpus (the normal command)
|
|
rebuild drop the collection and rebuild it from content/ (recovery)
|
|
audit freshness audit; reports stale documents, index drift
|
|
verify does the index agree with content/ and with Qdrant?
|
|
demo-filter the same query with and without an `area` filter
|
|
hybrid vectors + FTS5 fused with reciprocal rank fusion
|
|
measure Qdrant memory at 500 / 5,000 / 20,000 chunks
|
|
info what this build thinks it is configured to do
|
|
|
|
Exit codes: 0 success, 1 findings or verification problems, 2 configuration or
|
|
usage error. Every command accepts `--json` where a machine-readable result is
|
|
useful (Prefect reads JSON, a human reads the table).
|
|
|
|
Configuration is environment-first (`corpus/config.py`); the flags exist so a
|
|
one-off run can point at a scratch Qdrant without exporting variables.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from . import config as config_module
|
|
from . import freshness, guides, pipeline, store as store_module
|
|
from .config import Config, ConfigError
|
|
from .qdrant import Qdrant, QdrantError
|
|
from .sources import graph as graph_module
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
parser = _build_parser()
|
|
args = parser.parse_args(argv)
|
|
if args.command is None:
|
|
parser.print_help()
|
|
return 2
|
|
|
|
overrides = {
|
|
"root": args.root,
|
|
"qdrant_url": args.qdrant_url,
|
|
"collection": args.collection,
|
|
"provider": args.provider,
|
|
"model": args.model,
|
|
"dim": args.dim,
|
|
"db_path": args.db,
|
|
"data_dir": args.data,
|
|
"offline": True if args.offline else None,
|
|
"platforms": tuple(part.strip() for part in args.platforms.split(",") if part.strip())
|
|
if args.platforms
|
|
else None,
|
|
"graph_path": args.graph_path or None,
|
|
}
|
|
try:
|
|
cfg = Config.from_env(**overrides)
|
|
except ConfigError as exc:
|
|
print(f"configuration error: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
log = (lambda _msg: None) if args.quiet else (lambda msg: print(msg, file=sys.stderr))
|
|
|
|
try:
|
|
handler = {
|
|
"sync": _cmd_sync,
|
|
"rebuild": _cmd_rebuild,
|
|
"audit": _cmd_audit,
|
|
"verify": _cmd_verify,
|
|
"demo-filter": _cmd_demo_filter,
|
|
"hybrid": _cmd_hybrid,
|
|
"measure": _cmd_measure,
|
|
"info": _cmd_info,
|
|
"graph": _cmd_graph,
|
|
"graph-neighbours": _cmd_graph_neighbours,
|
|
"scan": _cmd_scan,
|
|
"sources": _cmd_sources,
|
|
}[args.command]
|
|
return handler(cfg, args, log)
|
|
except (pipeline.PipelineError, ConfigError) as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 2
|
|
except QdrantError as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
def _build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="python3 -m corpus.cli",
|
|
description="Build and audit the Ai Resume retrieval corpus (docs/phase-4-corpus.md).",
|
|
)
|
|
parser.add_argument("--root", help="repository root (default: parent of corpus/)")
|
|
parser.add_argument("--qdrant-url", help="Qdrant REST URL (default http://127.0.0.1:6333)")
|
|
parser.add_argument("--collection", help="Qdrant collection name (default resume_corpus)")
|
|
parser.add_argument("--provider", choices=sorted(config_module.PROVIDERS), help="embedding provider")
|
|
parser.add_argument("--model", help="embedding model name")
|
|
parser.add_argument("--dim", type=int, help="embedding dimension")
|
|
parser.add_argument("--db", help="SQLite corpus database path")
|
|
parser.add_argument("--data", help="corpus data directory")
|
|
parser.add_argument("--quiet", action="store_true", help="progress on stderr only becomes silent")
|
|
parser.add_argument("--offline", action="store_true", help="read content/ only; do not harvest the estate")
|
|
parser.add_argument("--platforms", help="comma-separated platforms to harvest (default: all enabled)")
|
|
parser.add_argument("--graph-path", help="read graph.json from this path instead of Gitea")
|
|
|
|
sub = parser.add_subparsers(dest="command")
|
|
|
|
sync = sub.add_parser("sync", help="incremental update")
|
|
sync.add_argument("--json", action="store_true")
|
|
|
|
rebuild = sub.add_parser("rebuild", help="drop the collection and rebuild from content/")
|
|
rebuild.add_argument("--json", action="store_true")
|
|
|
|
audit = sub.add_parser("audit", help="freshness and drift audit")
|
|
audit.add_argument("--json", action="store_true")
|
|
audit.add_argument("--today", help="audit as of this date (YYYY-MM-DD), for tests")
|
|
audit.add_argument("--fail-on-findings", action="store_true", help="exit 1 when anything is found")
|
|
audit.add_argument("--alert", action="store_true", help="POST the report to RESUME_APPRISE_URL")
|
|
|
|
verify = sub.add_parser("verify", help="check the index against content/ and Qdrant")
|
|
verify.add_argument("--json", action="store_true")
|
|
|
|
demo = sub.add_parser("demo-filter", help="same query with and without an area filter")
|
|
demo.add_argument("--query", required=True)
|
|
demo.add_argument("--area", help="area to filter on (ai, platform, edge, products, history, about)")
|
|
demo.add_argument("--limit", type=int, default=5)
|
|
demo.add_argument("--json", action="store_true")
|
|
|
|
hybrid = sub.add_parser("hybrid", help="vectors + FTS5 fused with RRF")
|
|
hybrid.add_argument("--query", required=True)
|
|
hybrid.add_argument("--area")
|
|
hybrid.add_argument("--limit", type=int, default=5)
|
|
hybrid.add_argument("--json", action="store_true")
|
|
|
|
measure = sub.add_parser("measure", help="Qdrant memory at three corpus sizes")
|
|
measure.add_argument("--sizes", default="500,5000,20000")
|
|
measure.add_argument("--container", default="", help="Qdrant container name for docker stats")
|
|
measure.add_argument("--keep", action="store_true", help="keep the measurement collection")
|
|
measure.add_argument("--json", action="store_true")
|
|
|
|
graph = sub.add_parser("graph", help="query graph nodes and edges by keyword or vector")
|
|
graph.add_argument("--query", required=True)
|
|
graph.add_argument("--area")
|
|
graph.add_argument("--limit", type=int, default=10)
|
|
graph.add_argument("--json", action="store_true")
|
|
|
|
neighbours = sub.add_parser("graph-neighbours", help="edges attached to a graph node id")
|
|
neighbours.add_argument("--node", required=True)
|
|
neighbours.add_argument("--limit", type=int, default=50)
|
|
neighbours.add_argument("--json", action="store_true")
|
|
|
|
scan = sub.add_parser("scan", help="search the finished pool for credential-shaped content")
|
|
scan.add_argument("--json", action="store_true")
|
|
|
|
sources = sub.add_parser("sources", help="show the platform guides and the last harvest")
|
|
sources.add_argument("--json", action="store_true")
|
|
|
|
sub.add_parser("info", help="print the resolved configuration")
|
|
return parser
|
|
|
|
|
|
# -- commands --------------------------------------------------------------
|
|
|
|
|
|
def _cmd_sync(cfg, args, log) -> int:
|
|
stats = pipeline.run(cfg, mode="sync", log=log)
|
|
if args.json:
|
|
print(json.dumps(stats.to_dict(), indent=2, sort_keys=True))
|
|
else:
|
|
print(stats.summary())
|
|
return 0
|
|
|
|
|
|
def _cmd_rebuild(cfg, args, log) -> int:
|
|
stats = pipeline.run(cfg, mode="rebuild", log=log)
|
|
if args.json:
|
|
print(json.dumps(stats.to_dict(), indent=2, sort_keys=True))
|
|
else:
|
|
print(stats.summary())
|
|
return 0
|
|
|
|
|
|
def _cmd_audit(cfg, args, log) -> int:
|
|
today = datetime.date.fromisoformat(args.today) if args.today else None
|
|
report = freshness.audit(cfg, today=today, log=log)
|
|
if args.json:
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
else:
|
|
print(freshness.render_text(report))
|
|
if args.alert:
|
|
try:
|
|
sent = freshness.alert(report, cfg.apprise_url)
|
|
if not sent:
|
|
log("alert: RESUME_APPRISE_URL is not set; nothing sent")
|
|
except Exception as exc: # noqa: BLE001 - an alert failure must not hide the audit
|
|
print(f"warning: could not send the Apprise alert: {exc}", file=sys.stderr)
|
|
if args.fail_on_findings and not report["ok"]:
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def _cmd_verify(cfg, args, log) -> int:
|
|
problems = pipeline.verify(cfg, log=log)
|
|
if args.json:
|
|
print(json.dumps({"ok": not problems, "problems": problems}, indent=2))
|
|
else:
|
|
if problems:
|
|
print(f"FAIL: {len(problems)} problem(s)")
|
|
for problem in problems:
|
|
print(f" - {problem}")
|
|
else:
|
|
print("OK: the index agrees with content/ and Qdrant")
|
|
return 1 if problems else 0
|
|
|
|
|
|
def _cmd_demo_filter(cfg, args, log) -> int:
|
|
result = pipeline.demo_area_filter(cfg, args.query, area=args.area, limit=args.limit)
|
|
if args.json:
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
return 0
|
|
print(f"query: {result['query']!r}")
|
|
print("")
|
|
print(f"without a filter ({len(result['unfiltered'])} result(s)):")
|
|
for row in result["unfiltered"]:
|
|
print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}")
|
|
print("")
|
|
if not args.area:
|
|
print("no --area given, so the filtered set is empty; re-run with e.g. --area edge")
|
|
return 0
|
|
print(f"with area={args.area!r} ({len(result['filtered'])} result(s)):")
|
|
for row in result["filtered"]:
|
|
print(f" {row['score']:.3f} [{row['area']}/{row['confidence']}] {row['title']} {row['url']}")
|
|
wrong = [row for row in result["filtered"] if row["area"] != args.area]
|
|
if wrong:
|
|
print(f"FAIL: {len(wrong)} filtered result(s) are not in area {args.area!r}")
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def _cmd_hybrid(cfg, args, log) -> int:
|
|
rows = pipeline.hybrid_search(cfg, args.query, area=args.area, limit=args.limit)
|
|
if args.json:
|
|
print(json.dumps(rows, indent=2, sort_keys=True, default=str))
|
|
return 0
|
|
print(f"hybrid (RRF) query={args.query!r} area={args.area!r}: {len(rows)} result(s)")
|
|
for row in rows:
|
|
print(f" {row['score']:.4f} {row.get('title')} {row.get('url')} [{row['id'][:8]}]")
|
|
return 0
|
|
|
|
|
|
def _cmd_measure(cfg, args, log) -> int:
|
|
from . import measure as measure_module
|
|
|
|
sizes = [int(part) for part in args.sizes.split(",") if part.strip()]
|
|
report = measure_module.measure(
|
|
cfg, sizes=sizes, container=args.container, keep=args.keep, log=log
|
|
)
|
|
if args.json:
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
return 0
|
|
print(measure_module.render_text(report))
|
|
return 0
|
|
|
|
|
|
def _cmd_graph(cfg, args, log) -> int:
|
|
with store_module.Store(cfg.db_path) as store:
|
|
report = graph_module.query(cfg, store, args.query, area=args.area, limit=args.limit)
|
|
if args.json:
|
|
print(json.dumps(report, indent=2, sort_keys=True, default=str))
|
|
return 0
|
|
print(f"graph query {args.query!r} area={args.area!r}")
|
|
print("")
|
|
print(f"keyword ({len(report['keyword'])} row(s)):")
|
|
for row in report["keyword"]:
|
|
kind = row.get("kind")
|
|
print(f" [{kind}] {row.get('label')} -> {row.get('source_note')} {row.get('origin_url')}")
|
|
print("")
|
|
print(f"vector ({len(report['vector'])} row(s)):")
|
|
for row in report["vector"]:
|
|
print(f" {row['score']:.3f} [{row['kind']}] {row['label']} -> {row['source_note']}")
|
|
if report.get("vector_note"):
|
|
print(f" note: {report['vector_note']}")
|
|
return 0 if report["ok"] else 1
|
|
|
|
|
|
def _cmd_graph_neighbours(cfg, args, log) -> int:
|
|
with store_module.Store(cfg.db_path) as store:
|
|
node = store.graph_node(args.node)
|
|
edges = store.graph_neighbours(args.node, limit=args.limit)
|
|
if args.json:
|
|
print(json.dumps({"node": node, "edges": edges}, indent=2, sort_keys=True, default=str))
|
|
return 0
|
|
if not node:
|
|
print(f"no graph node with id {args.node!r}")
|
|
return 1
|
|
print(f"{node.get('label')} ({node.get('node_type')})")
|
|
print(f" note: {node.get('source_note')}")
|
|
print(f" origin: {node.get('origin_url')}")
|
|
for edge in edges:
|
|
print(
|
|
f" {edge.get('source_label')} --{edge.get('relation')}--> {edge.get('target_label')}"
|
|
f" [{edge.get('source_note')}]"
|
|
)
|
|
return 0
|
|
|
|
|
|
def _cmd_scan(cfg, args, log) -> int:
|
|
report = pipeline.scan_pool(cfg)
|
|
if args.json:
|
|
print(json.dumps(report, indent=2, sort_keys=True))
|
|
else:
|
|
print(
|
|
f"scanned {report['chunks']} chunk(s): "
|
|
+ ("clean" if report["ok"] else f"{len(report['hits'])} hit(s)")
|
|
)
|
|
for hit in report["hits"]:
|
|
print(f" {hit}")
|
|
return 0 if report["ok"] else 1
|
|
|
|
|
|
def _cmd_sources(cfg, args, log) -> int:
|
|
loaded = guides.load_guides(cfg.root)
|
|
public = guides.build_public_manifest(loaded)
|
|
last = {}
|
|
if os.path.exists(cfg.db_path):
|
|
with store_module.Store(cfg.db_path) as store:
|
|
last = store.get_meta("sources") or {}
|
|
if args.json:
|
|
print(json.dumps({"guides": public, "last_harvest": last}, indent=2, sort_keys=True))
|
|
return 0
|
|
for row in public["platforms"]:
|
|
state = "enabled" if row["enabled"] else "disabled"
|
|
print(f"{row['name']} ({state}) — {row['what']}")
|
|
print(f" base: {row['base']} fits: {', '.join(row['fits'])} read: {', '.join(row['read'])}")
|
|
print(f" include: {json.dumps(row['include'])}")
|
|
print(f" exclude: {json.dumps(row['exclude'])}")
|
|
seen = last.get(row["name"]) or {}
|
|
if seen:
|
|
print(
|
|
f" last harvest: {seen.get('documents', 0)} document(s), "
|
|
f"newest {seen.get('newest') or '—'}"
|
|
)
|
|
print("")
|
|
return 0
|
|
|
|
|
|
def _cmd_info(cfg, args, log) -> int:
|
|
print(cfg.describe())
|
|
qdrant = Qdrant(cfg.qdrant_url, timeout=cfg.timeout)
|
|
print("")
|
|
print(f"qdrant: {'answering, version ' + qdrant.version() if qdrant.health() else 'NOT answering'}")
|
|
if qdrant.collection_exists(cfg.collection):
|
|
print(f"collection {cfg.collection}: {qdrant.point_count(cfg.collection)} point(s)")
|
|
else:
|
|
print(f"collection {cfg.collection}: does not exist")
|
|
if os.path.exists(cfg.db_path):
|
|
with store_module.Store(cfg.db_path) as store:
|
|
print(f"store: {store.chunk_count()} chunk(s) across {store.source_count()} source(s)")
|
|
print(f"contract: {json.dumps(store.get_meta('collection'), sort_keys=True)}")
|
|
print(f"graph: {json.dumps(store.graph_counts(), sort_keys=True)}")
|
|
sources = store.get_meta("sources") or {}
|
|
for name in sorted(sources):
|
|
row = sources[name]
|
|
print(
|
|
f"source {name}: {row.get('documents', 0)} document(s), "
|
|
f"newest {row.get('newest') or '—'}"
|
|
)
|
|
else:
|
|
print(f"store: {cfg.db_path} does not exist")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|