WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

corpus/cli.py

@@ -28,9 +28,10 @@ import os
 import sys
 
 from . import config as config_module
-from . import freshness, pipeline, store as store_module
+from . import freshness, guides, pipeline, store as store_module
 from .config import Config, ConfigError
 from .qdrant import Qdrant, QdrantError
+from .sources import graph as graph_module
 
 
 def main(argv=None) -> int:
@@ -49,6 +50,11 @@ def main(argv=None) -> int:
         "dim": args.dim,
         "db_path": args.db,
         "data_dir": args.data,
+        "offline": True if args.offline else None,
+        "platforms": tuple(part.strip() for part in args.platforms.split(",") if part.strip())
+        if args.platforms
+        else None,
+        "graph_path": args.graph_path or None,
     }
     try:
         cfg = Config.from_env(**overrides)
@@ -68,6 +74,9 @@ def main(argv=None) -> int:
             "hybrid": _cmd_hybrid,
             "measure": _cmd_measure,
             "info": _cmd_info,
+            "graph": _cmd_graph,
+            "scan": _cmd_scan,
+            "sources": _cmd_sources,
         }[args.command]
         return handler(cfg, args, log)
     except (pipeline.PipelineError, ConfigError) as exc:
@@ -92,6 +101,9 @@ def _build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--db", help="SQLite corpus database path")
     parser.add_argument("--data", help="corpus data directory")
     parser.add_argument("--quiet", action="store_true", help="progress on stderr only becomes silent")
+    parser.add_argument("--offline", action="store_true", help="read content/ only; do not harvest the estate")
+    parser.add_argument("--platforms", help="comma-separated platforms to harvest (default: all enabled)")
+    parser.add_argument("--graph-path", help="read graph.json from this path instead of Gitea")
 
     sub = parser.add_subparsers(dest="command")
 
@@ -128,6 +140,23 @@ def _build_parser() -> argparse.ArgumentParser:
     measure.add_argument("--keep", action="store_true", help="keep the measurement collection")
     measure.add_argument("--json", action="store_true")
 
+    graph = sub.add_parser("graph", help="query graph nodes and edges by keyword or vector")
+    graph.add_argument("--query", required=True)
+    graph.add_argument("--area")
+    graph.add_argument("--limit", type=int, default=10)
+    graph.add_argument("--json", action="store_true")
+
+    neighbours = sub.add_parser("graph-neighbours", help="edges attached to a graph node id")
+    neighbours.add_argument("--node", required=True)
+    neighbours.add_argument("--limit", type=int, default=50)
+    neighbours.add_argument("--json", action="store_true")
+
+    scan = sub.add_parser("scan", help="search the finished pool for credential-shaped content")
+    scan.add_argument("--json", action="store_true")
+
+    sources = sub.add_parser("sources", help="show the platform guides and the last harvest")
+    sources.add_argument("--json", action="store_true")
+
     sub.add_parser("info", help="print the resolved configuration")
     return parser
 
@@ -235,6 +264,88 @@ def _cmd_measure(cfg, args, log) -> int:
     return 0
 
 
+def _cmd_graph(cfg, args, log) -> int:
+    with store_module.Store(cfg.db_path) as store:
+        report = graph_module.query(cfg, store, args.query, area=args.area, limit=args.limit)
+    if args.json:
+        print(json.dumps(report, indent=2, sort_keys=True, default=str))
+        return 0
+    print(f"graph query {args.query!r} area={args.area!r}")
+    print("")
+    print(f"keyword ({len(report['keyword'])} row(s)):")
+    for row in report["keyword"]:
+        kind = row.get("kind")
+        print(f"  [{kind}] {row.get('label')}  ->  {row.get('source_note')}  {row.get('origin_url')}")
+    print("")
+    print(f"vector ({len(report['vector'])} row(s)):")
+    for row in report["vector"]:
+        print(f"  {row['score']:.3f}  [{row['kind']}] {row['label']}  ->  {row['source_note']}")
+    if report.get("vector_note"):
+        print(f"  note: {report['vector_note']}")
+    return 0 if report["ok"] else 1
+
+
+def _cmd_graph_neighbours(cfg, args, log) -> int:
+    with store_module.Store(cfg.db_path) as store:
+        node = store.graph_node(args.node)
+        edges = store.graph_neighbours(args.node, limit=args.limit)
+    if args.json:
+        print(json.dumps({"node": node, "edges": edges}, indent=2, sort_keys=True, default=str))
+        return 0
+    if not node:
+        print(f"no graph node with id {args.node!r}")
+        return 1
+    print(f"{node.get('label')}  ({node.get('node_type')})")
+    print(f"  note: {node.get('source_note')}")
+    print(f"  origin: {node.get('origin_url')}")
+    for edge in edges:
+        print(
+            f"  {edge.get('source_label')} --{edge.get('relation')}--> {edge.get('target_label')}"
+            f"  [{edge.get('source_note')}]"
+        )
+    return 0
+
+
+def _cmd_scan(cfg, args, log) -> int:
+    report = pipeline.scan_pool(cfg)
+    if args.json:
+        print(json.dumps(report, indent=2, sort_keys=True))
+    else:
+        print(
+            f"scanned {report['chunks']} chunk(s): "
+            + ("clean" if report["ok"] else f"{len(report['hits'])} hit(s)")
+        )
+        for hit in report["hits"]:
+            print(f"  {hit}")
+    return 0 if report["ok"] else 1
+
+
+def _cmd_sources(cfg, args, log) -> int:
+    loaded = guides.load_guides(cfg.root)
+    public = guides.build_public_manifest(loaded)
+    last = {}
+    if os.path.exists(cfg.db_path):
+        with store_module.Store(cfg.db_path) as store:
+            last = store.get_meta("sources") or {}
+    if args.json:
+        print(json.dumps({"guides": public, "last_harvest": last}, indent=2, sort_keys=True))
+        return 0
+    for row in public["platforms"]:
+        state = "enabled" if row["enabled"] else "disabled"
+        print(f"{row['name']} ({state}) — {row['what']}")
+        print(f"  base: {row['base']}  fits: {', '.join(row['fits'])}  read: {', '.join(row['read'])}")
+        print(f"  include: {json.dumps(row['include'])}")
+        print(f"  exclude: {json.dumps(row['exclude'])}")
+        seen = last.get(row["name"]) or {}
+        if seen:
+            print(
+                f"  last harvest: {seen.get('documents', 0)} document(s), "
+                f"newest {seen.get('newest') or '—'}"
+            )
+        print("")
+    return 0
+
+
 def _cmd_info(cfg, args, log) -> int:
     print(cfg.describe())
     qdrant = Qdrant(cfg.qdrant_url, timeout=cfg.timeout)
@@ -248,6 +359,14 @@ def _cmd_info(cfg, args, log) -> int:
         with store_module.Store(cfg.db_path) as store:
             print(f"store: {store.chunk_count()} chunk(s) across {store.source_count()} source(s)")
             print(f"contract: {json.dumps(store.get_meta('collection'), sort_keys=True)}")
+            print(f"graph: {json.dumps(store.graph_counts(), sort_keys=True)}")
+            sources = store.get_meta("sources") or {}
+            for name in sorted(sources):
+                row = sources[name]
+                print(
+                    f"source {name}: {row.get('documents', 0)} document(s), "
+                    f"newest {row.get('newest') or '—'}"
+                )
     else:
         print(f"store: {cfg.db_path} does not exist")
     return 0