Phase 4 ingestion: exclusion/provenance/incremental/graph tests

flows/corpus.py

@@ -48,13 +48,49 @@ from corpus.config import Config  # noqa: E402
 
 @_flow(name="corpus-sync", log_prints=True)
 def sync_corpus() -> dict:
-    """Incremental refresh. Runs on every push that touches content/."""
+    """Incremental refresh of the whole estate. Runs on a push or on demand."""
     logger = _get_run_logger()
     cfg = Config.from_env()
     stats = pipeline.run(cfg, mode="sync", log=logger.info)
     return stats.to_dict()
 
 
+@_flow(name="corpus-scheduled-ingest", log_prints=True)
+def scheduled_ingest(alert: bool = True) -> dict:
+    """Nightly: harvest, report what changed, then audit and alert on staleness.
+
+    This is the scheduled run the DoD asks for: it reports documents added,
+    updated and removed per platform, and it raises an Apprise alert when a
+    source is missing, unavailable or has stopped changing.
+    """
+    logger = _get_run_logger()
+    cfg = Config.from_env()
+    stats = pipeline.run(cfg, mode="sync", log=logger.info)
+    logger.info(stats.summary())
+    report = freshness.audit(cfg, log=logger.info)
+    logger.info(freshness.render_text(report))
+    if alert and not report["ok"]:
+        try:
+            freshness.alert(report, cfg.apprise_url)
+        except Exception as exc:  # noqa: BLE001 - the alert must not fail the run
+            logger.warning(f"Apprise alert failed: {exc}")
+    return {
+        "changed": {
+            "documents": stats.documents,
+            "added": stats.added,
+            "updated": stats.updated,
+            "removed": stats.removed,
+            "unchanged": stats.unchanged,
+            "embedding_calls": stats.embed_calls,
+            "texts_embedded": stats.texts_embedded,
+            "platforms": stats.platforms,
+            "rejections": stats.rejection_summary,
+            "graph_rebuilt": stats.graph_rebuilt,
+        },
+        "audit": report,
+    }
+
+
 @_flow(name="corpus-rebuild", log_prints=True)
 def rebuild_corpus() -> dict:
     """From-scratch rebuild. The recovery path: deleting the volume is always safe."""
@@ -85,5 +121,7 @@ if __name__ == "__main__":  # pragma: no cover - convenience for a manual run
         print(freshness.render_text(freshness_audit(alert=False)))
     elif command == "rebuild":
         print(rebuild_corpus())
+    elif command == "scheduled":
+        print(scheduled_ingest(alert=False))
     else:
         print(sync_corpus())