128 lines
4.4 KiB
Python
128 lines
4.4 KiB
Python
"""Prefect entry points for the corpus pipeline.
|
|
|
|
`docs/phase-4-corpus.md` §4.4: *"Prefect deployment on the existing Prefect
|
|
server; worker only, do not deploy a second server. Trigger: git push to
|
|
content/, plus a nightly audit."*
|
|
|
|
Prefect is optional at runtime. If it is not installed, these functions are
|
|
plain callables and the same code runs straight from the CLI
|
|
(`python3 -m corpus.cli sync`), which is what `make corpus` does. Nothing about
|
|
the pipeline depends on the orchestrator, and the orchestrator depends on
|
|
nothing but the pipeline.
|
|
|
|
The deployment commands (run on `.13`, which already has the Prefect server) are
|
|
in `docs/phase-4-notes.md` §6. This file is only the work definition.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
try: # pragma: no cover - exercised only where Prefect is installed
|
|
from prefect import flow as _flow
|
|
from prefect import get_run_logger as _get_run_logger
|
|
|
|
PREFECT_AVAILABLE = True
|
|
except ImportError: # pragma: no cover
|
|
PREFECT_AVAILABLE = False
|
|
|
|
def _flow(*_args, **_kwargs):
|
|
def decorator(func):
|
|
return func
|
|
|
|
return decorator
|
|
|
|
def _get_run_logger():
|
|
class _Logger:
|
|
def info(self, message, *args, **kwargs):
|
|
print(message, file=sys.stderr)
|
|
|
|
warning = error = info
|
|
|
|
return _Logger()
|
|
|
|
|
|
from corpus import freshness, pipeline # noqa: E402 (after the Prefect shim)
|
|
from corpus.config import Config # noqa: E402
|
|
|
|
|
|
@_flow(name="corpus-sync", log_prints=True)
|
|
def sync_corpus() -> dict:
|
|
"""Incremental refresh of the whole estate. Runs on a push or on demand."""
|
|
logger = _get_run_logger()
|
|
cfg = Config.from_env()
|
|
stats = pipeline.run(cfg, mode="sync", log=logger.info)
|
|
return stats.to_dict()
|
|
|
|
|
|
@_flow(name="corpus-scheduled-ingest", log_prints=True)
|
|
def scheduled_ingest(alert: bool = True) -> dict:
|
|
"""Nightly: harvest, report what changed, then audit and alert on staleness.
|
|
|
|
This is the scheduled run the DoD asks for: it reports documents added,
|
|
updated and removed per platform, and it raises an Apprise alert when a
|
|
source is missing, unavailable or has stopped changing.
|
|
"""
|
|
logger = _get_run_logger()
|
|
cfg = Config.from_env()
|
|
stats = pipeline.run(cfg, mode="sync", log=logger.info)
|
|
logger.info(stats.summary())
|
|
report = freshness.audit(cfg, log=logger.info)
|
|
logger.info(freshness.render_text(report))
|
|
if alert and not report["ok"]:
|
|
try:
|
|
freshness.alert(report, cfg.apprise_url)
|
|
except Exception as exc: # noqa: BLE001 - the alert must not fail the run
|
|
logger.warning(f"Apprise alert failed: {exc}")
|
|
return {
|
|
"changed": {
|
|
"documents": stats.documents,
|
|
"added": stats.added,
|
|
"updated": stats.updated,
|
|
"removed": stats.removed,
|
|
"unchanged": stats.unchanged,
|
|
"embedding_calls": stats.embed_calls,
|
|
"texts_embedded": stats.texts_embedded,
|
|
"platforms": stats.platforms,
|
|
"rejections": stats.rejection_summary,
|
|
"graph_rebuilt": stats.graph_rebuilt,
|
|
},
|
|
"audit": report,
|
|
}
|
|
|
|
|
|
@_flow(name="corpus-rebuild", log_prints=True)
|
|
def rebuild_corpus() -> dict:
|
|
"""From-scratch rebuild. The recovery path: deleting the volume is always safe."""
|
|
logger = _get_run_logger()
|
|
cfg = Config.from_env()
|
|
stats = pipeline.run(cfg, mode="rebuild", log=logger.info)
|
|
return stats.to_dict()
|
|
|
|
|
|
@_flow(name="corpus-audit", log_prints=True)
|
|
def freshness_audit(alert: bool = True) -> dict:
|
|
"""Nightly audit. Reports stale documents and index drift, then alerts."""
|
|
logger = _get_run_logger()
|
|
cfg = Config.from_env()
|
|
report = freshness.audit(cfg, log=logger.info)
|
|
logger.info(freshness.render_text(report))
|
|
if alert and not report["ok"]:
|
|
try:
|
|
freshness.alert(report, cfg.apprise_url)
|
|
except Exception as exc: # noqa: BLE001 - the alert must not fail the audit
|
|
logger.warning(f"Apprise alert failed: {exc}")
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - convenience for a manual run
|
|
command = sys.argv[1] if len(sys.argv) > 1 else "sync"
|
|
if command == "audit":
|
|
print(freshness.render_text(freshness_audit(alert=False)))
|
|
elif command == "rebuild":
|
|
print(rebuild_corpus())
|
|
elif command == "scheduled":
|
|
print(scheduled_ingest(alert=False))
|
|
else:
|
|
print(sync_corpus())
|