sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 4 ingestion: exclusion/provenance/incremental/graph tests

Browse Source
This commit is contained in:
Paperclip Builder
2026-10-26 22:28:21 +11:00
parent 1e46ec009a
commit e3f54f58c1
5 changed files with 495 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
corpus/freshness.py
View File

@@ -30,15 +30,74 @@ from . import loader, store as store_module
from .config import CHUNKER_VERSION
from .pipeline import _contract
from .qdrant import Qdrant
from .sources import estate as estate_module
def audit(cfg, today: datetime.date = None, log=None) -> dict:
"""Return the audit report as a plain dict (JSON-serialisable)."""
log = log or (lambda _msg: None)
today = today or datetime.date.today()
documents = loader.load_documents(cfg)
findings = {"stale": [], "due_soon": [], "index_drift": [], "contract_drift": []}
# The audit reads the same estate the pipeline reads, so "the source has
# stopped changing" is measured against the source, not against the index.
harvest = None
try:
harvest = estate_module.harvest(cfg, log=log)
documents = harvest.documents
except estate_module.EstateError as exc:
documents = loader.load_documents(cfg)
estate_error = str(exc)
else:
estate_error = ""
findings = {
"stale": [],
"due_soon": [],
"index_drift": [],
"contract_drift": [],
"stale_source": [],
"source_missing": [],
"source_unavailable": [],
}
if estate_error:
findings["source_unavailable"].append({"platform": "estate", "problem": estate_error})
if harvest is not None:
for name, row in sorted((harvest.sources or {}).items()):
if not row.get("enabled"):
continue
# Curated content is measured by last_verified above; source silence
# is about the harvested estate, which should change on its own.
if name == "curated":
continue
if row.get("missing") or not row.get("documents"):
findings["source_missing"].append(
{
"platform": name,
"problem": "no documents were harvested",
"repo": row.get("repo") or row.get("base") or "",
}
)
continue
newest = row.get("newest") or ""
try:
changed = datetime.date.fromisoformat(newest[:10])
except ValueError:
findings["source_missing"].append(
{"platform": name, "problem": f"no usable last_modified ({newest!r})"}
)
continue
age = (today - changed).days
if age > cfg.source_silence_days:
findings["stale_source"].append(
{
"platform": name,
"repo": row.get("repo") or row.get("base") or "",
"newest": newest,
"age_days": age,
"problem": f"the source has not changed in {age} days",
}
)
for doc in documents:
try:
@@ -90,6 +149,7 @@ def audit(cfg, today: datetime.date = None, log=None) -> dict:
{"source_path": source, "problem": "indexed but no longer in content/"}
)
graph_counts = store.graph_counts()
stored_contract = store.get_meta("collection")
if stored_contract != _contract(cfg):
findings["contract_drift"].append(
@@ -120,23 +180,36 @@ def audit(cfg, today: datetime.date = None, log=None) -> dict:
"sources": store.source_count(),
"collection": cfg.collection,
"collection_points": points,
"graph_collection": cfg.graph_collection,
"graph_nodes": graph_counts["nodes"],
"graph_edges": graph_counts["edges"],
"provider": cfg.provider,
"model": cfg.model,
"dim": cfg.dim,
"chunker_version": CHUNKER_VERSION,
},
"findings": findings,
"sources": (harvest.sources if harvest is not None else {}),
"warnings": (harvest.warnings if harvest is not None else []),
"summary": {
"stale": len(findings["stale"]),
"due_soon": len(findings["due_soon"]),
"index_drift": len(findings["index_drift"]),
"contract_drift": len(findings["contract_drift"]),
"stale_source": len(findings["stale_source"]),
"source_missing": len(findings["source_missing"]),
"source_unavailable": len(findings["source_unavailable"]),
},
# `due_soon` is a warning, not a failure: the document is still
# inside its verification horizon. Stale, drift and contract
# problems are failures.
"ok": not (
findings["stale"] or findings["index_drift"] or findings["contract_drift"]
findings["stale"]
or findings["index_drift"]
or findings["contract_drift"]
or findings["stale_source"]
or findings["source_missing"]
or findings["source_unavailable"]
),
}
log(f"audit: {json.dumps(report['summary'], sort_keys=True)}")
@@ -147,9 +220,12 @@ def render_text(report: dict) -> str:
"""Human-readable audit, for a terminal or an Apprise body."""
lines = []
corpus = report["corpus"]
graph = ""
if corpus.get("graph_nodes") is not None:
graph = f", graph {corpus['graph_nodes']} node(s) / {corpus['graph_edges']} edge(s)"
lines.append(
f"corpus: {corpus['documents']} document(s), {corpus['chunks']} chunk(s), "
f"{corpus['collection_points']} vector(s) in {corpus['collection']}"
f"{corpus['collection_points']} vector(s) in {corpus['collection']}{graph}"
)
lines.append(
f"horizon: last_verified within {report['horizon']['max_age_days']} days "
@@ -159,10 +235,13 @@ def render_text(report: dict) -> str:
for kind, label in (
("stale", "STALE — past the verification horizon"),
("due_soon", "DUE SOON — verify before it goes stale"),
("index_drift", "INDEX DRIFT — the index does not match content/"),
("index_drift", "INDEX DRIFT — the index does not match the sources"),
("contract_drift", "CONTRACT DRIFT — rebuild required"),
("stale_source", "STALE SOURCE — the estate has stopped changing"),
("source_missing", "SOURCE MISSING — an enabled platform produced nothing"),
("source_unavailable", "SOURCE UNAVAILABLE — the estate could not be read"),
):
rows = findings[kind]
rows = findings.get(kind) or []
if not rows:
continue
lines.append("")
@@ -177,7 +256,10 @@ def render_text(report: dict) -> str:
)
if report["ok"]:
lines.append("")
lines.append("OK: nothing stale, no drift." + (" (see warnings above)" if findings["due_soon"] else ""))
lines.append(
"OK: nothing stale, no drift, every source current."
+ (" (see warnings above)" if findings["due_soon"] else "")
)
return "\n".join(lines)