WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

corpus/payloads.py

@@ -16,8 +16,14 @@ import datetime
 
 AREAS = ("ai", "platform", "edge", "products", "history", "about")
 LEVELS = (1, 2, 3)
-SOURCE_TYPES = ("doc", "code", "adr", "metric")
+# docs/phase-4-corpus.md §3 fixes the harvested types; adr and metric were
+# already used by curated content and stay valid.
+SOURCE_TYPES = (
+    "doc", "code", "commit", "graph_node", "graph_edge", "claim", "adr", "metric",
+)
 CONFIDENCE = ("delivered", "running", "in-progress", "abandoned", "historic")
+PLATFORMS = ("curated", "gitea", "obsidian", "graphify")
+TIERS = (1, 2)
 
 # Fields every payload must carry (docs/phase-4-corpus.md §4.2), plus the ones
 # the index needs to answer "where did this come from?" without a second lookup.
@@ -30,6 +36,13 @@ REQUIRED_FIELDS = (
     "artefacts",
     "last_verified",
     "text",
+    # docs/phase-4-corpus.md §3: provenance is part of the payload, not an
+    # optional extra. A chunk that cannot say where it came from is not admitted.
+    "platform",
+    "tier",
+    "origin_url",
+    "path",
+    "last_modified",
 )
 PROVENANCE_FIELDS = (
     "id",
@@ -51,11 +64,15 @@ class PayloadError(RuntimeError):
 
 
 def source_type_for(doc) -> str:
-    """Classify a document. Everything in content/ is a doc unless it says so.
+    """Classify a document.
 
-    The schema has no `source_type` field and the schema is frozen, so the
-    classification is derived from tags the schema does allow.
+    Harvested documents carry an explicit `source_type`. Curated content does
+    not: the schema is frozen and has no such field, so it is derived from the
+    tags the schema does allow.
     """
+    explicit = str(getattr(doc, "source_type", "") or "").strip()
+    if explicit:
+        return explicit
     tags = {str(tag).lower() for tag in doc.tags}
     if "adr" in tags:
         return "adr"
@@ -82,6 +99,12 @@ def build_payload(doc, chunk, cfg) -> dict:
         "artefacts": [str(a.get("url", "")) for a in doc.artefacts],
         "last_verified": doc.last_verified,
         "text": chunk.text,
+        # -- provenance (docs/phase-4-corpus.md §3) ------------------------
+        "platform": doc.platform,
+        "tier": int(doc.tier),
+        "origin_url": doc.origin_url,
+        "path": getattr(doc, "repo_path", "") or doc.path,
+        "last_modified": doc.last_modified,
         # -- provenance ----------------------------------------------------
         "id": None,  # filled by the pipeline once the id is computed
         "source_path": doc.path,
@@ -125,7 +148,17 @@ def validate_payload(payload: dict, where: str = "chunk") -> None:
             f"{where}: confidence {payload['confidence']!r} is not one of {list(CONFIDENCE)}"
         )
 
-    for field in ("period", "title", "claim", "url", "source_path", "source_id", "embedding_model"):
+    if payload["platform"] not in PLATFORMS:
+        raise PayloadError(
+            f"{where}: platform {payload['platform']!r} is not one of {list(PLATFORMS)}"
+        )
+    if payload["tier"] not in TIERS:
+        raise PayloadError(f"{where}: tier {payload['tier']!r} is not one of {list(TIERS)}")
+
+    for field in (
+        "period", "title", "claim", "url", "source_path", "source_id",
+        "embedding_model", "origin_url", "path",
+    ):
         value = payload[field]
         if not isinstance(value, str) or not value.strip():
             raise PayloadError(f"{where}: field {field!r} must be a non-empty string")
@@ -138,6 +171,7 @@ def validate_payload(payload: dict, where: str = "chunk") -> None:
         raise PayloadError(f"{where}: field 'artefacts' must be a list (empty is allowed)")
 
     _check_date(payload["last_verified"], where)
+    _check_timestamp(payload["last_modified"], where)
     if not isinstance(payload["embedding_dim"], int) or payload["embedding_dim"] <= 0:
         raise PayloadError(f"{where}: field 'embedding_dim' must be a positive integer")
     if not isinstance(payload["chunk_index"], int) or payload["chunk_index"] < 0:
@@ -146,6 +180,19 @@ def validate_payload(payload: dict, where: str = "chunk") -> None:
         raise PayloadError(f"{where}: field 'content_hash' must be a sha256 hex digest")
 
 
+def _check_timestamp(value: str, where: str) -> None:
+    """`last_modified` is an ISO-8601 instant, e.g. 2026-10-14T05:00:00Z."""
+    text = str(value)
+    if text.endswith("Z"):
+        text = text[:-1] + "+00:00"
+    try:
+        datetime.datetime.fromisoformat(text)
+    except ValueError as exc:
+        raise PayloadError(
+            f"{where}: field 'last_modified' must be an ISO-8601 timestamp, got {value!r}"
+        ) from exc
+
+
 def _check_date(value: str, where: str) -> None:
     try:
         datetime.date.fromisoformat(str(value))