"""Chunk payloads: the thing the retrieval filter runs against. `docs/phase-4-corpus.md` §4.2 fixes the payload. `area` and `confidence` are the two that matter: they let the query path narrow *before* it searches, and they let an answer qualify unfinished work instead of presenting it as delivered. **A missing or invalid field fails the run.** Nothing in this module supplies a default value: `validate_payload` raises on the first field that is absent, empty or out of enum, and the pipeline reports which document and chunk it was. That is the DoD item "a missing field fails the run rather than defaulting". """ from __future__ import annotations import datetime AREAS = ("ai", "platform", "edge", "products", "history", "about") LEVELS = (1, 2, 3) # docs/phase-4-corpus.md §3 fixes the harvested types; adr and metric were # already used by curated content and stay valid. SOURCE_TYPES = ( "doc", "code", "commit", "graph_node", "graph_edge", "claim", "adr", "metric", ) CONFIDENCE = ("delivered", "running", "in-progress", "abandoned", "historic") PLATFORMS = ("curated", "gitea", "obsidian", "graphify") TIERS = (1, 2) # Fields every payload must carry (docs/phase-4-corpus.md §4.2), plus the ones # the index needs to answer "where did this come from?" without a second lookup. REQUIRED_FIELDS = ( "area", "level", "source_type", "confidence", "period", "artefacts", "last_verified", "text", # docs/phase-4-corpus.md §3: provenance is part of the payload, not an # optional extra. A chunk that cannot say where it came from is not admitted. "platform", "tier", "origin_url", "path", "last_modified", ) PROVENANCE_FIELDS = ( "id", "source_path", "source_id", "chunk_index", "title", "claim", "url", "content_hash", "embedding_model", "embedding_dim", "chunker_version", ) class PayloadError(RuntimeError): """A payload is incomplete or out of enum. The run must stop.""" def source_type_for(doc) -> str: """Classify a document. Harvested documents carry an explicit `source_type`. Curated content does not: the schema is frozen and has no such field, so it is derived from the tags the schema does allow. """ explicit = str(getattr(doc, "source_type", "") or "").strip() if explicit: return explicit tags = {str(tag).lower() for tag in doc.tags} if "adr" in tags: return "adr" if "metric" in tags: return "metric" if "code" in tags: return "code" return "doc" def build_payload(doc, chunk, cfg) -> dict: """Build one chunk payload from a document and a chunk. Values are copied, never guessed: if the document does not carry a field, the payload does not gain one. """ payload = { # -- the retrieval filter (docs/phase-4-corpus.md §4.2) ------------ "area": doc.area, "level": doc.level, "source_type": source_type_for(doc), "confidence": doc.confidence, "period": doc.period, "artefacts": [str(a.get("url", "")) for a in doc.artefacts], "last_verified": doc.last_verified, "text": chunk.text, # -- provenance (docs/phase-4-corpus.md §3) ------------------------ "platform": doc.platform, "tier": int(doc.tier), "origin_url": doc.origin_url, "path": getattr(doc, "repo_path", "") or doc.path, "last_modified": doc.last_modified, # -- provenance ---------------------------------------------------- "id": None, # filled by the pipeline once the id is computed "source_path": doc.path, "source_id": doc.source_id, "chunk_index": chunk.index, "title": doc.title, "claim": doc.claim, "url": doc.url, "content_hash": doc.digest, "embedding_model": cfg.model, "embedding_dim": cfg.dim, "chunker_version": cfg.chunker_version, # -- supporting evidence, used by the answer to cite --------------- "public": bool(doc.public), "tech": list(doc.tech), "tags": list(doc.tags), "chunk_tokens": chunk.tokens, } return payload def validate_payload(payload: dict, where: str = "chunk") -> None: """Raise PayloadError if a required field is missing, empty or out of enum.""" for field in REQUIRED_FIELDS + PROVENANCE_FIELDS: if field == "id" and payload.get("id") is None: raise PayloadError(f"{where}: payload field 'id' was not set") if field not in payload: raise PayloadError(f"{where}: payload is missing required field {field!r}") area = payload["area"] if area not in AREAS: raise PayloadError(f"{where}: area {area!r} is not one of {list(AREAS)}") if payload["level"] not in LEVELS: raise PayloadError(f"{where}: level {payload['level']!r} is not one of {list(LEVELS)}") if payload["source_type"] not in SOURCE_TYPES: raise PayloadError( f"{where}: source_type {payload['source_type']!r} is not one of {list(SOURCE_TYPES)}" ) if payload["confidence"] not in CONFIDENCE: raise PayloadError( f"{where}: confidence {payload['confidence']!r} is not one of {list(CONFIDENCE)}" ) if payload["platform"] not in PLATFORMS: raise PayloadError( f"{where}: platform {payload['platform']!r} is not one of {list(PLATFORMS)}" ) if payload["tier"] not in TIERS: raise PayloadError(f"{where}: tier {payload['tier']!r} is not one of {list(TIERS)}") for field in ( "period", "title", "claim", "url", "source_path", "source_id", "embedding_model", "origin_url", "path", ): value = payload[field] if not isinstance(value, str) or not value.strip(): raise PayloadError(f"{where}: field {field!r} must be a non-empty string") text = payload["text"] if not isinstance(text, str) or not text.strip(): raise PayloadError(f"{where}: field 'text' must be non-empty") if not isinstance(payload["artefacts"], list): raise PayloadError(f"{where}: field 'artefacts' must be a list (empty is allowed)") _check_date(payload["last_verified"], where) _check_timestamp(payload["last_modified"], where) if not isinstance(payload["embedding_dim"], int) or payload["embedding_dim"] <= 0: raise PayloadError(f"{where}: field 'embedding_dim' must be a positive integer") if not isinstance(payload["chunk_index"], int) or payload["chunk_index"] < 0: raise PayloadError(f"{where}: field 'chunk_index' must be a non-negative integer") if not isinstance(payload["content_hash"], str) or len(payload["content_hash"]) != 64: raise PayloadError(f"{where}: field 'content_hash' must be a sha256 hex digest") def _check_timestamp(value: str, where: str) -> None: """`last_modified` is an ISO-8601 instant, e.g. 2026-10-14T05:00:00Z.""" text = str(value) if text.endswith("Z"): text = text[:-1] + "+00:00" try: datetime.datetime.fromisoformat(text) except ValueError as exc: raise PayloadError( f"{where}: field 'last_modified' must be an ISO-8601 timestamp, got {value!r}" ) from exc def _check_date(value: str, where: str) -> None: try: datetime.date.fromisoformat(str(value)) except ValueError as exc: raise PayloadError( f"{where}: field 'last_verified' must be an ISO date (YYYY-MM-DD), got {value!r}" ) from exc