ai-resume/corpus/payloads.py

"""Chunk payloads: the thing the retrieval filter runs against.

`docs/phase-4-corpus.md` §4.2 fixes the payload. `area` and `confidence` are the
two that matter: they let the query path narrow *before* it searches, and they
let an answer qualify unfinished work instead of presenting it as delivered.

**A missing or invalid field fails the run.** Nothing in this module supplies a
default value: `validate_payload` raises on the first field that is absent,
empty or out of enum, and the pipeline reports which document and chunk it was.
That is the DoD item "a missing field fails the run rather than defaulting".
"""

from __future__ import annotations

import datetime

AREAS = ("ai", "platform", "edge", "products", "history", "about")
LEVELS = (1, 2, 3)
SOURCE_TYPES = ("doc", "code", "adr", "metric")
CONFIDENCE = ("delivered", "running", "in-progress", "abandoned", "historic")

# Fields every payload must carry (docs/phase-4-corpus.md §4.2), plus the ones
# the index needs to answer "where did this come from?" without a second lookup.
REQUIRED_FIELDS = (
    "area",
    "level",
    "source_type",
    "confidence",
    "period",
    "artefacts",
    "last_verified",
    "text",
)
PROVENANCE_FIELDS = (
    "id",
    "source_path",
    "source_id",
    "chunk_index",
    "title",
    "claim",
    "url",
    "content_hash",
    "embedding_model",
    "embedding_dim",
    "chunker_version",
)


class PayloadError(RuntimeError):
    """A payload is incomplete or out of enum. The run must stop."""


def source_type_for(doc) -> str:
    """Classify a document. Everything in content/ is a doc unless it says so.

    The schema has no `source_type` field and the schema is frozen, so the
    classification is derived from tags the schema does allow.
    """
    tags = {str(tag).lower() for tag in doc.tags}
    if "adr" in tags:
        return "adr"
    if "metric" in tags:
        return "metric"
    if "code" in tags:
        return "code"
    return "doc"


def build_payload(doc, chunk, cfg) -> dict:
    """Build one chunk payload from a document and a chunk.

    Values are copied, never guessed: if the document does not carry a field,
    the payload does not gain one.
    """
    payload = {
        # -- the retrieval filter (docs/phase-4-corpus.md §4.2) ------------
        "area": doc.area,
        "level": doc.level,
        "source_type": source_type_for(doc),
        "confidence": doc.confidence,
        "period": doc.period,
        "artefacts": [str(a.get("url", "")) for a in doc.artefacts],
        "last_verified": doc.last_verified,
        "text": chunk.text,
        # -- provenance ----------------------------------------------------
        "id": None,  # filled by the pipeline once the id is computed
        "source_path": doc.path,
        "source_id": doc.source_id,
        "chunk_index": chunk.index,
        "title": doc.title,
        "claim": doc.claim,
        "url": doc.url,
        "content_hash": doc.digest,
        "embedding_model": cfg.model,
        "embedding_dim": cfg.dim,
        "chunker_version": cfg.chunker_version,
        # -- supporting evidence, used by the answer to cite ---------------
        "public": bool(doc.public),
        "tech": list(doc.tech),
        "tags": list(doc.tags),
        "chunk_tokens": chunk.tokens,
    }
    return payload


def validate_payload(payload: dict, where: str = "chunk") -> None:
    """Raise PayloadError if a required field is missing, empty or out of enum."""
    for field in REQUIRED_FIELDS + PROVENANCE_FIELDS:
        if field == "id" and payload.get("id") is None:
            raise PayloadError(f"{where}: payload field 'id' was not set")
        if field not in payload:
            raise PayloadError(f"{where}: payload is missing required field {field!r}")

    area = payload["area"]
    if area not in AREAS:
        raise PayloadError(f"{where}: area {area!r} is not one of {list(AREAS)}")
    if payload["level"] not in LEVELS:
        raise PayloadError(f"{where}: level {payload['level']!r} is not one of {list(LEVELS)}")
    if payload["source_type"] not in SOURCE_TYPES:
        raise PayloadError(
            f"{where}: source_type {payload['source_type']!r} is not one of {list(SOURCE_TYPES)}"
        )
    if payload["confidence"] not in CONFIDENCE:
        raise PayloadError(
            f"{where}: confidence {payload['confidence']!r} is not one of {list(CONFIDENCE)}"
        )

    for field in ("period", "title", "claim", "url", "source_path", "source_id", "embedding_model"):
        value = payload[field]
        if not isinstance(value, str) or not value.strip():
            raise PayloadError(f"{where}: field {field!r} must be a non-empty string")

    text = payload["text"]
    if not isinstance(text, str) or not text.strip():
        raise PayloadError(f"{where}: field 'text' must be non-empty")

    if not isinstance(payload["artefacts"], list):
        raise PayloadError(f"{where}: field 'artefacts' must be a list (empty is allowed)")

    _check_date(payload["last_verified"], where)
    if not isinstance(payload["embedding_dim"], int) or payload["embedding_dim"] <= 0:
        raise PayloadError(f"{where}: field 'embedding_dim' must be a positive integer")
    if not isinstance(payload["chunk_index"], int) or payload["chunk_index"] < 0:
        raise PayloadError(f"{where}: field 'chunk_index' must be a non-negative integer")
    if not isinstance(payload["content_hash"], str) or len(payload["content_hash"]) != 64:
        raise PayloadError(f"{where}: field 'content_hash' must be a sha256 hex digest")


def _check_date(value: str, where: str) -> None:
    try:
        datetime.date.fromisoformat(str(value))
    except ValueError as exc:
        raise PayloadError(
            f"{where}: field 'last_verified' must be an ISO date (YYYY-MM-DD), got {value!r}"
        ) from exc