WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

corpus/guides.py

@@ -0,0 +1,227 @@
+"""Platform guides: one small file per platform, configuration and public copy.
+
+`platforms/<name>.yml` says what a platform is, where it lives, what to include,
+what to exclude, which resume area it feeds, and whether it is enabled. The same
+wording is shown on the public site, so every value here is written to be read
+by a visitor, not just by the pipeline.
+
+The schema is fixed and validated at load time. An unknown area, a missing
+`what`, or a guide whose filename and `name` disagree is a configuration error,
+not a default. A disabled guide is loaded and shown, but not harvested.
+
+There is exactly one YAML dialect in this repository and one parser for it
+(`tools/validate_content.py`); this module imports that parser rather than
+adding a second one.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+from . import loader
+from .payloads import AREAS  # one area enum for guides and payloads
+
+# The platforms the pipeline knows how to harvest. `curated` is content/ and is
+# always on: it is tier 1 and has no guide because it is not an external estate.
+KINDS = ("gitea", "obsidian", "graphify")
+REQUIRED = ("name", "enabled", "what", "fits", "read")
+DEFAULT_LIMITS = {
+    "max_documents": 4000,
+    "max_file_kb": 200,
+    "max_files_per_repo": 40,
+    "max_commits": 20,
+    "max_tree_entries": 3000,
+    "max_nodes": 20000,
+    "max_edges": 40000,
+}
+
+
+class GuideError(RuntimeError):
+    """A platform guide is missing, malformed, or inconsistent."""
+
+
+@dataclass
+class Guide:
+    name: str
+    enabled: bool
+    what: str
+    fits: list
+    read: list
+    base: str = ""
+    auth: str = ""
+    include: dict = field(default_factory=dict)
+    exclude: dict = field(default_factory=dict)
+    limits: dict = field(default_factory=dict)
+    public: bool = True
+    path: str = ""  # repository-relative, for the public page and error messages
+
+    # -- derived ----------------------------------------------------------
+
+    @property
+    def kind(self) -> str:
+        return self.name
+
+    @property
+    def primary_area(self) -> str:
+        return self.fits[0]
+
+    def limit(self, name: str) -> int:
+        return int(self.limits.get(name) or DEFAULT_LIMITS.get(name) or 0)
+
+    def to_public_dict(self) -> dict:
+        """What the site shows. No auth variable values, only its name."""
+        return {
+            "name": self.name,
+            "enabled": self.enabled,
+            "what": self.what,
+            "fits": list(self.fits),
+            "read": list(self.read),
+            "base": self.base,
+            "include": {
+                "repos": _as_list((self.include or {}).get("repos")),
+                "paths": _as_list((self.include or {}).get("paths")),
+            },
+            "exclude": {
+                "repos": _as_list((self.exclude or {}).get("repos")),
+                "paths": _as_list((self.exclude or {}).get("paths")),
+            },
+            "public": self.public,
+            "path": self.path,
+        }
+
+    def to_dict(self) -> dict:
+        data = self.to_public_dict()
+        data["limits"] = dict(self.limits)
+        data["auth"] = self.auth  # the variable name, never its value
+        return data
+
+
+def load_guides(root: str, directory: str = None) -> dict:
+    """Load every guide under platforms/, keyed by name. Sorted, deterministic."""
+    directory = directory or os.path.join(root, "platforms")
+    if not os.path.isdir(directory):
+        raise GuideError(f"platform guide directory not found: {directory}")
+
+    validator = loader.load_validator(root)
+    guides = {}
+    for filename in sorted(os.listdir(directory)):
+        if not filename.endswith((".yml", ".yaml")):
+            continue
+        path = os.path.join(directory, filename)
+        rel = os.path.relpath(path, root).replace(os.sep, "/")
+        guide = _load_one(validator, path, rel, os.path.splitext(filename)[0])
+        if guide.name in guides:
+            raise GuideError(f"{rel}: duplicate platform name {guide.name!r}")
+        guides[guide.name] = guide
+
+    missing = sorted(set(KINDS) - set(guides))
+    if missing:
+        raise GuideError(
+            f"no platform guide for {', '.join(missing)} under {directory}. "
+            "One small file per platform is required (docs/phase-4-corpus.md §2)."
+        )
+    return guides
+
+
+def _load_one(validator, path: str, rel: str, stem: str) -> Guide:
+    with open(path, "r", encoding="utf-8") as handle:
+        text = handle.read()
+    lines = [(number, line) for number, line in enumerate(text.splitlines(), start=1)]
+    try:
+        node = validator.parse_front_matter(lines)
+    except validator.YamlError as exc:
+        raise GuideError(f"{rel}:{exc.line}: invalid YAML: {exc.msg}") from exc
+    values = loader.to_python(node)
+    if not isinstance(values, dict):
+        raise GuideError(f"{rel}: a platform guide must be a mapping")
+
+    for key in REQUIRED:
+        if values.get(key) in (None, "", []):
+            raise GuideError(f"{rel}: missing required key {key!r}")
+
+    name = str(values["name"]).strip()
+    if name != stem:
+        raise GuideError(f"{rel}: name {name!r} does not match the filename {stem!r}")
+    if name not in KINDS:
+        raise GuideError(f"{rel}: unknown platform {name!r}; expected one of {list(KINDS)}")
+
+    fits = [str(area) for area in values["fits"]]
+    unknown = [area for area in fits if area not in AREAS]
+    if unknown:
+        raise GuideError(f"{rel}: fits contains unknown area(s) {unknown}; allowed {list(AREAS)}")
+
+    read = [str(item) for item in values["read"]]
+    if not read:
+        raise GuideError(f"{rel}: read must name at least one thing to harvest")
+
+    include = _mapping(values.get("include"), f"{rel}: include")
+    exclude = _mapping(values.get("exclude"), f"{rel}: exclude")
+    limits = _mapping(values.get("limits"), f"{rel}: limits")
+    for key, value in limits.items():
+        if isinstance(value, list):
+            # `graph_paths` is the one list-valued limit: the places a graph may
+            # be committed, tried in order.
+            continue
+        try:
+            limits[key] = int(value)
+        except (TypeError, ValueError):
+            raise GuideError(f"{rel}: limit {key!r} must be an integer, got {value!r}") from None
+
+    return Guide(
+        name=name,
+        enabled=bool(values.get("enabled")),
+        what=str(values["what"]).strip(),
+        fits=fits,
+        read=read,
+        base=str(values.get("base") or "").strip(),
+        auth=str(values.get("auth") or "").strip(),
+        include=include,
+        exclude=exclude,
+        limits=limits,
+        public=bool(values.get("public", True)),
+        path=rel,
+    )
+
+
+def _mapping(value, where: str) -> dict:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise GuideError(f"{where} must be a mapping")
+    out = {}
+    for key, item in value.items():
+        if isinstance(item, list):
+            out[key] = [str(entry) for entry in item]
+        else:
+            out[key] = item
+    return out
+
+
+def _as_list(value) -> list:
+    """`repos: all` is a scalar; the public manifest still emits a list."""
+    if value in (None, ""):
+        return []
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    return [str(value)]
+
+
+def enabled_guides(guides: dict) -> list:
+    return [guide for _, guide in sorted(guides.items()) if guide.enabled]
+
+
+def auth_value(guide: Guide, environ=None) -> str:
+    """Read a guide's token from the environment. Never from a file in the repo."""
+    if not guide.auth:
+        return ""
+    environ = os.environ if environ is None else environ
+    return environ.get(guide.auth, "")
+
+
+def build_public_manifest(guides: dict) -> dict:
+    """The guides as the public site shows them. Deterministic JSON shape."""
+    return {
+        "version": 1,
+        "platforms": [guide.to_public_dict() for _, guide in sorted(guides.items())],
+    }