228 lines
7.8 KiB
Python
228 lines
7.8 KiB
Python
"""Platform guides: one small file per platform, configuration and public copy.
|
|
|
|
`platforms/<name>.yml` says what a platform is, where it lives, what to include,
|
|
what to exclude, which resume area it feeds, and whether it is enabled. The same
|
|
wording is shown on the public site, so every value here is written to be read
|
|
by a visitor, not just by the pipeline.
|
|
|
|
The schema is fixed and validated at load time. An unknown area, a missing
|
|
`what`, or a guide whose filename and `name` disagree is a configuration error,
|
|
not a default. A disabled guide is loaded and shown, but not harvested.
|
|
|
|
There is exactly one YAML dialect in this repository and one parser for it
|
|
(`tools/validate_content.py`); this module imports that parser rather than
|
|
adding a second one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
|
|
from . import loader
|
|
from .payloads import AREAS # one area enum for guides and payloads
|
|
|
|
# The platforms the pipeline knows how to harvest. `curated` is content/ and is
|
|
# always on: it is tier 1 and has no guide because it is not an external estate.
|
|
KINDS = ("gitea", "obsidian", "graphify")
|
|
REQUIRED = ("name", "enabled", "what", "fits", "read")
|
|
DEFAULT_LIMITS = {
|
|
"max_documents": 4000,
|
|
"max_file_kb": 200,
|
|
"max_files_per_repo": 40,
|
|
"max_commits": 20,
|
|
"max_tree_entries": 3000,
|
|
"max_nodes": 20000,
|
|
"max_edges": 40000,
|
|
}
|
|
|
|
|
|
class GuideError(RuntimeError):
|
|
"""A platform guide is missing, malformed, or inconsistent."""
|
|
|
|
|
|
@dataclass
|
|
class Guide:
|
|
name: str
|
|
enabled: bool
|
|
what: str
|
|
fits: list
|
|
read: list
|
|
base: str = ""
|
|
auth: str = ""
|
|
include: dict = field(default_factory=dict)
|
|
exclude: dict = field(default_factory=dict)
|
|
limits: dict = field(default_factory=dict)
|
|
public: bool = True
|
|
path: str = "" # repository-relative, for the public page and error messages
|
|
|
|
# -- derived ----------------------------------------------------------
|
|
|
|
@property
|
|
def kind(self) -> str:
|
|
return self.name
|
|
|
|
@property
|
|
def primary_area(self) -> str:
|
|
return self.fits[0]
|
|
|
|
def limit(self, name: str) -> int:
|
|
return int(self.limits.get(name) or DEFAULT_LIMITS.get(name) or 0)
|
|
|
|
def to_public_dict(self) -> dict:
|
|
"""What the site shows. No auth variable values, only its name."""
|
|
return {
|
|
"name": self.name,
|
|
"enabled": self.enabled,
|
|
"what": self.what,
|
|
"fits": list(self.fits),
|
|
"read": list(self.read),
|
|
"base": self.base,
|
|
"include": {
|
|
"repos": _as_list((self.include or {}).get("repos")),
|
|
"paths": _as_list((self.include or {}).get("paths")),
|
|
},
|
|
"exclude": {
|
|
"repos": _as_list((self.exclude or {}).get("repos")),
|
|
"paths": _as_list((self.exclude or {}).get("paths")),
|
|
},
|
|
"public": self.public,
|
|
"path": self.path,
|
|
}
|
|
|
|
def to_dict(self) -> dict:
|
|
data = self.to_public_dict()
|
|
data["limits"] = dict(self.limits)
|
|
data["auth"] = self.auth # the variable name, never its value
|
|
return data
|
|
|
|
|
|
def load_guides(root: str, directory: str = None) -> dict:
|
|
"""Load every guide under platforms/, keyed by name. Sorted, deterministic."""
|
|
directory = directory or os.path.join(root, "platforms")
|
|
if not os.path.isdir(directory):
|
|
raise GuideError(f"platform guide directory not found: {directory}")
|
|
|
|
validator = loader.load_validator(root)
|
|
guides = {}
|
|
for filename in sorted(os.listdir(directory)):
|
|
if not filename.endswith((".yml", ".yaml")):
|
|
continue
|
|
path = os.path.join(directory, filename)
|
|
rel = os.path.relpath(path, root).replace(os.sep, "/")
|
|
guide = _load_one(validator, path, rel, os.path.splitext(filename)[0])
|
|
if guide.name in guides:
|
|
raise GuideError(f"{rel}: duplicate platform name {guide.name!r}")
|
|
guides[guide.name] = guide
|
|
|
|
missing = sorted(set(KINDS) - set(guides))
|
|
if missing:
|
|
raise GuideError(
|
|
f"no platform guide for {', '.join(missing)} under {directory}. "
|
|
"One small file per platform is required (docs/phase-4-corpus.md §2)."
|
|
)
|
|
return guides
|
|
|
|
|
|
def _load_one(validator, path: str, rel: str, stem: str) -> Guide:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
text = handle.read()
|
|
lines = [(number, line) for number, line in enumerate(text.splitlines(), start=1)]
|
|
try:
|
|
node = validator.parse_front_matter(lines)
|
|
except validator.YamlError as exc:
|
|
raise GuideError(f"{rel}:{exc.line}: invalid YAML: {exc.msg}") from exc
|
|
values = loader.to_python(node)
|
|
if not isinstance(values, dict):
|
|
raise GuideError(f"{rel}: a platform guide must be a mapping")
|
|
|
|
for key in REQUIRED:
|
|
if values.get(key) in (None, "", []):
|
|
raise GuideError(f"{rel}: missing required key {key!r}")
|
|
|
|
name = str(values["name"]).strip()
|
|
if name != stem:
|
|
raise GuideError(f"{rel}: name {name!r} does not match the filename {stem!r}")
|
|
if name not in KINDS:
|
|
raise GuideError(f"{rel}: unknown platform {name!r}; expected one of {list(KINDS)}")
|
|
|
|
fits = [str(area) for area in values["fits"]]
|
|
unknown = [area for area in fits if area not in AREAS]
|
|
if unknown:
|
|
raise GuideError(f"{rel}: fits contains unknown area(s) {unknown}; allowed {list(AREAS)}")
|
|
|
|
read = [str(item) for item in values["read"]]
|
|
if not read:
|
|
raise GuideError(f"{rel}: read must name at least one thing to harvest")
|
|
|
|
include = _mapping(values.get("include"), f"{rel}: include")
|
|
exclude = _mapping(values.get("exclude"), f"{rel}: exclude")
|
|
limits = _mapping(values.get("limits"), f"{rel}: limits")
|
|
for key, value in limits.items():
|
|
if isinstance(value, list):
|
|
# `graph_paths` is the one list-valued limit: the places a graph may
|
|
# be committed, tried in order.
|
|
continue
|
|
try:
|
|
limits[key] = int(value)
|
|
except (TypeError, ValueError):
|
|
raise GuideError(f"{rel}: limit {key!r} must be an integer, got {value!r}") from None
|
|
|
|
return Guide(
|
|
name=name,
|
|
enabled=bool(values.get("enabled")),
|
|
what=str(values["what"]).strip(),
|
|
fits=fits,
|
|
read=read,
|
|
base=str(values.get("base") or "").strip(),
|
|
auth=str(values.get("auth") or "").strip(),
|
|
include=include,
|
|
exclude=exclude,
|
|
limits=limits,
|
|
public=bool(values.get("public", True)),
|
|
path=rel,
|
|
)
|
|
|
|
|
|
def _mapping(value, where: str) -> dict:
|
|
if value is None:
|
|
return {}
|
|
if not isinstance(value, dict):
|
|
raise GuideError(f"{where} must be a mapping")
|
|
out = {}
|
|
for key, item in value.items():
|
|
if isinstance(item, list):
|
|
out[key] = [str(entry) for entry in item]
|
|
else:
|
|
out[key] = item
|
|
return out
|
|
|
|
|
|
def _as_list(value) -> list:
|
|
"""`repos: all` is a scalar; the public manifest still emits a list."""
|
|
if value in (None, ""):
|
|
return []
|
|
if isinstance(value, list):
|
|
return [str(item) for item in value]
|
|
return [str(value)]
|
|
|
|
|
|
def enabled_guides(guides: dict) -> list:
|
|
return [guide for _, guide in sorted(guides.items()) if guide.enabled]
|
|
|
|
|
|
def auth_value(guide: Guide, environ=None) -> str:
|
|
"""Read a guide's token from the environment. Never from a file in the repo."""
|
|
if not guide.auth:
|
|
return ""
|
|
environ = os.environ if environ is None else environ
|
|
return environ.get(guide.auth, "")
|
|
|
|
|
|
def build_public_manifest(guides: dict) -> dict:
|
|
"""The guides as the public site shows them. Deterministic JSON shape."""
|
|
return {
|
|
"version": 1,
|
|
"platforms": [guide.to_public_dict() for _, guide in sorted(guides.items())],
|
|
}
|