WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

corpus/loader.py

@@ -78,6 +78,16 @@ class Document:
     digest: str  # sha256 of the file bytes: the incremental-mode key
     url: str  # canonical page URL, matching internal/content.PageURL
 
+    # Provenance (docs/phase-4-corpus.md §3). Curated content is tier 1 and has
+    # no external origin; harvested documents set these in corpus/sources/.
+    platform: str = "curated"
+    tier: int = 1
+    origin_url: str = ""
+    last_modified: str = ""
+    source_type: str = ""  # empty means: derive from tags (payloads.source_type_for)
+    kind: str = "markdown"  # "markdown" or "code", selects the chunker
+    repo_path: str = ""  # origin-relative path; `path` above is the unique source key
+
     # Filled in by the pipeline as it goes; not part of the schema.
     chunk_count: int = field(default=0)
 
@@ -158,5 +168,13 @@ def _load_one(cfg, validator, path: str) -> Document:
         body=body,
         digest=digest_of(path),
         url=page_url(str(fields["area"]), str(fields["id"]), level),
+        platform="curated",
+        tier=1,
+        # A curated claim is traceable to the page it appears on. The URL is
+        # site-relative because the public origin is a deployment concern.
+        origin_url=page_url(str(fields["area"]), str(fields["id"]), level),
+        last_modified=f"{fields['last_verified']}T00:00:00Z",
+        kind="markdown",
+        repo_path=rel,
     )
     return doc