From 4896e18a4133618f3f2dacda17974c7154c6b645 Mon Sep 17 00:00:00 2001 From: Paperclip Builder Date: Mon, 26 Oct 2026 22:32:14 +1100 Subject: [PATCH] ingestion: strip front-matter from harvested markdown --- corpus/sources/estate.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/corpus/sources/estate.py b/corpus/sources/estate.py index 09e095b..0a87e90 100644 --- a/corpus/sources/estate.py +++ b/corpus/sources/estate.py @@ -284,7 +284,10 @@ def _file_document(result, client, guide, repo, branch, area, updated, created, kind = "code" if exclusions.extension(path) not in ("md", "markdown", "mdx", "txt", "rst") else "markdown" title = f"{full_name} — {path}" - body = text + body = _strip_front_matter(text) if kind == "markdown" else text + if not body.strip(): + result.add_rejection(exclusions.Rejection(path, "gitea", "front-matter only", full_name)) + return None if description or topics: header = [f"Repository: {full_name}"] if description: @@ -688,6 +691,17 @@ def _make_document(**values) -> loader.Document: ) +def _strip_front_matter(text: str) -> str: + """Drop a leading YAML block. The Note path parses it; files only need it gone.""" + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return text + for index in range(1, len(lines)): + if lines[index].strip() in ("---", "..."): + return "\n".join(lines[index + 1 :]).strip() + return text + + def _split_note(text: str, validator) -> tuple: """Tolerant front-matter split. A note without front-matter is still a note.""" lines = text.splitlines()