"""Harvest the estate into one set of documents. Four sources, one shape (`loader.Document`), one tier rule: | Platform | Tier | Reached by | |---|---|---| | `curated` (`content/`) | 1 | the local repository, already validated | | `gitea` | 2 | the Gitea API | | `obsidian` | 2 | the same Gitea API, one repository | | `graphify` | 2 | `graph.json` committed to the vault repository | `content/` is loaded first and is never displaced: the query layer uses `tier` to prefer a curated claim over harvested text that disagrees with it. Every refused file is recorded with the rule that refused it. The DoD asks for this explicitly ("the run reports the rejection"), and it is the difference between an exclusion list that works and one nobody can audit. """ from __future__ import annotations import datetime import hashlib import json import os import re from .. import areas, exclusions, globs, guides, loader from . import graph as graph_module from .gitea import GiteaClient, GiteaError, GiteaNotFound, file_url, repo_url PLATFORM_ORDER = ("curated", "gitea", "obsidian", "graphify") class EstateError(RuntimeError): """The estate could not be read. The run stops rather than indexing half of it.""" class Harvest: def __init__(self) -> None: self.documents: list = [] self.rejections: list = [] self.images: list = [] self.warnings: list = [] self.sources: dict = {} self.graph = None # graph_module.GraphResult | None self.offline = False # -- reporting -------------------------------------------------------- def add_rejection(self, rejection) -> None: self.rejections.append(rejection.to_dict() if hasattr(rejection, "to_dict") else rejection) def rejection_summary(self) -> dict: counts = {} for row in self.rejections: key = f"{row.get('platform')}/{row.get('reason')}" counts[key] = counts.get(key, 0) + 1 return dict(sorted(counts.items())) def to_dict(self) -> dict: return { "documents": len(self.documents), "images": len(self.images), "rejections": len(self.rejections), "rejection_summary": self.rejection_summary(), "warnings": list(self.warnings), "sources": self.sources, "graph": self.graph.counts() if self.graph else None, "offline": self.offline, } def harvest(cfg, log=None) -> Harvest: """Read every enabled platform. Curated content is always read.""" log = log or (lambda _msg: None) result = Harvest() result.offline = bool(cfg.offline) curated = loader.load_documents(cfg) result.documents.extend(curated) result.sources["curated"] = _source_row( "curated", enabled=True, documents=curated, extra={"path": "content/"} ) log(f"curated: {len(curated)} document(s) from content/") if cfg.offline: # Not a warning: offline is a deliberate mode, not a failure. log("offline mode: only content/ was read (RESUME_OFFLINE=1)") return result all_guides = guides.load_guides(cfg.root) selected = { name: guide for name, guide in all_guides.items() if guide.enabled and name in cfg.platforms } missing = [name for name in cfg.platforms if name not in all_guides] if missing: result.warnings.append(f"platform(s) {missing} have no guide and were skipped") needs_gitea = {"gitea", "obsidian", "graphify"} & set(selected) if not needs_gitea: return result base = cfg.gitea_url or selected.get("gitea", all_guides.get("gitea")).base client = GiteaClient(base, cfg.gitea_token, timeout=cfg.timeout) try: repos = client.repos() except GiteaError as exc: raise EstateError(f"cannot list Gitea repositories: {exc}") from exc log(f"gitea: {len(repos)} repositories visible") vault_repo = _vault_repo(selected) vault_tree = None if vault_repo: try: vault_tree = client.tree(vault_repo, "main") except GiteaNotFound: try: vault_tree = client.tree(vault_repo, "master") except GiteaError as exc: raise EstateError(f"cannot read {vault_repo}: {exc}") from exc except GiteaError as exc: raise EstateError(f"cannot read {vault_repo}: {exc}") from exc if "gitea" in selected: _harvest_repos(result, client, selected["gitea"], repos, base, log) if "obsidian" in selected: _harvest_obsidian(result, client, selected["obsidian"], repos, vault_repo, vault_tree, cfg, base, log) if "graphify" in selected: _harvest_graph(result, client, selected["graphify"], repos, vault_repo, vault_tree, cfg, base, log) return result # -- gitea repositories --------------------------------------------------- def _harvest_repos(result: Harvest, client, guide, repos, base_url, log) -> None: excluded = set(_as_list((guide.exclude or {}).get("repos"))) include = (guide.include or {}).get("repos") or "all" wanted = None if include == "all" or _as_list(include) == [] else set(_as_list(include)) documents = [] seen_repos = set() for repo in repos: full_name = repo.get("full_name") or "" if not full_name or full_name in excluded: continue if wanted is not None and full_name not in wanted: continue seen_repos.add(full_name) try: documents.extend(_harvest_one_repo(result, client, guide, repo, base_url)) except GiteaError as exc: result.warnings.append(f"gitea/{full_name}: {exc}") continue if len(documents) >= guide.limit("max_documents"): result.warnings.append( f"gitea: document limit {guide.limit('max_documents')} reached; " "remaining repositories were skipped" ) break result.documents.extend(documents) result.sources["gitea"] = _source_row( "gitea", enabled=True, documents=documents, extra={"repos": len(seen_repos), "base": base_url}, ) log(f"gitea: {len(documents)} document(s), {len(result.rejections)} rejection(s) so far") def _harvest_one_repo(result: Harvest, client, guide, repo, base_url) -> list: full_name = repo["full_name"] branch = repo.get("default_branch") or "main" updated = _timestamp(repo.get("updated_at")) created = _timestamp(repo.get("created_at")) area = areas.area_for_repo(repo, guide.fits) topics = [str(topic) for topic in (repo.get("topics") or [])] description = str(repo.get("description") or "").strip() confidence = "abandoned" if repo.get("archived") else "historic" documents = [] try: entries = client.tree(full_name, branch) except GiteaNotFound: entries = [] blobs = [entry for entry in entries if entry.get("type") == "blob"] admitted = [] for entry in blobs: path = globs.normalise(entry.get("path") or "") kind, reason = exclusions.classify(path, guide, size=entry.get("size")) if kind == "reject": result.add_rejection(exclusions.Rejection(path, "gitea", reason, full_name)) continue if kind == "image": result.images.append( {"path": path, "platform": "gitea", "repo": full_name, "origin_url": file_url(base_url, full_name, branch, path)} ) continue admitted.append(entry) # The file tree is paths only. It is what lets a question like "does he have # a Prometheus config anywhere?" find a repository whose README never says so. tree_doc = _tree_document(guide, repo, area, updated, admitted, base_url) if tree_doc: documents.append(tree_doc) # README first: it is the document a reader would open. readmes = [ entry for entry in admitted if _is_readme(entry.get("path") or "") and globs.normalise(entry.get("path") or "").count("/") <= 1 ][:3] used = set() for entry in readmes: doc = _file_document( result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url, ) if doc: documents.append(doc) used.add(globs.normalise(entry["path"])) # Selected source files: the evidence behind the README's claims. candidates = [ entry for entry in admitted if globs.normalise(entry.get("path") or "") not in used and not _is_readme(entry.get("path") or "") and _source_rank(entry.get("path") or "") ] candidates.sort(key=lambda entry: (_depth(entry.get("path") or ""), globs.normalise(entry["path"]))) for entry in candidates[: guide.limit("max_files_per_repo")]: doc = _file_document( result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url, ) if doc: documents.append(doc) # One document per repository for commits and one for languages. These are # short, stable, and answer "what is being worked on?" without a filesystem. commits = _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url) if commits: documents.append(commits) languages = _language_document(result, client, guide, repo, area, updated, created, confidence, base_url) if languages: documents.append(languages) if not any(not doc.source_id.endswith(("@tree", "@commits", "@languages")) for doc in documents): # No README and no admitted source file: still admit a summary so the # repository is discoverable and its silence is visible, not invisible. summary = _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url) if summary: documents.insert(0, summary) return documents def _file_document(result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url) -> object: path = globs.normalise(entry.get("path") or "") full_name = repo["full_name"] try: raw = client.raw(full_name, branch, path) except GiteaError as exc: result.warnings.append(f"gitea/{full_name}/{path}: {exc}") return None if exclusions.looks_binary(raw): result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name)) return None text = raw.decode("utf-8", "replace") if not text.strip(): result.add_rejection(exclusions.Rejection(path, "gitea", "empty file", full_name)) return None hits = exclusions.scan_text(text) if hits: kinds = sorted({hit[0] for hit in hits}) result.add_rejection( exclusions.Rejection(path, "gitea", "credential-shaped content", full_name + ": " + ", ".join(kinds)) ) return None kind = "code" if exclusions.extension(path) not in ("md", "markdown", "mdx", "txt", "rst") else "markdown" title = f"{full_name} — {path}" body = _strip_front_matter(text) if kind == "markdown" else text if not body.strip(): result.add_rejection(exclusions.Rejection(path, "gitea", "front-matter only", full_name)) return None if description or topics: header = [f"Repository: {full_name}"] if description: header.append(description) if topics: header.append("Topics: " + ", ".join(topics)) body = "\n\n".join(["\n".join(header), text]) return _make_document( key=f"gitea:{full_name}/{path}", platform="gitea", area=area, title=title, claim=_claim(text, title), body=body, digest=hashlib.sha256(raw).hexdigest(), origin_url=file_url(base_url, full_name, branch, path), path=path, repo_path=path, source_id=f"{full_name}/{path}", source_type="code" if kind == "code" else "doc", last_modified=updated, confidence=confidence, period=_period(created, updated), tags=topics + ["gitea", area], tech=[], public=guide.public, kind=kind, ) def _tree_document(guide, repo, area, updated, blobs, base_url) -> object: if not blobs: return None full_name = repo["full_name"] branch = repo.get("default_branch") or "main" limit = guide.limit("max_tree_entries") paths = [globs.normalise(entry.get("path") or "") for entry in blobs][:limit] body = "\n".join( [f"Repository: {full_name}", f"Branch: {branch}", f"Files: {len(blobs)}", "", "File tree:"] + [f"- {path}" for path in paths] ) if len(blobs) > limit: body += f"\n- … {len(blobs) - limit} more" digest = hashlib.sha256(body.encode("utf-8")).hexdigest() return _make_document( key=f"gitea:{full_name}@tree", platform="gitea", area=area, title=f"{full_name} — file tree", claim=f"File tree of {full_name}, {len(blobs)} files.", body=body, digest=digest, origin_url=repo_url(base_url, full_name), path="", repo_path="", source_id=f"{full_name}@tree", source_type="doc", last_modified=updated, confidence="abandoned" if repo.get("archived") else "historic", period="", tags=["gitea", "tree", area], tech=[], public=guide.public, kind="markdown", ) def _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object: full_name = repo["full_name"] branch = repo.get("default_branch") or "main" try: commits = client.commits(full_name, branch, limit=guide.limit("max_commits")) except GiteaError: return None if not commits: return None lines = [f"Repository: {full_name}", f"Recent commits on {branch}:", ""] newest = updated for item in commits: commit = item.get("commit") or {} message = (commit.get("message") or "").strip().splitlines() headline = message[0] if message else "(no message)" date = _timestamp(((commit.get("author") or {}).get("date")) or ((commit.get("committer") or {}).get("date"))) if date: newest = max(newest, date) lines.append(f"- {date[:10]} {headline}") body = "\n".join(lines) digest = hashlib.sha256(body.encode("utf-8")).hexdigest() return _make_document( key=f"gitea:{full_name}@commits", platform="gitea", area=area, title=f"{full_name} — recent commits", claim=f"Recent commit history for {full_name}.", body=body, digest=digest, origin_url=f"{repo_url(base_url, full_name)}/commits/branch/{branch}", path="", repo_path="", source_id=f"{full_name}@commits", source_type="commit", last_modified=newest, confidence=confidence, period=_period(created, newest), tags=["gitea", "commits", area], tech=[], public=guide.public, kind="markdown", ) def _language_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object: full_name = repo["full_name"] try: languages = client.languages(full_name) except GiteaError: return None if not languages: return None total = sum(int(value) for value in languages.values()) or 1 rows = sorted(languages.items(), key=lambda item: int(item[1]), reverse=True) lines = [f"Repository: {full_name}", "", "Language statistics:"] for name, size in rows: lines.append(f"- {name}: {int(size)} bytes ({100.0 * int(size) / total:.1f}%)") body = "\n".join(lines) digest = hashlib.sha256(body.encode("utf-8")).hexdigest() return _make_document( key=f"gitea:{full_name}@languages", platform="gitea", area=area, title=f"{full_name} — languages", claim=f"Language breakdown for {full_name}: " + ", ".join(name for name, _ in rows[:4]) + ".", body=body, digest=digest, origin_url=repo_url(base_url, full_name), path="", repo_path="", source_id=f"{full_name}@languages", source_type="metric", last_modified=updated, confidence=confidence, period=_period(created, updated), tags=["gitea", "languages", area], tech=[name for name, _ in rows[:6]], public=guide.public, kind="markdown", ) def _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url) -> object: full_name = repo["full_name"] body = "\n".join( [f"Repository: {full_name}", f"Description: {description or '(none)'}", f"Topics: {', '.join(topics) if topics else '(none)'}"] ) digest = hashlib.sha256(body.encode("utf-8")).hexdigest() return _make_document( key=f"gitea:{full_name}@summary", platform="gitea", area=area, title=full_name, claim=description or f"Repository {full_name}.", body=body, digest=digest, origin_url=repo_url(base_url, full_name), path="", repo_path="", source_id=f"{full_name}@summary", source_type="doc", last_modified=updated, confidence=confidence, period=_period(created, updated), tags=topics + ["gitea", area], tech=[], public=guide.public, kind="markdown", ) # -- obsidian vault ------------------------------------------------------- def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None: if not vault_repo or tree is None: result.warnings.append("obsidian: the vault repository could not be listed") result.sources["obsidian"] = _source_row("obsidian", enabled=True, documents=[], extra={"missing": True}) return repo = next((row for row in repos if row.get("full_name") == vault_repo), {}) branch = repo.get("default_branch") or "main" updated = _timestamp(repo.get("updated_at")) created = _timestamp(repo.get("created_at")) validator = loader.load_validator(cfg.root) documents = [] blobs = [entry for entry in tree if entry.get("type") == "blob"] for entry in blobs: if len(documents) >= guide.limit("max_documents"): result.warnings.append( f"obsidian: document limit {guide.limit('max_documents')} reached" ) break path = globs.normalise(entry.get("path") or "") kind, reason = exclusions.classify(path, guide, size=entry.get("size")) if kind == "reject": # A path outside the include list is expected and is not a finding; # only hard exclusions are reported, so the report stays readable. if reason != "not under an included path": result.add_rejection(exclusions.Rejection(path, "obsidian", reason)) continue if kind == "image": result.images.append( {"path": path, "platform": "obsidian", "origin_url": file_url(base_url, vault_repo, branch, path)} ) continue try: raw = client.raw(vault_repo, branch, path) except GiteaError as exc: result.warnings.append(f"obsidian/{path}: {exc}") continue if exclusions.looks_binary(raw): result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content")) continue text = raw.decode("utf-8", "replace") if not text.strip(): result.add_rejection(exclusions.Rejection(path, "obsidian", "empty file")) continue hits = exclusions.scan_text(text) if hits: result.add_rejection( exclusions.Rejection( path, "obsidian", "credential-shaped content", ", ".join(sorted({hit[0] for hit in hits})), ) ) continue fields, body = _split_note(text, validator) if not body.strip(): result.add_rejection(exclusions.Rejection(path, "obsidian", "empty note")) continue title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0]) tag_values = fields.get("tags") or [] if isinstance(tag_values, str): tag_values = [tag_values] tags = [str(tag) for tag in tag_values] + _folder_tags(path) area = areas.area_for_note(path, guide.fits) documents.append( _make_document( key=f"obsidian:{path}", platform="obsidian", area=area, title=title, claim=_claim(body, title), body=body, digest=hashlib.sha256(raw).hexdigest(), origin_url=file_url(base_url, vault_repo, branch, path), path=path, repo_path=path, source_id=path, source_type="doc", last_modified=updated, confidence="historic", period=_period(created, updated), tags=tags + ["obsidian", area], tech=[], public=guide.public, kind="markdown", ) ) result.documents.extend(documents) result.sources["obsidian"] = _source_row( "obsidian", enabled=True, documents=documents, extra={"repo": vault_repo, "base": base_url}, ) log(f"obsidian: {len(documents)} note(s)") # -- graphify ------------------------------------------------------------- def _harvest_graph(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None: if not vault_repo: result.warnings.append("graphify: no vault repository configured in the guide") result.sources["graphify"] = _source_row("graphify", enabled=True, documents=[], extra={"missing": True}) return repo = next((row for row in repos if row.get("full_name") == vault_repo), {}) branch = repo.get("default_branch") or "main" updated = _timestamp(repo.get("updated_at")) vault_paths = { globs.normalise(entry.get("path") or "") for entry in (tree or []) if entry.get("type") == "blob" } raw = None source_path = "" if cfg.graph_path: if not os.path.exists(cfg.graph_path): raise EstateError(f"RESUME_GRAPH_PATH is set but {cfg.graph_path} does not exist") with open(cfg.graph_path, "rb") as handle: raw = handle.read() source_path = cfg.graph_path else: for candidate in guide.limits.get("graph_paths") or ["graph.json"]: try: raw = client.raw(vault_repo, branch, candidate) source_path = f"gitea:{vault_repo}/{candidate}" break except GiteaNotFound: continue except GiteaError as exc: raise EstateError(f"cannot read graph.json from {vault_repo}: {exc}") from exc if raw is None: result.warnings.append( f"graphify: graph.json is not committed to {vault_repo} " f"(looked in {guide.limits.get('graph_paths')}). The graph was not ingested; " "the nightly audit will keep reporting this until it is committed." ) result.sources["graphify"] = _source_row( "graphify", enabled=True, documents=[], extra={"missing": True, "repo": vault_repo} ) return graph = graph_module.parse( raw, platform="graphify", base=base_url, repo=vault_repo, ref=branch, last_modified=updated, area=guide.primary_area, vault_paths=vault_paths, max_nodes=guide.limit("max_nodes"), max_edges=guide.limit("max_edges"), source_path=source_path, ) result.graph = graph for rejection in graph.rejections: result.add_rejection(rejection) result.images.extend(graph.images) result.sources["graphify"] = _source_row( "graphify", enabled=True, documents=[], extra={ "repo": vault_repo, "path": source_path, "nodes": len(graph.nodes), "edges": len(graph.edges), "digest": graph.digest, "built_at_commit": graph.built_at_commit, }, ) if graph.warnings: result.warnings.extend(f"graphify: {warning}" for warning in graph.warnings) log( f"graphify: {len(graph.nodes)} node(s), {len(graph.edges)} edge(s), " f"{len(graph.images)} image node(s)" ) # -- shared helpers ------------------------------------------------------- def _make_document(**values) -> loader.Document: """One Document constructor, so every source produces the same shape.""" defaults = dict( metrics=[], artefacts=[], tech=[], tags=[], public=True, kind="markdown", platform="gitea", tier=2, level=3, period="", source_type="doc", ) defaults.update(values) title = defaults["title"] origin = defaults["origin_url"] path = defaults.get("path") or "" return loader.Document( path=defaults["key"], source_id=defaults.get("source_id") or (path or defaults["key"]), repo_path=defaults.get("repo_path") or path, area=defaults["area"], level=defaults.get("level", 3), title=title, claim=defaults.get("claim") or title, confidence=defaults.get("confidence") or "historic", period=defaults.get("period") or "—", tech=defaults.get("tech") or [], metrics=defaults.get("metrics") or [], artefacts=defaults.get("artefacts") or [], public=bool(defaults.get("public", True)), last_verified=str(defaults["last_modified"])[:10], tags=defaults.get("tags") or [], body=defaults["body"], digest=defaults["digest"], url=origin, platform=defaults["platform"], tier=int(defaults.get("tier", 2)), origin_url=origin, last_modified=defaults["last_modified"], source_type=defaults.get("source_type") or "doc", kind=defaults.get("kind") or "markdown", ) def _strip_front_matter(text: str) -> str: """Drop leading YAML blocks. The Note path parses them; files only need them gone. Some notes carry two blocks in a row (a stale export, or a block followed by a horizontal rule that closes), so this repeats. It never strips a body that does not have a closing delimiter. """ body = text for _ in range(3): lines = body.splitlines() if not lines or lines[0].strip() != "---": return body closing = None for index in range(1, len(lines)): if lines[index].strip() in ("---", "..."): closing = index break if closing is None: return body body = "\n".join(lines[closing + 1 :]).strip() return body def _split_note(text: str, validator) -> tuple: """Tolerant front-matter split. A note without front-matter is still a note.""" lines = text.splitlines() if not lines or lines[0].strip() != "---": return {}, text end = None for index in range(1, len(lines)): if lines[index].strip() in ("---", "..."): end = index break if end is None: return {}, text fields = {} try: node = validator.parse_front_matter( [(number, line) for number, line in enumerate(lines[1:end], start=2)] ) parsed = loader.to_python(node) if isinstance(parsed, dict): fields = parsed except validator.YamlError: fields = {} # Strip any further leading block as well, so a duplicate front-matter block # never reaches the chunker. return fields, _strip_front_matter("\n".join(lines[end + 1 :]).strip()) def _source_row(name: str, enabled: bool, documents: list, extra: dict = None) -> dict: stamps = sorted(doc.last_modified for doc in documents if doc.last_modified) row = { "platform": name, "enabled": enabled, "documents": len(documents), "newest": stamps[-1] if stamps else "", "oldest": stamps[0] if stamps else "", "fetched_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), } row.update(extra or {}) return row def _vault_repo(selected: dict) -> str: for name in ("obsidian", "graphify"): guide = selected.get(name) if guide and guide.base and "/" in guide.base and not guide.base.startswith("http"): return guide.base return "" def _as_list(value) -> list: if value in (None, ""): return [] if isinstance(value, list): return [str(item) for item in value] return [str(value)] def _timestamp(value) -> str: """Normalise any RFC3339 timestamp to UTC `…Z`. Empty stays empty.""" if not value: return "1970-01-01T00:00:00Z" text = str(value).strip() if text.endswith("Z"): text = text[:-1] + "+00:00" try: moment = datetime.datetime.fromisoformat(text) except ValueError: return "1970-01-01T00:00:00Z" if moment.tzinfo is None: moment = moment.replace(tzinfo=datetime.timezone.utc) return moment.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _period(created: str, updated: str) -> str: start = (created or "")[:4] end = (updated or "")[:4] if not start or start == "1970": return end or "—" if start == end: return start return f"{start}–{end}" def _claim(text: str, title: str) -> str: for line in (text or "").splitlines(): stripped = line.strip() if not stripped or stripped.startswith(("#", "|", "```", "---", ">")): continue cleaned = re.sub(r"[*_`\[\]]", "", stripped) if len(cleaned) > 20: return cleaned[:180] return title[:180] def _heading(text: str) -> str: for line in (text or "").splitlines(): if line.startswith("# "): return line[2:].strip() return "" def _folder_tags(path: str) -> list: parts = [part for part in path.split("/")[:-1] if part and not part[0].isdigit()] return [part.lower() for part in parts[-2:]] def _is_readme(path: str) -> bool: return os.path.basename(path).lower().startswith("readme") def _depth(path: str) -> int: return globs.normalise(path).count("/") def _source_rank(path: str) -> bool: ext = exclusions.extension(path) if ext not in exclusions.TEXT_EXTENSIONS: return False if ext in ("json", "csv", "tsv", "svg"): return False # Prefer files that carry decisions over generated or vendored files. return not any( part.lower() in ("vendor", "third_party", "examples", "testdata", "fixtures", "migrations") for part in globs.normalise(path).split("/")[:-1] )