849 lines
31 KiB
Python
849 lines
31 KiB
Python
"""Harvest the estate into one set of documents.
|
||
|
||
Four sources, one shape (`loader.Document`), one tier rule:
|
||
|
||
| Platform | Tier | Reached by |
|
||
|---|---|---|
|
||
| `curated` (`content/`) | 1 | the local repository, already validated |
|
||
| `gitea` | 2 | the Gitea API |
|
||
| `obsidian` | 2 | the same Gitea API, one repository |
|
||
| `graphify` | 2 | `graph.json` committed to the vault repository |
|
||
|
||
`content/` is loaded first and is never displaced: the query layer uses `tier`
|
||
to prefer a curated claim over harvested text that disagrees with it.
|
||
|
||
Every refused file is recorded with the rule that refused it. The DoD asks for
|
||
this explicitly ("the run reports the rejection"), and it is the difference
|
||
between an exclusion list that works and one nobody can audit.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import datetime
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
|
||
from .. import areas, exclusions, globs, guides, loader
|
||
from . import graph as graph_module
|
||
from .gitea import GiteaClient, GiteaError, GiteaNotFound, file_url, repo_url
|
||
|
||
PLATFORM_ORDER = ("curated", "gitea", "obsidian", "graphify")
|
||
|
||
|
||
class EstateError(RuntimeError):
|
||
"""The estate could not be read. The run stops rather than indexing half of it."""
|
||
|
||
|
||
class Harvest:
|
||
def __init__(self) -> None:
|
||
self.documents: list = []
|
||
self.rejections: list = []
|
||
self.images: list = []
|
||
self.warnings: list = []
|
||
self.sources: dict = {}
|
||
self.graph = None # graph_module.GraphResult | None
|
||
self.offline = False
|
||
|
||
# -- reporting --------------------------------------------------------
|
||
|
||
def add_rejection(self, rejection) -> None:
|
||
self.rejections.append(rejection.to_dict() if hasattr(rejection, "to_dict") else rejection)
|
||
|
||
def rejection_summary(self) -> dict:
|
||
counts = {}
|
||
for row in self.rejections:
|
||
reason = str(row.get("reason") or "unknown")
|
||
# Keep the summary readable: 200 distinct oversize byte counts are
|
||
# one finding, not two hundred.
|
||
if reason.startswith("oversize"):
|
||
reason = "oversize"
|
||
elif reason.startswith("binary"):
|
||
reason = "binary"
|
||
key = f"{row.get('platform')}/{reason}"
|
||
counts[key] = counts.get(key, 0) + 1
|
||
return dict(sorted(counts.items()))
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"documents": len(self.documents),
|
||
"images": len(self.images),
|
||
"rejections": len(self.rejections),
|
||
"rejection_summary": self.rejection_summary(),
|
||
"warnings": list(self.warnings),
|
||
"sources": self.sources,
|
||
"graph": self.graph.counts() if self.graph else None,
|
||
"offline": self.offline,
|
||
}
|
||
|
||
|
||
def harvest(cfg, log=None) -> Harvest:
|
||
"""Read every enabled platform. Curated content is always read."""
|
||
log = log or (lambda _msg: None)
|
||
result = Harvest()
|
||
result.offline = bool(cfg.offline)
|
||
|
||
curated = loader.load_documents(cfg)
|
||
result.documents.extend(curated)
|
||
result.sources["curated"] = _source_row(
|
||
"curated", enabled=True, documents=curated, extra={"path": "content/"}
|
||
)
|
||
log(f"curated: {len(curated)} document(s) from content/")
|
||
|
||
if cfg.offline:
|
||
# Not a warning: offline is a deliberate mode, not a failure.
|
||
log("offline mode: only content/ was read (RESUME_OFFLINE=1)")
|
||
return result
|
||
|
||
all_guides = guides.load_guides(cfg.root)
|
||
selected = {
|
||
name: guide
|
||
for name, guide in all_guides.items()
|
||
if guide.enabled and name in cfg.platforms
|
||
}
|
||
missing = [name for name in cfg.platforms if name not in all_guides]
|
||
if missing:
|
||
result.warnings.append(f"platform(s) {missing} have no guide and were skipped")
|
||
|
||
needs_gitea = {"gitea", "obsidian", "graphify"} & set(selected)
|
||
if not needs_gitea:
|
||
return result
|
||
|
||
base = cfg.gitea_url or selected.get("gitea", all_guides.get("gitea")).base
|
||
client = GiteaClient(base, cfg.gitea_token, timeout=cfg.timeout)
|
||
try:
|
||
repos = client.repos()
|
||
except GiteaError as exc:
|
||
raise EstateError(f"cannot list Gitea repositories: {exc}") from exc
|
||
log(f"gitea: {len(repos)} repositories visible")
|
||
|
||
vault_repo = _vault_repo(selected)
|
||
vault_tree = None
|
||
if vault_repo:
|
||
try:
|
||
vault_tree = client.tree(vault_repo, "main")
|
||
except GiteaNotFound:
|
||
try:
|
||
vault_tree = client.tree(vault_repo, "master")
|
||
except GiteaError as exc:
|
||
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
|
||
except GiteaError as exc:
|
||
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
|
||
|
||
if "gitea" in selected:
|
||
_harvest_repos(result, client, selected["gitea"], repos, base, log)
|
||
if "obsidian" in selected:
|
||
_harvest_obsidian(result, client, selected["obsidian"], repos, vault_repo, vault_tree, cfg, base, log)
|
||
if "graphify" in selected:
|
||
_harvest_graph(result, client, selected["graphify"], repos, vault_repo, vault_tree, cfg, base, log)
|
||
|
||
return result
|
||
|
||
|
||
# -- gitea repositories ---------------------------------------------------
|
||
|
||
|
||
def _harvest_repos(result: Harvest, client, guide, repos, base_url, log) -> None:
|
||
excluded = set(_as_list((guide.exclude or {}).get("repos")))
|
||
include = (guide.include or {}).get("repos") or "all"
|
||
wanted = None if include == "all" or _as_list(include) == [] else set(_as_list(include))
|
||
|
||
documents = []
|
||
seen_repos = set()
|
||
for repo in repos:
|
||
full_name = repo.get("full_name") or ""
|
||
if not full_name or full_name in excluded:
|
||
continue
|
||
if wanted is not None and full_name not in wanted:
|
||
continue
|
||
seen_repos.add(full_name)
|
||
try:
|
||
documents.extend(_harvest_one_repo(result, client, guide, repo, base_url))
|
||
except GiteaError as exc:
|
||
result.warnings.append(f"gitea/{full_name}: {exc}")
|
||
continue
|
||
if len(documents) >= guide.limit("max_documents"):
|
||
result.warnings.append(
|
||
f"gitea: document limit {guide.limit('max_documents')} reached; "
|
||
"remaining repositories were skipped"
|
||
)
|
||
break
|
||
|
||
result.documents.extend(documents)
|
||
result.sources["gitea"] = _source_row(
|
||
"gitea", enabled=True, documents=documents,
|
||
extra={"repos": len(seen_repos), "base": base_url},
|
||
)
|
||
log(f"gitea: {len(documents)} document(s), {len(result.rejections)} rejection(s) so far")
|
||
|
||
|
||
def _harvest_one_repo(result: Harvest, client, guide, repo, base_url) -> list:
|
||
full_name = repo["full_name"]
|
||
branch = repo.get("default_branch") or "main"
|
||
updated = _timestamp(repo.get("updated_at"))
|
||
created = _timestamp(repo.get("created_at"))
|
||
area = areas.area_for_repo(repo, guide.fits)
|
||
topics = [str(topic) for topic in (repo.get("topics") or [])]
|
||
description = str(repo.get("description") or "").strip()
|
||
confidence = "abandoned" if repo.get("archived") else "historic"
|
||
|
||
documents = []
|
||
try:
|
||
entries = client.tree(full_name, branch)
|
||
except GiteaNotFound:
|
||
entries = []
|
||
|
||
blobs = [entry for entry in entries if entry.get("type") == "blob"]
|
||
admitted = []
|
||
for entry in blobs:
|
||
path = globs.normalise(entry.get("path") or "")
|
||
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
|
||
if kind == "reject":
|
||
result.add_rejection(exclusions.Rejection(path, "gitea", reason, full_name))
|
||
continue
|
||
if kind == "image":
|
||
result.images.append(
|
||
{"path": path, "platform": "gitea", "repo": full_name,
|
||
"origin_url": file_url(base_url, full_name, branch, path)}
|
||
)
|
||
continue
|
||
admitted.append(entry)
|
||
|
||
# The file tree is paths only. It is what lets a question like "does he have
|
||
# a Prometheus config anywhere?" find a repository whose README never says so.
|
||
tree_doc = _tree_document(guide, repo, area, updated, admitted, base_url)
|
||
if tree_doc:
|
||
documents.append(tree_doc)
|
||
|
||
# README first: it is the document a reader would open.
|
||
readmes = [
|
||
entry for entry in admitted
|
||
if _is_readme(entry.get("path") or "") and globs.normalise(entry.get("path") or "").count("/") <= 1
|
||
][:3]
|
||
used = set()
|
||
for entry in readmes:
|
||
doc = _file_document(
|
||
result, client, guide, repo, branch, area, updated, created,
|
||
confidence, description, topics, entry, base_url,
|
||
)
|
||
if doc:
|
||
documents.append(doc)
|
||
used.add(globs.normalise(entry["path"]))
|
||
|
||
# Selected source files: the evidence behind the README's claims.
|
||
candidates = [
|
||
entry for entry in admitted
|
||
if globs.normalise(entry.get("path") or "") not in used
|
||
and not _is_readme(entry.get("path") or "")
|
||
and _source_rank(entry.get("path") or "")
|
||
]
|
||
candidates.sort(key=lambda entry: (_depth(entry.get("path") or ""), globs.normalise(entry["path"])))
|
||
for entry in candidates[: guide.limit("max_files_per_repo")]:
|
||
doc = _file_document(
|
||
result, client, guide, repo, branch, area, updated, created,
|
||
confidence, description, topics, entry, base_url,
|
||
)
|
||
if doc:
|
||
documents.append(doc)
|
||
|
||
# One document per repository for commits and one for languages. These are
|
||
# short, stable, and answer "what is being worked on?" without a filesystem.
|
||
commits = _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url)
|
||
if commits:
|
||
documents.append(commits)
|
||
languages = _language_document(result, client, guide, repo, area, updated, created, confidence, base_url)
|
||
if languages:
|
||
documents.append(languages)
|
||
|
||
if not any(not doc.source_id.endswith(("@tree", "@commits", "@languages")) for doc in documents):
|
||
# No README and no admitted source file: still admit a summary so the
|
||
# repository is discoverable and its silence is visible, not invisible.
|
||
summary = _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url)
|
||
if summary:
|
||
documents.insert(0, summary)
|
||
|
||
return documents
|
||
|
||
|
||
def _file_document(result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url) -> object:
|
||
path = globs.normalise(entry.get("path") or "")
|
||
full_name = repo["full_name"]
|
||
try:
|
||
raw = client.raw(full_name, branch, path)
|
||
except GiteaError as exc:
|
||
result.warnings.append(f"gitea/{full_name}/{path}: {exc}")
|
||
return None
|
||
if exclusions.looks_binary(raw):
|
||
result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name))
|
||
return None
|
||
text = raw.decode("utf-8", "replace")
|
||
if not text.strip():
|
||
result.add_rejection(exclusions.Rejection(path, "gitea", "empty file", full_name))
|
||
return None
|
||
hits = exclusions.scan_text(text)
|
||
if hits:
|
||
kinds = sorted({hit[0] for hit in hits})
|
||
result.add_rejection(
|
||
exclusions.Rejection(path, "gitea", "credential-shaped content", full_name + ": " + ", ".join(kinds))
|
||
)
|
||
return None
|
||
|
||
kind = "code" if exclusions.extension(path) not in ("md", "markdown", "mdx", "txt", "rst") else "markdown"
|
||
title = f"{full_name} — {path}"
|
||
body = _strip_front_matter(text) if kind == "markdown" else text
|
||
if not body.strip():
|
||
result.add_rejection(exclusions.Rejection(path, "gitea", "front-matter only", full_name))
|
||
return None
|
||
if description or topics:
|
||
header = [f"Repository: {full_name}"]
|
||
if description:
|
||
header.append(description)
|
||
if topics:
|
||
header.append("Topics: " + ", ".join(topics))
|
||
body = "\n\n".join(["\n".join(header), text])
|
||
return _make_document(
|
||
key=f"gitea:{full_name}/{path}",
|
||
platform="gitea",
|
||
area=area,
|
||
title=title,
|
||
claim=_claim(text, title),
|
||
body=body,
|
||
digest=hashlib.sha256(raw).hexdigest(),
|
||
origin_url=file_url(base_url, full_name, branch, path),
|
||
path=path,
|
||
repo_path=path,
|
||
source_id=f"{full_name}/{path}",
|
||
source_type="code" if kind == "code" else "doc",
|
||
last_modified=updated,
|
||
confidence=confidence,
|
||
period=_period(created, updated),
|
||
tags=topics + ["gitea", area],
|
||
tech=[],
|
||
public=guide.public,
|
||
kind=kind,
|
||
)
|
||
|
||
|
||
def _tree_document(guide, repo, area, updated, blobs, base_url) -> object:
|
||
if not blobs:
|
||
return None
|
||
full_name = repo["full_name"]
|
||
branch = repo.get("default_branch") or "main"
|
||
limit = guide.limit("max_tree_entries")
|
||
paths = [globs.normalise(entry.get("path") or "") for entry in blobs][:limit]
|
||
body = "\n".join(
|
||
[f"Repository: {full_name}", f"Branch: {branch}", f"Files: {len(blobs)}", "", "File tree:"]
|
||
+ [f"- {path}" for path in paths]
|
||
)
|
||
if len(blobs) > limit:
|
||
body += f"\n- … {len(blobs) - limit} more"
|
||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
return _make_document(
|
||
key=f"gitea:{full_name}@tree",
|
||
platform="gitea",
|
||
area=area,
|
||
title=f"{full_name} — file tree",
|
||
claim=f"File tree of {full_name}, {len(blobs)} files.",
|
||
body=body,
|
||
digest=digest,
|
||
origin_url=repo_url(base_url, full_name),
|
||
path="",
|
||
repo_path="",
|
||
source_id=f"{full_name}@tree",
|
||
source_type="doc",
|
||
last_modified=updated,
|
||
confidence="abandoned" if repo.get("archived") else "historic",
|
||
period="",
|
||
tags=["gitea", "tree", area],
|
||
tech=[],
|
||
public=guide.public,
|
||
kind="markdown",
|
||
)
|
||
|
||
|
||
def _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
|
||
full_name = repo["full_name"]
|
||
branch = repo.get("default_branch") or "main"
|
||
try:
|
||
commits = client.commits(full_name, branch, limit=guide.limit("max_commits"))
|
||
except GiteaError:
|
||
return None
|
||
if not commits:
|
||
return None
|
||
lines = [f"Repository: {full_name}", f"Recent commits on {branch}:", ""]
|
||
newest = updated
|
||
for item in commits:
|
||
commit = item.get("commit") or {}
|
||
message = (commit.get("message") or "").strip().splitlines()
|
||
headline = message[0] if message else "(no message)"
|
||
date = _timestamp(((commit.get("author") or {}).get("date")) or ((commit.get("committer") or {}).get("date")))
|
||
if date:
|
||
newest = max(newest, date)
|
||
lines.append(f"- {date[:10]} {headline}")
|
||
body = "\n".join(lines)
|
||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
return _make_document(
|
||
key=f"gitea:{full_name}@commits",
|
||
platform="gitea",
|
||
area=area,
|
||
title=f"{full_name} — recent commits",
|
||
claim=f"Recent commit history for {full_name}.",
|
||
body=body,
|
||
digest=digest,
|
||
origin_url=f"{repo_url(base_url, full_name)}/commits/branch/{branch}",
|
||
path="",
|
||
repo_path="",
|
||
source_id=f"{full_name}@commits",
|
||
source_type="commit",
|
||
last_modified=newest,
|
||
confidence=confidence,
|
||
period=_period(created, newest),
|
||
tags=["gitea", "commits", area],
|
||
tech=[],
|
||
public=guide.public,
|
||
kind="markdown",
|
||
)
|
||
|
||
|
||
def _language_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
|
||
full_name = repo["full_name"]
|
||
try:
|
||
languages = client.languages(full_name)
|
||
except GiteaError:
|
||
return None
|
||
if not languages:
|
||
return None
|
||
total = sum(int(value) for value in languages.values()) or 1
|
||
rows = sorted(languages.items(), key=lambda item: int(item[1]), reverse=True)
|
||
lines = [f"Repository: {full_name}", "", "Language statistics:"]
|
||
for name, size in rows:
|
||
lines.append(f"- {name}: {int(size)} bytes ({100.0 * int(size) / total:.1f}%)")
|
||
body = "\n".join(lines)
|
||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
return _make_document(
|
||
key=f"gitea:{full_name}@languages",
|
||
platform="gitea",
|
||
area=area,
|
||
title=f"{full_name} — languages",
|
||
claim=f"Language breakdown for {full_name}: " + ", ".join(name for name, _ in rows[:4]) + ".",
|
||
body=body,
|
||
digest=digest,
|
||
origin_url=repo_url(base_url, full_name),
|
||
path="",
|
||
repo_path="",
|
||
source_id=f"{full_name}@languages",
|
||
source_type="metric",
|
||
last_modified=updated,
|
||
confidence=confidence,
|
||
period=_period(created, updated),
|
||
tags=["gitea", "languages", area],
|
||
tech=[name for name, _ in rows[:6]],
|
||
public=guide.public,
|
||
kind="markdown",
|
||
)
|
||
|
||
|
||
def _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url) -> object:
|
||
full_name = repo["full_name"]
|
||
body = "\n".join(
|
||
[f"Repository: {full_name}", f"Description: {description or '(none)'}",
|
||
f"Topics: {', '.join(topics) if topics else '(none)'}"]
|
||
)
|
||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
return _make_document(
|
||
key=f"gitea:{full_name}@summary",
|
||
platform="gitea",
|
||
area=area,
|
||
title=full_name,
|
||
claim=description or f"Repository {full_name}.",
|
||
body=body,
|
||
digest=digest,
|
||
origin_url=repo_url(base_url, full_name),
|
||
path="",
|
||
repo_path="",
|
||
source_id=f"{full_name}@summary",
|
||
source_type="doc",
|
||
last_modified=updated,
|
||
confidence=confidence,
|
||
period=_period(created, updated),
|
||
tags=topics + ["gitea", area],
|
||
tech=[],
|
||
public=guide.public,
|
||
kind="markdown",
|
||
)
|
||
|
||
|
||
# -- obsidian vault -------------------------------------------------------
|
||
|
||
|
||
def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
|
||
if not vault_repo or tree is None:
|
||
result.warnings.append("obsidian: the vault repository could not be listed")
|
||
result.sources["obsidian"] = _source_row("obsidian", enabled=True, documents=[], extra={"missing": True})
|
||
return
|
||
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
|
||
branch = repo.get("default_branch") or "main"
|
||
updated = _timestamp(repo.get("updated_at"))
|
||
created = _timestamp(repo.get("created_at"))
|
||
validator = loader.load_validator(cfg.root)
|
||
documents = []
|
||
|
||
blobs = [entry for entry in tree if entry.get("type") == "blob"]
|
||
for entry in blobs:
|
||
if len(documents) >= guide.limit("max_documents"):
|
||
result.warnings.append(
|
||
f"obsidian: document limit {guide.limit('max_documents')} reached"
|
||
)
|
||
break
|
||
path = globs.normalise(entry.get("path") or "")
|
||
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
|
||
if kind == "reject":
|
||
# A path outside the include list is expected and is not a finding;
|
||
# only hard exclusions are reported, so the report stays readable.
|
||
if reason != "not under an included path":
|
||
result.add_rejection(exclusions.Rejection(path, "obsidian", reason))
|
||
continue
|
||
if kind == "image":
|
||
result.images.append(
|
||
{"path": path, "platform": "obsidian",
|
||
"origin_url": file_url(base_url, vault_repo, branch, path)}
|
||
)
|
||
continue
|
||
try:
|
||
raw = client.raw(vault_repo, branch, path)
|
||
except GiteaError as exc:
|
||
result.warnings.append(f"obsidian/{path}: {exc}")
|
||
continue
|
||
if exclusions.looks_binary(raw):
|
||
result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content"))
|
||
continue
|
||
text = raw.decode("utf-8", "replace")
|
||
if not text.strip():
|
||
result.add_rejection(exclusions.Rejection(path, "obsidian", "empty file"))
|
||
continue
|
||
hits = exclusions.scan_text(text)
|
||
if hits:
|
||
result.add_rejection(
|
||
exclusions.Rejection(
|
||
path, "obsidian", "credential-shaped content",
|
||
", ".join(sorted({hit[0] for hit in hits})),
|
||
)
|
||
)
|
||
continue
|
||
|
||
fields, body = _split_note(text, validator)
|
||
if not body.strip():
|
||
result.add_rejection(exclusions.Rejection(path, "obsidian", "empty note"))
|
||
continue
|
||
title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0])
|
||
tag_values = fields.get("tags") or []
|
||
if isinstance(tag_values, str):
|
||
tag_values = [tag_values]
|
||
tags = [str(tag) for tag in tag_values] + _folder_tags(path)
|
||
area = areas.area_for_note(path, guide.fits)
|
||
documents.append(
|
||
_make_document(
|
||
key=f"obsidian:{path}",
|
||
platform="obsidian",
|
||
area=area,
|
||
title=title,
|
||
claim=_claim(body, title),
|
||
body=body,
|
||
digest=hashlib.sha256(raw).hexdigest(),
|
||
origin_url=file_url(base_url, vault_repo, branch, path),
|
||
path=path,
|
||
repo_path=path,
|
||
source_id=path,
|
||
source_type="doc",
|
||
last_modified=updated,
|
||
confidence="historic",
|
||
period=_period(created, updated),
|
||
tags=tags + ["obsidian", area],
|
||
tech=[],
|
||
public=guide.public,
|
||
kind="markdown",
|
||
)
|
||
)
|
||
|
||
result.documents.extend(documents)
|
||
result.sources["obsidian"] = _source_row(
|
||
"obsidian", enabled=True, documents=documents,
|
||
extra={"repo": vault_repo, "base": base_url},
|
||
)
|
||
log(f"obsidian: {len(documents)} note(s)")
|
||
|
||
|
||
# -- graphify -------------------------------------------------------------
|
||
|
||
|
||
def _harvest_graph(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
|
||
if not vault_repo:
|
||
result.warnings.append("graphify: no vault repository configured in the guide")
|
||
result.sources["graphify"] = _source_row("graphify", enabled=True, documents=[], extra={"missing": True})
|
||
return
|
||
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
|
||
branch = repo.get("default_branch") or "main"
|
||
updated = _timestamp(repo.get("updated_at"))
|
||
vault_paths = {
|
||
globs.normalise(entry.get("path") or "")
|
||
for entry in (tree or [])
|
||
if entry.get("type") == "blob"
|
||
}
|
||
|
||
raw = None
|
||
source_path = ""
|
||
if cfg.graph_path:
|
||
if not os.path.exists(cfg.graph_path):
|
||
raise EstateError(f"RESUME_GRAPH_PATH is set but {cfg.graph_path} does not exist")
|
||
with open(cfg.graph_path, "rb") as handle:
|
||
raw = handle.read()
|
||
source_path = cfg.graph_path
|
||
else:
|
||
for candidate in guide.limits.get("graph_paths") or ["graph.json"]:
|
||
try:
|
||
raw = client.raw(vault_repo, branch, candidate)
|
||
source_path = f"gitea:{vault_repo}/{candidate}"
|
||
break
|
||
except GiteaNotFound:
|
||
continue
|
||
except GiteaError as exc:
|
||
raise EstateError(f"cannot read graph.json from {vault_repo}: {exc}") from exc
|
||
|
||
if raw is None:
|
||
result.warnings.append(
|
||
f"graphify: graph.json is not committed to {vault_repo} "
|
||
f"(looked in {guide.limits.get('graph_paths')}). The graph was not ingested; "
|
||
"the nightly audit will keep reporting this until it is committed."
|
||
)
|
||
result.sources["graphify"] = _source_row(
|
||
"graphify", enabled=True, documents=[], extra={"missing": True, "repo": vault_repo}
|
||
)
|
||
return
|
||
|
||
graph = graph_module.parse(
|
||
raw,
|
||
platform="graphify",
|
||
base=base_url,
|
||
repo=vault_repo,
|
||
ref=branch,
|
||
last_modified=updated,
|
||
area=guide.primary_area,
|
||
vault_paths=vault_paths,
|
||
max_nodes=guide.limit("max_nodes"),
|
||
max_edges=guide.limit("max_edges"),
|
||
source_path=source_path,
|
||
)
|
||
result.graph = graph
|
||
for rejection in graph.rejections:
|
||
result.add_rejection(rejection)
|
||
result.images.extend(graph.images)
|
||
result.sources["graphify"] = _source_row(
|
||
"graphify", enabled=True, documents=[],
|
||
extra={
|
||
"repo": vault_repo,
|
||
"path": source_path,
|
||
"nodes": len(graph.nodes),
|
||
"edges": len(graph.edges),
|
||
"digest": graph.digest,
|
||
"built_at_commit": graph.built_at_commit,
|
||
},
|
||
)
|
||
if graph.warnings:
|
||
result.warnings.extend(f"graphify: {warning}" for warning in graph.warnings)
|
||
log(
|
||
f"graphify: {len(graph.nodes)} node(s), {len(graph.edges)} edge(s), "
|
||
f"{len(graph.images)} image node(s)"
|
||
)
|
||
|
||
|
||
# -- shared helpers -------------------------------------------------------
|
||
|
||
|
||
def _make_document(**values) -> loader.Document:
|
||
"""One Document constructor, so every source produces the same shape."""
|
||
defaults = dict(
|
||
metrics=[], artefacts=[], tech=[], tags=[], public=True, kind="markdown",
|
||
platform="gitea", tier=2, level=3, period="", source_type="doc",
|
||
)
|
||
defaults.update(values)
|
||
title = defaults["title"]
|
||
origin = defaults["origin_url"]
|
||
path = defaults.get("path") or ""
|
||
return loader.Document(
|
||
path=defaults["key"],
|
||
source_id=defaults.get("source_id") or (path or defaults["key"]),
|
||
repo_path=defaults.get("repo_path") or path,
|
||
area=defaults["area"],
|
||
level=defaults.get("level", 3),
|
||
title=title,
|
||
claim=defaults.get("claim") or title,
|
||
confidence=defaults.get("confidence") or "historic",
|
||
period=defaults.get("period") or "—",
|
||
tech=defaults.get("tech") or [],
|
||
metrics=defaults.get("metrics") or [],
|
||
artefacts=defaults.get("artefacts") or [],
|
||
public=bool(defaults.get("public", True)),
|
||
last_verified=str(defaults["last_modified"])[:10],
|
||
tags=defaults.get("tags") or [],
|
||
body=defaults["body"],
|
||
digest=defaults["digest"],
|
||
url=origin,
|
||
platform=defaults["platform"],
|
||
tier=int(defaults.get("tier", 2)),
|
||
origin_url=origin,
|
||
last_modified=defaults["last_modified"],
|
||
source_type=defaults.get("source_type") or "doc",
|
||
kind=defaults.get("kind") or "markdown",
|
||
)
|
||
|
||
|
||
def _strip_front_matter(text: str) -> str:
|
||
"""Drop leading YAML blocks. The Note path parses them; files only need them gone.
|
||
|
||
Some notes carry two blocks in a row (a stale export, or a block followed by
|
||
a horizontal rule that closes), so this repeats. It never strips a body that
|
||
does not have a closing delimiter.
|
||
"""
|
||
body = text
|
||
for _ in range(3):
|
||
lines = body.splitlines()
|
||
if not lines or lines[0].strip() != "---":
|
||
return body
|
||
closing = None
|
||
for index in range(1, len(lines)):
|
||
if lines[index].strip() in ("---", "..."):
|
||
closing = index
|
||
break
|
||
if closing is None:
|
||
return body
|
||
body = "\n".join(lines[closing + 1 :]).strip()
|
||
return body
|
||
|
||
|
||
def _split_note(text: str, validator) -> tuple:
|
||
"""Tolerant front-matter split. A note without front-matter is still a note."""
|
||
lines = text.splitlines()
|
||
if not lines or lines[0].strip() != "---":
|
||
return {}, text
|
||
end = None
|
||
for index in range(1, len(lines)):
|
||
if lines[index].strip() in ("---", "..."):
|
||
end = index
|
||
break
|
||
if end is None:
|
||
return {}, text
|
||
fields = {}
|
||
try:
|
||
node = validator.parse_front_matter(
|
||
[(number, line) for number, line in enumerate(lines[1:end], start=2)]
|
||
)
|
||
parsed = loader.to_python(node)
|
||
if isinstance(parsed, dict):
|
||
fields = parsed
|
||
except validator.YamlError:
|
||
fields = {}
|
||
# Strip any further leading block as well, so a duplicate front-matter block
|
||
# never reaches the chunker.
|
||
return fields, _strip_front_matter("\n".join(lines[end + 1 :]).strip())
|
||
|
||
|
||
def _source_row(name: str, enabled: bool, documents: list, extra: dict = None) -> dict:
|
||
stamps = sorted(doc.last_modified for doc in documents if doc.last_modified)
|
||
row = {
|
||
"platform": name,
|
||
"enabled": enabled,
|
||
"documents": len(documents),
|
||
"newest": stamps[-1] if stamps else "",
|
||
"oldest": stamps[0] if stamps else "",
|
||
"fetched_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||
}
|
||
row.update(extra or {})
|
||
return row
|
||
|
||
|
||
def _vault_repo(selected: dict) -> str:
|
||
for name in ("obsidian", "graphify"):
|
||
guide = selected.get(name)
|
||
if guide and guide.base and "/" in guide.base and not guide.base.startswith("http"):
|
||
return guide.base
|
||
return ""
|
||
|
||
|
||
def _as_list(value) -> list:
|
||
if value in (None, ""):
|
||
return []
|
||
if isinstance(value, list):
|
||
return [str(item) for item in value]
|
||
return [str(value)]
|
||
|
||
|
||
def _timestamp(value) -> str:
|
||
"""Normalise any RFC3339 timestamp to UTC `…Z`. Empty stays empty."""
|
||
if not value:
|
||
return "1970-01-01T00:00:00Z"
|
||
text = str(value).strip()
|
||
if text.endswith("Z"):
|
||
text = text[:-1] + "+00:00"
|
||
try:
|
||
moment = datetime.datetime.fromisoformat(text)
|
||
except ValueError:
|
||
return "1970-01-01T00:00:00Z"
|
||
if moment.tzinfo is None:
|
||
moment = moment.replace(tzinfo=datetime.timezone.utc)
|
||
return moment.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def _period(created: str, updated: str) -> str:
|
||
start = (created or "")[:4]
|
||
end = (updated or "")[:4]
|
||
if not start or start == "1970":
|
||
return end or "—"
|
||
if start == end:
|
||
return start
|
||
return f"{start}–{end}"
|
||
|
||
|
||
def _claim(text: str, title: str) -> str:
|
||
for line in (text or "").splitlines():
|
||
stripped = line.strip()
|
||
if not stripped or stripped.startswith(("#", "|", "```", "---", ">")):
|
||
continue
|
||
cleaned = re.sub(r"[*_`\[\]]", "", stripped)
|
||
if len(cleaned) > 20:
|
||
return cleaned[:180]
|
||
return title[:180]
|
||
|
||
|
||
def _heading(text: str) -> str:
|
||
for line in (text or "").splitlines():
|
||
if line.startswith("# "):
|
||
return line[2:].strip()
|
||
return ""
|
||
|
||
|
||
def _folder_tags(path: str) -> list:
|
||
parts = [part for part in path.split("/")[:-1] if part and not part[0].isdigit()]
|
||
return [part.lower() for part in parts[-2:]]
|
||
|
||
|
||
def _is_readme(path: str) -> bool:
|
||
return os.path.basename(path).lower().startswith("readme")
|
||
|
||
|
||
def _depth(path: str) -> int:
|
||
return globs.normalise(path).count("/")
|
||
|
||
|
||
def _source_rank(path: str) -> bool:
|
||
ext = exclusions.extension(path)
|
||
if ext not in exclusions.TEXT_EXTENSIONS:
|
||
return False
|
||
if ext in ("json", "csv", "tsv", "svg"):
|
||
return False
|
||
# Prefer files that carry decisions over generated or vendored files.
|
||
return not any(
|
||
part.lower() in ("vendor", "third_party", "examples", "testdata", "fixtures", "migrations")
|
||
for part in globs.normalise(path).split("/")[:-1]
|
||
)
|