WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance
This commit is contained in:
Paperclip Builder
804
corpus/sources/estate.py
Normal file
804
corpus/sources/estate.py
Normal file
@@ -0,0 +1,804 @@
|
||||
"""Harvest the estate into one set of documents.
|
||||
|
||||
Four sources, one shape (`loader.Document`), one tier rule:
|
||||
|
||||
| Platform | Tier | Reached by |
|
||||
|---|---|---|
|
||||
| `curated` (`content/`) | 1 | the local repository, already validated |
|
||||
| `gitea` | 2 | the Gitea API |
|
||||
| `obsidian` | 2 | the same Gitea API, one repository |
|
||||
| `graphify` | 2 | `graph.json` committed to the vault repository |
|
||||
|
||||
`content/` is loaded first and is never displaced: the query layer uses `tier`
|
||||
to prefer a curated claim over harvested text that disagrees with it.
|
||||
|
||||
Every refused file is recorded with the rule that refused it. The DoD asks for
|
||||
this explicitly ("the run reports the rejection"), and it is the difference
|
||||
between an exclusion list that works and one nobody can audit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .. import areas, exclusions, globs, guides, loader
|
||||
from . import graph as graph_module
|
||||
from .gitea import GiteaClient, GiteaError, GiteaNotFound, file_url, repo_url
|
||||
|
||||
PLATFORM_ORDER = ("curated", "gitea", "obsidian", "graphify")
|
||||
|
||||
|
||||
class EstateError(RuntimeError):
|
||||
"""The estate could not be read. The run stops rather than indexing half of it."""
|
||||
|
||||
|
||||
class Harvest:
|
||||
def __init__(self) -> None:
|
||||
self.documents: list = []
|
||||
self.rejections: list = []
|
||||
self.images: list = []
|
||||
self.warnings: list = []
|
||||
self.sources: dict = {}
|
||||
self.graph = None # graph_module.GraphResult | None
|
||||
self.offline = False
|
||||
|
||||
# -- reporting --------------------------------------------------------
|
||||
|
||||
def add_rejection(self, rejection) -> None:
|
||||
self.rejections.append(rejection.to_dict() if hasattr(rejection, "to_dict") else rejection)
|
||||
|
||||
def rejection_summary(self) -> dict:
|
||||
counts = {}
|
||||
for row in self.rejections:
|
||||
key = f"{row.get('platform')}/{row.get('reason')}"
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
return dict(sorted(counts.items()))
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"documents": len(self.documents),
|
||||
"images": len(self.images),
|
||||
"rejections": len(self.rejections),
|
||||
"rejection_summary": self.rejection_summary(),
|
||||
"warnings": list(self.warnings),
|
||||
"sources": self.sources,
|
||||
"graph": self.graph.counts() if self.graph else None,
|
||||
"offline": self.offline,
|
||||
}
|
||||
|
||||
|
||||
def harvest(cfg, log=None) -> Harvest:
|
||||
"""Read every enabled platform. Curated content is always read."""
|
||||
log = log or (lambda _msg: None)
|
||||
result = Harvest()
|
||||
result.offline = bool(cfg.offline)
|
||||
|
||||
curated = loader.load_documents(cfg)
|
||||
result.documents.extend(curated)
|
||||
result.sources["curated"] = _source_row(
|
||||
"curated", enabled=True, documents=curated, extra={"path": "content/"}
|
||||
)
|
||||
log(f"curated: {len(curated)} document(s) from content/")
|
||||
|
||||
if cfg.offline:
|
||||
# Not a warning: offline is a deliberate mode, not a failure.
|
||||
log("offline mode: only content/ was read (RESUME_OFFLINE=1)")
|
||||
return result
|
||||
|
||||
all_guides = guides.load_guides(cfg.root)
|
||||
selected = {
|
||||
name: guide
|
||||
for name, guide in all_guides.items()
|
||||
if guide.enabled and name in cfg.platforms
|
||||
}
|
||||
missing = [name for name in cfg.platforms if name not in all_guides]
|
||||
if missing:
|
||||
result.warnings.append(f"platform(s) {missing} have no guide and were skipped")
|
||||
|
||||
needs_gitea = {"gitea", "obsidian", "graphify"} & set(selected)
|
||||
if not needs_gitea:
|
||||
return result
|
||||
|
||||
base = cfg.gitea_url or selected.get("gitea", all_guides.get("gitea")).base
|
||||
client = GiteaClient(base, cfg.gitea_token, timeout=cfg.timeout)
|
||||
try:
|
||||
repos = client.repos()
|
||||
except GiteaError as exc:
|
||||
raise EstateError(f"cannot list Gitea repositories: {exc}") from exc
|
||||
log(f"gitea: {len(repos)} repositories visible")
|
||||
|
||||
vault_repo = _vault_repo(selected)
|
||||
vault_tree = None
|
||||
if vault_repo:
|
||||
try:
|
||||
vault_tree = client.tree(vault_repo, "main")
|
||||
except GiteaNotFound:
|
||||
try:
|
||||
vault_tree = client.tree(vault_repo, "master")
|
||||
except GiteaError as exc:
|
||||
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
|
||||
except GiteaError as exc:
|
||||
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
|
||||
|
||||
if "gitea" in selected:
|
||||
_harvest_repos(result, client, selected["gitea"], repos, base, log)
|
||||
if "obsidian" in selected:
|
||||
_harvest_obsidian(result, client, selected["obsidian"], repos, vault_repo, vault_tree, cfg, base, log)
|
||||
if "graphify" in selected:
|
||||
_harvest_graph(result, client, selected["graphify"], repos, vault_repo, vault_tree, cfg, base, log)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# -- gitea repositories ---------------------------------------------------
|
||||
|
||||
|
||||
def _harvest_repos(result: Harvest, client, guide, repos, base_url, log) -> None:
|
||||
excluded = set(_as_list((guide.exclude or {}).get("repos")))
|
||||
include = (guide.include or {}).get("repos") or "all"
|
||||
wanted = None if include == "all" or _as_list(include) == [] else set(_as_list(include))
|
||||
|
||||
documents = []
|
||||
seen_repos = set()
|
||||
for repo in repos:
|
||||
full_name = repo.get("full_name") or ""
|
||||
if not full_name or full_name in excluded:
|
||||
continue
|
||||
if wanted is not None and full_name not in wanted:
|
||||
continue
|
||||
seen_repos.add(full_name)
|
||||
try:
|
||||
documents.extend(_harvest_one_repo(result, client, guide, repo, base_url))
|
||||
except GiteaError as exc:
|
||||
result.warnings.append(f"gitea/{full_name}: {exc}")
|
||||
continue
|
||||
if len(documents) >= guide.limit("max_documents"):
|
||||
result.warnings.append(
|
||||
f"gitea: document limit {guide.limit('max_documents')} reached; "
|
||||
"remaining repositories were skipped"
|
||||
)
|
||||
break
|
||||
|
||||
result.documents.extend(documents)
|
||||
result.sources["gitea"] = _source_row(
|
||||
"gitea", enabled=True, documents=documents,
|
||||
extra={"repos": len(seen_repos), "base": base_url},
|
||||
)
|
||||
log(f"gitea: {len(documents)} document(s), {len(result.rejections)} rejection(s) so far")
|
||||
|
||||
|
||||
def _harvest_one_repo(result: Harvest, client, guide, repo, base_url) -> list:
|
||||
full_name = repo["full_name"]
|
||||
branch = repo.get("default_branch") or "main"
|
||||
updated = _timestamp(repo.get("updated_at"))
|
||||
created = _timestamp(repo.get("created_at"))
|
||||
area = areas.area_for_repo(repo, guide.fits)
|
||||
topics = [str(topic) for topic in (repo.get("topics") or [])]
|
||||
description = str(repo.get("description") or "").strip()
|
||||
confidence = "abandoned" if repo.get("archived") else "historic"
|
||||
|
||||
documents = []
|
||||
try:
|
||||
entries = client.tree(full_name, branch)
|
||||
except GiteaNotFound:
|
||||
entries = []
|
||||
|
||||
blobs = [entry for entry in entries if entry.get("type") == "blob"]
|
||||
admitted = []
|
||||
for entry in blobs:
|
||||
path = globs.normalise(entry.get("path") or "")
|
||||
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
|
||||
if kind == "reject":
|
||||
result.add_rejection(exclusions.Rejection(path, "gitea", reason, full_name))
|
||||
continue
|
||||
if kind == "image":
|
||||
result.images.append(
|
||||
{"path": path, "platform": "gitea", "repo": full_name,
|
||||
"origin_url": file_url(base_url, full_name, branch, path)}
|
||||
)
|
||||
continue
|
||||
admitted.append(entry)
|
||||
|
||||
# The file tree is paths only. It is what lets a question like "does he have
|
||||
# a Prometheus config anywhere?" find a repository whose README never says so.
|
||||
tree_doc = _tree_document(guide, repo, area, updated, admitted, base_url)
|
||||
if tree_doc:
|
||||
documents.append(tree_doc)
|
||||
|
||||
# README first: it is the document a reader would open.
|
||||
readmes = [
|
||||
entry for entry in admitted
|
||||
if _is_readme(entry.get("path") or "") and globs.normalise(entry.get("path") or "").count("/") <= 1
|
||||
][:3]
|
||||
used = set()
|
||||
for entry in readmes:
|
||||
doc = _file_document(
|
||||
result, client, guide, repo, branch, area, updated, created,
|
||||
confidence, description, topics, entry, base_url,
|
||||
)
|
||||
if doc:
|
||||
documents.append(doc)
|
||||
used.add(globs.normalise(entry["path"]))
|
||||
|
||||
# Selected source files: the evidence behind the README's claims.
|
||||
candidates = [
|
||||
entry for entry in admitted
|
||||
if globs.normalise(entry.get("path") or "") not in used
|
||||
and not _is_readme(entry.get("path") or "")
|
||||
and _source_rank(entry.get("path") or "")
|
||||
]
|
||||
candidates.sort(key=lambda entry: (_depth(entry.get("path") or ""), globs.normalise(entry["path"])))
|
||||
for entry in candidates[: guide.limit("max_files_per_repo")]:
|
||||
doc = _file_document(
|
||||
result, client, guide, repo, branch, area, updated, created,
|
||||
confidence, description, topics, entry, base_url,
|
||||
)
|
||||
if doc:
|
||||
documents.append(doc)
|
||||
|
||||
# One document per repository for commits and one for languages. These are
|
||||
# short, stable, and answer "what is being worked on?" without a filesystem.
|
||||
commits = _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url)
|
||||
if commits:
|
||||
documents.append(commits)
|
||||
languages = _language_document(result, client, guide, repo, area, updated, created, confidence, base_url)
|
||||
if languages:
|
||||
documents.append(languages)
|
||||
|
||||
if not any(not doc.source_id.endswith(("@tree", "@commits", "@languages")) for doc in documents):
|
||||
# No README and no admitted source file: still admit a summary so the
|
||||
# repository is discoverable and its silence is visible, not invisible.
|
||||
summary = _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url)
|
||||
if summary:
|
||||
documents.insert(0, summary)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def _file_document(result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url) -> object:
|
||||
path = globs.normalise(entry.get("path") or "")
|
||||
full_name = repo["full_name"]
|
||||
try:
|
||||
raw = client.raw(full_name, branch, path)
|
||||
except GiteaError as exc:
|
||||
result.warnings.append(f"gitea/{full_name}/{path}: {exc}")
|
||||
return None
|
||||
if exclusions.looks_binary(raw):
|
||||
result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name))
|
||||
return None
|
||||
text = raw.decode("utf-8", "replace")
|
||||
hits = exclusions.scan_text(text)
|
||||
if hits:
|
||||
kinds = sorted({hit[0] for hit in hits})
|
||||
result.add_rejection(
|
||||
exclusions.Rejection(path, "gitea", "credential-shaped content", full_name + ": " + ", ".join(kinds))
|
||||
)
|
||||
return None
|
||||
|
||||
kind = "code" if exclusions.extension(path) not in ("md", "markdown", "mdx", "txt", "rst") else "markdown"
|
||||
title = f"{full_name} — {path}"
|
||||
body = text
|
||||
if description or topics:
|
||||
header = [f"Repository: {full_name}"]
|
||||
if description:
|
||||
header.append(description)
|
||||
if topics:
|
||||
header.append("Topics: " + ", ".join(topics))
|
||||
body = "\n\n".join(["\n".join(header), text])
|
||||
return _make_document(
|
||||
key=f"gitea:{full_name}/{path}",
|
||||
platform="gitea",
|
||||
area=area,
|
||||
title=title,
|
||||
claim=_claim(text, title),
|
||||
body=body,
|
||||
digest=hashlib.sha256(raw).hexdigest(),
|
||||
origin_url=file_url(base_url, full_name, branch, path),
|
||||
path=path,
|
||||
repo_path=path,
|
||||
source_id=f"{full_name}/{path}",
|
||||
source_type="code" if kind == "code" else "doc",
|
||||
last_modified=updated,
|
||||
confidence=confidence,
|
||||
period=_period(created, updated),
|
||||
tags=topics + ["gitea", area],
|
||||
tech=[],
|
||||
public=guide.public,
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
|
||||
def _tree_document(guide, repo, area, updated, blobs, base_url) -> object:
|
||||
if not blobs:
|
||||
return None
|
||||
full_name = repo["full_name"]
|
||||
branch = repo.get("default_branch") or "main"
|
||||
limit = guide.limit("max_tree_entries")
|
||||
paths = [globs.normalise(entry.get("path") or "") for entry in blobs][:limit]
|
||||
body = "\n".join(
|
||||
[f"Repository: {full_name}", f"Branch: {branch}", f"Files: {len(blobs)}", "", "File tree:"]
|
||||
+ [f"- {path}" for path in paths]
|
||||
)
|
||||
if len(blobs) > limit:
|
||||
body += f"\n- … {len(blobs) - limit} more"
|
||||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
return _make_document(
|
||||
key=f"gitea:{full_name}@tree",
|
||||
platform="gitea",
|
||||
area=area,
|
||||
title=f"{full_name} — file tree",
|
||||
claim=f"File tree of {full_name}, {len(blobs)} files.",
|
||||
body=body,
|
||||
digest=digest,
|
||||
origin_url=repo_url(base_url, full_name),
|
||||
path="",
|
||||
repo_path="",
|
||||
source_id=f"{full_name}@tree",
|
||||
source_type="doc",
|
||||
last_modified=updated,
|
||||
confidence="abandoned" if repo.get("archived") else "historic",
|
||||
period="",
|
||||
tags=["gitea", "tree", area],
|
||||
tech=[],
|
||||
public=guide.public,
|
||||
kind="markdown",
|
||||
)
|
||||
|
||||
|
||||
def _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
|
||||
full_name = repo["full_name"]
|
||||
branch = repo.get("default_branch") or "main"
|
||||
try:
|
||||
commits = client.commits(full_name, branch, limit=guide.limit("max_commits"))
|
||||
except GiteaError:
|
||||
return None
|
||||
if not commits:
|
||||
return None
|
||||
lines = [f"Repository: {full_name}", f"Recent commits on {branch}:", ""]
|
||||
newest = updated
|
||||
for item in commits:
|
||||
commit = item.get("commit") or {}
|
||||
message = (commit.get("message") or "").strip().splitlines()
|
||||
headline = message[0] if message else "(no message)"
|
||||
date = _timestamp(((commit.get("author") or {}).get("date")) or ((commit.get("committer") or {}).get("date")))
|
||||
if date:
|
||||
newest = max(newest, date)
|
||||
lines.append(f"- {date[:10]} {headline}")
|
||||
body = "\n".join(lines)
|
||||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
return _make_document(
|
||||
key=f"gitea:{full_name}@commits",
|
||||
platform="gitea",
|
||||
area=area,
|
||||
title=f"{full_name} — recent commits",
|
||||
claim=f"Recent commit history for {full_name}.",
|
||||
body=body,
|
||||
digest=digest,
|
||||
origin_url=f"{repo_url(base_url, full_name)}/commits/branch/{branch}",
|
||||
path="",
|
||||
repo_path="",
|
||||
source_id=f"{full_name}@commits",
|
||||
source_type="commit",
|
||||
last_modified=newest,
|
||||
confidence=confidence,
|
||||
period=_period(created, newest),
|
||||
tags=["gitea", "commits", area],
|
||||
tech=[],
|
||||
public=guide.public,
|
||||
kind="markdown",
|
||||
)
|
||||
|
||||
|
||||
def _language_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
|
||||
full_name = repo["full_name"]
|
||||
try:
|
||||
languages = client.languages(full_name)
|
||||
except GiteaError:
|
||||
return None
|
||||
if not languages:
|
||||
return None
|
||||
total = sum(int(value) for value in languages.values()) or 1
|
||||
rows = sorted(languages.items(), key=lambda item: int(item[1]), reverse=True)
|
||||
lines = [f"Repository: {full_name}", "", "Language statistics:"]
|
||||
for name, size in rows:
|
||||
lines.append(f"- {name}: {int(size)} bytes ({100.0 * int(size) / total:.1f}%)")
|
||||
body = "\n".join(lines)
|
||||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
return _make_document(
|
||||
key=f"gitea:{full_name}@languages",
|
||||
platform="gitea",
|
||||
area=area,
|
||||
title=f"{full_name} — languages",
|
||||
claim=f"Language breakdown for {full_name}: " + ", ".join(name for name, _ in rows[:4]) + ".",
|
||||
body=body,
|
||||
digest=digest,
|
||||
origin_url=repo_url(base_url, full_name),
|
||||
path="",
|
||||
repo_path="",
|
||||
source_id=f"{full_name}@languages",
|
||||
source_type="metric",
|
||||
last_modified=updated,
|
||||
confidence=confidence,
|
||||
period=_period(created, updated),
|
||||
tags=["gitea", "languages", area],
|
||||
tech=[name for name, _ in rows[:6]],
|
||||
public=guide.public,
|
||||
kind="markdown",
|
||||
)
|
||||
|
||||
|
||||
def _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url) -> object:
|
||||
full_name = repo["full_name"]
|
||||
body = "\n".join(
|
||||
[f"Repository: {full_name}", f"Description: {description or '(none)'}",
|
||||
f"Topics: {', '.join(topics) if topics else '(none)'}"]
|
||||
)
|
||||
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
return _make_document(
|
||||
key=f"gitea:{full_name}@summary",
|
||||
platform="gitea",
|
||||
area=area,
|
||||
title=full_name,
|
||||
claim=description or f"Repository {full_name}.",
|
||||
body=body,
|
||||
digest=digest,
|
||||
origin_url=repo_url(base_url, full_name),
|
||||
path="",
|
||||
repo_path="",
|
||||
source_id=f"{full_name}@summary",
|
||||
source_type="doc",
|
||||
last_modified=updated,
|
||||
confidence=confidence,
|
||||
period=_period(created, updated),
|
||||
tags=topics + ["gitea", area],
|
||||
tech=[],
|
||||
public=guide.public,
|
||||
kind="markdown",
|
||||
)
|
||||
|
||||
|
||||
# -- obsidian vault -------------------------------------------------------
|
||||
|
||||
|
||||
def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
|
||||
if not vault_repo or tree is None:
|
||||
result.warnings.append("obsidian: the vault repository could not be listed")
|
||||
result.sources["obsidian"] = _source_row("obsidian", enabled=True, documents=[], extra={"missing": True})
|
||||
return
|
||||
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
|
||||
branch = repo.get("default_branch") or "main"
|
||||
updated = _timestamp(repo.get("updated_at"))
|
||||
created = _timestamp(repo.get("created_at"))
|
||||
validator = loader.load_validator(cfg.root)
|
||||
documents = []
|
||||
|
||||
blobs = [entry for entry in tree if entry.get("type") == "blob"]
|
||||
for entry in blobs:
|
||||
if len(documents) >= guide.limit("max_documents"):
|
||||
result.warnings.append(
|
||||
f"obsidian: document limit {guide.limit('max_documents')} reached"
|
||||
)
|
||||
break
|
||||
path = globs.normalise(entry.get("path") or "")
|
||||
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
|
||||
if kind == "reject":
|
||||
# A path outside the include list is expected and is not a finding;
|
||||
# only hard exclusions are reported, so the report stays readable.
|
||||
if reason != "not under an included path":
|
||||
result.add_rejection(exclusions.Rejection(path, "obsidian", reason))
|
||||
continue
|
||||
if kind == "image":
|
||||
result.images.append(
|
||||
{"path": path, "platform": "obsidian",
|
||||
"origin_url": file_url(base_url, vault_repo, branch, path)}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
raw = client.raw(vault_repo, branch, path)
|
||||
except GiteaError as exc:
|
||||
result.warnings.append(f"obsidian/{path}: {exc}")
|
||||
continue
|
||||
if exclusions.looks_binary(raw):
|
||||
result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content"))
|
||||
continue
|
||||
text = raw.decode("utf-8", "replace")
|
||||
hits = exclusions.scan_text(text)
|
||||
if hits:
|
||||
result.add_rejection(
|
||||
exclusions.Rejection(
|
||||
path, "obsidian", "credential-shaped content",
|
||||
", ".join(sorted({hit[0] for hit in hits})),
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
fields, body = _split_note(text, validator)
|
||||
title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0])
|
||||
tag_values = fields.get("tags") or []
|
||||
if isinstance(tag_values, str):
|
||||
tag_values = [tag_values]
|
||||
tags = [str(tag) for tag in tag_values] + _folder_tags(path)
|
||||
area = areas.area_for_note(path, guide.fits)
|
||||
documents.append(
|
||||
_make_document(
|
||||
key=f"obsidian:{path}",
|
||||
platform="obsidian",
|
||||
area=area,
|
||||
title=title,
|
||||
claim=_claim(body, title),
|
||||
body=body,
|
||||
digest=hashlib.sha256(raw).hexdigest(),
|
||||
origin_url=file_url(base_url, vault_repo, branch, path),
|
||||
path=path,
|
||||
repo_path=path,
|
||||
source_id=path,
|
||||
source_type="doc",
|
||||
last_modified=updated,
|
||||
confidence="historic",
|
||||
period=_period(created, updated),
|
||||
tags=tags + ["obsidian", area],
|
||||
tech=[],
|
||||
public=guide.public,
|
||||
kind="markdown",
|
||||
)
|
||||
)
|
||||
|
||||
result.documents.extend(documents)
|
||||
result.sources["obsidian"] = _source_row(
|
||||
"obsidian", enabled=True, documents=documents,
|
||||
extra={"repo": vault_repo, "base": base_url},
|
||||
)
|
||||
log(f"obsidian: {len(documents)} note(s)")
|
||||
|
||||
|
||||
# -- graphify -------------------------------------------------------------
|
||||
|
||||
|
||||
def _harvest_graph(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
|
||||
if not vault_repo:
|
||||
result.warnings.append("graphify: no vault repository configured in the guide")
|
||||
result.sources["graphify"] = _source_row("graphify", enabled=True, documents=[], extra={"missing": True})
|
||||
return
|
||||
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
|
||||
branch = repo.get("default_branch") or "main"
|
||||
updated = _timestamp(repo.get("updated_at"))
|
||||
vault_paths = {
|
||||
globs.normalise(entry.get("path") or "")
|
||||
for entry in (tree or [])
|
||||
if entry.get("type") == "blob"
|
||||
}
|
||||
|
||||
raw = None
|
||||
source_path = ""
|
||||
if cfg.graph_path:
|
||||
if not os.path.exists(cfg.graph_path):
|
||||
raise EstateError(f"RESUME_GRAPH_PATH is set but {cfg.graph_path} does not exist")
|
||||
with open(cfg.graph_path, "rb") as handle:
|
||||
raw = handle.read()
|
||||
source_path = cfg.graph_path
|
||||
else:
|
||||
for candidate in guide.limits.get("graph_paths") or ["graph.json"]:
|
||||
try:
|
||||
raw = client.raw(vault_repo, branch, candidate)
|
||||
source_path = f"gitea:{vault_repo}/{candidate}"
|
||||
break
|
||||
except GiteaNotFound:
|
||||
continue
|
||||
except GiteaError as exc:
|
||||
raise EstateError(f"cannot read graph.json from {vault_repo}: {exc}") from exc
|
||||
|
||||
if raw is None:
|
||||
result.warnings.append(
|
||||
f"graphify: graph.json is not committed to {vault_repo} "
|
||||
f"(looked in {guide.limits.get('graph_paths')}). The graph was not ingested; "
|
||||
"the nightly audit will keep reporting this until it is committed."
|
||||
)
|
||||
result.sources["graphify"] = _source_row(
|
||||
"graphify", enabled=True, documents=[], extra={"missing": True, "repo": vault_repo}
|
||||
)
|
||||
return
|
||||
|
||||
graph = graph_module.parse(
|
||||
raw,
|
||||
platform="graphify",
|
||||
base=base_url,
|
||||
repo=vault_repo,
|
||||
ref=branch,
|
||||
last_modified=updated,
|
||||
area=guide.primary_area,
|
||||
vault_paths=vault_paths,
|
||||
max_nodes=guide.limit("max_nodes"),
|
||||
max_edges=guide.limit("max_edges"),
|
||||
source_path=source_path,
|
||||
)
|
||||
result.graph = graph
|
||||
for rejection in graph.rejections:
|
||||
result.add_rejection(rejection)
|
||||
result.images.extend(graph.images)
|
||||
result.sources["graphify"] = _source_row(
|
||||
"graphify", enabled=True, documents=[],
|
||||
extra={
|
||||
"repo": vault_repo,
|
||||
"path": source_path,
|
||||
"nodes": len(graph.nodes),
|
||||
"edges": len(graph.edges),
|
||||
"digest": graph.digest,
|
||||
"built_at_commit": graph.built_at_commit,
|
||||
},
|
||||
)
|
||||
if graph.warnings:
|
||||
result.warnings.extend(f"graphify: {warning}" for warning in graph.warnings)
|
||||
log(
|
||||
f"graphify: {len(graph.nodes)} node(s), {len(graph.edges)} edge(s), "
|
||||
f"{len(graph.images)} image node(s)"
|
||||
)
|
||||
|
||||
|
||||
# -- shared helpers -------------------------------------------------------
|
||||
|
||||
|
||||
def _make_document(**values) -> loader.Document:
|
||||
"""One Document constructor, so every source produces the same shape."""
|
||||
defaults = dict(
|
||||
metrics=[], artefacts=[], tech=[], tags=[], public=True, kind="markdown",
|
||||
platform="gitea", tier=2, level=3, period="", source_type="doc",
|
||||
)
|
||||
defaults.update(values)
|
||||
title = defaults["title"]
|
||||
origin = defaults["origin_url"]
|
||||
path = defaults.get("path") or ""
|
||||
return loader.Document(
|
||||
path=defaults["key"],
|
||||
source_id=defaults.get("source_id") or (path or defaults["key"]),
|
||||
repo_path=defaults.get("repo_path") or path,
|
||||
area=defaults["area"],
|
||||
level=defaults.get("level", 3),
|
||||
title=title,
|
||||
claim=defaults.get("claim") or title,
|
||||
confidence=defaults.get("confidence") or "historic",
|
||||
period=defaults.get("period") or "—",
|
||||
tech=defaults.get("tech") or [],
|
||||
metrics=defaults.get("metrics") or [],
|
||||
artefacts=defaults.get("artefacts") or [],
|
||||
public=bool(defaults.get("public", True)),
|
||||
last_verified=str(defaults["last_modified"])[:10],
|
||||
tags=defaults.get("tags") or [],
|
||||
body=defaults["body"],
|
||||
digest=defaults["digest"],
|
||||
url=origin,
|
||||
platform=defaults["platform"],
|
||||
tier=int(defaults.get("tier", 2)),
|
||||
origin_url=origin,
|
||||
last_modified=defaults["last_modified"],
|
||||
source_type=defaults.get("source_type") or "doc",
|
||||
kind=defaults.get("kind") or "markdown",
|
||||
)
|
||||
|
||||
|
||||
def _split_note(text: str, validator) -> tuple:
|
||||
"""Tolerant front-matter split. A note without front-matter is still a note."""
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}, text
|
||||
end = None
|
||||
for index in range(1, len(lines)):
|
||||
if lines[index].strip() in ("---", "..."):
|
||||
end = index
|
||||
break
|
||||
if end is None:
|
||||
return {}, text
|
||||
fields = {}
|
||||
try:
|
||||
node = validator.parse_front_matter(
|
||||
[(number, line) for number, line in enumerate(lines[1:end], start=2)]
|
||||
)
|
||||
parsed = loader.to_python(node)
|
||||
if isinstance(parsed, dict):
|
||||
fields = parsed
|
||||
except validator.YamlError:
|
||||
fields = {}
|
||||
return fields, "\n".join(lines[end + 1 :]).strip()
|
||||
|
||||
|
||||
def _source_row(name: str, enabled: bool, documents: list, extra: dict = None) -> dict:
|
||||
stamps = sorted(doc.last_modified for doc in documents if doc.last_modified)
|
||||
row = {
|
||||
"platform": name,
|
||||
"enabled": enabled,
|
||||
"documents": len(documents),
|
||||
"newest": stamps[-1] if stamps else "",
|
||||
"oldest": stamps[0] if stamps else "",
|
||||
"fetched_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
row.update(extra or {})
|
||||
return row
|
||||
|
||||
|
||||
def _vault_repo(selected: dict) -> str:
|
||||
for name in ("obsidian", "graphify"):
|
||||
guide = selected.get(name)
|
||||
if guide and guide.base and "/" in guide.base and not guide.base.startswith("http"):
|
||||
return guide.base
|
||||
return ""
|
||||
|
||||
|
||||
def _as_list(value) -> list:
|
||||
if value in (None, ""):
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [str(item) for item in value]
|
||||
return [str(value)]
|
||||
|
||||
|
||||
def _timestamp(value) -> str:
|
||||
"""Normalise any RFC3339 timestamp to UTC `…Z`. Empty stays empty."""
|
||||
if not value:
|
||||
return "1970-01-01T00:00:00Z"
|
||||
text = str(value).strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
moment = datetime.datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return "1970-01-01T00:00:00Z"
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=datetime.timezone.utc)
|
||||
return moment.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _period(created: str, updated: str) -> str:
|
||||
start = (created or "")[:4]
|
||||
end = (updated or "")[:4]
|
||||
if not start or start == "1970":
|
||||
return end or "—"
|
||||
if start == end:
|
||||
return start
|
||||
return f"{start}–{end}"
|
||||
|
||||
|
||||
def _claim(text: str, title: str) -> str:
|
||||
for line in (text or "").splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith(("#", "|", "```", "---", ">")):
|
||||
continue
|
||||
cleaned = re.sub(r"[*_`\[\]]", "", stripped)
|
||||
if len(cleaned) > 20:
|
||||
return cleaned[:180]
|
||||
return title[:180]
|
||||
|
||||
|
||||
def _heading(text: str) -> str:
|
||||
for line in (text or "").splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _folder_tags(path: str) -> list:
|
||||
parts = [part for part in path.split("/")[:-1] if part and not part[0].isdigit()]
|
||||
return [part.lower() for part in parts[-2:]]
|
||||
|
||||
|
||||
def _is_readme(path: str) -> bool:
|
||||
return os.path.basename(path).lower().startswith("readme")
|
||||
|
||||
|
||||
def _depth(path: str) -> int:
|
||||
return globs.normalise(path).count("/")
|
||||
|
||||
|
||||
def _source_rank(path: str) -> bool:
|
||||
ext = exclusions.extension(path)
|
||||
if ext not in exclusions.TEXT_EXTENSIONS:
|
||||
return False
|
||||
if ext in ("json", "csv", "tsv", "svg"):
|
||||
return False
|
||||
# Prefer files that carry decisions over generated or vendored files.
|
||||
return not any(
|
||||
part.lower() in ("vendor", "third_party", "examples", "testdata", "fixtures", "migrations")
|
||||
for part in globs.normalise(path).split("/")[:-1]
|
||||
)
|
||||
Reference in New Issue
Block a user