sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

Browse Source
This commit is contained in:
Paperclip Builder
2026-10-26 22:26:45 +11:00
parent 5c0622c1b1
commit 1e46ec009a
19 changed files with 2952 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

804
corpus/sources/estate.py Normal file
View File

@@ -0,0 +1,804 @@
"""Harvest the estate into one set of documents.
Four sources, one shape (`loader.Document`), one tier rule:
| Platform | Tier | Reached by |
|---|---|---|
| `curated` (`content/`) | 1 | the local repository, already validated |
| `gitea` | 2 | the Gitea API |
| `obsidian` | 2 | the same Gitea API, one repository |
| `graphify` | 2 | `graph.json` committed to the vault repository |
`content/` is loaded first and is never displaced: the query layer uses `tier`
to prefer a curated claim over harvested text that disagrees with it.
Every refused file is recorded with the rule that refused it. The DoD asks for
this explicitly ("the run reports the rejection"), and it is the difference
between an exclusion list that works and one nobody can audit.
"""
from __future__ import annotations
import datetime
import hashlib
import json
import os
import re
from .. import areas, exclusions, globs, guides, loader
from . import graph as graph_module
from .gitea import GiteaClient, GiteaError, GiteaNotFound, file_url, repo_url
PLATFORM_ORDER = ("curated", "gitea", "obsidian", "graphify")
class EstateError(RuntimeError):
"""The estate could not be read. The run stops rather than indexing half of it."""
class Harvest:
def __init__(self) -> None:
self.documents: list = []
self.rejections: list = []
self.images: list = []
self.warnings: list = []
self.sources: dict = {}
self.graph = None # graph_module.GraphResult | None
self.offline = False
# -- reporting --------------------------------------------------------
def add_rejection(self, rejection) -> None:
self.rejections.append(rejection.to_dict() if hasattr(rejection, "to_dict") else rejection)
def rejection_summary(self) -> dict:
counts = {}
for row in self.rejections:
key = f"{row.get('platform')}/{row.get('reason')}"
counts[key] = counts.get(key, 0) + 1
return dict(sorted(counts.items()))
def to_dict(self) -> dict:
return {
"documents": len(self.documents),
"images": len(self.images),
"rejections": len(self.rejections),
"rejection_summary": self.rejection_summary(),
"warnings": list(self.warnings),
"sources": self.sources,
"graph": self.graph.counts() if self.graph else None,
"offline": self.offline,
}
def harvest(cfg, log=None) -> Harvest:
"""Read every enabled platform. Curated content is always read."""
log = log or (lambda _msg: None)
result = Harvest()
result.offline = bool(cfg.offline)
curated = loader.load_documents(cfg)
result.documents.extend(curated)
result.sources["curated"] = _source_row(
"curated", enabled=True, documents=curated, extra={"path": "content/"}
)
log(f"curated: {len(curated)} document(s) from content/")
if cfg.offline:
# Not a warning: offline is a deliberate mode, not a failure.
log("offline mode: only content/ was read (RESUME_OFFLINE=1)")
return result
all_guides = guides.load_guides(cfg.root)
selected = {
name: guide
for name, guide in all_guides.items()
if guide.enabled and name in cfg.platforms
}
missing = [name for name in cfg.platforms if name not in all_guides]
if missing:
result.warnings.append(f"platform(s) {missing} have no guide and were skipped")
needs_gitea = {"gitea", "obsidian", "graphify"} & set(selected)
if not needs_gitea:
return result
base = cfg.gitea_url or selected.get("gitea", all_guides.get("gitea")).base
client = GiteaClient(base, cfg.gitea_token, timeout=cfg.timeout)
try:
repos = client.repos()
except GiteaError as exc:
raise EstateError(f"cannot list Gitea repositories: {exc}") from exc
log(f"gitea: {len(repos)} repositories visible")
vault_repo = _vault_repo(selected)
vault_tree = None
if vault_repo:
try:
vault_tree = client.tree(vault_repo, "main")
except GiteaNotFound:
try:
vault_tree = client.tree(vault_repo, "master")
except GiteaError as exc:
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
except GiteaError as exc:
raise EstateError(f"cannot read {vault_repo}: {exc}") from exc
if "gitea" in selected:
_harvest_repos(result, client, selected["gitea"], repos, base, log)
if "obsidian" in selected:
_harvest_obsidian(result, client, selected["obsidian"], repos, vault_repo, vault_tree, cfg, base, log)
if "graphify" in selected:
_harvest_graph(result, client, selected["graphify"], repos, vault_repo, vault_tree, cfg, base, log)
return result
# -- gitea repositories ---------------------------------------------------
def _harvest_repos(result: Harvest, client, guide, repos, base_url, log) -> None:
excluded = set(_as_list((guide.exclude or {}).get("repos")))
include = (guide.include or {}).get("repos") or "all"
wanted = None if include == "all" or _as_list(include) == [] else set(_as_list(include))
documents = []
seen_repos = set()
for repo in repos:
full_name = repo.get("full_name") or ""
if not full_name or full_name in excluded:
continue
if wanted is not None and full_name not in wanted:
continue
seen_repos.add(full_name)
try:
documents.extend(_harvest_one_repo(result, client, guide, repo, base_url))
except GiteaError as exc:
result.warnings.append(f"gitea/{full_name}: {exc}")
continue
if len(documents) >= guide.limit("max_documents"):
result.warnings.append(
f"gitea: document limit {guide.limit('max_documents')} reached; "
"remaining repositories were skipped"
)
break
result.documents.extend(documents)
result.sources["gitea"] = _source_row(
"gitea", enabled=True, documents=documents,
extra={"repos": len(seen_repos), "base": base_url},
)
log(f"gitea: {len(documents)} document(s), {len(result.rejections)} rejection(s) so far")
def _harvest_one_repo(result: Harvest, client, guide, repo, base_url) -> list:
full_name = repo["full_name"]
branch = repo.get("default_branch") or "main"
updated = _timestamp(repo.get("updated_at"))
created = _timestamp(repo.get("created_at"))
area = areas.area_for_repo(repo, guide.fits)
topics = [str(topic) for topic in (repo.get("topics") or [])]
description = str(repo.get("description") or "").strip()
confidence = "abandoned" if repo.get("archived") else "historic"
documents = []
try:
entries = client.tree(full_name, branch)
except GiteaNotFound:
entries = []
blobs = [entry for entry in entries if entry.get("type") == "blob"]
admitted = []
for entry in blobs:
path = globs.normalise(entry.get("path") or "")
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
if kind == "reject":
result.add_rejection(exclusions.Rejection(path, "gitea", reason, full_name))
continue
if kind == "image":
result.images.append(
{"path": path, "platform": "gitea", "repo": full_name,
"origin_url": file_url(base_url, full_name, branch, path)}
)
continue
admitted.append(entry)
# The file tree is paths only. It is what lets a question like "does he have
# a Prometheus config anywhere?" find a repository whose README never says so.
tree_doc = _tree_document(guide, repo, area, updated, admitted, base_url)
if tree_doc:
documents.append(tree_doc)
# README first: it is the document a reader would open.
readmes = [
entry for entry in admitted
if _is_readme(entry.get("path") or "") and globs.normalise(entry.get("path") or "").count("/") <= 1
][:3]
used = set()
for entry in readmes:
doc = _file_document(
result, client, guide, repo, branch, area, updated, created,
confidence, description, topics, entry, base_url,
)
if doc:
documents.append(doc)
used.add(globs.normalise(entry["path"]))
# Selected source files: the evidence behind the README's claims.
candidates = [
entry for entry in admitted
if globs.normalise(entry.get("path") or "") not in used
and not _is_readme(entry.get("path") or "")
and _source_rank(entry.get("path") or "")
]
candidates.sort(key=lambda entry: (_depth(entry.get("path") or ""), globs.normalise(entry["path"])))
for entry in candidates[: guide.limit("max_files_per_repo")]:
doc = _file_document(
result, client, guide, repo, branch, area, updated, created,
confidence, description, topics, entry, base_url,
)
if doc:
documents.append(doc)
# One document per repository for commits and one for languages. These are
# short, stable, and answer "what is being worked on?" without a filesystem.
commits = _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url)
if commits:
documents.append(commits)
languages = _language_document(result, client, guide, repo, area, updated, created, confidence, base_url)
if languages:
documents.append(languages)
if not any(not doc.source_id.endswith(("@tree", "@commits", "@languages")) for doc in documents):
# No README and no admitted source file: still admit a summary so the
# repository is discoverable and its silence is visible, not invisible.
summary = _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url)
if summary:
documents.insert(0, summary)
return documents
def _file_document(result, client, guide, repo, branch, area, updated, created, confidence, description, topics, entry, base_url) -> object:
path = globs.normalise(entry.get("path") or "")
full_name = repo["full_name"]
try:
raw = client.raw(full_name, branch, path)
except GiteaError as exc:
result.warnings.append(f"gitea/{full_name}/{path}: {exc}")
return None
if exclusions.looks_binary(raw):
result.add_rejection(exclusions.Rejection(path, "gitea", "binary content", full_name))
return None
text = raw.decode("utf-8", "replace")
hits = exclusions.scan_text(text)
if hits:
kinds = sorted({hit[0] for hit in hits})
result.add_rejection(
exclusions.Rejection(path, "gitea", "credential-shaped content", full_name + ": " + ", ".join(kinds))
)
return None
kind = "code" if exclusions.extension(path) not in ("md", "markdown", "mdx", "txt", "rst") else "markdown"
title = f"{full_name} — {path}"
body = text
if description or topics:
header = [f"Repository: {full_name}"]
if description:
header.append(description)
if topics:
header.append("Topics: " + ", ".join(topics))
body = "\n\n".join(["\n".join(header), text])
return _make_document(
key=f"gitea:{full_name}/{path}",
platform="gitea",
area=area,
title=title,
claim=_claim(text, title),
body=body,
digest=hashlib.sha256(raw).hexdigest(),
origin_url=file_url(base_url, full_name, branch, path),
path=path,
repo_path=path,
source_id=f"{full_name}/{path}",
source_type="code" if kind == "code" else "doc",
last_modified=updated,
confidence=confidence,
period=_period(created, updated),
tags=topics + ["gitea", area],
tech=[],
public=guide.public,
kind=kind,
)
def _tree_document(guide, repo, area, updated, blobs, base_url) -> object:
if not blobs:
return None
full_name = repo["full_name"]
branch = repo.get("default_branch") or "main"
limit = guide.limit("max_tree_entries")
paths = [globs.normalise(entry.get("path") or "") for entry in blobs][:limit]
body = "\n".join(
[f"Repository: {full_name}", f"Branch: {branch}", f"Files: {len(blobs)}", "", "File tree:"]
+ [f"- {path}" for path in paths]
)
if len(blobs) > limit:
body += f"\n- … {len(blobs) - limit} more"
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
return _make_document(
key=f"gitea:{full_name}@tree",
platform="gitea",
area=area,
title=f"{full_name} — file tree",
claim=f"File tree of {full_name}, {len(blobs)} files.",
body=body,
digest=digest,
origin_url=repo_url(base_url, full_name),
path="",
repo_path="",
source_id=f"{full_name}@tree",
source_type="doc",
last_modified=updated,
confidence="abandoned" if repo.get("archived") else "historic",
period="",
tags=["gitea", "tree", area],
tech=[],
public=guide.public,
kind="markdown",
)
def _commit_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
full_name = repo["full_name"]
branch = repo.get("default_branch") or "main"
try:
commits = client.commits(full_name, branch, limit=guide.limit("max_commits"))
except GiteaError:
return None
if not commits:
return None
lines = [f"Repository: {full_name}", f"Recent commits on {branch}:", ""]
newest = updated
for item in commits:
commit = item.get("commit") or {}
message = (commit.get("message") or "").strip().splitlines()
headline = message[0] if message else "(no message)"
date = _timestamp(((commit.get("author") or {}).get("date")) or ((commit.get("committer") or {}).get("date")))
if date:
newest = max(newest, date)
lines.append(f"- {date[:10]} {headline}")
body = "\n".join(lines)
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
return _make_document(
key=f"gitea:{full_name}@commits",
platform="gitea",
area=area,
title=f"{full_name} — recent commits",
claim=f"Recent commit history for {full_name}.",
body=body,
digest=digest,
origin_url=f"{repo_url(base_url, full_name)}/commits/branch/{branch}",
path="",
repo_path="",
source_id=f"{full_name}@commits",
source_type="commit",
last_modified=newest,
confidence=confidence,
period=_period(created, newest),
tags=["gitea", "commits", area],
tech=[],
public=guide.public,
kind="markdown",
)
def _language_document(result, client, guide, repo, area, updated, created, confidence, base_url) -> object:
full_name = repo["full_name"]
try:
languages = client.languages(full_name)
except GiteaError:
return None
if not languages:
return None
total = sum(int(value) for value in languages.values()) or 1
rows = sorted(languages.items(), key=lambda item: int(item[1]), reverse=True)
lines = [f"Repository: {full_name}", "", "Language statistics:"]
for name, size in rows:
lines.append(f"- {name}: {int(size)} bytes ({100.0 * int(size) / total:.1f}%)")
body = "\n".join(lines)
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
return _make_document(
key=f"gitea:{full_name}@languages",
platform="gitea",
area=area,
title=f"{full_name} — languages",
claim=f"Language breakdown for {full_name}: " + ", ".join(name for name, _ in rows[:4]) + ".",
body=body,
digest=digest,
origin_url=repo_url(base_url, full_name),
path="",
repo_path="",
source_id=f"{full_name}@languages",
source_type="metric",
last_modified=updated,
confidence=confidence,
period=_period(created, updated),
tags=["gitea", "languages", area],
tech=[name for name, _ in rows[:6]],
public=guide.public,
kind="markdown",
)
def _summary_document(guide, repo, area, updated, created, confidence, description, topics, base_url) -> object:
full_name = repo["full_name"]
body = "\n".join(
[f"Repository: {full_name}", f"Description: {description or '(none)'}",
f"Topics: {', '.join(topics) if topics else '(none)'}"]
)
digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
return _make_document(
key=f"gitea:{full_name}@summary",
platform="gitea",
area=area,
title=full_name,
claim=description or f"Repository {full_name}.",
body=body,
digest=digest,
origin_url=repo_url(base_url, full_name),
path="",
repo_path="",
source_id=f"{full_name}@summary",
source_type="doc",
last_modified=updated,
confidence=confidence,
period=_period(created, updated),
tags=topics + ["gitea", area],
tech=[],
public=guide.public,
kind="markdown",
)
# -- obsidian vault -------------------------------------------------------
def _harvest_obsidian(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
if not vault_repo or tree is None:
result.warnings.append("obsidian: the vault repository could not be listed")
result.sources["obsidian"] = _source_row("obsidian", enabled=True, documents=[], extra={"missing": True})
return
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
branch = repo.get("default_branch") or "main"
updated = _timestamp(repo.get("updated_at"))
created = _timestamp(repo.get("created_at"))
validator = loader.load_validator(cfg.root)
documents = []
blobs = [entry for entry in tree if entry.get("type") == "blob"]
for entry in blobs:
if len(documents) >= guide.limit("max_documents"):
result.warnings.append(
f"obsidian: document limit {guide.limit('max_documents')} reached"
)
break
path = globs.normalise(entry.get("path") or "")
kind, reason = exclusions.classify(path, guide, size=entry.get("size"))
if kind == "reject":
# A path outside the include list is expected and is not a finding;
# only hard exclusions are reported, so the report stays readable.
if reason != "not under an included path":
result.add_rejection(exclusions.Rejection(path, "obsidian", reason))
continue
if kind == "image":
result.images.append(
{"path": path, "platform": "obsidian",
"origin_url": file_url(base_url, vault_repo, branch, path)}
)
continue
try:
raw = client.raw(vault_repo, branch, path)
except GiteaError as exc:
result.warnings.append(f"obsidian/{path}: {exc}")
continue
if exclusions.looks_binary(raw):
result.add_rejection(exclusions.Rejection(path, "obsidian", "binary content"))
continue
text = raw.decode("utf-8", "replace")
hits = exclusions.scan_text(text)
if hits:
result.add_rejection(
exclusions.Rejection(
path, "obsidian", "credential-shaped content",
", ".join(sorted({hit[0] for hit in hits})),
)
)
continue
fields, body = _split_note(text, validator)
title = str(fields.get("title") or _heading(body) or os.path.splitext(os.path.basename(path))[0])
tag_values = fields.get("tags") or []
if isinstance(tag_values, str):
tag_values = [tag_values]
tags = [str(tag) for tag in tag_values] + _folder_tags(path)
area = areas.area_for_note(path, guide.fits)
documents.append(
_make_document(
key=f"obsidian:{path}",
platform="obsidian",
area=area,
title=title,
claim=_claim(body, title),
body=body,
digest=hashlib.sha256(raw).hexdigest(),
origin_url=file_url(base_url, vault_repo, branch, path),
path=path,
repo_path=path,
source_id=path,
source_type="doc",
last_modified=updated,
confidence="historic",
period=_period(created, updated),
tags=tags + ["obsidian", area],
tech=[],
public=guide.public,
kind="markdown",
)
)
result.documents.extend(documents)
result.sources["obsidian"] = _source_row(
"obsidian", enabled=True, documents=documents,
extra={"repo": vault_repo, "base": base_url},
)
log(f"obsidian: {len(documents)} note(s)")
# -- graphify -------------------------------------------------------------
def _harvest_graph(result, client, guide, repos, vault_repo, tree, cfg, base_url, log) -> None:
if not vault_repo:
result.warnings.append("graphify: no vault repository configured in the guide")
result.sources["graphify"] = _source_row("graphify", enabled=True, documents=[], extra={"missing": True})
return
repo = next((row for row in repos if row.get("full_name") == vault_repo), {})
branch = repo.get("default_branch") or "main"
updated = _timestamp(repo.get("updated_at"))
vault_paths = {
globs.normalise(entry.get("path") or "")
for entry in (tree or [])
if entry.get("type") == "blob"
}
raw = None
source_path = ""
if cfg.graph_path:
if not os.path.exists(cfg.graph_path):
raise EstateError(f"RESUME_GRAPH_PATH is set but {cfg.graph_path} does not exist")
with open(cfg.graph_path, "rb") as handle:
raw = handle.read()
source_path = cfg.graph_path
else:
for candidate in guide.limits.get("graph_paths") or ["graph.json"]:
try:
raw = client.raw(vault_repo, branch, candidate)
source_path = f"gitea:{vault_repo}/{candidate}"
break
except GiteaNotFound:
continue
except GiteaError as exc:
raise EstateError(f"cannot read graph.json from {vault_repo}: {exc}") from exc
if raw is None:
result.warnings.append(
f"graphify: graph.json is not committed to {vault_repo} "
f"(looked in {guide.limits.get('graph_paths')}). The graph was not ingested; "
"the nightly audit will keep reporting this until it is committed."
)
result.sources["graphify"] = _source_row(
"graphify", enabled=True, documents=[], extra={"missing": True, "repo": vault_repo}
)
return
graph = graph_module.parse(
raw,
platform="graphify",
base=base_url,
repo=vault_repo,
ref=branch,
last_modified=updated,
area=guide.primary_area,
vault_paths=vault_paths,
max_nodes=guide.limit("max_nodes"),
max_edges=guide.limit("max_edges"),
source_path=source_path,
)
result.graph = graph
for rejection in graph.rejections:
result.add_rejection(rejection)
result.images.extend(graph.images)
result.sources["graphify"] = _source_row(
"graphify", enabled=True, documents=[],
extra={
"repo": vault_repo,
"path": source_path,
"nodes": len(graph.nodes),
"edges": len(graph.edges),
"digest": graph.digest,
"built_at_commit": graph.built_at_commit,
},
)
if graph.warnings:
result.warnings.extend(f"graphify: {warning}" for warning in graph.warnings)
log(
f"graphify: {len(graph.nodes)} node(s), {len(graph.edges)} edge(s), "
f"{len(graph.images)} image node(s)"
)
# -- shared helpers -------------------------------------------------------
def _make_document(**values) -> loader.Document:
"""One Document constructor, so every source produces the same shape."""
defaults = dict(
metrics=[], artefacts=[], tech=[], tags=[], public=True, kind="markdown",
platform="gitea", tier=2, level=3, period="", source_type="doc",
)
defaults.update(values)
title = defaults["title"]
origin = defaults["origin_url"]
path = defaults.get("path") or ""
return loader.Document(
path=defaults["key"],
source_id=defaults.get("source_id") or (path or defaults["key"]),
repo_path=defaults.get("repo_path") or path,
area=defaults["area"],
level=defaults.get("level", 3),
title=title,
claim=defaults.get("claim") or title,
confidence=defaults.get("confidence") or "historic",
period=defaults.get("period") or "—",
tech=defaults.get("tech") or [],
metrics=defaults.get("metrics") or [],
artefacts=defaults.get("artefacts") or [],
public=bool(defaults.get("public", True)),
last_verified=str(defaults["last_modified"])[:10],
tags=defaults.get("tags") or [],
body=defaults["body"],
digest=defaults["digest"],
url=origin,
platform=defaults["platform"],
tier=int(defaults.get("tier", 2)),
origin_url=origin,
last_modified=defaults["last_modified"],
source_type=defaults.get("source_type") or "doc",
kind=defaults.get("kind") or "markdown",
)
def _split_note(text: str, validator) -> tuple:
"""Tolerant front-matter split. A note without front-matter is still a note."""
lines = text.splitlines()
if not lines or lines[0].strip() != "---":
return {}, text
end = None
for index in range(1, len(lines)):
if lines[index].strip() in ("---", "..."):
end = index
break
if end is None:
return {}, text
fields = {}
try:
node = validator.parse_front_matter(
[(number, line) for number, line in enumerate(lines[1:end], start=2)]
)
parsed = loader.to_python(node)
if isinstance(parsed, dict):
fields = parsed
except validator.YamlError:
fields = {}
return fields, "\n".join(lines[end + 1 :]).strip()
def _source_row(name: str, enabled: bool, documents: list, extra: dict = None) -> dict:
stamps = sorted(doc.last_modified for doc in documents if doc.last_modified)
row = {
"platform": name,
"enabled": enabled,
"documents": len(documents),
"newest": stamps[-1] if stamps else "",
"oldest": stamps[0] if stamps else "",
"fetched_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
}
row.update(extra or {})
return row
def _vault_repo(selected: dict) -> str:
for name in ("obsidian", "graphify"):
guide = selected.get(name)
if guide and guide.base and "/" in guide.base and not guide.base.startswith("http"):
return guide.base
return ""
def _as_list(value) -> list:
if value in (None, ""):
return []
if isinstance(value, list):
return [str(item) for item in value]
return [str(value)]
def _timestamp(value) -> str:
"""Normalise any RFC3339 timestamp to UTC `…Z`. Empty stays empty."""
if not value:
return "1970-01-01T00:00:00Z"
text = str(value).strip()
if text.endswith("Z"):
text = text[:-1] + "+00:00"
try:
moment = datetime.datetime.fromisoformat(text)
except ValueError:
return "1970-01-01T00:00:00Z"
if moment.tzinfo is None:
moment = moment.replace(tzinfo=datetime.timezone.utc)
return moment.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _period(created: str, updated: str) -> str:
start = (created or "")[:4]
end = (updated or "")[:4]
if not start or start == "1970":
return end or "—"
if start == end:
return start
return f"{start}–{end}"
def _claim(text: str, title: str) -> str:
for line in (text or "").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith(("#", "|", "```", "---", ">")):
continue
cleaned = re.sub(r"[*_`\[\]]", "", stripped)
if len(cleaned) > 20:
return cleaned[:180]
return title[:180]
def _heading(text: str) -> str:
for line in (text or "").splitlines():
if line.startswith("# "):
return line[2:].strip()
return ""
def _folder_tags(path: str) -> list:
parts = [part for part in path.split("/")[:-1] if part and not part[0].isdigit()]
return [part.lower() for part in parts[-2:]]
def _is_readme(path: str) -> bool:
return os.path.basename(path).lower().startswith("readme")
def _depth(path: str) -> int:
return globs.normalise(path).count("/")
def _source_rank(path: str) -> bool:
ext = exclusions.extension(path)
if ext not in exclusions.TEXT_EXTENSIONS:
return False
if ext in ("json", "csv", "tsv", "svg"):
return False
# Prefer files that carry decisions over generated or vendored files.
return not any(
part.lower() in ("vendor", "third_party", "examples", "testdata", "fixtures", "migrations")
for part in globs.normalise(path).split("/")[:-1]
)

165
corpus/sources/gitea.py Normal file
View File

@@ -0,0 +1,165 @@
"""Gitea API client — the only network dependency in the pipeline.
`docs/phase-4-corpus.md` §1: Gitea is reached over HTTPS, and the vault and the
graph arrive through the same API, so there is no cross-machine copy and no SSH
from the runtime host. This client is standard library only (`urllib`), it never
prints or stores the token, and every call names only the path it requested.
Read-only by construction: there is no method here that writes, creates, deletes
or changes anything in Gitea. The token only needs read scope.
"""
from __future__ import annotations
import http.client
import json
import time
import urllib.error
import urllib.parse
import urllib.request
class GiteaError(RuntimeError):
"""A Gitea call failed. The message never contains the token."""
class GiteaNotFound(GiteaError):
"""The requested repository, branch or file does not exist."""
class GiteaClient:
def __init__(self, base: str, token: str, timeout: float = 30.0, attempts: int = 3) -> None:
if not base:
raise GiteaError("a Gitea base URL is required (GITEA_URL or the guide's base)")
if not token:
raise GiteaError(
"a Gitea token is required. It is read from the environment variable "
"named by the platform guide (GITEA_TOKEN) and never from this repository."
)
self.base = base.rstrip("/")
self._token = token
self.timeout = timeout
self.attempts = max(1, attempts)
self.calls = 0
# -- transport --------------------------------------------------------
def _request(self, path: str, expect_bytes: bool = False):
"""One GET with retry/backoff. The only place that touches the network."""
url = f"{self.base}/api/v1{path}"
last_error = None
for attempt in range(self.attempts):
request = urllib.request.Request(
url,
method="GET",
headers={
"Authorization": f"token {self._token}",
"Accept": "application/vnd.gitea+json, application/json",
"User-Agent": "ai-resume-corpus/1",
},
)
try:
self.calls += 1
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read()
if expect_bytes:
return raw
return json.loads(raw.decode("utf-8")) if raw.strip() else {}
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", "replace")[:200]
exc.close()
if exc.code == 404:
raise GiteaNotFound(f"GET {path}: not found") from None
if exc.code == 401:
raise GiteaError(
f"GET {path}: authentication failed. Check the token named by the "
"platform guide; it is not printed here."
) from None
if exc.code == 403:
raise GiteaError(f"GET {path}: forbidden (the token may lack read scope)") from None
if exc.code != 429 and exc.code < 500:
raise GiteaError(f"GET {path}: HTTP {exc.code} {detail}") from None
last_error = f"HTTP {exc.code}"
except (urllib.error.URLError, TimeoutError, http.client.HTTPException, ConnectionError) as exc:
last_error = f"cannot reach {self.base}: {getattr(exc, 'reason', exc)}"
if attempt + 1 < self.attempts:
time.sleep(0.5 * (2 ** attempt))
raise GiteaError(f"GET {path} failed after {self.attempts} attempts: {last_error}")
# -- repositories -----------------------------------------------------
def repos(self, limit: int = 50) -> list:
"""Every repository the token can read, sorted by full name."""
out = []
page = 1
while True:
response = self._request(f"/repos/search?limit={limit}&page={page}")
rows = response.get("data") or []
out.extend(rows)
if len(rows) < limit:
break
page += 1
if page > 200: # a guard, not a limit anyone should reach
break
return sorted(out, key=lambda repo: repo.get("full_name", ""))
def repo(self, full_name: str) -> dict:
return self._request(f"/repos/{_repo_path(full_name)}")
# -- contents ---------------------------------------------------------
def tree(self, full_name: str, ref: str) -> list:
"""Recursive blob/tree listing. Returns every entry, following pages."""
entries = []
page = 1
while True:
response = self._request(
f"/repos/{_repo_path(full_name)}/git/trees/{urllib.parse.quote(ref, safe='')}"
f"?recursive=true&per_page=1000&page={page}"
)
rows = response.get("tree") or []
entries.extend(rows)
if not rows or not response.get("truncated") or len(rows) < 1000:
break
page += 1
if page > 50:
break
return entries
def raw(self, full_name: str, ref: str, path: str) -> bytes:
quoted = urllib.parse.quote(path, safe="/")
return self._request(
f"/repos/{_repo_path(full_name)}/raw/{quoted}?ref={urllib.parse.quote(ref, safe='')}",
expect_bytes=True,
)
def commits(self, full_name: str, ref: str, limit: int = 20) -> list:
return self._request(
f"/repos/{_repo_path(full_name)}/commits?sha={urllib.parse.quote(ref, safe='')}"
f"&limit={int(limit)}&page=1"
)
def languages(self, full_name: str) -> dict:
return self._request(f"/repos/{_repo_path(full_name)}/languages")
# -- presentation -----------------------------------------------------
def describe(self) -> dict:
return {"base": self.base, "token": "set", "calls": self.calls}
def file_url(base: str, full_name: str, ref: str, path: str) -> str:
"""The URL a reader can open: the file at its branch, in Gitea."""
return (
f"{base.rstrip('/')}/{full_name}/src/branch/{urllib.parse.quote(ref, safe='')}/"
f"{urllib.parse.quote(path, safe='/')}"
)
def repo_url(base: str, full_name: str) -> str:
return f"{base.rstrip('/')}/{full_name}"
def _repo_path(full_name: str) -> str:
owner, _, name = full_name.partition("/")
return f"{urllib.parse.quote(owner, safe='')}/{urllib.parse.quote(name, safe='')}"

335
corpus/sources/graph.py Normal file
View File

@@ -0,0 +1,335 @@
"""Graphify output: nodes and edges, each joined back to the note it came from.
`docs/phase-4-corpus.md` §4: *"Graph is not chunked. Nodes and edges go into
their own collection, joined to note chunks by `path`, so a graph answer can name
the note it came from."*
The graph is not rebuilt here. It already exists and is rebuilt on the build host
by its own skill; this module consumes `graph.json`.
Two rules from §3 drive the whole file:
* **A node with no resolvable source note is refused, not defaulted.** Graphify
emits nodes for files outside the vault (`settings.json`, `.pi/…`). Those are
not traceable to a note, so they are rejected and counted.
* **Media never becomes text.** A media node becomes an image reference: path,
origin URL and note, and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from .. import areas, exclusions, globs
from ..sources.gitea import file_url
class GraphError(RuntimeError):
"""graph.json is not a graph this pipeline can admit."""
class GraphResult:
def __init__(self, platform: str = "graphify"):
self.platform = platform
self.nodes: list = []
self.edges: list = []
self.images: list = []
self.rejections: list = []
self.warnings: list = []
self.digest = ""
self.source_path = ""
self.built_at_commit = ""
def counts(self) -> dict:
return {
"nodes": len(self.nodes),
"edges": len(self.edges),
"images": len(self.images),
"rejected": len(self.rejections),
}
def parse(
raw: bytes,
*,
platform: str,
base: str,
repo: str,
ref: str,
last_modified: str,
area: str,
vault_paths: set,
max_nodes: int = 20000,
max_edges: int = 40000,
source_path: str = "",
) -> GraphResult:
"""Turn graph.json bytes into rows that carry their note. Never guesses."""
result = GraphResult(platform)
result.digest = hashlib.sha256(raw).hexdigest()
result.source_path = source_path
try:
data = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise GraphError(f"graph.json is not valid UTF-8 JSON: {exc}") from exc
if not isinstance(data, dict):
raise GraphError("graph.json must be a JSON object")
result.built_at_commit = str(data.get("built_at_commit") or "")
raw_nodes = data.get("nodes") or []
raw_links = data.get("links") or data.get("edges") or []
if not isinstance(raw_nodes, list) or not isinstance(raw_links, list):
raise GraphError("graph.json must contain `nodes` and `links` arrays")
labels = {}
notes = {}
for item in raw_nodes:
if not isinstance(item, dict):
continue
node_id = str(item.get("id") or "").strip()
if not node_id:
continue
labels[node_id] = str(item.get("label") or node_id)
notes[node_id] = _note_for(item, vault_paths)
for item in raw_nodes:
if len(result.nodes) >= max_nodes:
result.warnings.append(f"graph node limit {max_nodes} reached; remaining nodes skipped")
break
if not isinstance(item, dict):
continue
node_id = str(item.get("id") or "").strip()
if not node_id:
result.rejections.append(
_rejection("graph.json", platform, "graph node has no id")
)
continue
source_file = globs.normalise(item.get("source_file") or "")
note = notes.get(node_id)
if not note:
result.rejections.append(
_rejection(
source_file or f"node:{node_id}", platform,
"no resolvable source note", "graph node does not point at a file in the vault",
)
)
continue
if globs.normalise(source_file) != note:
# A node can be derived from one note while naming another file; the
# note is the traceable origin either way.
pass
if exclusions.classify(note, None)[0] == "image" or exclusions.extension(note) in exclusions.MEDIA_EXTENSIONS:
result.images.append(
{
"path": note,
"label": str(item.get("label") or node_id),
"origin_url": file_url(base, repo, ref, note),
"platform": platform,
"note": note,
}
)
continue
node_area = areas.area_for(note, (area,), fallback=area)
label = str(item.get("label") or node_id)
content_hash = _hash(
{"kind": "node", "id": node_id, "label": label, "note": note,
"community": item.get("community"), "type": item.get("node_type")}
)
text = (
f"{label}. Knowledge graph node"
+ (f" in community {item.get('community')}" if item.get("community") is not None else "")
+ f", from the note {note}."
)
result.nodes.append(
{
"id": node_id,
"point_id": _point_id("node", node_id, content_hash),
"graph_id": node_id,
"label": label,
"norm_label": str(item.get("norm_label") or _norm(label)),
"node_type": str(item.get("node_type") or item.get("file_type") or "entity"),
"community": item.get("community"),
"source_note": note,
"origin_url": file_url(base, repo, ref, note),
"area": node_area,
"tier": 2,
"platform": platform,
"last_modified": last_modified,
"content_hash": content_hash,
"text": text,
}
)
admitted_nodes = {row["graph_id"]: row for row in result.nodes}
for item in raw_links:
if len(result.edges) >= max_edges:
result.warnings.append(f"graph edge limit {max_edges} reached; remaining edges skipped")
break
if not isinstance(item, dict):
continue
source = str(item.get("source") or "").strip()
target = str(item.get("target") or "").strip()
if not source or not target:
result.rejections.append(_rejection("graph.json", platform, "graph edge has no endpoints"))
continue
note = notes.get(source) or _note_for(item, vault_paths)
if not note:
result.rejections.append(
_rejection(
f"edge:{source}->{target}", platform,
"no resolvable source note", "graph edge does not point at a file in the vault",
)
)
continue
source_label = labels.get(source, source)
target_label = labels.get(target, target)
relation = str(item.get("relation") or item.get("rel") or "related_to")
area_name = areas.area_for(note, (area,), fallback=area)
content_hash = _hash(
{"kind": "edge", "source": source, "target": target,
"relation": relation, "note": note}
)
text = f"{source_label} {relation} {target_label}, from the note {note}."
result.edges.append(
{
"id": _edge_id(source, relation, target, content_hash),
"point_id": _point_id("edge", f"{source}>{relation}>{target}", content_hash),
"source_id": source,
"target_id": target,
"relation": relation,
"confidence": str(item.get("confidence") or "INFERRED"),
"confidence_score": item.get("confidence_score"),
"source_note": note,
"origin_url": file_url(base, repo, ref, note),
"area": area_name,
"tier": 2,
"platform": platform,
"last_modified": last_modified,
"content_hash": content_hash,
"text": text,
}
)
# Edges whose endpoints were not admitted are still traceable and are kept;
# an answer can name the relation and its note even when the node itself had
# no origin. `admitted_node_ids` is exposed for verify, which reports the gap.
result.admitted_node_ids = set(admitted_nodes)
result.orphan_edges = [
edge for edge in result.edges
if edge["source_id"] not in admitted_nodes or edge["target_id"] not in admitted_nodes
]
return result
def point_payload(row: dict, kind: str) -> dict:
"""One Qdrant point payload for a node or edge. Same provenance as the row."""
payload = {
"platform": row["platform"],
"tier": row["tier"],
"source_type": "graph_node" if kind == "node" else "graph_edge",
"area": row["area"],
"origin_url": row["origin_url"],
"path": row["source_note"],
"source_note": row["source_note"],
"title": row.get("label") or row.get("relation") or row["source_note"],
"confidence": row.get("confidence") or "INFERRED",
"last_modified": row["last_modified"],
"content_hash": row["content_hash"],
"text": row["text"],
"kind": kind,
"graph_id": row.get("graph_id") or row.get("id"),
}
if kind == "node":
payload["community"] = row.get("community")
payload["node_type"] = row.get("node_type")
else:
payload["relation"] = row.get("relation")
payload["source_id"] = row.get("source_id")
payload["target_id"] = row.get("target_id")
payload["confidence_score"] = row.get("confidence_score")
return payload
def query(cfg, store, text: str, area: str = None, limit: int = 10) -> dict:
"""Graph query: keyword hits from SQLite, vector hits from Qdrant when up.
Returning both is deliberate: the keyword side always works, and the vector
side proves the graph collection is live. Each row carries `source_note`.
"""
from ..qdrant import Qdrant, QdrantError
report = {"query": text, "area": area, "keyword": [], "vector": [], "ok": True}
report["keyword"] = store.graph_search(text, area=area, limit=limit)
if not report["keyword"]:
report["ok"] = False
report["reason"] = "no graph rows matched the query"
qdrant = Qdrant(cfg.qdrant_url, timeout=cfg.timeout)
if not qdrant.health() or not qdrant.collection_exists(cfg.graph_collection):
report["vector"] = []
report["vector_note"] = f"graph collection {cfg.graph_collection} is not available"
return report
from .. import embeddings
provider = embeddings.build_provider(cfg)
try:
vector = provider.embed([text])[0]
hits = qdrant.search(cfg.graph_collection, vector, limit=limit, area=area)
except (QdrantError, Exception) as exc: # noqa: BLE001 - a graph query must not crash a shell
report["vector_note"] = f"graph vector search unavailable: {exc}"
return report
report["vector"] = [
{
"score": round(hit.get("score", 0.0), 4),
"kind": (hit.get("payload") or {}).get("kind"),
"label": (hit.get("payload") or {}).get("title"),
"source_note": (hit.get("payload") or {}).get("source_note"),
"origin_url": (hit.get("payload") or {}).get("origin_url"),
}
for hit in hits
]
return report
# -- helpers -------------------------------------------------------------
def _note_for(item: dict, vault_paths: set) -> str:
source = globs.normalise(item.get("source_file") or "")
if not source:
return ""
if source in vault_paths:
return source
# Graphify sometimes prefixes `./` or uses an absolute scan root. A suffix
# match is only accepted when it is unique, so a note is never guessed.
candidates = [path for path in vault_paths if path.endswith("/" + source)]
if len(candidates) == 1:
return candidates[0]
return ""
def _rejection(path: str, platform: str, reason: str, detail: str = ""):
rejection = exclusions.Rejection(path, platform, reason, detail)
return rejection.to_dict()
def _hash(value) -> str:
body = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(body).hexdigest()
def _point_id(kind: str, key: str, content_hash: str) -> str:
digest = hashlib.sha256(f"graph\x00{kind}\x00{key}\x00{content_hash}".encode("utf-8")).hexdigest()
return str(uuid.UUID(hex=digest[:32]))
def _edge_id(source: str, relation: str, target: str, content_hash: str) -> str:
return hashlib.sha256(
f"edge\x00{source}\x00{relation}\x00{target}\x00{content_hash}".encode("utf-8")
).hexdigest()[:32]
def _norm(label: str) -> str:
return re.sub(r"[^0-9a-z]+", "_", (label or "").lower()).strip("_")