347 lines
15 KiB
Python
347 lines
15 KiB
Python
"""Ingestion from the estate: exclusions, provenance, incremental work, graph.
|
||
|
||
These tests never touch the network. `FakeGitea` implements the four read calls
|
||
the harvester uses; `FakeQdrant` implements the collection routes the pipeline
|
||
uses. The real estate is exercised separately (docs/phase-4-notes.md), so a
|
||
failure here is a logic failure, not a connectivity one.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import unittest
|
||
from unittest import mock
|
||
|
||
from corpus import exclusions, guides, loader, pipeline, store as store_module
|
||
from corpus.sources import estate, graph as graph_module
|
||
from corpus.tests import support
|
||
|
||
|
||
def gitea_guide(**overrides) -> guides.Guide:
|
||
values = dict(
|
||
name="gitea", enabled=True, what="Self-hosted Git.", fits=["ai", "platform", "edge", "products"],
|
||
read=["readme", "tree", "selected_source", "commits", "languages"],
|
||
base="https://gitea.example", auth="GITEA_TOKEN",
|
||
include={"repos": "all", "paths": []},
|
||
exclude={"repos": [], "paths": []},
|
||
limits={"max_file_kb": 200, "max_files_per_repo": 10, "max_commits": 5, "max_tree_entries": 100},
|
||
)
|
||
values.update(overrides)
|
||
return guides.Guide(**values)
|
||
|
||
|
||
class FakeGitea:
|
||
def __init__(self, repos=None, trees=None, files=None, commits=None, languages=None):
|
||
self.repos_list = repos or []
|
||
self.trees = trees or {}
|
||
self.files = files or {}
|
||
self.commit_list = commits or []
|
||
self.language_map = languages or {}
|
||
self.calls = []
|
||
|
||
def repos(self):
|
||
return list(self.repos_list)
|
||
|
||
def tree(self, full_name, ref):
|
||
self.calls.append(("tree", full_name, ref))
|
||
return list(self.trees.get(full_name, []))
|
||
|
||
def raw(self, full_name, ref, path):
|
||
self.calls.append(("raw", full_name, ref, path))
|
||
try:
|
||
return self.files[(full_name, path)]
|
||
except KeyError:
|
||
from corpus.sources.gitea import GiteaNotFound
|
||
|
||
raise GiteaNotFound(f"{full_name}/{path}") from None
|
||
|
||
def commits(self, full_name, ref, limit=20):
|
||
return list(self.commit_list)
|
||
|
||
def languages(self, full_name):
|
||
return dict(self.language_map)
|
||
|
||
|
||
def repo(full_name="sam/photo-pipeline", **overrides):
|
||
values = dict(
|
||
full_name=full_name, name=full_name.split("/")[-1], description="Photo ingestion pipeline",
|
||
topics=["photo", "python"], default_branch="main", archived=False,
|
||
created_at="2024-01-01T00:00:00Z", updated_at="2026-10-14T05:00:00Z",
|
||
html_url=f"https://gitea.example/{full_name}",
|
||
)
|
||
values.update(overrides)
|
||
return values
|
||
|
||
|
||
def blob(path, size=None):
|
||
return {"type": "blob", "path": path, "size": len(path) if size is None else size}
|
||
|
||
|
||
class ExclusionTest(unittest.TestCase):
|
||
"""The DoD probe: a file in an excluded path never appears in the pool."""
|
||
|
||
def setUp(self):
|
||
self.guide = gitea_guide()
|
||
self.result = estate.Harvest()
|
||
tree = [
|
||
blob("README.md"),
|
||
blob("node_modules/left-pad/probe.md"),
|
||
blob(".venv/lib/probe.py"),
|
||
blob(".git/config"),
|
||
blob(".trash/old.md"),
|
||
blob("docs/.silverbullet.auth.json"),
|
||
blob("src/main.py"),
|
||
blob("assets/photo.png"),
|
||
]
|
||
files = {
|
||
("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n\nDeduplicates by hash.\n",
|
||
("sam/photo-pipeline", "node_modules/left-pad/probe.md"): b"# PROBE\n",
|
||
("sam/photo-pipeline", ".venv/lib/probe.py"): b"PROBE = 1\n",
|
||
("sam/photo-pipeline", ".git/config"): b"[core]\n",
|
||
("sam/photo-pipeline", ".trash/old.md"): b"# old\n",
|
||
("sam/photo-pipeline", "docs/.silverbullet.auth.json"): b'{"secret_key": "x"}\n',
|
||
("sam/photo-pipeline", "src/main.py"): b"def dedup():\n return True\n",
|
||
}
|
||
self.client = FakeGitea(
|
||
repos=[repo()],
|
||
trees={"sam/photo-pipeline": tree},
|
||
files=files,
|
||
commits=[{"commit": {"message": "dedup by hash", "author": {"date": "2026-10-14T05:00:00Z"}}}],
|
||
languages={"Python": 900, "Shell": 100},
|
||
)
|
||
self.documents = estate._harvest_one_repo(
|
||
self.result, self.client, self.guide, repo(), "https://gitea.example"
|
||
)
|
||
|
||
def test_probe_in_excluded_paths_never_appears(self):
|
||
paths = {doc.repo_path for doc in self.documents}
|
||
for probe in (
|
||
"node_modules/left-pad/probe.md",
|
||
".venv/lib/probe.py",
|
||
".git/config",
|
||
".trash/old.md",
|
||
"docs/.silverbullet.auth.json",
|
||
):
|
||
self.assertNotIn(probe, paths)
|
||
self.assertIn("README.md", paths)
|
||
self.assertIn("src/main.py", paths)
|
||
|
||
def test_rejections_are_reported_with_a_reason(self):
|
||
reasons = {(row["path"], row["reason"]) for row in self.result.rejections}
|
||
self.assertIn(("node_modules/left-pad/probe.md", "excluded directory 'node_modules'"), reasons)
|
||
self.assertIn(("docs/.silverbullet.auth.json", "credential-shaped file name"), reasons)
|
||
self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])
|
||
|
||
def test_media_is_an_image_reference_not_text(self):
|
||
self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])
|
||
for doc in self.documents:
|
||
self.assertNotIn("assets/photo.png", doc.body)
|
||
|
||
def test_every_document_carries_provenance(self):
|
||
for doc in self.documents:
|
||
self.assertEqual(doc.platform, "gitea")
|
||
self.assertEqual(doc.tier, 2)
|
||
self.assertTrue(doc.origin_url.startswith("https://gitea.example/sam/photo-pipeline"))
|
||
self.assertTrue(doc.last_modified)
|
||
self.assertIn(doc.area, self.guide.fits)
|
||
|
||
|
||
class CredentialContentTest(unittest.TestCase):
|
||
def test_a_valid_path_with_a_private_key_is_rejected(self):
|
||
result = estate.Harvest()
|
||
guide = gitea_guide()
|
||
client = FakeGitea(
|
||
repos=[repo()],
|
||
trees={"sam/photo-pipeline": [blob("README.md"), blob("src/leak.py")]},
|
||
files={
|
||
("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n",
|
||
("sam/photo-pipeline", "src/leak.py"): (
|
||
("KEY = '" + "-----BEGIN " + "PRIVATE KEY-----'\n").encode("utf-8")
|
||
),
|
||
},
|
||
commits=[],
|
||
languages={"Python": 10},
|
||
)
|
||
estate._harvest_one_repo(result, client, guide, repo(), "https://gitea.example")
|
||
reasons = {row["reason"]: row for row in result.rejections}
|
||
self.assertIn("credential-shaped content", reasons)
|
||
self.assertEqual(reasons["credential-shaped content"]["path"], "src/leak.py")
|
||
|
||
|
||
class PayloadProvenanceTest(unittest.TestCase):
|
||
def test_missing_provenance_field_fails_the_payload(self):
|
||
root = support.temp_repo()
|
||
self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
|
||
docs = loader.load_documents(support.make_config(root))
|
||
chunk = __import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(docs[0].body)[0]
|
||
payload = __import__("corpus.payloads", fromlist=["build_payload"]).build_payload(
|
||
docs[0], chunk, support.make_config(root)
|
||
)
|
||
payload["id"] = pipeline.chunk_id(docs[0].path, 0, docs[0].digest)
|
||
for field in ("platform", "tier", "origin_url", "path", "last_modified"):
|
||
broken = dict(payload)
|
||
broken.pop(field)
|
||
with self.assertRaises(Exception):
|
||
__import__("corpus.payloads", fromlist=["validate_payload"]).validate_payload(broken)
|
||
|
||
|
||
GRAPH = {
|
||
"built_at_commit": "abc123",
|
||
"nodes": [
|
||
{"id": "photo_ingest", "label": "Photo ingest", "source_file": "300 areas/Photo.md",
|
||
"community": 7, "node_type": "entity"},
|
||
{"id": "settings", "label": "settings.json", "source_file": "settings.json",
|
||
"community": 118, "node_type": "code"},
|
||
{"id": "image_node", "label": "diagram", "source_file": "300 areas/diagram.png",
|
||
"community": 7, "node_type": "image"},
|
||
],
|
||
"links": [
|
||
{"source": "photo_ingest", "target": "settings", "relation": "configured_by",
|
||
"source_file": "300 areas/Photo.md", "confidence": "EXTRACTED", "confidence_score": 0.9},
|
||
{"source": "settings", "target": "photo_ingest", "relation": "configures",
|
||
"source_file": "settings.json", "confidence": "INFERRED"},
|
||
],
|
||
}
|
||
|
||
|
||
class GraphTest(unittest.TestCase):
|
||
def parse(self):
|
||
return graph_module.parse(
|
||
json.dumps(GRAPH).encode("utf-8"),
|
||
platform="graphify", base="https://gitea.example", repo="sam/obsidian-vault",
|
||
ref="main", last_modified="2026-10-14T05:00:00Z", area="ai",
|
||
vault_paths={"300 areas/Photo.md", "300 areas/diagram.png"},
|
||
)
|
||
|
||
def test_nodes_and_edges_carry_their_source_note(self):
|
||
result = self.parse()
|
||
self.assertEqual(len(result.nodes), 1)
|
||
self.assertEqual(result.nodes[0]["id"], "photo_ingest")
|
||
self.assertEqual(result.nodes[0]["source_note"], "300 areas/Photo.md")
|
||
self.assertIn("/sam/obsidian-vault/src/branch/main/300%20areas/Photo.md", result.nodes[0]["origin_url"])
|
||
self.assertTrue(result.edges)
|
||
for edge in result.edges:
|
||
self.assertTrue(edge["source_note"])
|
||
|
||
def test_a_node_without_a_resolvable_note_is_refused(self):
|
||
result = self.parse()
|
||
rejected = {row["path"] for row in result.rejections}
|
||
self.assertIn("settings.json", rejected)
|
||
self.assertNotIn("settings", {row["id"] for row in result.nodes})
|
||
|
||
def test_media_becomes_an_image_reference(self):
|
||
result = self.parse()
|
||
self.assertEqual([image["path"] for image in result.images], ["300 areas/diagram.png"])
|
||
self.assertNotIn("image_node", {row["id"] for row in result.nodes})
|
||
|
||
def test_graph_query_returns_the_note(self):
|
||
root = support.temp_repo()
|
||
self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
|
||
cfg = support.make_config(root)
|
||
harvest = estate.Harvest()
|
||
harvest.documents = loader.load_documents(cfg)
|
||
harvest.graph = self.parse()
|
||
harvest.sources["graphify"] = {"platform": "graphify", "enabled": True, "documents": 0}
|
||
fake = support.FakeQdrant(cfg.dim)
|
||
with mock.patch("corpus.sources.estate.harvest", return_value=harvest):
|
||
with support.fake_qdrant(cfg, fake):
|
||
pipeline.run(cfg, mode="rebuild")
|
||
with store_module.Store(cfg.db_path) as store:
|
||
rows = store.graph_search("photo")
|
||
self.assertTrue(rows)
|
||
self.assertTrue(all(row["source_note"] for row in rows))
|
||
self.assertEqual(store.graph_node("photo_ingest")["source_note"], "300 areas/Photo.md")
|
||
self.assertTrue(store.graph_neighbours("photo_ingest"))
|
||
|
||
|
||
class IncrementalEstateTest(unittest.TestCase):
|
||
def setUp(self):
|
||
self.root = support.temp_repo()
|
||
self.addCleanup(lambda: __import__("shutil").rmtree(self.root, ignore_errors=True))
|
||
self.cfg = support.make_config(self.root)
|
||
self.fake = support.FakeQdrant(self.cfg.dim)
|
||
self.harvest = estate.Harvest()
|
||
self.harvest.documents = loader.load_documents(self.cfg)
|
||
self.harvest.documents.append(self.harvested("a" * 64, "First version of the repository."))
|
||
self.harvest.sources["gitea"] = {"platform": "gitea", "enabled": True, "documents": 1}
|
||
|
||
def harvested(self, digest, body):
|
||
return loader.Document(
|
||
path="gitea:sam/photo-pipeline/README.md", source_id="sam/photo-pipeline/README.md",
|
||
area="platform", level=3, title="sam/photo-pipeline — README.md",
|
||
claim=body, confidence="historic", period="2024–2026", tech=[], metrics=[], artefacts=[],
|
||
public=True, last_verified="2026-10-14", tags=["gitea"], body=body, digest=digest,
|
||
url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
|
||
platform="gitea", tier=2, origin_url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
|
||
last_modified="2026-10-14T05:00:00Z", source_type="doc", kind="markdown",
|
||
repo_path="README.md",
|
||
)
|
||
|
||
def run_pipeline(self, mode="sync"):
|
||
with mock.patch("corpus.sources.estate.harvest", return_value=self.harvest):
|
||
with support.fake_qdrant(self.cfg, self.fake):
|
||
return pipeline.run(self.cfg, mode=mode)
|
||
|
||
def test_rebuild_and_second_run_do_not_duplicate(self):
|
||
first = self.run_pipeline("rebuild")
|
||
points = len(self.fake.collections[self.cfg.collection])
|
||
self.assertEqual(first.platforms["gitea"]["documents"], 1)
|
||
second = self.run_pipeline("sync")
|
||
self.assertEqual(len(self.fake.collections[self.cfg.collection]), points)
|
||
self.assertEqual(second.added, 0)
|
||
self.assertEqual(second.updated, 0)
|
||
self.assertEqual(second.texts_embedded, 0)
|
||
|
||
def test_only_the_changed_harvested_document_is_reembedded(self):
|
||
self.run_pipeline("rebuild")
|
||
self.harvest.documents[-1] = self.harvested("b" * 64, "Second version, one line changed.")
|
||
stats = self.run_pipeline("sync")
|
||
self.assertEqual(stats.updated, 1)
|
||
expected = len(__import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(
|
||
"Second version, one line changed."
|
||
))
|
||
self.assertEqual(stats.texts_embedded, expected)
|
||
self.assertEqual(stats.embed_calls, 1)
|
||
self.assertEqual(stats.unchanged, 3)
|
||
|
||
def test_pool_scan_finds_a_planted_secret(self):
|
||
self.harvest.documents[-1] = self.harvested(
|
||
"c" * 64,
|
||
"AWS_ACCESS_KEY_ID=" + "AKIA" + "ABCDEFGHIJKLMNOP" + "\nThis line is a planted credential.",
|
||
)
|
||
with self.assertRaises(pipeline.PipelineError) as caught:
|
||
self.run_pipeline("rebuild")
|
||
self.assertIn("credential-shaped content", str(caught.exception))
|
||
|
||
|
||
class CodeChunkerTest(unittest.TestCase):
|
||
def test_a_function_is_never_cut(self):
|
||
from corpus import chunker
|
||
|
||
body = "\n\n".join(
|
||
f"def function_{index}():\n value = {{'index': {index}}}\n return value"
|
||
for index in range(120)
|
||
)
|
||
chunks = chunker.chunk_code(body)
|
||
self.assertGreater(len(chunks), 1)
|
||
for chunk in chunks:
|
||
depth = 0
|
||
for line in chunk.text.splitlines():
|
||
depth = chunker._code_depth(line, depth)
|
||
self.assertEqual(depth, 0, f"chunk {chunk.index} ends inside a block")
|
||
self.assertIn("def function_", chunk.text)
|
||
|
||
def test_oversize_single_function_is_flagged_not_cut(self):
|
||
from corpus import chunker
|
||
|
||
body = "def huge():\n" + "\n".join(f" x{index} = {index}" for index in range(2000))
|
||
chunks = chunker.chunk_code(body)
|
||
self.assertEqual(len(chunks), 1)
|
||
self.assertTrue(chunks[0].oversize)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|