Phase 4 ingestion: exclusion/provenance/incremental/graph tests
This commit is contained in:
Paperclip Builder
346
corpus/tests/test_ingestion.py
Normal file
346
corpus/tests/test_ingestion.py
Normal file
@@ -0,0 +1,346 @@
|
||||
"""Ingestion from the estate: exclusions, provenance, incremental work, graph.
|
||||
|
||||
These tests never touch the network. `FakeGitea` implements the four read calls
|
||||
the harvester uses; `FakeQdrant` implements the collection routes the pipeline
|
||||
uses. The real estate is exercised separately (docs/phase-4-notes.md), so a
|
||||
failure here is a logic failure, not a connectivity one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from corpus import exclusions, guides, loader, pipeline, store as store_module
|
||||
from corpus.sources import estate, graph as graph_module
|
||||
from corpus.tests import support
|
||||
|
||||
|
||||
def gitea_guide(**overrides) -> guides.Guide:
|
||||
values = dict(
|
||||
name="gitea", enabled=True, what="Self-hosted Git.", fits=["ai", "platform", "edge", "products"],
|
||||
read=["readme", "tree", "selected_source", "commits", "languages"],
|
||||
base="https://gitea.example", auth="GITEA_TOKEN",
|
||||
include={"repos": "all", "paths": []},
|
||||
exclude={"repos": [], "paths": []},
|
||||
limits={"max_file_kb": 200, "max_files_per_repo": 10, "max_commits": 5, "max_tree_entries": 100},
|
||||
)
|
||||
values.update(overrides)
|
||||
return guides.Guide(**values)
|
||||
|
||||
|
||||
class FakeGitea:
|
||||
def __init__(self, repos=None, trees=None, files=None, commits=None, languages=None):
|
||||
self.repos_list = repos or []
|
||||
self.trees = trees or {}
|
||||
self.files = files or {}
|
||||
self.commit_list = commits or []
|
||||
self.language_map = languages or {}
|
||||
self.calls = []
|
||||
|
||||
def repos(self):
|
||||
return list(self.repos_list)
|
||||
|
||||
def tree(self, full_name, ref):
|
||||
self.calls.append(("tree", full_name, ref))
|
||||
return list(self.trees.get(full_name, []))
|
||||
|
||||
def raw(self, full_name, ref, path):
|
||||
self.calls.append(("raw", full_name, ref, path))
|
||||
try:
|
||||
return self.files[(full_name, path)]
|
||||
except KeyError:
|
||||
from corpus.sources.gitea import GiteaNotFound
|
||||
|
||||
raise GiteaNotFound(f"{full_name}/{path}") from None
|
||||
|
||||
def commits(self, full_name, ref, limit=20):
|
||||
return list(self.commit_list)
|
||||
|
||||
def languages(self, full_name):
|
||||
return dict(self.language_map)
|
||||
|
||||
|
||||
def repo(full_name="sam/photo-pipeline", **overrides):
|
||||
values = dict(
|
||||
full_name=full_name, name=full_name.split("/")[-1], description="Photo ingestion pipeline",
|
||||
topics=["photo", "python"], default_branch="main", archived=False,
|
||||
created_at="2024-01-01T00:00:00Z", updated_at="2026-10-14T05:00:00Z",
|
||||
html_url=f"https://gitea.example/{full_name}",
|
||||
)
|
||||
values.update(overrides)
|
||||
return values
|
||||
|
||||
|
||||
def blob(path, size=None):
|
||||
return {"type": "blob", "path": path, "size": len(path) if size is None else size}
|
||||
|
||||
|
||||
class ExclusionTest(unittest.TestCase):
|
||||
"""The DoD probe: a file in an excluded path never appears in the pool."""
|
||||
|
||||
def setUp(self):
|
||||
self.guide = gitea_guide()
|
||||
self.result = estate.Harvest()
|
||||
tree = [
|
||||
blob("README.md"),
|
||||
blob("node_modules/left-pad/probe.md"),
|
||||
blob(".venv/lib/probe.py"),
|
||||
blob(".git/config"),
|
||||
blob(".trash/old.md"),
|
||||
blob("docs/.silverbullet.auth.json"),
|
||||
blob("src/main.py"),
|
||||
blob("assets/photo.png"),
|
||||
]
|
||||
files = {
|
||||
("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n\nDeduplicates by hash.\n",
|
||||
("sam/photo-pipeline", "node_modules/left-pad/probe.md"): b"# PROBE\n",
|
||||
("sam/photo-pipeline", ".venv/lib/probe.py"): b"PROBE = 1\n",
|
||||
("sam/photo-pipeline", ".git/config"): b"[core]\n",
|
||||
("sam/photo-pipeline", ".trash/old.md"): b"# old\n",
|
||||
("sam/photo-pipeline", "docs/.silverbullet.auth.json"): b'{"secret_key": "x"}\n',
|
||||
("sam/photo-pipeline", "src/main.py"): b"def dedup():\n return True\n",
|
||||
}
|
||||
self.client = FakeGitea(
|
||||
repos=[repo()],
|
||||
trees={"sam/photo-pipeline": tree},
|
||||
files=files,
|
||||
commits=[{"commit": {"message": "dedup by hash", "author": {"date": "2026-10-14T05:00:00Z"}}}],
|
||||
languages={"Python": 900, "Shell": 100},
|
||||
)
|
||||
self.documents = estate._harvest_one_repo(
|
||||
self.result, self.client, self.guide, repo(), "https://gitea.example"
|
||||
)
|
||||
|
||||
def test_probe_in_excluded_paths_never_appears(self):
|
||||
paths = {doc.repo_path for doc in self.documents}
|
||||
for probe in (
|
||||
"node_modules/left-pad/probe.md",
|
||||
".venv/lib/probe.py",
|
||||
".git/config",
|
||||
".trash/old.md",
|
||||
"docs/.silverbullet.auth.json",
|
||||
):
|
||||
self.assertNotIn(probe, paths)
|
||||
self.assertIn("README.md", paths)
|
||||
self.assertIn("src/main.py", paths)
|
||||
|
||||
def test_rejections_are_reported_with_a_reason(self):
|
||||
reasons = {(row["path"], row["reason"]) for row in self.result.rejections}
|
||||
self.assertIn(("node_modules/left-pad/probe.md", "excluded directory 'node_modules'"), reasons)
|
||||
self.assertIn(("docs/.silverbullet.auth.json", "credential-shaped file name"), reasons)
|
||||
self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])
|
||||
|
||||
def test_media_is_an_image_reference_not_text(self):
|
||||
self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])
|
||||
for doc in self.documents:
|
||||
self.assertNotIn("assets/photo.png", doc.body)
|
||||
|
||||
def test_every_document_carries_provenance(self):
|
||||
for doc in self.documents:
|
||||
self.assertEqual(doc.platform, "gitea")
|
||||
self.assertEqual(doc.tier, 2)
|
||||
self.assertTrue(doc.origin_url.startswith("https://gitea.example/sam/photo-pipeline"))
|
||||
self.assertTrue(doc.last_modified)
|
||||
self.assertIn(doc.area, self.guide.fits)
|
||||
|
||||
|
||||
class CredentialContentTest(unittest.TestCase):
|
||||
def test_a_valid_path_with_a_private_key_is_rejected(self):
|
||||
result = estate.Harvest()
|
||||
guide = gitea_guide()
|
||||
client = FakeGitea(
|
||||
repos=[repo()],
|
||||
trees={"sam/photo-pipeline": [blob("README.md"), blob("src/leak.py")]},
|
||||
files={
|
||||
("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n",
|
||||
("sam/photo-pipeline", "src/leak.py"): (
|
||||
("KEY = '" + "-----BEGIN " + "PRIVATE KEY-----'\n").encode("utf-8")
|
||||
),
|
||||
},
|
||||
commits=[],
|
||||
languages={"Python": 10},
|
||||
)
|
||||
estate._harvest_one_repo(result, client, guide, repo(), "https://gitea.example")
|
||||
reasons = {row["reason"]: row for row in result.rejections}
|
||||
self.assertIn("credential-shaped content", reasons)
|
||||
self.assertEqual(reasons["credential-shaped content"]["path"], "src/leak.py")
|
||||
|
||||
|
||||
class PayloadProvenanceTest(unittest.TestCase):
|
||||
def test_missing_provenance_field_fails_the_payload(self):
|
||||
root = support.temp_repo()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
|
||||
docs = loader.load_documents(support.make_config(root))
|
||||
chunk = __import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(docs[0].body)[0]
|
||||
payload = __import__("corpus.payloads", fromlist=["build_payload"]).build_payload(
|
||||
docs[0], chunk, support.make_config(root)
|
||||
)
|
||||
payload["id"] = pipeline.chunk_id(docs[0].path, 0, docs[0].digest)
|
||||
for field in ("platform", "tier", "origin_url", "path", "last_modified"):
|
||||
broken = dict(payload)
|
||||
broken.pop(field)
|
||||
with self.assertRaises(Exception):
|
||||
__import__("corpus.payloads", fromlist=["validate_payload"]).validate_payload(broken)
|
||||
|
||||
|
||||
GRAPH = {
|
||||
"built_at_commit": "abc123",
|
||||
"nodes": [
|
||||
{"id": "photo_ingest", "label": "Photo ingest", "source_file": "300 areas/Photo.md",
|
||||
"community": 7, "node_type": "entity"},
|
||||
{"id": "settings", "label": "settings.json", "source_file": "settings.json",
|
||||
"community": 118, "node_type": "code"},
|
||||
{"id": "image_node", "label": "diagram", "source_file": "300 areas/diagram.png",
|
||||
"community": 7, "node_type": "image"},
|
||||
],
|
||||
"links": [
|
||||
{"source": "photo_ingest", "target": "settings", "relation": "configured_by",
|
||||
"source_file": "300 areas/Photo.md", "confidence": "EXTRACTED", "confidence_score": 0.9},
|
||||
{"source": "settings", "target": "photo_ingest", "relation": "configures",
|
||||
"source_file": "settings.json", "confidence": "INFERRED"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class GraphTest(unittest.TestCase):
|
||||
def parse(self):
|
||||
return graph_module.parse(
|
||||
json.dumps(GRAPH).encode("utf-8"),
|
||||
platform="graphify", base="https://gitea.example", repo="sam/obsidian-vault",
|
||||
ref="main", last_modified="2026-10-14T05:00:00Z", area="ai",
|
||||
vault_paths={"300 areas/Photo.md", "300 areas/diagram.png"},
|
||||
)
|
||||
|
||||
def test_nodes_and_edges_carry_their_source_note(self):
|
||||
result = self.parse()
|
||||
self.assertEqual(len(result.nodes), 1)
|
||||
self.assertEqual(result.nodes[0]["id"], "photo_ingest")
|
||||
self.assertEqual(result.nodes[0]["source_note"], "300 areas/Photo.md")
|
||||
self.assertIn("/sam/obsidian-vault/src/branch/main/300%20areas/Photo.md", result.nodes[0]["origin_url"])
|
||||
self.assertTrue(result.edges)
|
||||
for edge in result.edges:
|
||||
self.assertTrue(edge["source_note"])
|
||||
|
||||
def test_a_node_without_a_resolvable_note_is_refused(self):
|
||||
result = self.parse()
|
||||
rejected = {row["path"] for row in result.rejections}
|
||||
self.assertIn("settings.json", rejected)
|
||||
self.assertNotIn("settings", {row["id"] for row in result.nodes})
|
||||
|
||||
def test_media_becomes_an_image_reference(self):
|
||||
result = self.parse()
|
||||
self.assertEqual([image["path"] for image in result.images], ["300 areas/diagram.png"])
|
||||
self.assertNotIn("image_node", {row["id"] for row in result.nodes})
|
||||
|
||||
def test_graph_query_returns_the_note(self):
|
||||
root = support.temp_repo()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
|
||||
cfg = support.make_config(root)
|
||||
harvest = estate.Harvest()
|
||||
harvest.documents = loader.load_documents(cfg)
|
||||
harvest.graph = self.parse()
|
||||
harvest.sources["graphify"] = {"platform": "graphify", "enabled": True, "documents": 0}
|
||||
fake = support.FakeQdrant(cfg.dim)
|
||||
with mock.patch("corpus.sources.estate.harvest", return_value=harvest):
|
||||
with support.fake_qdrant(cfg, fake):
|
||||
pipeline.run(cfg, mode="rebuild")
|
||||
with store_module.Store(cfg.db_path) as store:
|
||||
rows = store.graph_search("photo")
|
||||
self.assertTrue(rows)
|
||||
self.assertTrue(all(row["source_note"] for row in rows))
|
||||
self.assertEqual(store.graph_node("photo_ingest")["source_note"], "300 areas/Photo.md")
|
||||
self.assertTrue(store.graph_neighbours("photo_ingest"))
|
||||
|
||||
|
||||
class IncrementalEstateTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = support.temp_repo()
|
||||
self.addCleanup(lambda: __import__("shutil").rmtree(self.root, ignore_errors=True))
|
||||
self.cfg = support.make_config(self.root)
|
||||
self.fake = support.FakeQdrant(self.cfg.dim)
|
||||
self.harvest = estate.Harvest()
|
||||
self.harvest.documents = loader.load_documents(self.cfg)
|
||||
self.harvest.documents.append(self.harvested("a" * 64, "First version of the repository."))
|
||||
self.harvest.sources["gitea"] = {"platform": "gitea", "enabled": True, "documents": 1}
|
||||
|
||||
def harvested(self, digest, body):
|
||||
return loader.Document(
|
||||
path="gitea:sam/photo-pipeline/README.md", source_id="sam/photo-pipeline/README.md",
|
||||
area="platform", level=3, title="sam/photo-pipeline — README.md",
|
||||
claim=body, confidence="historic", period="2024–2026", tech=[], metrics=[], artefacts=[],
|
||||
public=True, last_verified="2026-10-14", tags=["gitea"], body=body, digest=digest,
|
||||
url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
|
||||
platform="gitea", tier=2, origin_url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
|
||||
last_modified="2026-10-14T05:00:00Z", source_type="doc", kind="markdown",
|
||||
repo_path="README.md",
|
||||
)
|
||||
|
||||
def run_pipeline(self, mode="sync"):
|
||||
with mock.patch("corpus.sources.estate.harvest", return_value=self.harvest):
|
||||
with support.fake_qdrant(self.cfg, self.fake):
|
||||
return pipeline.run(self.cfg, mode=mode)
|
||||
|
||||
def test_rebuild_and_second_run_do_not_duplicate(self):
|
||||
first = self.run_pipeline("rebuild")
|
||||
points = len(self.fake.collections[self.cfg.collection])
|
||||
self.assertEqual(first.platforms["gitea"]["documents"], 1)
|
||||
second = self.run_pipeline("sync")
|
||||
self.assertEqual(len(self.fake.collections[self.cfg.collection]), points)
|
||||
self.assertEqual(second.added, 0)
|
||||
self.assertEqual(second.updated, 0)
|
||||
self.assertEqual(second.texts_embedded, 0)
|
||||
|
||||
def test_only_the_changed_harvested_document_is_reembedded(self):
|
||||
self.run_pipeline("rebuild")
|
||||
self.harvest.documents[-1] = self.harvested("b" * 64, "Second version, one line changed.")
|
||||
stats = self.run_pipeline("sync")
|
||||
self.assertEqual(stats.updated, 1)
|
||||
expected = len(__import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(
|
||||
"Second version, one line changed."
|
||||
))
|
||||
self.assertEqual(stats.texts_embedded, expected)
|
||||
self.assertEqual(stats.embed_calls, 1)
|
||||
self.assertEqual(stats.unchanged, 3)
|
||||
|
||||
def test_pool_scan_finds_a_planted_secret(self):
|
||||
self.harvest.documents[-1] = self.harvested(
|
||||
"c" * 64,
|
||||
"AWS_ACCESS_KEY_ID=" + "AKIA" + "ABCDEFGHIJKLMNOP" + "\nThis line is a planted credential.",
|
||||
)
|
||||
with self.assertRaises(pipeline.PipelineError) as caught:
|
||||
self.run_pipeline("rebuild")
|
||||
self.assertIn("credential-shaped content", str(caught.exception))
|
||||
|
||||
|
||||
class CodeChunkerTest(unittest.TestCase):
|
||||
def test_a_function_is_never_cut(self):
|
||||
from corpus import chunker
|
||||
|
||||
body = "\n\n".join(
|
||||
f"def function_{index}():\n value = {{'index': {index}}}\n return value"
|
||||
for index in range(120)
|
||||
)
|
||||
chunks = chunker.chunk_code(body)
|
||||
self.assertGreater(len(chunks), 1)
|
||||
for chunk in chunks:
|
||||
depth = 0
|
||||
for line in chunk.text.splitlines():
|
||||
depth = chunker._code_depth(line, depth)
|
||||
self.assertEqual(depth, 0, f"chunk {chunk.index} ends inside a block")
|
||||
self.assertIn("def function_", chunk.text)
|
||||
|
||||
def test_oversize_single_function_is_flagged_not_cut(self):
|
||||
from corpus import chunker
|
||||
|
||||
body = "def huge():\n" + "\n".join(f" x{index} = {index}" for index in range(2000))
|
||||
chunks = chunker.chunk_code(body)
|
||||
self.assertEqual(len(chunks), 1)
|
||||
self.assertTrue(chunks[0].oversize)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user