"""Ingestion from the estate: exclusions, provenance, incremental work, graph.

These tests never touch the network. `FakeGitea` implements the four read calls
the harvester uses; `FakeQdrant` implements the collection routes the pipeline
uses. The real estate is exercised separately (docs/phase-4-notes.md), so a
failure here is a logic failure, not a connectivity one.
"""

from __future__ import annotations

import hashlib
import json
import os
import unittest
from unittest import mock

from corpus import exclusions, guides, loader, pipeline, store as store_module
from corpus.sources import estate, graph as graph_module
from corpus.tests import support


def gitea_guide(**overrides) -> guides.Guide:
    values = dict(
        name="gitea", enabled=True, what="Self-hosted Git.", fits=["ai", "platform", "edge", "products"],
        read=["readme", "tree", "selected_source", "commits", "languages"],
        base="https://gitea.example", auth="GITEA_TOKEN",
        include={"repos": "all", "paths": []},
        exclude={"repos": [], "paths": []},
        limits={"max_file_kb": 200, "max_files_per_repo": 10, "max_commits": 5, "max_tree_entries": 100},
    )
    values.update(overrides)
    return guides.Guide(**values)


class FakeGitea:
    def __init__(self, repos=None, trees=None, files=None, commits=None, languages=None):
        self.repos_list = repos or []
        self.trees = trees or {}
        self.files = files or {}
        self.commit_list = commits or []
        self.language_map = languages or {}
        self.calls = []

    def repos(self):
        return list(self.repos_list)

    def tree(self, full_name, ref):
        self.calls.append(("tree", full_name, ref))
        return list(self.trees.get(full_name, []))

    def raw(self, full_name, ref, path):
        self.calls.append(("raw", full_name, ref, path))
        try:
            return self.files[(full_name, path)]
        except KeyError:
            from corpus.sources.gitea import GiteaNotFound

            raise GiteaNotFound(f"{full_name}/{path}") from None

    def commits(self, full_name, ref, limit=20):
        return list(self.commit_list)

    def languages(self, full_name):
        return dict(self.language_map)


def repo(full_name="sam/photo-pipeline", **overrides):
    values = dict(
        full_name=full_name, name=full_name.split("/")[-1], description="Photo ingestion pipeline",
        topics=["photo", "python"], default_branch="main", archived=False,
        created_at="2024-01-01T00:00:00Z", updated_at="2026-10-14T05:00:00Z",
        html_url=f"https://gitea.example/{full_name}",
    )
    values.update(overrides)
    return values


def blob(path, size=None):
    return {"type": "blob", "path": path, "size": len(path) if size is None else size}


class ExclusionTest(unittest.TestCase):
    """The DoD probe: a file in an excluded path never appears in the pool."""

    def setUp(self):
        self.guide = gitea_guide()
        self.result = estate.Harvest()
        tree = [
            blob("README.md"),
            blob("node_modules/left-pad/probe.md"),
            blob(".venv/lib/probe.py"),
            blob(".git/config"),
            blob(".trash/old.md"),
            blob("docs/.silverbullet.auth.json"),
            blob("src/main.py"),
            blob("assets/photo.png"),
        ]
        files = {
            ("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n\nDeduplicates by hash.\n",
            ("sam/photo-pipeline", "node_modules/left-pad/probe.md"): b"# PROBE\n",
            ("sam/photo-pipeline", ".venv/lib/probe.py"): b"PROBE = 1\n",
            ("sam/photo-pipeline", ".git/config"): b"[core]\n",
            ("sam/photo-pipeline", ".trash/old.md"): b"# old\n",
            ("sam/photo-pipeline", "docs/.silverbullet.auth.json"): b'{"secret_key": "x"}\n',
            ("sam/photo-pipeline", "src/main.py"): b"def dedup():\n    return True\n",
        }
        self.client = FakeGitea(
            repos=[repo()],
            trees={"sam/photo-pipeline": tree},
            files=files,
            commits=[{"commit": {"message": "dedup by hash", "author": {"date": "2026-10-14T05:00:00Z"}}}],
            languages={"Python": 900, "Shell": 100},
        )
        self.documents = estate._harvest_one_repo(
            self.result, self.client, self.guide, repo(), "https://gitea.example"
        )

    def test_probe_in_excluded_paths_never_appears(self):
        paths = {doc.repo_path for doc in self.documents}
        for probe in (
            "node_modules/left-pad/probe.md",
            ".venv/lib/probe.py",
            ".git/config",
            ".trash/old.md",
            "docs/.silverbullet.auth.json",
        ):
            self.assertNotIn(probe, paths)
        self.assertIn("README.md", paths)
        self.assertIn("src/main.py", paths)

    def test_rejections_are_reported_with_a_reason(self):
        reasons = {(row["path"], row["reason"]) for row in self.result.rejections}
        self.assertIn(("node_modules/left-pad/probe.md", "excluded directory 'node_modules'"), reasons)
        self.assertIn(("docs/.silverbullet.auth.json", "credential-shaped file name"), reasons)
        self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])

    def test_media_is_an_image_reference_not_text(self):
        self.assertEqual([image["path"] for image in self.result.images], ["assets/photo.png"])
        for doc in self.documents:
            self.assertNotIn("assets/photo.png", doc.body)

    def test_every_document_carries_provenance(self):
        for doc in self.documents:
            self.assertEqual(doc.platform, "gitea")
            self.assertEqual(doc.tier, 2)
            self.assertTrue(doc.origin_url.startswith("https://gitea.example/sam/photo-pipeline"))
            self.assertTrue(doc.last_modified)
            self.assertIn(doc.area, self.guide.fits)


class CredentialContentTest(unittest.TestCase):
    def test_a_valid_path_with_a_private_key_is_rejected(self):
        result = estate.Harvest()
        guide = gitea_guide()
        client = FakeGitea(
            repos=[repo()],
            trees={"sam/photo-pipeline": [blob("README.md"), blob("src/leak.py")]},
            files={
                ("sam/photo-pipeline", "README.md"): b"# Photo pipeline\n",
                ("sam/photo-pipeline", "src/leak.py"): (
                    ("KEY = '" + "-----BEGIN " + "PRIVATE KEY-----'\n").encode("utf-8")
                ),
            },
            commits=[],
            languages={"Python": 10},
        )
        estate._harvest_one_repo(result, client, guide, repo(), "https://gitea.example")
        reasons = {row["reason"]: row for row in result.rejections}
        self.assertIn("credential-shaped content", reasons)
        self.assertEqual(reasons["credential-shaped content"]["path"], "src/leak.py")


class PayloadProvenanceTest(unittest.TestCase):
    def test_missing_provenance_field_fails_the_payload(self):
        root = support.temp_repo()
        self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
        docs = loader.load_documents(support.make_config(root))
        chunk = __import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(docs[0].body)[0]
        payload = __import__("corpus.payloads", fromlist=["build_payload"]).build_payload(
            docs[0], chunk, support.make_config(root)
        )
        payload["id"] = pipeline.chunk_id(docs[0].path, 0, docs[0].digest)
        for field in ("platform", "tier", "origin_url", "path", "last_modified"):
            broken = dict(payload)
            broken.pop(field)
            with self.assertRaises(Exception):
                __import__("corpus.payloads", fromlist=["validate_payload"]).validate_payload(broken)


GRAPH = {
    "built_at_commit": "abc123",
    "nodes": [
        {"id": "photo_ingest", "label": "Photo ingest", "source_file": "300 areas/Photo.md",
         "community": 7, "node_type": "entity"},
        {"id": "settings", "label": "settings.json", "source_file": "settings.json",
         "community": 118, "node_type": "code"},
        {"id": "image_node", "label": "diagram", "source_file": "300 areas/diagram.png",
         "community": 7, "node_type": "image"},
    ],
    "links": [
        {"source": "photo_ingest", "target": "settings", "relation": "configured_by",
         "source_file": "300 areas/Photo.md", "confidence": "EXTRACTED", "confidence_score": 0.9},
        {"source": "settings", "target": "photo_ingest", "relation": "configures",
         "source_file": "settings.json", "confidence": "INFERRED"},
    ],
}


class GraphTest(unittest.TestCase):
    def parse(self):
        return graph_module.parse(
            json.dumps(GRAPH).encode("utf-8"),
            platform="graphify", base="https://gitea.example", repo="sam/obsidian-vault",
            ref="main", last_modified="2026-10-14T05:00:00Z", area="ai",
            vault_paths={"300 areas/Photo.md", "300 areas/diagram.png"},
        )

    def test_nodes_and_edges_carry_their_source_note(self):
        result = self.parse()
        self.assertEqual(len(result.nodes), 1)
        self.assertEqual(result.nodes[0]["id"], "photo_ingest")
        self.assertEqual(result.nodes[0]["source_note"], "300 areas/Photo.md")
        self.assertIn("/sam/obsidian-vault/src/branch/main/300%20areas/Photo.md", result.nodes[0]["origin_url"])
        self.assertTrue(result.edges)
        for edge in result.edges:
            self.assertTrue(edge["source_note"])

    def test_a_node_without_a_resolvable_note_is_refused(self):
        result = self.parse()
        rejected = {row["path"] for row in result.rejections}
        self.assertIn("settings.json", rejected)
        self.assertNotIn("settings", {row["id"] for row in result.nodes})

    def test_media_becomes_an_image_reference(self):
        result = self.parse()
        self.assertEqual([image["path"] for image in result.images], ["300 areas/diagram.png"])
        self.assertNotIn("image_node", {row["id"] for row in result.nodes})

    def test_graph_query_returns_the_note(self):
        root = support.temp_repo()
        self.addCleanup(lambda: __import__("shutil").rmtree(root, ignore_errors=True))
        cfg = support.make_config(root)
        harvest = estate.Harvest()
        harvest.documents = loader.load_documents(cfg)
        harvest.graph = self.parse()
        harvest.sources["graphify"] = {"platform": "graphify", "enabled": True, "documents": 0}
        fake = support.FakeQdrant(cfg.dim)
        with mock.patch("corpus.sources.estate.harvest", return_value=harvest):
            with support.fake_qdrant(cfg, fake):
                pipeline.run(cfg, mode="rebuild")
                with store_module.Store(cfg.db_path) as store:
                    rows = store.graph_search("photo")
                    self.assertTrue(rows)
                    self.assertTrue(all(row["source_note"] for row in rows))
                    self.assertEqual(store.graph_node("photo_ingest")["source_note"], "300 areas/Photo.md")
                    self.assertTrue(store.graph_neighbours("photo_ingest"))


class IncrementalEstateTest(unittest.TestCase):
    def setUp(self):
        self.root = support.temp_repo()
        self.addCleanup(lambda: __import__("shutil").rmtree(self.root, ignore_errors=True))
        self.cfg = support.make_config(self.root)
        self.fake = support.FakeQdrant(self.cfg.dim)
        self.harvest = estate.Harvest()
        self.harvest.documents = loader.load_documents(self.cfg)
        self.harvest.documents.append(self.harvested("a" * 64, "First version of the repository."))
        self.harvest.sources["gitea"] = {"platform": "gitea", "enabled": True, "documents": 1}

    def harvested(self, digest, body):
        return loader.Document(
            path="gitea:sam/photo-pipeline/README.md", source_id="sam/photo-pipeline/README.md",
            area="platform", level=3, title="sam/photo-pipeline — README.md",
            claim=body, confidence="historic", period="2024–2026", tech=[], metrics=[], artefacts=[],
            public=True, last_verified="2026-10-14", tags=["gitea"], body=body, digest=digest,
            url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
            platform="gitea", tier=2, origin_url="https://gitea.example/sam/photo-pipeline/src/branch/main/README.md",
            last_modified="2026-10-14T05:00:00Z", source_type="doc", kind="markdown",
            repo_path="README.md",
        )

    def run_pipeline(self, mode="sync"):
        with mock.patch("corpus.sources.estate.harvest", return_value=self.harvest):
            with support.fake_qdrant(self.cfg, self.fake):
                return pipeline.run(self.cfg, mode=mode)

    def test_rebuild_and_second_run_do_not_duplicate(self):
        first = self.run_pipeline("rebuild")
        points = len(self.fake.collections[self.cfg.collection])
        self.assertEqual(first.platforms["gitea"]["documents"], 1)
        second = self.run_pipeline("sync")
        self.assertEqual(len(self.fake.collections[self.cfg.collection]), points)
        self.assertEqual(second.added, 0)
        self.assertEqual(second.updated, 0)
        self.assertEqual(second.texts_embedded, 0)

    def test_only_the_changed_harvested_document_is_reembedded(self):
        self.run_pipeline("rebuild")
        self.harvest.documents[-1] = self.harvested("b" * 64, "Second version, one line changed.")
        stats = self.run_pipeline("sync")
        self.assertEqual(stats.updated, 1)
        expected = len(__import__("corpus.chunker", fromlist=["chunk_document"]).chunk_document(
            "Second version, one line changed."
        ))
        self.assertEqual(stats.texts_embedded, expected)
        self.assertEqual(stats.embed_calls, 1)
        self.assertEqual(stats.unchanged, 3)

    def test_pool_scan_finds_a_planted_secret(self):
        self.harvest.documents[-1] = self.harvested(
            "c" * 64,
            "AWS_ACCESS_KEY_ID=" + "AKIA" + "ABCDEFGHIJKLMNOP" + "\nThis line is a planted credential.",
        )
        with self.assertRaises(pipeline.PipelineError) as caught:
            self.run_pipeline("rebuild")
        self.assertIn("credential-shaped content", str(caught.exception))


class CodeChunkerTest(unittest.TestCase):
    def test_a_function_is_never_cut(self):
        from corpus import chunker

        body = "\n\n".join(
            f"def function_{index}():\n    value = {{'index': {index}}}\n    return value"
            for index in range(120)
        )
        chunks = chunker.chunk_code(body)
        self.assertGreater(len(chunks), 1)
        for chunk in chunks:
            depth = 0
            for line in chunk.text.splitlines():
                depth = chunker._code_depth(line, depth)
            self.assertEqual(depth, 0, f"chunk {chunk.index} ends inside a block")
            self.assertIn("def function_", chunk.text)

    def test_oversize_single_function_is_flagged_not_cut(self):
        from corpus import chunker

        body = "def huge():\n" + "\n".join(f"    x{index} = {index}" for index in range(2000))
        chunks = chunker.chunk_code(body)
        self.assertEqual(len(chunks), 1)
        self.assertTrue(chunks[0].oversize)


if __name__ == "__main__":
    unittest.main()