WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

corpus/chunker.py

@@ -285,3 +285,124 @@ def verify_no_split(chunks: list, regions: list) -> None:
                     f"chunk {chunk.index} splits protected lines {first}-{last} "
                     f"(chunk covers {chunk.line_start}-{chunk.line_end})"
                 )
+
+
+# -- code -----------------------------------------------------------------
+
+
+def _code_depth(line: str, depth: int) -> int:
+    """Approximate bracket depth for one line, ignoring strings and comments.
+
+    This is a heuristic lexer, not a parser: it is here to answer one question —
+    "is this a safe line to break after?" — and it errs towards *not* breaking.
+    A line with an ambiguous quote or an unterminated bracket simply keeps the
+    chunk open, which can only produce a larger chunk, never a broken one.
+    """
+    index = 0
+    quote = None
+    length = len(line)
+    while index < length:
+        char = line[index]
+        if quote:
+            if char == "\\":
+                index += 2
+                continue
+            if line.startswith(quote, index):
+                index += len(quote)
+                quote = None
+                continue
+            index += 1
+            continue
+        if line.startswith("//", index) or line.startswith("#", index) and line.lstrip().startswith("#"):
+            break
+        if char in "\"'`":
+            triple = char * 3
+            if line.startswith(triple, index):
+                quote = triple
+                index += 3
+                continue
+            quote = char
+            index += 1
+            continue
+        if char in "([{":
+            depth += 1
+        elif char in ")]}":
+            depth = max(0, depth - 1)
+        index += 1
+    return depth
+
+
+def code_segments(text: str) -> list:
+    """Split code into whole top-level segments. A function is never cut."""
+    lines = text.split("\n")
+    segments = []
+    current = []
+    depth = 0
+    start = 1
+    for number, line in enumerate(lines, start=1):
+        if not current:
+            start = number
+        depth = _code_depth(line, depth)
+        current.append(line)
+        # A blank line at bracket depth zero is the only automatic boundary.
+        if depth == 0 and not line.strip() and len(current) > 1:
+            segments.append((start, number, "\n".join(current)))
+            current = []
+    if current:
+        segments.append((start, len(lines), "\n".join(current)))
+    return [(start, end, body) for start, end, body in segments if body.strip()]
+
+
+def chunk_code(text: str) -> list:
+    """Chunk a source file without cutting a function, class or block.
+
+    Segments are whole top-level units. A unit larger than the ceiling is kept
+    as one oversize chunk and flagged, exactly like an oversized Markdown fence:
+    cutting it would break the rule this function exists to keep.
+
+    Overlap repeats the previous unit, never part of one, so an overlapping
+    chunk is still syntactically whole.
+    """
+    segments = code_segments(text)
+    if not segments:
+        raise ChunkError("source file produced no segments (empty?)")
+
+    chunks = []
+    current = []  # list of (start, end, body)
+    tokens = 0
+
+    def flush():
+        nonlocal current, tokens
+        if not current:
+            return
+        body = "\n".join(segment[2] for segment in current).strip("\n")
+        chunks.append(
+            Chunk(
+                index=len(chunks),
+                text=body,
+                tokens=estimate_tokens(body),
+                line_start=current[0][0],
+                line_end=current[-1][1],
+                blocks=tuple(
+                    Block("code", segment[2], estimate_tokens(segment[2]), segment[0], segment[1])
+                    for segment in current
+                ),
+                oversize=len(current) == 1 and estimate_tokens(current[0][2]) > TOKEN_MAX,
+            )
+        )
+        current = []
+        tokens = 0
+
+    for segment in segments:
+        segment_tokens = estimate_tokens(segment[2])
+        if current and tokens + segment_tokens > TOKEN_MAX:
+            previous = current[-1]
+            flush()
+            budget = int(round(TOKEN_TARGET * OVERLAP_RATIO))
+            if estimate_tokens(previous[2]) <= budget:
+                current = [previous]
+                tokens = estimate_tokens(previous[2])
+        current.append(segment)
+        tokens += segment_tokens
+    flush()
+    return chunks