sam/ai-resume
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP Phase 4 ingestion: platform guides, exclusion gate, gitea/vault/graph harvest, payload provenance

Browse Source
This commit is contained in:
Paperclip Builder
2026-10-26 22:26:45 +11:00
parent 5c0622c1b1
commit 1e46ec009a
19 changed files with 2952 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
corpus/store.py
View File

@@ -27,7 +27,7 @@ import sqlite3
import time
import unicodedata
SCHEMA_VERSION = "1"
SCHEMA_VERSION = "2"
SCHEMA = """
CREATE TABLE IF NOT EXISTS chunks (
@@ -49,6 +49,11 @@ CREATE TABLE IF NOT EXISTS chunks (
embedding_dim INTEGER NOT NULL,
chunker_version TEXT NOT NULL,
chunk_tokens INTEGER NOT NULL,
platform TEXT NOT NULL,
tier INTEGER NOT NULL,
origin_url TEXT NOT NULL,
path TEXT NOT NULL,
last_modified TEXT NOT NULL,
public INTEGER NOT NULL,
artefacts TEXT NOT NULL,
tech TEXT NOT NULL,
@@ -66,6 +71,8 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5(
text,
area UNINDEXED,
confidence UNINDEXED,
platform UNINDEXED,
tier UNINDEXED,
tokenize = 'porter unicode61 remove_diacritics 2'
);
@@ -84,6 +91,55 @@ CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
-- The knowledge graph is not chunked (docs/phase-4-corpus.md §4). Nodes and
-- edges live in their own tables and their own Qdrant collection, joined to note
-- chunks by source_note. Every row carries the note it came from.
CREATE TABLE IF NOT EXISTS graph_nodes (
id TEXT PRIMARY KEY,
label TEXT NOT NULL,
norm_label TEXT NOT NULL,
node_type TEXT NOT NULL,
community INTEGER,
source_note TEXT NOT NULL,
origin_url TEXT NOT NULL,
area TEXT NOT NULL,
tier INTEGER NOT NULL,
platform TEXT NOT NULL,
last_modified TEXT NOT NULL,
content_hash TEXT NOT NULL,
text TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS graph_nodes_note ON graph_nodes(source_note);
CREATE TABLE IF NOT EXISTS graph_edges (
id TEXT PRIMARY KEY,
source_id TEXT NOT NULL,
target_id TEXT NOT NULL,
relation TEXT NOT NULL,
confidence TEXT NOT NULL,
confidence_score REAL,
source_note TEXT NOT NULL,
origin_url TEXT NOT NULL,
area TEXT NOT NULL,
tier INTEGER NOT NULL,
platform TEXT NOT NULL,
last_modified TEXT NOT NULL,
content_hash TEXT NOT NULL,
text TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS graph_edges_source ON graph_edges(source_id);
CREATE INDEX IF NOT EXISTS graph_edges_target ON graph_edges(target_id);
CREATE INDEX IF NOT EXISTS graph_edges_note ON graph_edges(source_note);
CREATE VIRTUAL TABLE IF NOT EXISTS graph_fts USING fts5(
id UNINDEXED,
kind UNINDEXED,
label,
text,
area UNINDEXED,
tokenize = 'porter unicode61 remove_diacritics 2'
);
"""
# Column order used by every INSERT in this module.
@@ -91,6 +147,7 @@ _CHUNK_COLUMNS = (
"id", "source_path", "source_id", "chunk_index", "area", "level", "source_type",
"confidence", "period", "last_verified", "title", "claim", "url", "content_hash",
"embedding_model", "embedding_dim", "chunker_version", "chunk_tokens",
"platform", "tier", "origin_url", "path", "last_modified",
"public", "artefacts", "tech", "tags", "text",
)
@@ -110,7 +167,13 @@ class Store:
self.db.execute("PRAGMA journal_mode = WAL")
self.db.execute("PRAGMA synchronous = NORMAL")
self.db.executescript(SCHEMA)
self.set_meta("schema_version", SCHEMA_VERSION)
stored = self.get_meta("schema_version")
# A database from an older payload shape has no provenance columns. It is
# derived data, so the fix is a rebuild, but it must never be silently
# mixed with the new shape: `rebuild_schema` is called by a rebuild only.
self.schema_stale = stored not in (None, SCHEMA_VERSION)
if not self.schema_stale:
self.set_meta("schema_version", SCHEMA_VERSION)
# -- lifecycle --------------------------------------------------------
@@ -125,10 +188,22 @@ class Store:
def reset(self) -> None:
"""Empty every table. Used by a from-scratch rebuild."""
for table in ("chunk_fts", "chunks", "manifest", "meta"):
for table in ("chunk_fts", "chunks", "manifest", "meta", "graph_fts", "graph_nodes", "graph_edges"):
self.db.execute(f"DELETE FROM {table}")
self.db.commit()
def rebuild_schema(self) -> None:
"""Drop and recreate every table. Only a rebuild may call this."""
for table in (
"chunk_fts", "chunks", "manifest", "meta",
"graph_fts", "graph_nodes", "graph_edges",
):
self.db.execute(f"DROP TABLE IF EXISTS {table}")
self.db.commit()
self.db.executescript(SCHEMA)
self.schema_stale = False
self.set_meta("schema_version", SCHEMA_VERSION)
# -- writing ----------------------------------------------------------
def write_document(self, doc, payloads: list) -> None:
@@ -161,8 +236,9 @@ class Store:
if value
)
self.db.execute(
"INSERT INTO chunk_fts (rowid, id, source_path, title, terms, text, area, confidence) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO chunk_fts "
"(rowid, id, source_path, title, terms, text, area, confidence, platform, tier) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
cursor.lastrowid,
payload["id"],
@@ -172,6 +248,8 @@ class Store:
payload["text"],
payload["area"],
payload["confidence"],
payload["platform"],
payload["tier"],
),
)
self.db.execute(
@@ -237,7 +315,10 @@ class Store:
out.append(payload)
return out
def keyword_search(self, query: str, area: str = None, confidence: str = None, limit: int = 10) -> list:
def keyword_search(
self, query: str, area: str = None, confidence: str = None,
platform: str = None, tier: int = None, limit: int = 10,
) -> list:
"""BM25 over chunk text, optionally filtered before ranking."""
match = match_expression(query)
if not match:
@@ -256,11 +337,110 @@ class Store:
if confidence:
sql.append("AND c.confidence = ?")
args.append(confidence)
if platform:
sql.append("AND c.platform = ?")
args.append(platform)
if tier is not None:
sql.append("AND c.tier = ?")
args.append(int(tier))
sql.append("ORDER BY bm25(chunk_fts) LIMIT ?")
args.append(int(limit))
rows = self.db.execute(" ".join(sql), args)
return [dict(row) for row in rows]
def pool_texts(self) -> list:
"""Every chunk text in the pool. The credential scan reads this."""
return [
(row["source_path"], row["chunk_index"], row["text"])
for row in self.db.execute("SELECT source_path, chunk_index, text FROM chunks")
]
# -- graph ------------------------------------------------------------
def write_graph(self, nodes: list, edges: list) -> None:
"""Replace the whole graph. Nodes and edges are rebuilt together."""
self.db.execute("DELETE FROM graph_fts")
self.db.execute("DELETE FROM graph_nodes")
self.db.execute("DELETE FROM graph_edges")
for node in nodes:
self.db.execute(
"INSERT INTO graph_nodes (id, label, norm_label, node_type, community, "
"source_note, origin_url, area, tier, platform, last_modified, content_hash, text) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
node["id"], node["label"], node.get("norm_label") or _norm(node["label"]),
node.get("node_type") or "entity", node.get("community"),
node["source_note"], node["origin_url"], node["area"], node["tier"],
node["platform"], node["last_modified"], node["content_hash"], node["text"],
),
)
self.db.execute(
"INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'node', ?, ?, ?)",
(node["id"], node["label"], node["text"], node["area"]),
)
for edge in edges:
self.db.execute(
"INSERT INTO graph_edges (id, source_id, target_id, relation, confidence, "
"confidence_score, source_note, origin_url, area, tier, platform, "
"last_modified, content_hash, text) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
edge["id"], edge["source_id"], edge["target_id"], edge["relation"],
edge["confidence"], edge.get("confidence_score"), edge["source_note"],
edge["origin_url"], edge["area"], edge["tier"], edge["platform"],
edge["last_modified"], edge["content_hash"], edge["text"],
),
)
self.db.execute(
"INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'edge', ?, ?, ?)",
(edge["id"], edge["relation"], edge["text"], edge["area"]),
)
self.db.commit()
def graph_counts(self) -> dict:
return {
"nodes": int(self.db.execute("SELECT count(*) FROM graph_nodes").fetchone()[0]),
"edges": int(self.db.execute("SELECT count(*) FROM graph_edges").fetchone()[0]),
}
def graph_node(self, node_id: str) -> dict:
row = self.db.execute("SELECT * FROM graph_nodes WHERE id = ?", (node_id,)).fetchone()
return dict(row) if row else {}
def graph_neighbours(self, node_id: str, limit: int = 50) -> list:
rows = self.db.execute(
"SELECT e.*, sn.label AS source_label, tn.label AS target_label "
"FROM graph_edges e "
"LEFT JOIN graph_nodes sn ON sn.id = e.source_id "
"LEFT JOIN graph_nodes tn ON tn.id = e.target_id "
"WHERE e.source_id = ? OR e.target_id = ? LIMIT ?",
(node_id, node_id, int(limit)),
)
return [dict(row) for row in rows]
def graph_search(self, query: str, area: str = None, limit: int = 10) -> list:
"""Keyword search over node labels and edge relations, with provenance."""
match = match_expression(query)
if not match:
return []
sql = [
"SELECT f.id, f.kind, f.label, f.text, f.area,",
" COALESCE(n.source_note, e.source_note) AS source_note,",
" COALESCE(n.origin_url, e.origin_url) AS origin_url,",
" n.community, e.relation, e.source_id, e.target_id",
"FROM graph_fts f",
"LEFT JOIN graph_nodes n ON f.kind = 'node' AND n.id = f.id",
"LEFT JOIN graph_edges e ON f.kind = 'edge' AND e.id = f.id",
"WHERE graph_fts MATCH ?",
]
args = [match]
if area:
sql.append("AND f.area = ?")
args.append(area)
sql.append("ORDER BY bm25(graph_fts) LIMIT ?")
args.append(int(limit))
return [dict(row) for row in self.db.execute(" ".join(sql), args)]
# -- meta -------------------------------------------------------------
def set_meta(self, key: str, value) -> None:
@@ -280,6 +460,10 @@ class Store:
return {row["key"]: json.loads(row["value"]) for row in self.db.execute("SELECT key, value FROM meta")}
def _norm(label: str) -> str:
return re.sub(r"[^0-9a-z]+", "_", (label or "").lower()).strip("_")
def match_expression(query: str) -> str:
"""Free text -> a safe FTS5 expression: bare words, each a prefix, all required.