"""SQLite: the keyword index and the incremental state. Two jobs, one file (`RESUME_CORPUS_DB`, default `data/corpus/corpus.db`): * **`chunk_fts`** — an FTS5 index over chunk text with the same filter columns the vectors carry. This is the keyword half of hybrid retrieval (decision D10). Phase 2 indexes *pages* for the site's own search box; this indexes *chunks*, because retrieval hands the model a passage, not a page. * **`manifest`** — one row per source document: its content hash, its chunk count and the embedding model/dimension it was embedded with. Incremental mode is a hash comparison against this table, and nothing else. `meta` records the collection contract (provider, model, dimension, chunker version) so a provider change is detected instead of silently corrupting the index. Standard library `sqlite3` only. The same file is opened read-only by the Phase 5 retrieval service. """ from __future__ import annotations import json import os import re import sqlite3 import time import unicodedata SCHEMA_VERSION = "2" SCHEMA = """ CREATE TABLE IF NOT EXISTS chunks ( id TEXT PRIMARY KEY, source_path TEXT NOT NULL, source_id TEXT NOT NULL, chunk_index INTEGER NOT NULL, area TEXT NOT NULL, level INTEGER NOT NULL, source_type TEXT NOT NULL, confidence TEXT NOT NULL, period TEXT NOT NULL, last_verified TEXT NOT NULL, title TEXT NOT NULL, claim TEXT NOT NULL, url TEXT NOT NULL, content_hash TEXT NOT NULL, embedding_model TEXT NOT NULL, embedding_dim INTEGER NOT NULL, chunker_version TEXT NOT NULL, chunk_tokens INTEGER NOT NULL, platform TEXT NOT NULL, tier INTEGER NOT NULL, origin_url TEXT NOT NULL, path TEXT NOT NULL, last_modified TEXT NOT NULL, public INTEGER NOT NULL, artefacts TEXT NOT NULL, tech TEXT NOT NULL, tags TEXT NOT NULL, text TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS chunks_source ON chunks(source_path); CREATE INDEX IF NOT EXISTS chunks_area ON chunks(area); CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5( id UNINDEXED, source_path UNINDEXED, title, terms, text, area UNINDEXED, confidence UNINDEXED, platform UNINDEXED, tier UNINDEXED, tokenize = 'porter unicode61 remove_diacritics 2' ); CREATE TABLE IF NOT EXISTS manifest ( source_path TEXT PRIMARY KEY, content_hash TEXT NOT NULL, chunk_count INTEGER NOT NULL, embedding_model TEXT NOT NULL, embedding_dim INTEGER NOT NULL, chunker_version TEXT NOT NULL, last_verified TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); -- The knowledge graph is not chunked (docs/phase-4-corpus.md §4). Nodes and -- edges live in their own tables and their own Qdrant collection, joined to note -- chunks by source_note. Every row carries the note it came from. CREATE TABLE IF NOT EXISTS graph_nodes ( id TEXT PRIMARY KEY, label TEXT NOT NULL, norm_label TEXT NOT NULL, node_type TEXT NOT NULL, community INTEGER, source_note TEXT NOT NULL, origin_url TEXT NOT NULL, area TEXT NOT NULL, tier INTEGER NOT NULL, platform TEXT NOT NULL, last_modified TEXT NOT NULL, content_hash TEXT NOT NULL, text TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS graph_nodes_note ON graph_nodes(source_note); CREATE TABLE IF NOT EXISTS graph_edges ( id TEXT PRIMARY KEY, source_id TEXT NOT NULL, target_id TEXT NOT NULL, relation TEXT NOT NULL, confidence TEXT NOT NULL, confidence_score REAL, source_note TEXT NOT NULL, origin_url TEXT NOT NULL, area TEXT NOT NULL, tier INTEGER NOT NULL, platform TEXT NOT NULL, last_modified TEXT NOT NULL, content_hash TEXT NOT NULL, text TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS graph_edges_source ON graph_edges(source_id); CREATE INDEX IF NOT EXISTS graph_edges_target ON graph_edges(target_id); CREATE INDEX IF NOT EXISTS graph_edges_note ON graph_edges(source_note); CREATE VIRTUAL TABLE IF NOT EXISTS graph_fts USING fts5( id UNINDEXED, kind UNINDEXED, label, text, area UNINDEXED, tokenize = 'porter unicode61 remove_diacritics 2' ); """ # Column order used by every INSERT in this module. _CHUNK_COLUMNS = ( "id", "source_path", "source_id", "chunk_index", "area", "level", "source_type", "confidence", "period", "last_verified", "title", "claim", "url", "content_hash", "embedding_model", "embedding_dim", "chunker_version", "chunk_tokens", "platform", "tier", "origin_url", "path", "last_modified", "public", "artefacts", "tech", "tags", "text", ) _JSON_FIELDS = ("artefacts", "tech", "tags") class StoreError(RuntimeError): """The corpus database is unreadable or inconsistent.""" class Store: def __init__(self, path: str) -> None: self.path = path os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True) self.db = sqlite3.connect(path) self.db.row_factory = sqlite3.Row self.db.execute("PRAGMA journal_mode = WAL") self.db.execute("PRAGMA synchronous = NORMAL") self.db.executescript(SCHEMA) stored = self.get_meta("schema_version") # A database from an older payload shape has no provenance columns. It is # derived data, so the fix is a rebuild, but it must never be silently # mixed with the new shape: `rebuild_schema` is called by a rebuild only. self.schema_stale = stored not in (None, SCHEMA_VERSION) if not self.schema_stale: self.set_meta("schema_version", SCHEMA_VERSION) # -- lifecycle -------------------------------------------------------- def close(self) -> None: self.db.close() def __enter__(self) -> "Store": return self def __exit__(self, *_exc) -> None: self.close() def reset(self) -> None: """Empty every table. Used by a from-scratch rebuild.""" for table in ("chunk_fts", "chunks", "manifest", "meta", "graph_fts", "graph_nodes", "graph_edges"): self.db.execute(f"DELETE FROM {table}") self.db.commit() def rebuild_schema(self) -> None: """Drop and recreate every table. Only a rebuild may call this.""" for table in ( "chunk_fts", "chunks", "manifest", "meta", "graph_fts", "graph_nodes", "graph_edges", ): self.db.execute(f"DROP TABLE IF EXISTS {table}") self.db.commit() self.db.executescript(SCHEMA) self.schema_stale = False self.set_meta("schema_version", SCHEMA_VERSION) # -- writing ---------------------------------------------------------- def write_document(self, doc, payloads: list) -> None: """Replace all rows for one document: chunks, FTS rows and manifest.""" self.delete_document(doc.path, keep_manifest=True) for payload in payloads: values = [] for column in _CHUNK_COLUMNS: value = payload[column] if column in _JSON_FIELDS: value = json.dumps(value, sort_keys=True) values.append(value) cursor = self.db.execute( f"INSERT INTO chunks ({', '.join(_CHUNK_COLUMNS)}) " f"VALUES ({', '.join('?' for _ in _CHUNK_COLUMNS)})", values, ) # `terms` is the keyword surface: the words a visitor or a recruiter # would type that live in front-matter rather than the body — tech # names, tags, the one-line claim. Without it, a search for "zigbee" # misses a page whose body says "mesh". It is indexed, never retrieved. terms = " ".join( str(value) for value in ( payload["claim"], payload["title"], " ".join(payload["tech"]), " ".join(payload["tags"]), ) if value ) self.db.execute( "INSERT INTO chunk_fts " "(rowid, id, source_path, title, terms, text, area, confidence, platform, tier) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( cursor.lastrowid, payload["id"], payload["source_path"], payload["title"], terms, payload["text"], payload["area"], payload["confidence"], payload["platform"], payload["tier"], ), ) self.db.execute( "INSERT OR REPLACE INTO manifest " "(source_path, content_hash, chunk_count, embedding_model, embedding_dim, " " chunker_version, last_verified, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ( doc.path, doc.digest, len(payloads), payloads[0]["embedding_model"] if payloads else "", payloads[0]["embedding_dim"] if payloads else 0, payloads[0]["chunker_version"] if payloads else "", doc.last_verified, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), ), ) self.db.commit() def delete_document(self, source_path: str, keep_manifest: bool = False) -> int: rowids = [ row["rowid"] for row in self.db.execute("SELECT rowid FROM chunks WHERE source_path = ?", (source_path,)) ] for rowid in rowids: self.db.execute("DELETE FROM chunk_fts WHERE rowid = ?", (rowid,)) self.db.execute("DELETE FROM chunks WHERE source_path = ?", (source_path,)) if not keep_manifest: self.db.execute("DELETE FROM manifest WHERE source_path = ?", (source_path,)) self.db.commit() return len(rowids) # -- reading ---------------------------------------------------------- def manifest(self) -> dict: return { row["source_path"]: dict(row) for row in self.db.execute("SELECT * FROM manifest") } def chunk_count(self) -> int: return int(self.db.execute("SELECT count(*) FROM chunks").fetchone()[0]) def source_count(self) -> int: return int(self.db.execute("SELECT count(DISTINCT source_path) FROM chunks").fetchone()[0]) def chunk_ids(self) -> list: return [row["id"] for row in self.db.execute("SELECT id FROM chunks ORDER BY id")] def chunk_payloads(self, source_path: str = None) -> list: """Every stored payload, optionally for one document. Used by verify().""" sql = "SELECT * FROM chunks" args = () if source_path: sql += " WHERE source_path = ?" args = (source_path,) sql += " ORDER BY source_path, chunk_index" out = [] for row in self.db.execute(sql, args): payload = dict(row) for field in _JSON_FIELDS: payload[field] = json.loads(payload[field]) out.append(payload) return out def keyword_search( self, query: str, area: str = None, confidence: str = None, platform: str = None, tier: int = None, limit: int = 10, ) -> list: """BM25 over chunk text, optionally filtered before ranking.""" match = match_expression(query) if not match: return [] sql = [ "SELECT c.id, c.url, c.title, c.area, c.confidence, c.source_path,", " c.chunk_index, snippet(chunk_fts, 4, '', '', '…', 18) AS snippet,", " bm25(chunk_fts) AS score", "FROM chunk_fts JOIN chunks c ON c.rowid = chunk_fts.rowid", "WHERE chunk_fts MATCH ?", ] args = [match] if area: sql.append("AND c.area = ?") args.append(area) if confidence: sql.append("AND c.confidence = ?") args.append(confidence) if platform: sql.append("AND c.platform = ?") args.append(platform) if tier is not None: sql.append("AND c.tier = ?") args.append(int(tier)) sql.append("ORDER BY bm25(chunk_fts) LIMIT ?") args.append(int(limit)) rows = self.db.execute(" ".join(sql), args) return [dict(row) for row in rows] def pool_texts(self) -> list: """Every chunk text in the pool. The credential scan reads this.""" return [ (row["source_path"], row["chunk_index"], row["text"]) for row in self.db.execute("SELECT source_path, chunk_index, text FROM chunks") ] # -- graph ------------------------------------------------------------ def write_graph(self, nodes: list, edges: list) -> None: """Replace the whole graph. Nodes and edges are rebuilt together.""" self.db.execute("DELETE FROM graph_fts") self.db.execute("DELETE FROM graph_nodes") self.db.execute("DELETE FROM graph_edges") for node in nodes: self.db.execute( "INSERT INTO graph_nodes (id, label, norm_label, node_type, community, " "source_note, origin_url, area, tier, platform, last_modified, content_hash, text) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( node["id"], node["label"], node.get("norm_label") or _norm(node["label"]), node.get("node_type") or "entity", node.get("community"), node["source_note"], node["origin_url"], node["area"], node["tier"], node["platform"], node["last_modified"], node["content_hash"], node["text"], ), ) self.db.execute( "INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'node', ?, ?, ?)", (node["id"], node["label"], node["text"], node["area"]), ) for edge in edges: self.db.execute( "INSERT INTO graph_edges (id, source_id, target_id, relation, confidence, " "confidence_score, source_note, origin_url, area, tier, platform, " "last_modified, content_hash, text) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( edge["id"], edge["source_id"], edge["target_id"], edge["relation"], edge["confidence"], edge.get("confidence_score"), edge["source_note"], edge["origin_url"], edge["area"], edge["tier"], edge["platform"], edge["last_modified"], edge["content_hash"], edge["text"], ), ) self.db.execute( "INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'edge', ?, ?, ?)", (edge["id"], edge["relation"], edge["text"], edge["area"]), ) self.db.commit() def graph_counts(self) -> dict: return { "nodes": int(self.db.execute("SELECT count(*) FROM graph_nodes").fetchone()[0]), "edges": int(self.db.execute("SELECT count(*) FROM graph_edges").fetchone()[0]), } def graph_node(self, node_id: str) -> dict: row = self.db.execute("SELECT * FROM graph_nodes WHERE id = ?", (node_id,)).fetchone() return dict(row) if row else {} def graph_neighbours(self, node_id: str, limit: int = 50) -> list: rows = self.db.execute( "SELECT e.*, sn.label AS source_label, tn.label AS target_label " "FROM graph_edges e " "LEFT JOIN graph_nodes sn ON sn.id = e.source_id " "LEFT JOIN graph_nodes tn ON tn.id = e.target_id " "WHERE e.source_id = ? OR e.target_id = ? LIMIT ?", (node_id, node_id, int(limit)), ) return [dict(row) for row in rows] def graph_search(self, query: str, area: str = None, limit: int = 10) -> list: """Keyword search over node labels and edge relations, with provenance.""" match = match_expression(query) if not match: return [] sql = [ "SELECT f.id, f.kind, f.label, f.text, f.area,", " COALESCE(n.source_note, e.source_note) AS source_note,", " COALESCE(n.origin_url, e.origin_url) AS origin_url,", " n.community, e.relation, e.source_id, e.target_id", "FROM graph_fts f", "LEFT JOIN graph_nodes n ON f.kind = 'node' AND n.id = f.id", "LEFT JOIN graph_edges e ON f.kind = 'edge' AND e.id = f.id", "WHERE graph_fts MATCH ?", ] args = [match] if area: sql.append("AND f.area = ?") args.append(area) sql.append("ORDER BY bm25(graph_fts) LIMIT ?") args.append(int(limit)) return [dict(row) for row in self.db.execute(" ".join(sql), args)] # -- meta ------------------------------------------------------------- def set_meta(self, key: str, value) -> None: self.db.execute( "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (key, json.dumps(value, sort_keys=True)), ) self.db.commit() def get_meta(self, key: str, default=None): row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() if row is None: return default return json.loads(row["value"]) def meta(self) -> dict: return {row["key"]: json.loads(row["value"]) for row in self.db.execute("SELECT key, value FROM meta")} def _norm(label: str) -> str: return re.sub(r"[^0-9a-z]+", "_", (label or "").lower()).strip("_") def match_expression(query: str) -> str: """Free text -> a safe FTS5 expression: bare words, each a prefix, all required. Mirrors `MatchExpression` in internal/search/search.go so the site's search box and the retrieval path treat a visitor's words identically. FTS5 operators in the input are never honoured. """ terms = [] for raw in re.split(r"[^0-9a-zA-Z\-]+", unicodedata.normalize("NFKC", query or "").lower()): term = raw.strip("-") if term: terms.append('"' + term.replace('"', "") + '"*') return " AND ".join(terms)