480 lines
19 KiB
Python
480 lines
19 KiB
Python
"""SQLite: the keyword index and the incremental state.
|
|
|
|
Two jobs, one file (`RESUME_CORPUS_DB`, default `data/corpus/corpus.db`):
|
|
|
|
* **`chunk_fts`** — an FTS5 index over chunk text with the same filter columns
|
|
the vectors carry. This is the keyword half of hybrid retrieval (decision
|
|
D10). Phase 2 indexes *pages* for the site's own search box; this indexes
|
|
*chunks*, because retrieval hands the model a passage, not a page.
|
|
* **`manifest`** — one row per source document: its content hash, its chunk
|
|
count and the embedding model/dimension it was embedded with. Incremental mode
|
|
is a hash comparison against this table, and nothing else.
|
|
|
|
`meta` records the collection contract (provider, model, dimension, chunker
|
|
version) so a provider change is detected instead of silently corrupting the
|
|
index.
|
|
|
|
Standard library `sqlite3` only. The same file is opened read-only by the Phase
|
|
5 retrieval service.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
import unicodedata
|
|
|
|
SCHEMA_VERSION = "2"
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS chunks (
|
|
id TEXT PRIMARY KEY,
|
|
source_path TEXT NOT NULL,
|
|
source_id TEXT NOT NULL,
|
|
chunk_index INTEGER NOT NULL,
|
|
area TEXT NOT NULL,
|
|
level INTEGER NOT NULL,
|
|
source_type TEXT NOT NULL,
|
|
confidence TEXT NOT NULL,
|
|
period TEXT NOT NULL,
|
|
last_verified TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
claim TEXT NOT NULL,
|
|
url TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
embedding_model TEXT NOT NULL,
|
|
embedding_dim INTEGER NOT NULL,
|
|
chunker_version TEXT NOT NULL,
|
|
chunk_tokens INTEGER NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
tier INTEGER NOT NULL,
|
|
origin_url TEXT NOT NULL,
|
|
path TEXT NOT NULL,
|
|
last_modified TEXT NOT NULL,
|
|
public INTEGER NOT NULL,
|
|
artefacts TEXT NOT NULL,
|
|
tech TEXT NOT NULL,
|
|
tags TEXT NOT NULL,
|
|
text TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS chunks_source ON chunks(source_path);
|
|
CREATE INDEX IF NOT EXISTS chunks_area ON chunks(area);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5(
|
|
id UNINDEXED,
|
|
source_path UNINDEXED,
|
|
title,
|
|
terms,
|
|
text,
|
|
area UNINDEXED,
|
|
confidence UNINDEXED,
|
|
platform UNINDEXED,
|
|
tier UNINDEXED,
|
|
tokenize = 'porter unicode61 remove_diacritics 2'
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS manifest (
|
|
source_path TEXT PRIMARY KEY,
|
|
content_hash TEXT NOT NULL,
|
|
chunk_count INTEGER NOT NULL,
|
|
embedding_model TEXT NOT NULL,
|
|
embedding_dim INTEGER NOT NULL,
|
|
chunker_version TEXT NOT NULL,
|
|
last_verified TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
|
|
-- The knowledge graph is not chunked (docs/phase-4-corpus.md §4). Nodes and
|
|
-- edges live in their own tables and their own Qdrant collection, joined to note
|
|
-- chunks by source_note. Every row carries the note it came from.
|
|
CREATE TABLE IF NOT EXISTS graph_nodes (
|
|
id TEXT PRIMARY KEY,
|
|
label TEXT NOT NULL,
|
|
norm_label TEXT NOT NULL,
|
|
node_type TEXT NOT NULL,
|
|
community INTEGER,
|
|
source_note TEXT NOT NULL,
|
|
origin_url TEXT NOT NULL,
|
|
area TEXT NOT NULL,
|
|
tier INTEGER NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
last_modified TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
text TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS graph_nodes_note ON graph_nodes(source_note);
|
|
|
|
CREATE TABLE IF NOT EXISTS graph_edges (
|
|
id TEXT PRIMARY KEY,
|
|
source_id TEXT NOT NULL,
|
|
target_id TEXT NOT NULL,
|
|
relation TEXT NOT NULL,
|
|
confidence TEXT NOT NULL,
|
|
confidence_score REAL,
|
|
source_note TEXT NOT NULL,
|
|
origin_url TEXT NOT NULL,
|
|
area TEXT NOT NULL,
|
|
tier INTEGER NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
last_modified TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
text TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS graph_edges_source ON graph_edges(source_id);
|
|
CREATE INDEX IF NOT EXISTS graph_edges_target ON graph_edges(target_id);
|
|
CREATE INDEX IF NOT EXISTS graph_edges_note ON graph_edges(source_note);
|
|
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS graph_fts USING fts5(
|
|
id UNINDEXED,
|
|
kind UNINDEXED,
|
|
label,
|
|
text,
|
|
area UNINDEXED,
|
|
tokenize = 'porter unicode61 remove_diacritics 2'
|
|
);
|
|
"""
|
|
|
|
# Column order used by every INSERT in this module.
|
|
_CHUNK_COLUMNS = (
|
|
"id", "source_path", "source_id", "chunk_index", "area", "level", "source_type",
|
|
"confidence", "period", "last_verified", "title", "claim", "url", "content_hash",
|
|
"embedding_model", "embedding_dim", "chunker_version", "chunk_tokens",
|
|
"platform", "tier", "origin_url", "path", "last_modified",
|
|
"public", "artefacts", "tech", "tags", "text",
|
|
)
|
|
|
|
_JSON_FIELDS = ("artefacts", "tech", "tags")
|
|
|
|
|
|
class StoreError(RuntimeError):
|
|
"""The corpus database is unreadable or inconsistent."""
|
|
|
|
|
|
class Store:
|
|
def __init__(self, path: str) -> None:
|
|
self.path = path
|
|
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
|
|
self.db = sqlite3.connect(path)
|
|
self.db.row_factory = sqlite3.Row
|
|
self.db.execute("PRAGMA journal_mode = WAL")
|
|
self.db.execute("PRAGMA synchronous = NORMAL")
|
|
self.db.executescript(SCHEMA)
|
|
stored = self.get_meta("schema_version")
|
|
# A database from an older payload shape has no provenance columns. It is
|
|
# derived data, so the fix is a rebuild, but it must never be silently
|
|
# mixed with the new shape: `rebuild_schema` is called by a rebuild only.
|
|
self.schema_stale = stored not in (None, SCHEMA_VERSION)
|
|
if not self.schema_stale:
|
|
self.set_meta("schema_version", SCHEMA_VERSION)
|
|
|
|
# -- lifecycle --------------------------------------------------------
|
|
|
|
def close(self) -> None:
|
|
self.db.close()
|
|
|
|
def __enter__(self) -> "Store":
|
|
return self
|
|
|
|
def __exit__(self, *_exc) -> None:
|
|
self.close()
|
|
|
|
def reset(self) -> None:
|
|
"""Empty every table. Used by a from-scratch rebuild."""
|
|
for table in ("chunk_fts", "chunks", "manifest", "meta", "graph_fts", "graph_nodes", "graph_edges"):
|
|
self.db.execute(f"DELETE FROM {table}")
|
|
self.db.commit()
|
|
|
|
def rebuild_schema(self) -> None:
|
|
"""Drop and recreate every table. Only a rebuild may call this."""
|
|
for table in (
|
|
"chunk_fts", "chunks", "manifest", "meta",
|
|
"graph_fts", "graph_nodes", "graph_edges",
|
|
):
|
|
self.db.execute(f"DROP TABLE IF EXISTS {table}")
|
|
self.db.commit()
|
|
self.db.executescript(SCHEMA)
|
|
self.schema_stale = False
|
|
self.set_meta("schema_version", SCHEMA_VERSION)
|
|
|
|
# -- writing ----------------------------------------------------------
|
|
|
|
def write_document(self, doc, payloads: list) -> None:
|
|
"""Replace all rows for one document: chunks, FTS rows and manifest."""
|
|
self.delete_document(doc.path, keep_manifest=True)
|
|
for payload in payloads:
|
|
values = []
|
|
for column in _CHUNK_COLUMNS:
|
|
value = payload[column]
|
|
if column in _JSON_FIELDS:
|
|
value = json.dumps(value, sort_keys=True)
|
|
values.append(value)
|
|
cursor = self.db.execute(
|
|
f"INSERT INTO chunks ({', '.join(_CHUNK_COLUMNS)}) "
|
|
f"VALUES ({', '.join('?' for _ in _CHUNK_COLUMNS)})",
|
|
values,
|
|
)
|
|
# `terms` is the keyword surface: the words a visitor or a recruiter
|
|
# would type that live in front-matter rather than the body — tech
|
|
# names, tags, the one-line claim. Without it, a search for "zigbee"
|
|
# misses a page whose body says "mesh". It is indexed, never retrieved.
|
|
terms = " ".join(
|
|
str(value)
|
|
for value in (
|
|
payload["claim"],
|
|
payload["title"],
|
|
" ".join(payload["tech"]),
|
|
" ".join(payload["tags"]),
|
|
)
|
|
if value
|
|
)
|
|
self.db.execute(
|
|
"INSERT INTO chunk_fts "
|
|
"(rowid, id, source_path, title, terms, text, area, confidence, platform, tier) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
cursor.lastrowid,
|
|
payload["id"],
|
|
payload["source_path"],
|
|
payload["title"],
|
|
terms,
|
|
payload["text"],
|
|
payload["area"],
|
|
payload["confidence"],
|
|
payload["platform"],
|
|
payload["tier"],
|
|
),
|
|
)
|
|
self.db.execute(
|
|
"INSERT OR REPLACE INTO manifest "
|
|
"(source_path, content_hash, chunk_count, embedding_model, embedding_dim, "
|
|
" chunker_version, last_verified, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
doc.path,
|
|
doc.digest,
|
|
len(payloads),
|
|
payloads[0]["embedding_model"] if payloads else "",
|
|
payloads[0]["embedding_dim"] if payloads else 0,
|
|
payloads[0]["chunker_version"] if payloads else "",
|
|
doc.last_verified,
|
|
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
),
|
|
)
|
|
self.db.commit()
|
|
|
|
def delete_document(self, source_path: str, keep_manifest: bool = False) -> int:
|
|
rowids = [
|
|
row["rowid"]
|
|
for row in self.db.execute("SELECT rowid FROM chunks WHERE source_path = ?", (source_path,))
|
|
]
|
|
for rowid in rowids:
|
|
self.db.execute("DELETE FROM chunk_fts WHERE rowid = ?", (rowid,))
|
|
self.db.execute("DELETE FROM chunks WHERE source_path = ?", (source_path,))
|
|
if not keep_manifest:
|
|
self.db.execute("DELETE FROM manifest WHERE source_path = ?", (source_path,))
|
|
self.db.commit()
|
|
return len(rowids)
|
|
|
|
# -- reading ----------------------------------------------------------
|
|
|
|
def manifest(self) -> dict:
|
|
return {
|
|
row["source_path"]: dict(row)
|
|
for row in self.db.execute("SELECT * FROM manifest")
|
|
}
|
|
|
|
def chunk_count(self) -> int:
|
|
return int(self.db.execute("SELECT count(*) FROM chunks").fetchone()[0])
|
|
|
|
def source_count(self) -> int:
|
|
return int(self.db.execute("SELECT count(DISTINCT source_path) FROM chunks").fetchone()[0])
|
|
|
|
def chunk_ids(self) -> list:
|
|
return [row["id"] for row in self.db.execute("SELECT id FROM chunks ORDER BY id")]
|
|
|
|
def chunk_payloads(self, source_path: str = None) -> list:
|
|
"""Every stored payload, optionally for one document. Used by verify()."""
|
|
sql = "SELECT * FROM chunks"
|
|
args = ()
|
|
if source_path:
|
|
sql += " WHERE source_path = ?"
|
|
args = (source_path,)
|
|
sql += " ORDER BY source_path, chunk_index"
|
|
out = []
|
|
for row in self.db.execute(sql, args):
|
|
payload = dict(row)
|
|
for field in _JSON_FIELDS:
|
|
payload[field] = json.loads(payload[field])
|
|
out.append(payload)
|
|
return out
|
|
|
|
def keyword_search(
|
|
self, query: str, area: str = None, confidence: str = None,
|
|
platform: str = None, tier: int = None, limit: int = 10,
|
|
) -> list:
|
|
"""BM25 over chunk text, optionally filtered before ranking."""
|
|
match = match_expression(query)
|
|
if not match:
|
|
return []
|
|
sql = [
|
|
"SELECT c.id, c.url, c.title, c.area, c.confidence, c.source_path,",
|
|
" c.chunk_index, snippet(chunk_fts, 4, '<mark>', '</mark>', '…', 18) AS snippet,",
|
|
" bm25(chunk_fts) AS score",
|
|
"FROM chunk_fts JOIN chunks c ON c.rowid = chunk_fts.rowid",
|
|
"WHERE chunk_fts MATCH ?",
|
|
]
|
|
args = [match]
|
|
if area:
|
|
sql.append("AND c.area = ?")
|
|
args.append(area)
|
|
if confidence:
|
|
sql.append("AND c.confidence = ?")
|
|
args.append(confidence)
|
|
if platform:
|
|
sql.append("AND c.platform = ?")
|
|
args.append(platform)
|
|
if tier is not None:
|
|
sql.append("AND c.tier = ?")
|
|
args.append(int(tier))
|
|
sql.append("ORDER BY bm25(chunk_fts) LIMIT ?")
|
|
args.append(int(limit))
|
|
rows = self.db.execute(" ".join(sql), args)
|
|
return [dict(row) for row in rows]
|
|
|
|
def pool_texts(self) -> list:
|
|
"""Every chunk text in the pool. The credential scan reads this."""
|
|
return [
|
|
(row["source_path"], row["chunk_index"], row["text"])
|
|
for row in self.db.execute("SELECT source_path, chunk_index, text FROM chunks")
|
|
]
|
|
|
|
# -- graph ------------------------------------------------------------
|
|
|
|
def write_graph(self, nodes: list, edges: list) -> None:
|
|
"""Replace the whole graph. Nodes and edges are rebuilt together."""
|
|
self.db.execute("DELETE FROM graph_fts")
|
|
self.db.execute("DELETE FROM graph_nodes")
|
|
self.db.execute("DELETE FROM graph_edges")
|
|
for node in nodes:
|
|
self.db.execute(
|
|
"INSERT INTO graph_nodes (id, label, norm_label, node_type, community, "
|
|
"source_note, origin_url, area, tier, platform, last_modified, content_hash, text) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
node["id"], node["label"], node.get("norm_label") or _norm(node["label"]),
|
|
node.get("node_type") or "entity", node.get("community"),
|
|
node["source_note"], node["origin_url"], node["area"], node["tier"],
|
|
node["platform"], node["last_modified"], node["content_hash"], node["text"],
|
|
),
|
|
)
|
|
self.db.execute(
|
|
"INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'node', ?, ?, ?)",
|
|
(node["id"], node["label"], node["text"], node["area"]),
|
|
)
|
|
for edge in edges:
|
|
self.db.execute(
|
|
"INSERT INTO graph_edges (id, source_id, target_id, relation, confidence, "
|
|
"confidence_score, source_note, origin_url, area, tier, platform, "
|
|
"last_modified, content_hash, text) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
edge["id"], edge["source_id"], edge["target_id"], edge["relation"],
|
|
edge["confidence"], edge.get("confidence_score"), edge["source_note"],
|
|
edge["origin_url"], edge["area"], edge["tier"], edge["platform"],
|
|
edge["last_modified"], edge["content_hash"], edge["text"],
|
|
),
|
|
)
|
|
self.db.execute(
|
|
"INSERT INTO graph_fts (id, kind, label, text, area) VALUES (?, 'edge', ?, ?, ?)",
|
|
(edge["id"], edge["relation"], edge["text"], edge["area"]),
|
|
)
|
|
self.db.commit()
|
|
|
|
def graph_counts(self) -> dict:
|
|
return {
|
|
"nodes": int(self.db.execute("SELECT count(*) FROM graph_nodes").fetchone()[0]),
|
|
"edges": int(self.db.execute("SELECT count(*) FROM graph_edges").fetchone()[0]),
|
|
}
|
|
|
|
def graph_node(self, node_id: str) -> dict:
|
|
row = self.db.execute("SELECT * FROM graph_nodes WHERE id = ?", (node_id,)).fetchone()
|
|
return dict(row) if row else {}
|
|
|
|
def graph_neighbours(self, node_id: str, limit: int = 50) -> list:
|
|
rows = self.db.execute(
|
|
"SELECT e.*, sn.label AS source_label, tn.label AS target_label "
|
|
"FROM graph_edges e "
|
|
"LEFT JOIN graph_nodes sn ON sn.id = e.source_id "
|
|
"LEFT JOIN graph_nodes tn ON tn.id = e.target_id "
|
|
"WHERE e.source_id = ? OR e.target_id = ? LIMIT ?",
|
|
(node_id, node_id, int(limit)),
|
|
)
|
|
return [dict(row) for row in rows]
|
|
|
|
def graph_search(self, query: str, area: str = None, limit: int = 10) -> list:
|
|
"""Keyword search over node labels and edge relations, with provenance."""
|
|
match = match_expression(query)
|
|
if not match:
|
|
return []
|
|
sql = [
|
|
"SELECT f.id, f.kind, f.label, f.text, f.area,",
|
|
" COALESCE(n.source_note, e.source_note) AS source_note,",
|
|
" COALESCE(n.origin_url, e.origin_url) AS origin_url,",
|
|
" n.community, e.relation, e.source_id, e.target_id",
|
|
"FROM graph_fts f",
|
|
"LEFT JOIN graph_nodes n ON f.kind = 'node' AND n.id = f.id",
|
|
"LEFT JOIN graph_edges e ON f.kind = 'edge' AND e.id = f.id",
|
|
"WHERE graph_fts MATCH ?",
|
|
]
|
|
args = [match]
|
|
if area:
|
|
sql.append("AND f.area = ?")
|
|
args.append(area)
|
|
sql.append("ORDER BY bm25(graph_fts) LIMIT ?")
|
|
args.append(int(limit))
|
|
return [dict(row) for row in self.db.execute(" ".join(sql), args)]
|
|
|
|
# -- meta -------------------------------------------------------------
|
|
|
|
def set_meta(self, key: str, value) -> None:
|
|
self.db.execute(
|
|
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
|
|
(key, json.dumps(value, sort_keys=True)),
|
|
)
|
|
self.db.commit()
|
|
|
|
def get_meta(self, key: str, default=None):
|
|
row = self.db.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
|
if row is None:
|
|
return default
|
|
return json.loads(row["value"])
|
|
|
|
def meta(self) -> dict:
|
|
return {row["key"]: json.loads(row["value"]) for row in self.db.execute("SELECT key, value FROM meta")}
|
|
|
|
|
|
def _norm(label: str) -> str:
|
|
return re.sub(r"[^0-9a-z]+", "_", (label or "").lower()).strip("_")
|
|
|
|
|
|
def match_expression(query: str) -> str:
|
|
"""Free text -> a safe FTS5 expression: bare words, each a prefix, all required.
|
|
|
|
Mirrors `MatchExpression` in internal/search/search.go so the site's search
|
|
box and the retrieval path treat a visitor's words identically. FTS5
|
|
operators in the input are never honoured.
|
|
"""
|
|
terms = []
|
|
for raw in re.split(r"[^0-9a-zA-Z\-]+", unicodedata.normalize("NFKC", query or "").lower()):
|
|
term = raw.strip("-")
|
|
if term:
|
|
terms.append('"' + term.replace('"', "") + '"*')
|
|
return " AND ".join(terms)
|