Phase 4: corpus and retrieval — chunks, Qdrant, FTS5, freshness audit
Satisfies docs/phase-4-corpus.md. content/ stays the single source of truth:
one pass over it writes both indexes, so the static pages and the retrieval
corpus cannot drift.
- corpus/: stdlib-only pipeline. validate -> chunk -> enrich -> embed -> upsert.
Chunking is block-based, so a boundary cannot fall inside a fenced block, a
table or front-matter (verified structurally, not by inspection).
- Deterministic ids: sha256(source_path + chunk_index + content_hash), rendered
as a UUID. Re-running updates points; re-chunking deletes the old ones by
source filter, so duplicates and orphans are both impossible.
- Incremental by content hash against a SQLite manifest. A provider, dimension
or chunker change is refused instead of mixed: it needs a rebuild.
- Payloads per section 4.2 plus provenance, validated before any vector is
written; a missing field fails the run and names the document and chunk.
- SQLite FTS5 over chunks (the keyword half of hybrid retrieval, decision D10)
with area/confidence filters and a `terms` column so front-matter tech and
tags are findable.
- Freshness audit: stale documents, due soon, index drift, contract drift;
optional Apprise alert; exit code for Prefect.
- Embeddings behind one interface: omniroute (OpenAI-compatible, retries, key
from the environment only) and local-hashing (offline placeholder). The
provider decision is still Sam's; the pipeline does not depend on it.
- deploy/qdrant.yml: mem_limit = memswap_limit = 400m, loopback only.
- docs/phase-4-notes.md: operator manual, DoD evidence, measured memory.
- 76 unit tests, no container and no network.
Measured, Qdrant 1.20.0 under a 400 m cap at 1024 dimensions: 500 / 5,000 /
20,000 chunks settle at 128 / 138 / 151 MiB, peaking at 149 / 305 / 292 MiB
during ingest. Qdrant's default settings are OOM-killed between 5,000 and
20,000, so the collection is created with HNSW and payloads on disk; recorded in
reference/stack.md with the ceiling and the escalation order.
Note: the `make corpus*` targets were swept into the Phase 3 deploy commit
24590cc by concurrent work in this tree; the Makefile content is correct, and
this commit contains the pipeline they call.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
160
corpus/config.py
Normal file
160
corpus/config.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""Configuration for the corpus pipeline. Environment only.
|
||||
|
||||
Nothing in this file holds a credential. The OmniRoute key is read from the
|
||||
process environment, which systemd populates from
|
||||
`~/.config/environment.d/10-secrets.conf` (AGENTS.md section 4.4). `describe()`
|
||||
is the only way the pipeline prints its configuration, and it redacts the key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Bumped whenever chunk boundaries or payload shape change. A stored corpus
|
||||
# carrying a different version is stale by definition and gets rebuilt.
|
||||
CHUNKER_VERSION = "blocks-v1"
|
||||
|
||||
# Provider defaults. `local-hashing` is a deterministic, offline provider used
|
||||
# by the tests, by `make corpus-test`, and by any rebuild on a host with no
|
||||
# model gateway. It is NOT a semantic embedding model: it exists so the whole
|
||||
# pipeline can be exercised without a provider decision, and so a rebuild never
|
||||
# depends on the network. The production provider is `omniroute`.
|
||||
PROVIDERS = {
|
||||
"local-hashing": ("local-hashing-v1", 256),
|
||||
"omniroute": ("text-embedding-3-small", 1536),
|
||||
}
|
||||
|
||||
DEFAULT_QDRANT_URL = "http://127.0.0.1:6333"
|
||||
DEFAULT_COLLECTION = "resume_corpus"
|
||||
DEFAULT_PROVIDER = "local-hashing"
|
||||
DEFAULT_OMNIROUTE_URL = "https://omniroute.lab.audasmedia.com.au"
|
||||
|
||||
# Freshness horizon: a document whose `last_verified` is older than this is
|
||||
# reported stale by the nightly audit. docs/phase-4-notes.md section 5.
|
||||
DEFAULT_MAX_AGE_DAYS = 180
|
||||
DEFAULT_WARN_DAYS = 30
|
||||
|
||||
|
||||
class ConfigError(RuntimeError):
|
||||
"""A configuration value is missing or unusable."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
"""Everything the pipeline needs, resolved once."""
|
||||
|
||||
root: str
|
||||
content_dir: str
|
||||
data_dir: str
|
||||
db_path: str
|
||||
qdrant_url: str
|
||||
collection: str
|
||||
provider: str
|
||||
model: str
|
||||
dim: int
|
||||
api_url: str
|
||||
api_key: str
|
||||
batch_size: int
|
||||
timeout: float
|
||||
max_age_days: int
|
||||
warn_days: int
|
||||
apprise_url: str
|
||||
vectors_on_disk: bool = False
|
||||
|
||||
# -- construction -----------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, root: str | None = None, **overrides) -> "Config":
|
||||
"""Build a Config from the environment, then apply explicit overrides.
|
||||
|
||||
Overrides exist so the CLI can accept flags without a second code path;
|
||||
they are applied last so a flag always wins over the environment.
|
||||
"""
|
||||
root = os.path.abspath(root or os.environ.get("RESUME_ROOT") or _default_root())
|
||||
content_dir = os.environ.get("RESUME_CONTENT") or os.path.join(root, "content")
|
||||
data_dir = os.environ.get("RESUME_CORPUS_DATA") or os.path.join(root, "data", "corpus")
|
||||
provider = os.environ.get("RESUME_EMBED_PROVIDER") or DEFAULT_PROVIDER
|
||||
if provider not in PROVIDERS:
|
||||
raise ConfigError(
|
||||
f"RESUME_EMBED_PROVIDER={provider!r} is not one of {sorted(PROVIDERS)}"
|
||||
)
|
||||
default_model, default_dim = PROVIDERS[provider]
|
||||
|
||||
values = dict(
|
||||
root=root,
|
||||
content_dir=os.path.abspath(content_dir),
|
||||
data_dir=os.path.abspath(data_dir),
|
||||
db_path=os.path.abspath(
|
||||
os.environ.get("RESUME_CORPUS_DB") or os.path.join(data_dir, "corpus.db")
|
||||
),
|
||||
qdrant_url=(os.environ.get("RESUME_QDRANT_URL") or DEFAULT_QDRANT_URL).rstrip("/"),
|
||||
collection=os.environ.get("RESUME_COLLECTION") or DEFAULT_COLLECTION,
|
||||
provider=provider,
|
||||
model=os.environ.get("RESUME_EMBED_MODEL") or default_model,
|
||||
dim=int(os.environ.get("RESUME_EMBED_DIM") or default_dim),
|
||||
api_url=(os.environ.get("OMNIROUTE_URL") or DEFAULT_OMNIROUTE_URL).rstrip("/"),
|
||||
api_key=os.environ.get("OMNIROUTE_API_KEY") or "",
|
||||
batch_size=int(os.environ.get("RESUME_EMBED_BATCH") or 32),
|
||||
timeout=float(os.environ.get("RESUME_QDRANT_TIMEOUT") or 30.0),
|
||||
max_age_days=int(
|
||||
os.environ.get("RESUME_MAX_AGE_DAYS") or DEFAULT_MAX_AGE_DAYS
|
||||
),
|
||||
warn_days=int(os.environ.get("RESUME_WARN_DAYS") or DEFAULT_WARN_DAYS),
|
||||
apprise_url=os.environ.get("RESUME_APPRISE_URL") or "",
|
||||
vectors_on_disk=(os.environ.get("RESUME_QDRANT_VECTORS_ON_DISK") or "").lower()
|
||||
in ("1", "true", "yes"),
|
||||
)
|
||||
values.update({k: v for k, v in overrides.items() if v not in (None, "")})
|
||||
cfg = cls(**values)
|
||||
cfg.check()
|
||||
return cfg
|
||||
|
||||
# -- validation -------------------------------------------------------
|
||||
|
||||
def check(self) -> None:
|
||||
if self.dim <= 0:
|
||||
raise ConfigError(f"embedding dimension must be positive, got {self.dim}")
|
||||
if self.batch_size <= 0:
|
||||
raise ConfigError(f"batch size must be positive, got {self.batch_size}")
|
||||
if self.provider == "omniroute" and not self.api_key:
|
||||
raise ConfigError(
|
||||
"provider 'omniroute' needs OMNIROUTE_API_KEY in the environment "
|
||||
"(~/.config/environment.d/10-secrets.conf). It is never read from this repo."
|
||||
)
|
||||
if not os.path.isdir(self.content_dir):
|
||||
raise ConfigError(f"content directory not found: {self.content_dir}")
|
||||
|
||||
# -- presentation -----------------------------------------------------
|
||||
|
||||
def describe(self) -> str:
|
||||
"""Human-readable configuration. Never includes the API key value."""
|
||||
key = "set" if self.api_key else "unset"
|
||||
return (
|
||||
f"root={self.root}\n"
|
||||
f"content={self.content_dir}\n"
|
||||
f"data={self.data_dir}\n"
|
||||
f"db={self.db_path}\n"
|
||||
f"qdrant={self.qdrant_url} collection={self.collection}\n"
|
||||
f"provider={self.provider} model={self.model} dim={self.dim} "
|
||||
f"batch={self.batch_size}\n"
|
||||
f"api_url={self.api_url} api_key={key}\n"
|
||||
f"freshness=max_age_days:{self.max_age_days} warn_days:{self.warn_days}\n"
|
||||
f"qdrant vectors_on_disk={self.vectors_on_disk}\n"
|
||||
f"chunker={CHUNKER_VERSION}"
|
||||
)
|
||||
|
||||
# -- derived paths ----------------------------------------------------
|
||||
|
||||
@property
|
||||
def chunker_version(self) -> str:
|
||||
return CHUNKER_VERSION
|
||||
|
||||
@property
|
||||
def meta_path(self) -> str:
|
||||
return os.path.join(self.data_dir, "collection.json")
|
||||
|
||||
|
||||
def _default_root() -> str:
|
||||
"""The repository root: the parent of this package."""
|
||||
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
Reference in New Issue
Block a user