"""Configuration for the corpus pipeline. Environment only. Nothing in this file holds a credential. The OmniRoute key is read from the process environment, which systemd populates from `~/.config/environment.d/10-secrets.conf` (AGENTS.md section 4.4). `describe()` is the only way the pipeline prints its configuration, and it redacts the key. """ from __future__ import annotations import os from dataclasses import dataclass # Bumped whenever chunk boundaries or payload shape change. A stored corpus # carrying a different version is stale by definition and gets rebuilt. CHUNKER_VERSION = "blocks-v2-code-v1" # Provider defaults. `local-hashing` is a deterministic, offline provider used # by the tests, by `make corpus-test`, and by any rebuild on a host with no # model gateway. It is NOT a semantic embedding model: it exists so the whole # pipeline can be exercised without a provider decision, and so a rebuild never # depends on the network. The production provider is `omniroute`. PROVIDERS = { "local-hashing": ("local-hashing-v1", 256), "omniroute": ("text-embedding-3-small", 1536), } DEFAULT_QDRANT_URL = "http://127.0.0.1:6333" DEFAULT_COLLECTION = "resume_corpus" DEFAULT_PROVIDER = "local-hashing" DEFAULT_OMNIROUTE_URL = "https://omniroute.lab.audasmedia.com.au" DEFAULT_GITEA_URL = "https://gitea.lab.audasmedia.com.au" DEFAULT_PLATFORMS = ("gitea", "obsidian", "graphify") DEFAULT_GRAPH_COLLECTION = "resume_graph" # A source that has not changed in this window is reported by the audit. The # estate is active; silence is evidence that ingestion stopped, not that the # work stopped. DEFAULT_SOURCE_SILENCE_DAYS = 30 # Freshness horizon: a document whose `last_verified` is older than this is # reported stale by the nightly audit. docs/phase-4-notes.md section 5. DEFAULT_MAX_AGE_DAYS = 180 DEFAULT_WARN_DAYS = 30 class ConfigError(RuntimeError): """A configuration value is missing or unusable.""" @dataclass(frozen=True) class Config: """Everything the pipeline needs, resolved once.""" root: str content_dir: str data_dir: str db_path: str qdrant_url: str collection: str provider: str model: str dim: int api_url: str api_key: str batch_size: int timeout: float max_age_days: int warn_days: int apprise_url: str gitea_url: str = DEFAULT_GITEA_URL gitea_token: str = "" platforms: tuple = DEFAULT_PLATFORMS offline: bool = False graph_path: str = "" graph_collection: str = DEFAULT_GRAPH_COLLECTION source_silence_days: int = DEFAULT_SOURCE_SILENCE_DAYS vectors_on_disk: bool = False # -- construction ----------------------------------------------------- @classmethod def from_env(cls, root: str | None = None, **overrides) -> "Config": """Build a Config from the environment, then apply explicit overrides. Overrides exist so the CLI can accept flags without a second code path; they are applied last so a flag always wins over the environment. """ root = os.path.abspath(root or os.environ.get("RESUME_ROOT") or _default_root()) content_dir = os.environ.get("RESUME_CONTENT") or os.path.join(root, "content") data_dir = os.environ.get("RESUME_CORPUS_DATA") or os.path.join(root, "data", "corpus") provider = os.environ.get("RESUME_EMBED_PROVIDER") or DEFAULT_PROVIDER if provider not in PROVIDERS: raise ConfigError( f"RESUME_EMBED_PROVIDER={provider!r} is not one of {sorted(PROVIDERS)}" ) default_model, default_dim = PROVIDERS[provider] values = dict( root=root, content_dir=os.path.abspath(content_dir), data_dir=os.path.abspath(data_dir), db_path=os.path.abspath( os.environ.get("RESUME_CORPUS_DB") or os.path.join(data_dir, "corpus.db") ), qdrant_url=(os.environ.get("RESUME_QDRANT_URL") or DEFAULT_QDRANT_URL).rstrip("/"), collection=os.environ.get("RESUME_COLLECTION") or DEFAULT_COLLECTION, provider=provider, model=os.environ.get("RESUME_EMBED_MODEL") or default_model, dim=int(os.environ.get("RESUME_EMBED_DIM") or default_dim), api_url=(os.environ.get("OMNIROUTE_URL") or DEFAULT_OMNIROUTE_URL).rstrip("/"), api_key=os.environ.get("OMNIROUTE_API_KEY") or "", batch_size=int(os.environ.get("RESUME_EMBED_BATCH") or 32), timeout=float(os.environ.get("RESUME_QDRANT_TIMEOUT") or 30.0), max_age_days=int( os.environ.get("RESUME_MAX_AGE_DAYS") or DEFAULT_MAX_AGE_DAYS ), warn_days=int(os.environ.get("RESUME_WARN_DAYS") or DEFAULT_WARN_DAYS), apprise_url=os.environ.get("RESUME_APPRISE_URL") or "", gitea_url=(os.environ.get("GITEA_URL") or os.environ.get("RESUME_GITEA_URL") or DEFAULT_GITEA_URL).rstrip("/"), gitea_token=os.environ.get("GITEA_TOKEN") or "", platforms=tuple( part.strip() for part in (os.environ.get("RESUME_PLATFORMS") or ",".join(DEFAULT_PLATFORMS)).split(",") if part.strip() ), offline=(os.environ.get("RESUME_OFFLINE") or "").lower() in ("1", "true", "yes"), graph_path=os.environ.get("RESUME_GRAPH_PATH") or "", graph_collection=os.environ.get("RESUME_GRAPH_COLLECTION") or DEFAULT_GRAPH_COLLECTION, source_silence_days=int( os.environ.get("RESUME_SOURCE_SILENCE_DAYS") or DEFAULT_SOURCE_SILENCE_DAYS ), vectors_on_disk=(os.environ.get("RESUME_QDRANT_VECTORS_ON_DISK") or "").lower() in ("1", "true", "yes"), ) values.update({k: v for k, v in overrides.items() if v not in (None, "")}) cfg = cls(**values) cfg.check() return cfg # -- validation ------------------------------------------------------- def check(self) -> None: if self.dim <= 0: raise ConfigError(f"embedding dimension must be positive, got {self.dim}") if self.batch_size <= 0: raise ConfigError(f"batch size must be positive, got {self.batch_size}") if self.provider == "omniroute" and not self.api_key: raise ConfigError( "provider 'omniroute' needs OMNIROUTE_API_KEY in the environment " "(~/.config/environment.d/10-secrets.conf). It is never read from this repo." ) if not os.path.isdir(self.content_dir): raise ConfigError(f"content directory not found: {self.content_dir}") unknown = [name for name in self.platforms if name not in ("gitea", "obsidian", "graphify")] if unknown: raise ConfigError( f"RESUME_PLATFORMS contains unknown platform(s) {unknown}; " "expected gitea, obsidian, graphify" ) if not self.offline and self.gitea_token == "": # Only a hard error when a platform that needs Gitea is selected; the # token itself is never printed or stored. if set(self.platforms) & {"gitea", "obsidian", "graphify"}: raise ConfigError( "GITEA_TOKEN is not set. Harvesting needs read access to Gitea " "(~/.config/environment.d/10-secrets.conf), or set RESUME_OFFLINE=1 " "to rebuild only the curated tier." ) # -- presentation ----------------------------------------------------- def describe(self) -> str: """Human-readable configuration. Never includes the API key value.""" key = "set" if self.api_key else "unset" return ( f"root={self.root}\n" f"content={self.content_dir}\n" f"data={self.data_dir}\n" f"db={self.db_path}\n" f"qdrant={self.qdrant_url} collection={self.collection}\n" f"provider={self.provider} model={self.model} dim={self.dim} " f"batch={self.batch_size}\n" f"api_url={self.api_url} api_key={key}\n" f"freshness=max_age_days:{self.max_age_days} warn_days:{self.warn_days}\n" f"estate=platforms:{','.join(self.platforms)} offline:{self.offline} " f"gitea:{self.gitea_url} token:{'set' if self.gitea_token else 'unset'} " f"graph_path:{self.graph_path or 'from gitea'} silence_days:{self.source_silence_days}\n" f"qdrant vectors_on_disk={self.vectors_on_disk} graph_collection={self.graph_collection}\n" f"chunker={CHUNKER_VERSION}" ) # -- derived paths ---------------------------------------------------- @property def chunker_version(self) -> str: return CHUNKER_VERSION @property def meta_path(self) -> str: return os.path.join(self.data_dir, "collection.json") def _default_root() -> str: """The repository root: the parent of this package.""" return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))