33ec9b5bc5
Classifies notes from an inbox folder and files them, with a human approval step. Zero third-party dependencies. - type mirrors folder (200 projects / 300 areas / 400 Personal Family) - TypeSafe JEV for semantic classification (~$0.0001 per note) - watcher writes a proposal INTO the note; the owner ticks a checkbox - systemd user service on .27, 10s poll - 15/15 functional tests
290 lines
10 KiB
Python
290 lines
10 KiB
Python
"""
|
|
Tier 1 deterministic classifier.
|
|
|
|
Simplified 2026-09-24. The rule is now trivial:
|
|
|
|
type mirrors folder
|
|
|
|
200 projects/ -> project
|
|
300 areas/ -> area
|
|
400 Personal Family/ -> family
|
|
|
|
So the classifier's only real job is to choose a DESTINATION FOLDER. The type
|
|
then falls out of the folder automatically — nothing is inferred from content
|
|
beyond a weak first guess.
|
|
|
|
Because picking the right folder is genuinely hard, the rules here are
|
|
deliberately conservative: when they are not confident, the note escalates to
|
|
Tier 2 (JEV).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
from .models import (
|
|
BUCKETS,
|
|
Note,
|
|
Proposal,
|
|
PLACEHOLDER_TYPE,
|
|
VALID_TYPES,
|
|
type_from_folder,
|
|
)
|
|
from .vault import Vault
|
|
|
|
# --------------------------------------------------------------- TOC + templates
|
|
|
|
# Which index note lists each type.
|
|
TYPE_TO_TOC: dict[str, str] = {
|
|
"project": "100 Table of Contents/Projects.md",
|
|
"area": "100 Table of Contents/Areas.md",
|
|
"family": "100 Table of Contents/Personal Family.md",
|
|
"index": "100 Table of Contents/Table of Contents.md",
|
|
}
|
|
|
|
# The template a type should use.
|
|
TYPE_TO_TEMPLATE: dict[str, str] = {
|
|
"project": "Project.md",
|
|
"area": "Area.md",
|
|
"family": "Family.md",
|
|
"index": "Index.md",
|
|
}
|
|
|
|
# ------------------------------------------------------------------- hints
|
|
|
|
# Keywords that weakly suggest a bucket. Used only as a first guess; JEV does
|
|
# the real work. Order of checks: family, then project, else area.
|
|
FAMILY_HINTS = [
|
|
"recipe", "birthday", "holiday", "family", "movie", "film", "book",
|
|
"game", "travel", "ski", "camp", "garden", "aquaponic", "art", "music",
|
|
"photography", "investment", "business idea", "school", "basketball",
|
|
]
|
|
PROJECT_HINTS = [
|
|
"phase", "milestone", "build plan", "roadmap", "work log", "checklist",
|
|
"deliverable", "to do", "todo", "needs to be built", "not built",
|
|
"original plan was", "next steps", "in progress",
|
|
]
|
|
|
|
STOPWORDS = {
|
|
"the", "and", "for", "with", "from", "this", "that", "into", "your",
|
|
"notes", "note", "about", "overview", "setup", "guide", "using", "new",
|
|
}
|
|
|
|
|
|
def _tokens(text: str) -> set[str]:
|
|
return {
|
|
w for w in re.findall(r"[a-z0-9]+", text.lower())
|
|
if w not in STOPWORDS and len(w) > 2
|
|
}
|
|
|
|
|
|
def explicit_bucket_from_tags(note: "Note") -> str | None:
|
|
"""
|
|
If the author tagged the note with a bucket name, honour it.
|
|
|
|
Your own tag is the strongest signal available — a human already decided.
|
|
"""
|
|
tags = {t.lower() for t in (note.tags or [])}
|
|
if "project" in tags:
|
|
return "project"
|
|
if tags & {"family", "personal"}:
|
|
return "family"
|
|
return None
|
|
|
|
|
|
def explicit_bucket_from_tags(note: "Note") -> str | None:
|
|
"""
|
|
If the author tagged the note with a bucket name, honour it.
|
|
|
|
Your own tag is the strongest signal available — a human already decided.
|
|
"""
|
|
tags = {t.lower() for t in (note.tags or [])}
|
|
if "project" in tags:
|
|
return "project"
|
|
if tags & {"family", "personal"}:
|
|
return "family"
|
|
return None
|
|
|
|
|
|
# A tag can pin a note to a specific folder — but ONLY when the tag names a
|
|
# clear CATEGORY (where the note belongs), never when it describes the note's
|
|
# nature. 'recipe' is a category; 'business-idea' is a property, so it must not
|
|
# pin anything — an electronics business idea still belongs in projects.
|
|
TAG_FOLDER_HINTS: dict[str, str] = {
|
|
"recipe": "400 Personal Family/450 Recipes",
|
|
"holiday": "400 Personal Family/470 Holidays Travel",
|
|
"travel": "400 Personal Family/470 Holidays Travel",
|
|
"ski": "400 Personal Family/470 Holidays Travel",
|
|
"film": "400 Personal Family/430 Movies Books Plays Games",
|
|
}
|
|
|
|
|
|
def explicit_folder_from_tags(note: "Note") -> str | None:
|
|
"""Return a folder pinned by one of the note's tags, if any."""
|
|
tags = {t.lower() for t in (note.tags or [])}
|
|
for tag, folder in TAG_FOLDER_HINTS.items():
|
|
if tag in tags:
|
|
return folder
|
|
return None
|
|
|
|
|
|
@dataclass
|
|
class Guess:
|
|
bucket: str
|
|
score: float
|
|
reason: str
|
|
|
|
|
|
class Classifier:
|
|
"""Deterministic (Tier 1) folder classifier."""
|
|
|
|
def __init__(self, vault: Vault):
|
|
self.vault = vault
|
|
self._folders: list[str] | None = None
|
|
|
|
@property
|
|
def folders(self) -> list[str]:
|
|
if self._folders is None:
|
|
self._folders = self.vault.folders()
|
|
return self._folders
|
|
|
|
# ------------------------------------------------------------------ public
|
|
|
|
def classify(self, note: Note) -> Proposal:
|
|
"""Produce a Proposal for a note. Never writes anything."""
|
|
# --- guard: untitled notes are never moved -------------------------
|
|
if note.stem.lower().startswith("untitled"):
|
|
return Proposal(
|
|
source_rel=note.rel_path, note_type="", target_folder="",
|
|
toc_section="", template="", confidence=0.0, method="blocked",
|
|
reasons=["Filename begins with 'Untitled'"],
|
|
warnings=[
|
|
"BLOCKED: untitled notes are never moved automatically. "
|
|
"Rename it first if it is real content."
|
|
],
|
|
)
|
|
|
|
current = (note.note_type or "").strip().lower()
|
|
expected = note.expected_type
|
|
|
|
# --- a tag can pin the folder outright -----------------------------
|
|
pinned = explicit_folder_from_tags(note)
|
|
if pinned and (note.folder != pinned or current != type_from_folder(pinned)):
|
|
return Proposal(
|
|
source_rel=note.rel_path,
|
|
note_type=type_from_folder(pinned) or "family",
|
|
target_folder=pinned,
|
|
toc_section=TYPE_TO_TOC.get(type_from_folder(pinned) or "family", ""),
|
|
template=TYPE_TO_TEMPLATE.get(type_from_folder(pinned) or "family", "Family.md"),
|
|
confidence=0.95,
|
|
method="deterministic",
|
|
reasons=[f"your tag pinned this to {pinned}"],
|
|
)
|
|
|
|
# --- already correctly filed? --------------------------------------
|
|
if expected and current == expected:
|
|
toc = TYPE_TO_TOC.get(current, "")
|
|
return Proposal(
|
|
source_rel=note.rel_path, note_type=current,
|
|
target_folder=note.folder,
|
|
toc_section=toc, template=TYPE_TO_TEMPLATE.get(current, "Area.md"),
|
|
confidence=1.0, method="trusted",
|
|
reasons=[f"type matches its folder ({expected})"],
|
|
)
|
|
|
|
# --- weak first guess ----------------------------------------------
|
|
guess = self._guess_bucket(note)
|
|
if guess is None:
|
|
return Proposal(
|
|
source_rel=note.rel_path, note_type="", target_folder="",
|
|
toc_section="", template="", confidence=0.0, method="none",
|
|
reasons=["No deterministic match"],
|
|
warnings=["No rule matched — requires Tier 2 (JEV)"],
|
|
)
|
|
|
|
folder, matched = self._pick_folder(note, guess.bucket)
|
|
toc = TYPE_TO_TOC.get(guess.bucket, "")
|
|
|
|
# A blind folder guess (no keyword evidence) must NOT look confident, or
|
|
# the note never escalates to JEV and lands in an arbitrary folder.
|
|
confidence = guess.score if matched else min(guess.score, 0.25)
|
|
|
|
warnings: list[str] = []
|
|
if not matched:
|
|
warnings.append("No keyword evidence for a folder — likely needs Tier 2 (JEV)")
|
|
elif confidence < 0.6:
|
|
warnings.append("Low confidence — likely needs Tier 2 (JEV)")
|
|
|
|
reasons = [guess.reason]
|
|
if matched:
|
|
reasons.append(f"folder matched by keyword: {folder}")
|
|
|
|
return Proposal(
|
|
source_rel=note.rel_path,
|
|
note_type=guess.bucket,
|
|
target_folder=folder,
|
|
toc_section=toc,
|
|
template=TYPE_TO_TEMPLATE.get(guess.bucket, "Area.md"),
|
|
confidence=confidence,
|
|
method="deterministic",
|
|
reasons=reasons,
|
|
warnings=warnings,
|
|
)
|
|
|
|
# --------------------------------------------------------------- internals
|
|
|
|
def _guess_bucket(self, note: Note) -> Guess | None:
|
|
# Your own tags win: if you tagged it 'project' or 'family', believe you.
|
|
tagged = explicit_bucket_from_tags(note)
|
|
if tagged:
|
|
return Guess(tagged, 0.9, f"you tagged this note '{tagged}'")
|
|
|
|
text = " ".join(
|
|
[(note.title or note.stem).lower(), " ".join(t.lower() for t in note.tags)]
|
|
).lower()
|
|
|
|
fam = sum(1 for k in FAMILY_HINTS if k in text)
|
|
proj = sum(1 for k in PROJECT_HINTS if k in text)
|
|
|
|
# An explicit `type: note` says nothing useful; rely on keywords.
|
|
if current := (note.note_type or "").strip().lower():
|
|
if current in BUCKETS and current != PLACEHOLDER_TYPE:
|
|
return Guess(current, 0.5, f"existing type {current}")
|
|
|
|
if fam and fam >= proj:
|
|
return Guess("family", min(0.7, 0.35 + 0.15 * fam),
|
|
f"{fam} family keyword(s) matched")
|
|
if proj:
|
|
return Guess("project", min(0.7, 0.35 + 0.15 * proj),
|
|
f"{proj} project keyword(s) matched")
|
|
|
|
# Default bucket is area — the catch-all for ongoing material.
|
|
return Guess("area", 0.3, "defaulted to area (no strong signal)")
|
|
|
|
def _pick_folder(self, note: Note, bucket: str) -> tuple[str, bool]:
|
|
"""
|
|
Choose a folder inside the bucket.
|
|
|
|
Returns (folder, matched) where `matched` is False when there was no
|
|
keyword evidence and the choice is effectively arbitrary.
|
|
"""
|
|
candidates = [
|
|
f for f in self.folders
|
|
if type_from_folder(f) == bucket
|
|
]
|
|
if not candidates:
|
|
return "", False
|
|
|
|
target = _tokens((note.title or note.stem) + " " + " ".join(note.tags))
|
|
best, best_score = None, 0
|
|
for cand in candidates:
|
|
score = len(target & _tokens(cand))
|
|
if score > best_score or (score == best_score and best is None):
|
|
best, best_score = cand, score
|
|
|
|
if best_score == 0:
|
|
# No evidence — return a candidate but flag it as unmatched.
|
|
return candidates[0], False
|
|
return best or candidates[0], True
|