obsidian-sorter: automated note filing for an Obsidian vault
Classifies notes from an inbox folder and files them, with a human approval step. Zero third-party dependencies. - type mirrors folder (200 projects / 300 areas / 400 Personal Family) - TypeSafe JEV for semantic classification (~$0.0001 per note) - watcher writes a proposal INTO the note; the owner ticks a checkbox - systemd user service on .27, 10s poll - 15/15 functional tests
This commit is contained in:
sam
536
sorter/cli.py
Normal file
536
sorter/cli.py
Normal file
@@ -0,0 +1,536 @@
|
||||
"""Command-line entry point for obsidian-sorter.
|
||||
|
||||
Phase A commands (read-only — nothing here writes to the vault):
|
||||
|
||||
scan report vault shape
|
||||
propose <file> classify one note, print a proposal
|
||||
inbox classify every note in the inbox
|
||||
validate replay the classifier against known-good notes
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
|
||||
from .classify import Classifier
|
||||
from .jev import CostTracker, JevClient, JevError
|
||||
from .models import Note
|
||||
from .tier2 import HybridClassifier, JevClassifier
|
||||
from .vault import Vault
|
||||
from .watch import Watcher
|
||||
from .writer import Writer
|
||||
|
||||
|
||||
def _load_vault(args) -> Vault:
|
||||
return Vault(args.vault)
|
||||
|
||||
|
||||
def _print_proposal(p) -> None:
|
||||
if p.method == "blocked":
|
||||
print(f" ⛔ BLOCKED — {p.source_rel}")
|
||||
for w in p.warnings:
|
||||
print(f" {w}")
|
||||
return
|
||||
|
||||
print(f" Proposal for: {p.source_rel}")
|
||||
print(f" type : {p.note_type or '(none)'}")
|
||||
print(f" target folder : {p.target_folder or '(unknown)'}")
|
||||
print(f" toc : {p.toc_section or '(none)'}")
|
||||
print(f" template : {p.template or '(none)'}")
|
||||
print(f" confidence : {p.confidence:.2f} [{p.method}]")
|
||||
if p.reasons:
|
||||
print(f" why : {'; '.join(p.reasons)}")
|
||||
if p.alternatives:
|
||||
alts = ", ".join(f"{t}={s:.2f}" for t, s in p.alternatives)
|
||||
print(f" alternatives : {alts}")
|
||||
for w in p.warnings:
|
||||
print(f" ⚠ {w}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- commands
|
||||
|
||||
def cmd_scan(args) -> int:
|
||||
vault = _load_vault(args)
|
||||
notes = vault.scan()
|
||||
print(f"Vault : {vault.root}")
|
||||
print(f"Notes : {len(notes)}")
|
||||
print(f"Folders : {len(vault.folders())}")
|
||||
|
||||
types: dict[str, int] = {}
|
||||
for n in notes:
|
||||
key = (n.note_type or "(none)").lower()
|
||||
types[key] = types.get(key, 0) + 1
|
||||
print("Types :")
|
||||
for t, c in sorted(types.items(), key=lambda kv: -kv[1]):
|
||||
print(f" {t:20s} {c}")
|
||||
|
||||
needing = [n for n in notes if n.needs_classification]
|
||||
print(f"\nNeeding classification (no trusted type): {len(needing)}")
|
||||
for n in needing[:20]:
|
||||
print(f" - {n.rel_path}")
|
||||
if len(needing) > 20:
|
||||
print(f" ... +{len(needing) - 20} more")
|
||||
|
||||
inbox = vault.inbox_notes(args.inbox)
|
||||
print(f"\nInbox ({args.inbox}): {len(inbox)} note(s)")
|
||||
for n in inbox:
|
||||
print(f" - {n.filename}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_propose(args) -> int:
|
||||
vault = _load_vault(args)
|
||||
|
||||
if args.jev:
|
||||
jev, tracker = _make_jev(args)
|
||||
clf = HybridClassifier(vault, jev=jev, threshold=args.threshold)
|
||||
else:
|
||||
clf = Classifier(vault)
|
||||
tracker = None
|
||||
|
||||
from pathlib import Path
|
||||
path = Path(args.file).expanduser()
|
||||
if not path.is_absolute():
|
||||
candidate = vault.root / args.file
|
||||
path = candidate if candidate.exists() else path
|
||||
|
||||
note = vault.read_note(path)
|
||||
if note is None:
|
||||
print(f"error: cannot read note: {path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
p = clf.classify(note)
|
||||
_print_proposal(p)
|
||||
if tracker is not None:
|
||||
s = tracker.summary()
|
||||
print(f" cost : ${s['cost_usd']:.6f} ({s['input_tokens']} in tokens)")
|
||||
if args.json:
|
||||
print(json.dumps(p.as_dict(), indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_inbox(args) -> int:
|
||||
vault = _load_vault(args)
|
||||
|
||||
if getattr(args, "jev", False):
|
||||
jev, tracker = _make_jev(args)
|
||||
clf = HybridClassifier(vault, jev=jev, threshold=args.threshold)
|
||||
else:
|
||||
clf = Classifier(vault)
|
||||
tracker = None
|
||||
|
||||
notes = vault.inbox_notes(args.inbox)
|
||||
|
||||
if not notes:
|
||||
print(f"Inbox is empty ({args.inbox}).")
|
||||
return 0
|
||||
|
||||
print(f"Found {len(notes)} note(s) in {args.inbox}:\n")
|
||||
proposals = []
|
||||
for note in notes:
|
||||
p = clf.classify(note)
|
||||
proposals.append(p)
|
||||
_print_proposal(p)
|
||||
print()
|
||||
|
||||
if tracker is not None:
|
||||
s = tracker.summary()
|
||||
print(f"cost: ${s['cost_usd']:.6f} ({s['input_tokens']} in tokens, {s['calls']} calls)")
|
||||
|
||||
if args.json:
|
||||
print(json.dumps([p.as_dict() for p in proposals], indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_validate(args) -> int:
|
||||
"""
|
||||
Replay the classifier against notes that already have a known-good type
|
||||
(established during the Phase 3 cleanup) and measure agreement.
|
||||
|
||||
Two modes:
|
||||
--in-folder keep the note's real folder (folder evidence available)
|
||||
--as-inbox pretend the note lives in the inbox (no folder evidence)
|
||||
"""
|
||||
vault = _load_vault(args)
|
||||
clf = Classifier(vault)
|
||||
notes = vault.scan()
|
||||
|
||||
ground_truth = [
|
||||
n for n in notes
|
||||
if (n.note_type or "").strip().lower() in {
|
||||
"project", "area", "tool", "subject", "backup-system",
|
||||
"container", "machine", "website", "service",
|
||||
"network-component", "person", "storage",
|
||||
}
|
||||
]
|
||||
|
||||
correct = 0
|
||||
total = 0
|
||||
misses: list[tuple[str, str, str, float]] = []
|
||||
low_conf = 0
|
||||
|
||||
for note in ground_truth:
|
||||
expected = (note.note_type or "").strip().lower()
|
||||
|
||||
# Simulate an UNSORTED note: strip the known-good type/status so the
|
||||
# classifier must work from evidence rather than echoing it back.
|
||||
if args.in_folder:
|
||||
probe = replace(note, note_type=None, status=None)
|
||||
else:
|
||||
probe = replace(
|
||||
note, note_type=None, status=None,
|
||||
rel_path=f"{args.inbox}/{note.filename}",
|
||||
)
|
||||
|
||||
p = clf.classify(probe)
|
||||
if p.method in ("blocked", "none"):
|
||||
total += 1
|
||||
misses.append((note.rel_path, expected, p.note_type or "(none)", p.confidence))
|
||||
continue
|
||||
|
||||
total += 1
|
||||
if p.note_type == expected:
|
||||
correct += 1
|
||||
else:
|
||||
misses.append((note.rel_path, expected, p.note_type, p.confidence))
|
||||
if p.confidence < 0.45:
|
||||
low_conf += 1
|
||||
|
||||
acc = (correct / total * 100) if total else 0.0
|
||||
mode = "IN-FOLDER (folder evidence available)" if args.in_folder else "AS-INBOX (no folder evidence)"
|
||||
print(f"=== Validation — {mode} ===")
|
||||
print(f"Notes tested : {total}")
|
||||
print(f"Correct : {correct}")
|
||||
print(f"Accuracy : {acc:.1f}%")
|
||||
print(f"Low-conf : {low_conf} (would escalate to JEV)")
|
||||
|
||||
if misses:
|
||||
print(f"\nMisses ({len(misses)}):")
|
||||
for rel, exp, got, conf in sorted(misses, key=lambda m: -m[3]):
|
||||
print(f" expected={exp:16s} got={got:16s} conf={conf:.2f} {rel}")
|
||||
return 0
|
||||
|
||||
|
||||
def _make_jev(args):
|
||||
"""Build a JEV classifier, or None when unavailable."""
|
||||
try:
|
||||
client = JevClient(timeout=args.timeout)
|
||||
except JevError as e:
|
||||
print(f"warning: JEV unavailable ({e})", file=sys.stderr)
|
||||
return None, None
|
||||
vault = Vault(args.vault)
|
||||
tracker = CostTracker(max_calls_per_run=args.max_calls)
|
||||
jev = JevClassifier(vault, client, tracker=tracker)
|
||||
return jev, tracker
|
||||
|
||||
|
||||
def cmd_benchmark(args) -> int:
|
||||
"""Run the hybrid classifier against notes with known-good types.
|
||||
|
||||
Reports accuracy and real token cost. JEV calls cost money — keep --limit sane.
|
||||
"""
|
||||
vault = _load_vault(args)
|
||||
jev, tracker = _make_jev(args)
|
||||
if jev is None:
|
||||
return 1
|
||||
|
||||
clf = HybridClassifier(vault, jev=jev, threshold=args.threshold)
|
||||
|
||||
ground_truth = [
|
||||
n for n in vault.scan()
|
||||
if (n.note_type or "").strip().lower() in {
|
||||
"project", "area", "tool", "subject", "backup-system",
|
||||
"container", "machine", "website", "service",
|
||||
"network-component", "person", "storage",
|
||||
}
|
||||
]
|
||||
|
||||
if not args.all:
|
||||
ground_truth = ground_truth[: args.limit]
|
||||
|
||||
correct = 0
|
||||
folder_correct = 0
|
||||
misses: list[tuple[str, str, str, float, str]] = []
|
||||
|
||||
for i, note in enumerate(ground_truth, 1):
|
||||
expected = (note.note_type or "").strip().lower()
|
||||
# Simulate an unsorted inbox note.
|
||||
probe = replace(
|
||||
note, note_type=None, status=None,
|
||||
rel_path=f"{args.inbox}/{note.filename}",
|
||||
)
|
||||
|
||||
if tracker.budget_exceeded():
|
||||
print(f"\nBudget cap reached ({args.max_calls} calls) — stopping.")
|
||||
break
|
||||
|
||||
p = clf.classify(probe)
|
||||
|
||||
if p.note_type == expected:
|
||||
correct += 1
|
||||
else:
|
||||
misses.append((note.rel_path, expected, p.note_type, p.confidence, p.method))
|
||||
|
||||
if p.target_folder and note.folder and p.target_folder == note.folder:
|
||||
folder_correct += 1
|
||||
|
||||
if not args.quiet:
|
||||
mark = "✓" if p.note_type == expected else "✗"
|
||||
print(f" [{i:>3}/{len(ground_truth)}] {mark} {expected:14s} → {p.note_type:14s} "
|
||||
f"({p.confidence:.2f}, {p.method})")
|
||||
|
||||
tested = correct + len(misses)
|
||||
acc = (correct / tested * 100) if tested else 0.0
|
||||
facc = (folder_correct / tested * 100) if tested else 0.0
|
||||
|
||||
print(f"\n=== JEV Benchmark ===")
|
||||
print(f"Notes tested : {tested}")
|
||||
print(f"Type accuracy : {acc:.1f}% ({correct}/{tested})")
|
||||
print(f"Folder match : {facc:.1f}% ({folder_correct}/{tested})")
|
||||
s = tracker.summary()
|
||||
print(f"JEV calls : {s['calls']} ({s['errors']} errors)")
|
||||
print(f"Tokens : {s['input_tokens']} in / {s['output_tokens']} out")
|
||||
print(f"Cost : ${s['cost_usd']:.6f}")
|
||||
if tested:
|
||||
print(f"Cost / note : ${s['cost_usd'] / tested:.6f}")
|
||||
|
||||
if misses:
|
||||
print(f"\nMisses ({len(misses)}):")
|
||||
for rel, exp, got, conf, method in misses:
|
||||
print(f" expected={exp:14s} got={got:14s} conf={conf:.2f} [{method}] {rel[:70]}")
|
||||
return 0
|
||||
|
||||
|
||||
def _make_classifier(args, vault):
|
||||
"""Rules-only or hybrid, depending on --jev. Returns (classifier, tracker)."""
|
||||
if getattr(args, "jev", False):
|
||||
jev, tracker = _make_jev(args)
|
||||
return HybridClassifier(vault, jev=jev, threshold=args.threshold), tracker
|
||||
return Classifier(vault), None
|
||||
|
||||
|
||||
def _confirm(prompt: str) -> bool:
|
||||
try:
|
||||
return input(f"{prompt} [y/N] ").strip().lower() in ("y", "yes")
|
||||
except EOFError:
|
||||
return False
|
||||
|
||||
|
||||
def cmd_sort(args) -> int:
|
||||
"""Plan (and optionally apply) sorting of one note."""
|
||||
vault = _load_vault(args)
|
||||
clf, tracker = _make_classifier(args, vault)
|
||||
writer = Writer(vault)
|
||||
|
||||
from pathlib import Path
|
||||
path = Path(args.file).expanduser()
|
||||
if not path.is_absolute():
|
||||
candidate = vault.root / args.file
|
||||
path = candidate if candidate.exists() else path
|
||||
|
||||
note = vault.read_note(path)
|
||||
if note is None:
|
||||
print(f"error: cannot read note: {path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
proposal = clf.classify(note)
|
||||
plan = writer.plan(note, proposal)
|
||||
print(plan.render())
|
||||
|
||||
if tracker is not None:
|
||||
s = tracker.summary()
|
||||
print(f"\n cost: ${s['cost_usd']:.6f} ({s['input_tokens']} in tokens)")
|
||||
|
||||
if not args.apply:
|
||||
print("\n(dry run — pass --apply to write)")
|
||||
return 0
|
||||
|
||||
if not plan.writable:
|
||||
print(f"\nrefusing to apply: {plan.blocked_reason}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if not args.yes and not _confirm("\nApply these changes?"):
|
||||
print("aborted.")
|
||||
return 1
|
||||
|
||||
ok, log = writer.apply(plan, source_path=note.path)
|
||||
print()
|
||||
for line in log:
|
||||
print(f" {line}")
|
||||
return 0 if ok else 3
|
||||
|
||||
|
||||
def cmd_sort_inbox(args) -> int:
|
||||
"""Plan (and optionally apply) every note in the inbox."""
|
||||
vault = _load_vault(args)
|
||||
clf, tracker = _make_classifier(args, vault)
|
||||
writer = Writer(vault)
|
||||
|
||||
notes = vault.inbox_notes(args.inbox)
|
||||
if not notes:
|
||||
print(f"Inbox is empty ({args.inbox}).")
|
||||
return 0
|
||||
|
||||
plans = []
|
||||
for note in notes:
|
||||
proposal = clf.classify(note)
|
||||
plan = writer.plan(note, proposal)
|
||||
plans.append((note, plan))
|
||||
|
||||
print(f"Found {len(plans)} note(s) in {args.inbox}:\n")
|
||||
for _note, plan in plans:
|
||||
print(plan.render())
|
||||
print()
|
||||
|
||||
if tracker is not None:
|
||||
s = tracker.summary()
|
||||
print(f" cost: ${s['cost_usd']:.6f} ({s['input_tokens']} in tokens)\n")
|
||||
|
||||
if not args.apply:
|
||||
print("(dry run — pass --apply to write)")
|
||||
return 0
|
||||
|
||||
writable = [(n, p) for n, p in plans if p.writable]
|
||||
blocked = [(n, p) for n, p in plans if not p.writable]
|
||||
|
||||
for _n, p in blocked:
|
||||
print(f"skipping (blocked): {p.source_rel} — {p.blocked_reason}", file=sys.stderr)
|
||||
|
||||
if not writable:
|
||||
return 2
|
||||
|
||||
if not args.yes and not _confirm(f"Apply {len(writable)} change(s)?"):
|
||||
print("aborted.")
|
||||
return 1
|
||||
|
||||
failures = 0
|
||||
for note, plan in writable:
|
||||
ok, log = writer.apply(plan, source_path=note.path)
|
||||
print(f"\n{plan.source_rel}:")
|
||||
for line in log:
|
||||
print(f" {line}")
|
||||
if not ok:
|
||||
failures += 1
|
||||
|
||||
return 0 if failures == 0 else 3
|
||||
|
||||
|
||||
def cmd_watch(args) -> int:
|
||||
"""Watch the inbox: propose destinations, and file notes you approve."""
|
||||
vault = _load_vault(args)
|
||||
jev, tracker = _make_jev(args)
|
||||
clf = HybridClassifier(vault, jev=jev, threshold=args.threshold)
|
||||
watcher = Watcher(
|
||||
vault, clf, inbox=args.inbox,
|
||||
interval=args.interval, dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
if args.once:
|
||||
lines = watcher.run_once()
|
||||
if lines:
|
||||
for line in lines:
|
||||
print(line)
|
||||
else:
|
||||
print("nothing to do")
|
||||
if tracker is not None and tracker.calls:
|
||||
print(f"\ncost: ${tracker.summary()['cost_usd']:.6f}")
|
||||
return 0
|
||||
|
||||
mode = "DRY RUN" if args.dry_run else "live"
|
||||
print(f"watching '{args.inbox}' every {args.interval}s ({mode}) — Ctrl-C to stop")
|
||||
try:
|
||||
watcher.run_forever(max_iterations=args.iterations)
|
||||
except KeyboardInterrupt:
|
||||
print("\nstopped")
|
||||
return 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ main
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="obsidian-sort",
|
||||
description="Classify and file Obsidian notes (Phase A: read-only).",
|
||||
)
|
||||
p.add_argument("--vault", default="~/obsidian", help="vault root (default: ~/obsidian)")
|
||||
p.add_argument("--inbox", default="inbox", help="inbox folder name (default: inbox)")
|
||||
p.add_argument("--json", action="store_true", help="also emit JSON")
|
||||
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("scan", help="report vault shape").set_defaults(func=cmd_scan)
|
||||
|
||||
sp = sub.add_parser("propose", help="classify one note")
|
||||
sp.add_argument("file", help="path to a note (absolute or vault-relative)")
|
||||
sp.add_argument("--jev", action="store_true", help="escalate to JEV when rules are unsure")
|
||||
sp.add_argument("--threshold", type=float, default=0.45,
|
||||
help="escalate to JEV below this rules confidence")
|
||||
sp.add_argument("--timeout", type=float, default=90.0)
|
||||
sp.add_argument("--max-calls", type=int, default=500)
|
||||
sp.set_defaults(func=cmd_propose)
|
||||
|
||||
si = sub.add_parser("inbox", help="classify all inbox notes")
|
||||
si.add_argument("--jev", action="store_true", help="escalate to JEV when rules are unsure")
|
||||
si.add_argument("--threshold", type=float, default=0.45)
|
||||
si.add_argument("--timeout", type=float, default=90.0)
|
||||
si.add_argument("--max-calls", type=int, default=500)
|
||||
si.set_defaults(func=cmd_inbox)
|
||||
|
||||
sv = sub.add_parser("validate", help="replay classifier against known-good notes")
|
||||
sv.add_argument("--in-folder", action="store_true",
|
||||
help="use the note's real folder (default: simulate inbox)")
|
||||
sv.set_defaults(func=cmd_validate)
|
||||
|
||||
sb = sub.add_parser("benchmark", help="run the hybrid classifier and measure accuracy + cost")
|
||||
sb.add_argument("--limit", type=int, default=25, help="how many notes to test (default 25)")
|
||||
sb.add_argument("--all", action="store_true", help="test every known-good note")
|
||||
sb.add_argument("--threshold", type=float, default=0.45,
|
||||
help="escalate to JEV below this rules confidence")
|
||||
sb.add_argument("--quiet", action="store_true", help="only print the summary")
|
||||
sb.add_argument("--timeout", type=float, default=90.0)
|
||||
sb.add_argument("--max-calls", type=int, default=500)
|
||||
sb.set_defaults(func=cmd_benchmark)
|
||||
|
||||
# --- sort (Phase C: writes) -----------------------------------------
|
||||
ss = sub.add_parser("sort", help="plan and optionally apply sorting for one note")
|
||||
ss.add_argument("file", help="path to a note (absolute or vault-relative)")
|
||||
ss.add_argument("--apply", action="store_true", help="write the changes (default: dry run)")
|
||||
ss.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
|
||||
ss.add_argument("--jev", action="store_true", help="escalate to JEV when rules are unsure")
|
||||
ss.add_argument("--threshold", type=float, default=0.45)
|
||||
ss.add_argument("--timeout", type=float, default=90.0)
|
||||
ss.add_argument("--max-calls", type=int, default=500)
|
||||
ss.set_defaults(func=cmd_sort)
|
||||
|
||||
si = sub.add_parser("sort-inbox", help="plan and optionally apply sorting for the whole inbox")
|
||||
si.add_argument("--apply", action="store_true", help="write the changes (default: dry run)")
|
||||
si.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
|
||||
si.add_argument("--jev", action="store_true", help="escalate to JEV when rules are unsure")
|
||||
si.add_argument("--threshold", type=float, default=0.45)
|
||||
si.add_argument("--timeout", type=float, default=90.0)
|
||||
si.add_argument("--max-calls", type=int, default=500)
|
||||
si.set_defaults(func=cmd_sort_inbox)
|
||||
|
||||
sw = sub.add_parser("watch", help="watch the inbox and maintain proposals")
|
||||
sw.add_argument("--once", action="store_true", help="single pass, then exit")
|
||||
sw.add_argument("--interval", type=float, default=5.0, help="seconds between passes")
|
||||
sw.add_argument("--iterations", type=int, default=None, help="stop after N passes")
|
||||
sw.add_argument("--dry-run", action="store_true", help="report but never write")
|
||||
sw.add_argument("--threshold", type=float, default=0.45)
|
||||
sw.add_argument("--timeout", type=float, default=90.0)
|
||||
sw.add_argument("--max-calls", type=int, default=500)
|
||||
sw.set_defaults(func=cmd_watch)
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user