obsidian-sorter: automated note filing for an Obsidian vault
Classifies notes from an inbox folder and files them, with a human approval step. Zero third-party dependencies. - type mirrors folder (200 projects / 300 areas / 400 Personal Family) - TypeSafe JEV for semantic classification (~$0.0001 per note) - watcher writes a proposal INTO the note; the owner ticks a checkbox - systemd user service on .27, 10s poll - 15/15 functional tests
This commit is contained in:
sam
209
tools/normalize_tags.py
Normal file
209
tools/normalize_tags.py
Normal file
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
One-off tag normalisation for the Obsidian vault.
|
||||
|
||||
Rules (confirmed 2026-09-24):
|
||||
* lowercase everything
|
||||
* 'tools' -> 'tool'
|
||||
* split comma-joined pseudo-tags into real tags
|
||||
* fix known typos
|
||||
* spaces -> hyphens
|
||||
* drop retired-type tags
|
||||
|
||||
Usage:
|
||||
python3 tools/normalize_tags.py --vault ./test-vault # dry run
|
||||
python3 tools/normalize_tags.py --vault ~/obsidian --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
SKIP_DIRS = {".git", ".trash", ".venv", ".obsidian", "templates",
|
||||
"graphify-out", "graphify-out-temp", "__pycache__"}
|
||||
|
||||
# Tags that came from the retired type list — drop them.
|
||||
REMOVE = {
|
||||
"subject", "website", "overview", "log", "runbook", "media", "area",
|
||||
"machine", "service", "container", "storage", "backup-system",
|
||||
"network-component", "person", "note", "filesmd",
|
||||
}
|
||||
|
||||
# Renames (including merges and typo fixes).
|
||||
RENAME = {
|
||||
"tools": "tool",
|
||||
"exrcise": "exercise",
|
||||
"rasperry-pi": "raspberry-pi",
|
||||
"recipie": "recipe",
|
||||
"wikunja": "vikunja",
|
||||
"svelt": "svelte",
|
||||
}
|
||||
|
||||
|
||||
def fix_tag(raw: str):
|
||||
"""Normalise one tag. Returns the new tag, or None if it should be dropped."""
|
||||
t = raw.strip().strip('"').strip("'").strip()
|
||||
if not t:
|
||||
return None
|
||||
t = t.lower()
|
||||
t = RENAME.get(t, t)
|
||||
t = re.sub(r"\s+", "-", t) # spaces -> hyphens
|
||||
t = t.strip("-")
|
||||
if not t or t in REMOVE:
|
||||
return None
|
||||
return t
|
||||
|
||||
|
||||
def normalise(tags: list[str]) -> list[str]:
|
||||
"""Normalise a list, splitting comma-joined pseudo-tags, preserving order."""
|
||||
out: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in tags:
|
||||
parts = raw.split(",") if "," in raw else [raw]
|
||||
for part in parts:
|
||||
fixed = fix_tag(part)
|
||||
if fixed and fixed not in seen:
|
||||
seen.add(fixed)
|
||||
out.append(fixed)
|
||||
return out
|
||||
|
||||
|
||||
_TAGS_LINE = re.compile(r"^tags:\s*(.*)$")
|
||||
|
||||
|
||||
def rewrite_note(text: str) -> tuple[str, list[str], list[str]]:
|
||||
"""
|
||||
Rewrite the `tags:` field in a note's frontmatter.
|
||||
|
||||
Returns (new_text, old_tags, new_tags). new_text == text when nothing changed.
|
||||
"""
|
||||
m = re.match(r"^(---\s*\n)(.*?)(\n---\s*\n?)", text, re.DOTALL)
|
||||
if not m:
|
||||
return text, [], []
|
||||
|
||||
header, block, close = m.group(1), m.group(2), m.group(3)
|
||||
body = text[m.end():]
|
||||
lines = block.split("\n")
|
||||
|
||||
out: list[str] = []
|
||||
old: list[str] = []
|
||||
i = 0
|
||||
found = False
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
tm = _TAGS_LINE.match(line)
|
||||
if not tm:
|
||||
out.append(line)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
found = True
|
||||
value = tm.group(1).strip()
|
||||
if value.startswith("["):
|
||||
inner = value[1:-1]
|
||||
old = [p.strip().strip('"').strip("'") for p in inner.split(",") if p.strip()]
|
||||
i += 1
|
||||
else:
|
||||
j = i + 1
|
||||
while j < len(lines) and re.match(r"^\s*-\s", lines[j]):
|
||||
item = re.match(r"^\s*-\s*(.*)$", lines[j]).group(1)
|
||||
old.append(item.strip().strip('"').strip("'"))
|
||||
j += 1
|
||||
i = j
|
||||
|
||||
new = normalise(old)
|
||||
if new:
|
||||
out.append("tags:")
|
||||
for t in new:
|
||||
out.append(f" - {t}")
|
||||
else:
|
||||
out.append("tags: []")
|
||||
|
||||
if not found:
|
||||
return text, [], []
|
||||
|
||||
new_block = "\n".join(out)
|
||||
if new_block == block:
|
||||
return text, old, old
|
||||
return header + new_block + close + body, old, normalise(old)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vault", default="~/obsidian")
|
||||
ap.add_argument("--apply", action="store_true", help="write changes (default: dry run)")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
root = os.path.expanduser(args.vault)
|
||||
if not os.path.isdir(root):
|
||||
print(f"error: vault not found: {root}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
changed = 0
|
||||
scanned = 0
|
||||
removed = Counter()
|
||||
renamed = Counter()
|
||||
touched: list[str] = []
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
||||
for fn in sorted(filenames):
|
||||
if not fn.endswith(".md"):
|
||||
continue
|
||||
p = os.path.join(dirpath, fn)
|
||||
rel = os.path.relpath(p, root)
|
||||
try:
|
||||
text = open(p, encoding="utf-8").read()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
scanned += 1
|
||||
new_text, old, new = rewrite_note(text)
|
||||
if new_text == text:
|
||||
continue
|
||||
changed += 1
|
||||
touched.append(rel)
|
||||
|
||||
# Reporting: a tag is only "dropped" if none of its parts survived.
|
||||
new_set = set(new)
|
||||
for t in old:
|
||||
ts = t.strip()
|
||||
if not ts:
|
||||
continue
|
||||
parts = ts.split(",") if "," in ts else [ts]
|
||||
fixed_parts = [f for f in (fix_tag(p) for p in parts) if f]
|
||||
if not any(f in new_set for f in fixed_parts):
|
||||
removed[ts.lower()] += 1
|
||||
else:
|
||||
for f in fixed_parts:
|
||||
if f != ts.lower():
|
||||
renamed[f] += 1
|
||||
break
|
||||
|
||||
if not args.quiet:
|
||||
print(f" {rel}")
|
||||
print(f" before: {', '.join(old) if old else '(none)'}")
|
||||
print(f" after : {', '.join(new) if new else '(none)'}")
|
||||
if args.apply:
|
||||
with open(p, "w", encoding="utf-8") as fh:
|
||||
fh.write(new_text)
|
||||
|
||||
print(f"\n{'APPLIED' if args.apply else 'DRY RUN'} — {changed} of {scanned} notes changed")
|
||||
if removed:
|
||||
print("\ntags dropped:")
|
||||
for t, c in removed.most_common():
|
||||
print(f" {c:3d} {t}")
|
||||
if renamed:
|
||||
print("\ntags introduced by renaming:")
|
||||
for t, c in renamed.most_common(15):
|
||||
print(f" {c:3d} {t}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user