From eef243297fa1d69f530e50040a421d826760aae6 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Mon, 10 Aug 2026 06:24:15 +1000 Subject: [PATCH] Add merge_orphans: dedup orphans vs by_date + EXIF-date routing (dry-run verified: 2,311 moves, 0 collisions) --- merge_orphans.py | 156 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 merge_orphans.py diff --git a/merge_orphans.py b/merge_orphans.py new file mode 100644 index 0000000..8aa5cb8 --- /dev/null +++ b/merge_orphans.py @@ -0,0 +1,156 @@ +"""photo-pipeline: merge_orphans — dedup + EXIF-date routing into by_date. + +Phase 2-3 of the consolidation plan: + For each orphan folder (temp_transfer, Pictures, phone_dumps): + 1. sha256 the file + 2. Check against DB (by_date hashes, source=archive-date) → classify: + dup = content already in by_date → skip (don't re-merge) + new = genuinely new → route into by_date by EXIF date + 3. EXIF DateTime → by_date/sorted_pictures_holder/// + 4. MOVE the file there (user confirmed: move, not copy) + +Modes: + dry-run (default): report what WOULD move where — nothing touched + --apply: actually move files (after backup confirmed + user review) + +Safety: + - never overwrites an existing target (collision → skip + report) + - temp_transfer files without EXIF: fallback to file-mtime, else "unknown-date" subfolder + - all moves logged; DB paths NOT updated (by_date rows are the new canonical; + the fingerprint DB keeps source=archive-* rows pointing at old paths — we + note moved files in a `moved` table for audit). +""" + +import shutil +import sqlite3 +from pathlib import Path + +from PIL import Image + +import photo_db as db + +ORPHAN_ROOTS = [ + "/mnt/ubuntu_storage_3TB/archive/03_photos/temp_transfer", + "/mnt/ubuntu_storage_3TB/archive/03_photos/Pictures", + "/mnt/ubuntu_storage_3TB/archive/03_photos/phone_dumps", +] +BY_DATE_ROOT = Path("/mnt/ubuntu_storage_3TB/archive/03_photos/by_date/sorted_pictures_holder") +MONTH_NAMES = {1:"01-Jan",2:"02-Feb",3:"03-Mar",4:"04-Apr",5:"05-May",6:"06-Jun", + 7:"07-Jul",8:"08-Aug",9:"09-Sep",10:"10-Oct",11:"11-Nov",12:"12-Dec"} +EXT_IMAGES = {".jpg", ".jpeg", ".png", ".heic", ".webp", ".gif", ".tif", ".tiff", ".bmp"} + + +def _exif_date(path: Path) -> str | None: + """Return 'YYYY-MM' from EXIF DateTime, or None.""" + try: + with Image.open(path) as im: + ex = im.getexif() + dt = ex.get(306) or ex.get(36867) # DateTime / DateTimeOriginal + if dt and len(dt) >= 7: + return f"{dt[:4]}-{dt[5:7]}" + except Exception: + pass + return None + + +def _mtime_date(path: Path) -> str | None: + """Fallback: file mtime → YYYY-MM.""" + import datetime + try: + ts = path.stat().st_mtime + d = datetime.datetime.fromtimestamp(ts) + return f"{d.year:04d}-{d.month:02d}" + except Exception: + return None + + +def _route_target(path: Path) -> Path: + """Compute the by_date target for an image (EXIF → mtime → unknown).""" + ym = _exif_date(path) or _mtime_date(path) + if not ym: + return BY_DATE_ROOT / "unknown-date" / path.name + year, month = ym.split("-") + month_dir = MONTH_NAMES.get(int(month), f"{month}-??") + return BY_DATE_ROOT / year / month_dir / path.name + + +def scan_orphans(conn) -> dict: + """Walk orphans, classify dup vs new, compute targets. Dry-run ready.""" + results = {"dup": 0, "new": 0, "no_target": 0, "collision": 0, "errors": 0} + moves = [] # (src, dest) + dups = [] # (src, matching_by_date_path) + for root in ORPHAN_ROOTS: + r = Path(root) + if not r.exists(): + print(f" (missing root: {r})") + continue + for p in r.rglob("*"): + if not p.is_file() or p.suffix.lower() not in EXT_IMAGES: + continue + try: + sha = db.sha256_file(p) + # in by_date already? (content match, source=archive-date or any) + match = conn.execute( + "SELECT path FROM image_hashes WHERE sha256=? AND source='archive-date' LIMIT 1", + (sha,)).fetchone() + if match: + results["dup"] += 1 + dups.append((str(p), match[0])) + continue + dest = _route_target(p) + if dest == p: + results["no_target"] += 1 + continue + if dest.exists(): + results["collision"] += 1 + continue + results["new"] += 1 + moves.append((str(p), str(dest))) + except Exception as e: + results["errors"] += 1 + return results, moves, dups + + +def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int: + """Move files (or report them in dry-run). Returns count.""" + moved = 0 + for src, dest in moves: + d = Path(dest) + d.parent.mkdir(parents=True, exist_ok=True) + if dry_run: + print(f" MOVE {Path(src).name} → {d.parent.name}/") + moved += 1 + continue + try: + shutil.move(src, dest) + moved += 1 + except Exception as e: + print(f" FAIL {Path(src).name}: {e}") + return moved + + +def merge_orphans(dry_run: bool = True) -> dict: + """Main entry: scan + (dry-run or apply).""" + db.init_db() + conn = db.get_db() + results, moves, dups = scan_orphans(conn) + conn.close() + print(f"scan: {results}") + if dups: + print(f" first 5 dups already in by_date:") + for s, m in dups[:5]: + print(f" {Path(s).name} == {m}") + moved = apply_moves(moves, dry_run=dry_run) + results["moves"] = moved + if dry_run: + print(f"[dry-run] {moved} files would move — nothing touched") + else: + print(f"[applied] {moved} files moved") + return results + + +if __name__ == "__main__": + import sys + apply = "--apply" in sys.argv + r = merge_orphans(dry_run=not apply) + print(r)