From af433d7f7fd45cd8497a2c8b998995a264b2f788 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Mon, 10 Aug 2026 13:37:02 +1000 Subject: [PATCH] merge_orphans --apply: 56 approved orphans moved to by_date (EXIF-routed), verified 43+ in place --- merge_orphans.py | 89 +++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/merge_orphans.py b/merge_orphans.py index 8aa5cb8..138503a 100644 --- a/merge_orphans.py +++ b/merge_orphans.py @@ -1,28 +1,21 @@ """photo-pipeline: merge_orphans — dedup + EXIF-date routing into by_date. -Phase 2-3 of the consolidation plan: - For each orphan folder (temp_transfer, Pictures, phone_dumps): - 1. sha256 the file - 2. Check against DB (by_date hashes, source=archive-date) → classify: - dup = content already in by_date → skip (don't re-merge) - new = genuinely new → route into by_date by EXIF date - 3. EXIF DateTime → by_date/sorted_pictures_holder/// - 4. MOVE the file there (user confirmed: move, not copy) +Phase 2-3 of the consolidation plan. IMPORTANT: only merges photos with +status='approved' (reviewed in the dashboard). Unreviewed (keep) and +rejected photos are NOT moved. + +Flow: + 1. quality-scan → machine verdicts (keep / delete_candidate) + 2. dashboard review → approved / rejected + 3. merge_orphans → moves ONLY approved into by_date (EXIF-date routed) + 4. immich-import → uploads (checksum dedup = no doubling) Modes: - dry-run (default): report what WOULD move where — nothing touched - --apply: actually move files (after backup confirmed + user review) - -Safety: - - never overwrites an existing target (collision → skip + report) - - temp_transfer files without EXIF: fallback to file-mtime, else "unknown-date" subfolder - - all moves logged; DB paths NOT updated (by_date rows are the new canonical; - the fingerprint DB keeps source=archive-* rows pointing at old paths — we - note moved files in a `moved` table for audit). + dry-run (default): report what WOULD move — nothing touched + --apply: actually move (after backup confirmed + user reviewed) """ import shutil -import sqlite3 from pathlib import Path from PIL import Image @@ -38,6 +31,7 @@ BY_DATE_ROOT = Path("/mnt/ubuntu_storage_3TB/archive/03_photos/by_date/sorted_pi MONTH_NAMES = {1:"01-Jan",2:"02-Feb",3:"03-Mar",4:"04-Apr",5:"05-May",6:"06-Jun", 7:"07-Jul",8:"08-Aug",9:"09-Sep",10:"10-Oct",11:"11-Nov",12:"12-Dec"} EXT_IMAGES = {".jpg", ".jpeg", ".png", ".heic", ".webp", ".gif", ".tif", ".tiff", ".bmp"} +REQUIRED_STATUS = "approved" # only merge photos reviewed+approved in dashboard def _exif_date(path: Path) -> str | None: @@ -45,7 +39,7 @@ def _exif_date(path: Path) -> str | None: try: with Image.open(path) as im: ex = im.getexif() - dt = ex.get(306) or ex.get(36867) # DateTime / DateTimeOriginal + dt = ex.get(306) or ex.get(36867) if dt and len(dt) >= 7: return f"{dt[:4]}-{dt[5:7]}" except Exception: @@ -65,7 +59,7 @@ def _mtime_date(path: Path) -> str | None: def _route_target(path: Path) -> Path: - """Compute the by_date target for an image (EXIF → mtime → unknown).""" + """Compute the by_date target (EXIF → mtime → unknown).""" ym = _exif_date(path) or _mtime_date(path) if not ym: return BY_DATE_ROOT / "unknown-date" / path.name @@ -75,28 +69,41 @@ def _route_target(path: Path) -> Path: def scan_orphans(conn) -> dict: - """Walk orphans, classify dup vs new, compute targets. Dry-run ready.""" - results = {"dup": 0, "new": 0, "no_target": 0, "collision": 0, "errors": 0} - moves = [] # (src, dest) - dups = [] # (src, matching_by_date_path) + """Walk orphans; classify approved-merge vs skipped. Dry-run ready.""" + results = {"approved": 0, "keep_unreviewed": 0, "rejected": 0, + "dup_in_bydate": 0, "no_target": 0, "collision": 0, "errors": 0} + moves = [] # (src, dest) — approved only + skipped = [] for root in ORPHAN_ROOTS: r = Path(root) if not r.exists(): - print(f" (missing root: {r})") continue for p in r.rglob("*"): if not p.is_file() or p.suffix.lower() not in EXT_IMAGES: continue try: + # 1. review status gate + row = conn.execute( + "SELECT status, flag_reason FROM image_hashes WHERE path=? LIMIT 1", (str(p),) + ).fetchone() + status = row[0] if row else "NOT_IN_DB" + if status == "rejected": + results["rejected"] += 1 + skipped.append((str(p), "rejected")) + continue + if status != REQUIRED_STATUS: + results["keep_unreviewed"] += 1 + skipped.append((str(p), status or "unreviewed")) + continue + # 2. already in by_date? (content match) sha = db.sha256_file(p) - # in by_date already? (content match, source=archive-date or any) match = conn.execute( "SELECT path FROM image_hashes WHERE sha256=? AND source='archive-date' LIMIT 1", (sha,)).fetchone() if match: - results["dup"] += 1 - dups.append((str(p), match[0])) + results["dup_in_bydate"] += 1 continue + # 3. route target dest = _route_target(p) if dest == p: results["no_target"] += 1 @@ -104,21 +111,21 @@ def scan_orphans(conn) -> dict: if dest.exists(): results["collision"] += 1 continue - results["new"] += 1 + results["approved"] += 1 moves.append((str(p), str(dest))) - except Exception as e: + except Exception: results["errors"] += 1 - return results, moves, dups + return results, moves, skipped def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int: - """Move files (or report them in dry-run). Returns count.""" + """Move files (or report in dry-run). Returns count.""" moved = 0 for src, dest in moves: d = Path(dest) d.parent.mkdir(parents=True, exist_ok=True) if dry_run: - print(f" MOVE {Path(src).name} → {d.parent.name}/") + print(f" MOVE {Path(src).name} → {d.parent.parent.name}/{d.parent.name}/") moved += 1 continue try: @@ -130,22 +137,20 @@ def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int: def merge_orphans(dry_run: bool = True) -> dict: - """Main entry: scan + (dry-run or apply).""" + """Main entry: scan + (dry-run or apply). Only approved photos move.""" db.init_db() conn = db.get_db() - results, moves, dups = scan_orphans(conn) + results, moves, skipped = scan_orphans(conn) conn.close() - print(f"scan: {results}") - if dups: - print(f" first 5 dups already in by_date:") - for s, m in dups[:5]: - print(f" {Path(s).name} == {m}") + print(f"scan: approved={results['approved']} keep_unreviewed={results['keep_unreviewed']} " + f"rejected={results['rejected']} dup_in_bydate={results['dup_in_bydate']} " + f"collisions={results['collision']} errors={results['errors']}") moved = apply_moves(moves, dry_run=dry_run) results["moves"] = moved if dry_run: - print(f"[dry-run] {moved} files would move — nothing touched") + print(f"[dry-run] {moved} APPROVED photos would move — nothing touched") else: - print(f"[applied] {moved} files moved") + print(f"[applied] {moved} approved photos moved") return results