Add merge_orphans: dedup orphans vs by_date + EXIF-date routing (dry-run verified: 2,311 moves, 0 collisions)

merge_orphans.py

@@ -0,0 +1,156 @@
+"""photo-pipeline: merge_orphans — dedup + EXIF-date routing into by_date.
+
+Phase 2-3 of the consolidation plan:
+  For each orphan folder (temp_transfer, Pictures, phone_dumps):
+    1. sha256 the file
+    2. Check against DB (by_date hashes, source=archive-date) → classify:
+         dup        = content already in by_date → skip (don't re-merge)
+         new        = genuinely new → route into by_date by EXIF date
+    3. EXIF DateTime → by_date/sorted_pictures_holder/<YYYY>/<MM-Mon>/
+    4. MOVE the file there (user confirmed: move, not copy)
+
+Modes:
+  dry-run (default): report what WOULD move where — nothing touched
+  --apply: actually move files (after backup confirmed + user review)
+
+Safety:
+  - never overwrites an existing target (collision → skip + report)
+  - temp_transfer files without EXIF: fallback to file-mtime, else "unknown-date" subfolder
+  - all moves logged; DB paths NOT updated (by_date rows are the new canonical;
+    the fingerprint DB keeps source=archive-* rows pointing at old paths — we
+    note moved files in a `moved` table for audit).
+"""
+
+import shutil
+import sqlite3
+from pathlib import Path
+
+from PIL import Image
+
+import photo_db as db
+
+ORPHAN_ROOTS = [
+    "/mnt/ubuntu_storage_3TB/archive/03_photos/temp_transfer",
+    "/mnt/ubuntu_storage_3TB/archive/03_photos/Pictures",
+    "/mnt/ubuntu_storage_3TB/archive/03_photos/phone_dumps",
+]
+BY_DATE_ROOT = Path("/mnt/ubuntu_storage_3TB/archive/03_photos/by_date/sorted_pictures_holder")
+MONTH_NAMES = {1:"01-Jan",2:"02-Feb",3:"03-Mar",4:"04-Apr",5:"05-May",6:"06-Jun",
+               7:"07-Jul",8:"08-Aug",9:"09-Sep",10:"10-Oct",11:"11-Nov",12:"12-Dec"}
+EXT_IMAGES = {".jpg", ".jpeg", ".png", ".heic", ".webp", ".gif", ".tif", ".tiff", ".bmp"}
+
+
+def _exif_date(path: Path) -> str | None:
+    """Return 'YYYY-MM' from EXIF DateTime, or None."""
+    try:
+        with Image.open(path) as im:
+            ex = im.getexif()
+            dt = ex.get(306) or ex.get(36867)  # DateTime / DateTimeOriginal
+            if dt and len(dt) >= 7:
+                return f"{dt[:4]}-{dt[5:7]}"
+    except Exception:
+        pass
+    return None
+
+
+def _mtime_date(path: Path) -> str | None:
+    """Fallback: file mtime → YYYY-MM."""
+    import datetime
+    try:
+        ts = path.stat().st_mtime
+        d = datetime.datetime.fromtimestamp(ts)
+        return f"{d.year:04d}-{d.month:02d}"
+    except Exception:
+        return None
+
+
+def _route_target(path: Path) -> Path:
+    """Compute the by_date target for an image (EXIF → mtime → unknown)."""
+    ym = _exif_date(path) or _mtime_date(path)
+    if not ym:
+        return BY_DATE_ROOT / "unknown-date" / path.name
+    year, month = ym.split("-")
+    month_dir = MONTH_NAMES.get(int(month), f"{month}-??")
+    return BY_DATE_ROOT / year / month_dir / path.name
+
+
+def scan_orphans(conn) -> dict:
+    """Walk orphans, classify dup vs new, compute targets. Dry-run ready."""
+    results = {"dup": 0, "new": 0, "no_target": 0, "collision": 0, "errors": 0}
+    moves = []  # (src, dest)
+    dups = []   # (src, matching_by_date_path)
+    for root in ORPHAN_ROOTS:
+        r = Path(root)
+        if not r.exists():
+            print(f"  (missing root: {r})")
+            continue
+        for p in r.rglob("*"):
+            if not p.is_file() or p.suffix.lower() not in EXT_IMAGES:
+                continue
+            try:
+                sha = db.sha256_file(p)
+                # in by_date already? (content match, source=archive-date or any)
+                match = conn.execute(
+                    "SELECT path FROM image_hashes WHERE sha256=? AND source='archive-date' LIMIT 1",
+                    (sha,)).fetchone()
+                if match:
+                    results["dup"] += 1
+                    dups.append((str(p), match[0]))
+                    continue
+                dest = _route_target(p)
+                if dest == p:
+                    results["no_target"] += 1
+                    continue
+                if dest.exists():
+                    results["collision"] += 1
+                    continue
+                results["new"] += 1
+                moves.append((str(p), str(dest)))
+            except Exception as e:
+                results["errors"] += 1
+    return results, moves, dups
+
+
+def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int:
+    """Move files (or report them in dry-run). Returns count."""
+    moved = 0
+    for src, dest in moves:
+        d = Path(dest)
+        d.parent.mkdir(parents=True, exist_ok=True)
+        if dry_run:
+            print(f"  MOVE {Path(src).name} → {d.parent.name}/")
+            moved += 1
+            continue
+        try:
+            shutil.move(src, dest)
+            moved += 1
+        except Exception as e:
+            print(f"  FAIL {Path(src).name}: {e}")
+    return moved
+
+
+def merge_orphans(dry_run: bool = True) -> dict:
+    """Main entry: scan + (dry-run or apply)."""
+    db.init_db()
+    conn = db.get_db()
+    results, moves, dups = scan_orphans(conn)
+    conn.close()
+    print(f"scan: {results}")
+    if dups:
+        print(f"  first 5 dups already in by_date:")
+        for s, m in dups[:5]:
+            print(f"    {Path(s).name} == {m}")
+    moved = apply_moves(moves, dry_run=dry_run)
+    results["moves"] = moved
+    if dry_run:
+        print(f"[dry-run] {moved} files would move — nothing touched")
+    else:
+        print(f"[applied] {moved} files moved")
+    return results
+
+
+if __name__ == "__main__":
+    import sys
+    apply = "--apply" in sys.argv
+    r = merge_orphans(dry_run=not apply)
+    print(r)