merge_orphans --apply: 56 approved orphans moved to by_date (EXIF-routed), verified 43+ in place

merge_orphans.py

@@ -1,28 +1,21 @@
 """photo-pipeline: merge_orphans — dedup + EXIF-date routing into by_date.
 
-Phase 2-3 of the consolidation plan:
+Phase 2-3 of the consolidation plan. IMPORTANT: only merges photos with
-  For each orphan folder (temp_transfer, Pictures, phone_dumps):
+status='approved' (reviewed in the dashboard). Unreviewed (keep) and
-    1. sha256 the file
+rejected photos are NOT moved.
-    2. Check against DB (by_date hashes, source=archive-date) → classify:
+
-         dup        = content already in by_date → skip (don't re-merge)
+Flow:
-         new        = genuinely new → route into by_date by EXIF date
+  1. quality-scan → machine verdicts (keep / delete_candidate)
-    3. EXIF DateTime → by_date/sorted_pictures_holder/<YYYY>/<MM-Mon>/
+  2. dashboard review → approved / rejected
-    4. MOVE the file there (user confirmed: move, not copy)
+  3. merge_orphans → moves ONLY approved into by_date (EXIF-date routed)
+  4. immich-import → uploads (checksum dedup = no doubling)
 
 Modes:
-  dry-run (default): report what WOULD move where — nothing touched
+  dry-run (default): report what WOULD move — nothing touched
-  --apply: actually move files (after backup confirmed + user review)
+  --apply: actually move (after backup confirmed + user reviewed)
-
-Safety:
-  - never overwrites an existing target (collision → skip + report)
-  - temp_transfer files without EXIF: fallback to file-mtime, else "unknown-date" subfolder
-  - all moves logged; DB paths NOT updated (by_date rows are the new canonical;
-    the fingerprint DB keeps source=archive-* rows pointing at old paths — we
-    note moved files in a `moved` table for audit).
 """
 
 import shutil
-import sqlite3
 from pathlib import Path
 
 from PIL import Image
@@ -38,6 +31,7 @@ BY_DATE_ROOT = Path("/mnt/ubuntu_storage_3TB/archive/03_photos/by_date/sorted_pi
 MONTH_NAMES = {1:"01-Jan",2:"02-Feb",3:"03-Mar",4:"04-Apr",5:"05-May",6:"06-Jun",
                7:"07-Jul",8:"08-Aug",9:"09-Sep",10:"10-Oct",11:"11-Nov",12:"12-Dec"}
 EXT_IMAGES = {".jpg", ".jpeg", ".png", ".heic", ".webp", ".gif", ".tif", ".tiff", ".bmp"}
+REQUIRED_STATUS = "approved"  # only merge photos reviewed+approved in dashboard
 
 
 def _exif_date(path: Path) -> str | None:
@@ -45,7 +39,7 @@ def _exif_date(path: Path) -> str | None:
     try:
         with Image.open(path) as im:
             ex = im.getexif()
-            dt = ex.get(306) or ex.get(36867)  # DateTime / DateTimeOriginal
+            dt = ex.get(306) or ex.get(36867)
             if dt and len(dt) >= 7:
                 return f"{dt[:4]}-{dt[5:7]}"
     except Exception:
@@ -65,7 +59,7 @@ def _mtime_date(path: Path) -> str | None:
 
 
 def _route_target(path: Path) -> Path:
-    """Compute the by_date target for an image (EXIF → mtime → unknown)."""
+    """Compute the by_date target (EXIF → mtime → unknown)."""
     ym = _exif_date(path) or _mtime_date(path)
     if not ym:
         return BY_DATE_ROOT / "unknown-date" / path.name
@@ -75,28 +69,41 @@ def _route_target(path: Path) -> Path:
 
 
 def scan_orphans(conn) -> dict:
-    """Walk orphans, classify dup vs new, compute targets. Dry-run ready."""
+    """Walk orphans; classify approved-merge vs skipped. Dry-run ready."""
-    results = {"dup": 0, "new": 0, "no_target": 0, "collision": 0, "errors": 0}
+    results = {"approved": 0, "keep_unreviewed": 0, "rejected": 0,
-    moves = []  # (src, dest)
+               "dup_in_bydate": 0, "no_target": 0, "collision": 0, "errors": 0}
-    dups = []   # (src, matching_by_date_path)
+    moves = []   # (src, dest) — approved only
+    skipped = []
     for root in ORPHAN_ROOTS:
         r = Path(root)
         if not r.exists():
-            print(f"  (missing root: {r})")
             continue
         for p in r.rglob("*"):
             if not p.is_file() or p.suffix.lower() not in EXT_IMAGES:
                 continue
             try:
+                # 1. review status gate
+                row = conn.execute(
+                    "SELECT status, flag_reason FROM image_hashes WHERE path=? LIMIT 1", (str(p),)
+                ).fetchone()
+                status = row[0] if row else "NOT_IN_DB"
+                if status == "rejected":
+                    results["rejected"] += 1
+                    skipped.append((str(p), "rejected"))
+                    continue
+                if status != REQUIRED_STATUS:
+                    results["keep_unreviewed"] += 1
+                    skipped.append((str(p), status or "unreviewed"))
+                    continue
+                # 2. already in by_date? (content match)
                 sha = db.sha256_file(p)
-                # in by_date already? (content match, source=archive-date or any)
                 match = conn.execute(
                     "SELECT path FROM image_hashes WHERE sha256=? AND source='archive-date' LIMIT 1",
                     (sha,)).fetchone()
                 if match:
-                    results["dup"] += 1
+                    results["dup_in_bydate"] += 1
-                    dups.append((str(p), match[0]))
                     continue
+                # 3. route target
                 dest = _route_target(p)
                 if dest == p:
                     results["no_target"] += 1
@@ -104,21 +111,21 @@ def scan_orphans(conn) -> dict:
                 if dest.exists():
                     results["collision"] += 1
                     continue
-                results["new"] += 1
+                results["approved"] += 1
                 moves.append((str(p), str(dest)))
-            except Exception as e:
+            except Exception:
                 results["errors"] += 1
-    return results, moves, dups
+    return results, moves, skipped
 
 
 def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int:
-    """Move files (or report them in dry-run). Returns count."""
+    """Move files (or report in dry-run). Returns count."""
     moved = 0
     for src, dest in moves:
         d = Path(dest)
         d.parent.mkdir(parents=True, exist_ok=True)
         if dry_run:
-            print(f"  MOVE {Path(src).name} → {d.parent.name}/")
+            print(f"  MOVE {Path(src).name} → {d.parent.parent.name}/{d.parent.name}/")
             moved += 1
             continue
         try:
@@ -130,22 +137,20 @@ def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int:
 
 
 def merge_orphans(dry_run: bool = True) -> dict:
-    """Main entry: scan + (dry-run or apply)."""
+    """Main entry: scan + (dry-run or apply). Only approved photos move."""
     db.init_db()
     conn = db.get_db()
-    results, moves, dups = scan_orphans(conn)
+    results, moves, skipped = scan_orphans(conn)
     conn.close()
-    print(f"scan: {results}")
+    print(f"scan: approved={results['approved']} keep_unreviewed={results['keep_unreviewed']} "
-    if dups:
+          f"rejected={results['rejected']} dup_in_bydate={results['dup_in_bydate']} "
-        print(f"  first 5 dups already in by_date:")
+          f"collisions={results['collision']} errors={results['errors']}")
-        for s, m in dups[:5]:
-            print(f"    {Path(s).name} == {m}")
     moved = apply_moves(moves, dry_run=dry_run)
     results["moves"] = moved
     if dry_run:
-        print(f"[dry-run] {moved} files would move — nothing touched")
+        print(f"[dry-run] {moved} APPROVED photos would move — nothing touched")
     else:
-        print(f"[applied] {moved} files moved")
+        print(f"[applied] {moved} approved photos moved")
     return results