merge_orphans --apply: 56 approved orphans moved to by_date (EXIF-routed), verified 43+ in place
This commit is contained in:
@@ -1,28 +1,21 @@
|
||||
"""photo-pipeline: merge_orphans — dedup + EXIF-date routing into by_date.
|
||||
|
||||
Phase 2-3 of the consolidation plan:
|
||||
For each orphan folder (temp_transfer, Pictures, phone_dumps):
|
||||
1. sha256 the file
|
||||
2. Check against DB (by_date hashes, source=archive-date) → classify:
|
||||
dup = content already in by_date → skip (don't re-merge)
|
||||
new = genuinely new → route into by_date by EXIF date
|
||||
3. EXIF DateTime → by_date/sorted_pictures_holder/<YYYY>/<MM-Mon>/
|
||||
4. MOVE the file there (user confirmed: move, not copy)
|
||||
Phase 2-3 of the consolidation plan. IMPORTANT: only merges photos with
|
||||
status='approved' (reviewed in the dashboard). Unreviewed (keep) and
|
||||
rejected photos are NOT moved.
|
||||
|
||||
Flow:
|
||||
1. quality-scan → machine verdicts (keep / delete_candidate)
|
||||
2. dashboard review → approved / rejected
|
||||
3. merge_orphans → moves ONLY approved into by_date (EXIF-date routed)
|
||||
4. immich-import → uploads (checksum dedup = no doubling)
|
||||
|
||||
Modes:
|
||||
dry-run (default): report what WOULD move where — nothing touched
|
||||
--apply: actually move files (after backup confirmed + user review)
|
||||
|
||||
Safety:
|
||||
- never overwrites an existing target (collision → skip + report)
|
||||
- temp_transfer files without EXIF: fallback to file-mtime, else "unknown-date" subfolder
|
||||
- all moves logged; DB paths NOT updated (by_date rows are the new canonical;
|
||||
the fingerprint DB keeps source=archive-* rows pointing at old paths — we
|
||||
note moved files in a `moved` table for audit).
|
||||
dry-run (default): report what WOULD move — nothing touched
|
||||
--apply: actually move (after backup confirmed + user reviewed)
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
@@ -38,6 +31,7 @@ BY_DATE_ROOT = Path("/mnt/ubuntu_storage_3TB/archive/03_photos/by_date/sorted_pi
|
||||
MONTH_NAMES = {1:"01-Jan",2:"02-Feb",3:"03-Mar",4:"04-Apr",5:"05-May",6:"06-Jun",
|
||||
7:"07-Jul",8:"08-Aug",9:"09-Sep",10:"10-Oct",11:"11-Nov",12:"12-Dec"}
|
||||
EXT_IMAGES = {".jpg", ".jpeg", ".png", ".heic", ".webp", ".gif", ".tif", ".tiff", ".bmp"}
|
||||
REQUIRED_STATUS = "approved" # only merge photos reviewed+approved in dashboard
|
||||
|
||||
|
||||
def _exif_date(path: Path) -> str | None:
|
||||
@@ -45,7 +39,7 @@ def _exif_date(path: Path) -> str | None:
|
||||
try:
|
||||
with Image.open(path) as im:
|
||||
ex = im.getexif()
|
||||
dt = ex.get(306) or ex.get(36867) # DateTime / DateTimeOriginal
|
||||
dt = ex.get(306) or ex.get(36867)
|
||||
if dt and len(dt) >= 7:
|
||||
return f"{dt[:4]}-{dt[5:7]}"
|
||||
except Exception:
|
||||
@@ -65,7 +59,7 @@ def _mtime_date(path: Path) -> str | None:
|
||||
|
||||
|
||||
def _route_target(path: Path) -> Path:
|
||||
"""Compute the by_date target for an image (EXIF → mtime → unknown)."""
|
||||
"""Compute the by_date target (EXIF → mtime → unknown)."""
|
||||
ym = _exif_date(path) or _mtime_date(path)
|
||||
if not ym:
|
||||
return BY_DATE_ROOT / "unknown-date" / path.name
|
||||
@@ -75,28 +69,41 @@ def _route_target(path: Path) -> Path:
|
||||
|
||||
|
||||
def scan_orphans(conn) -> dict:
|
||||
"""Walk orphans, classify dup vs new, compute targets. Dry-run ready."""
|
||||
results = {"dup": 0, "new": 0, "no_target": 0, "collision": 0, "errors": 0}
|
||||
moves = [] # (src, dest)
|
||||
dups = [] # (src, matching_by_date_path)
|
||||
"""Walk orphans; classify approved-merge vs skipped. Dry-run ready."""
|
||||
results = {"approved": 0, "keep_unreviewed": 0, "rejected": 0,
|
||||
"dup_in_bydate": 0, "no_target": 0, "collision": 0, "errors": 0}
|
||||
moves = [] # (src, dest) — approved only
|
||||
skipped = []
|
||||
for root in ORPHAN_ROOTS:
|
||||
r = Path(root)
|
||||
if not r.exists():
|
||||
print(f" (missing root: {r})")
|
||||
continue
|
||||
for p in r.rglob("*"):
|
||||
if not p.is_file() or p.suffix.lower() not in EXT_IMAGES:
|
||||
continue
|
||||
try:
|
||||
# 1. review status gate
|
||||
row = conn.execute(
|
||||
"SELECT status, flag_reason FROM image_hashes WHERE path=? LIMIT 1", (str(p),)
|
||||
).fetchone()
|
||||
status = row[0] if row else "NOT_IN_DB"
|
||||
if status == "rejected":
|
||||
results["rejected"] += 1
|
||||
skipped.append((str(p), "rejected"))
|
||||
continue
|
||||
if status != REQUIRED_STATUS:
|
||||
results["keep_unreviewed"] += 1
|
||||
skipped.append((str(p), status or "unreviewed"))
|
||||
continue
|
||||
# 2. already in by_date? (content match)
|
||||
sha = db.sha256_file(p)
|
||||
# in by_date already? (content match, source=archive-date or any)
|
||||
match = conn.execute(
|
||||
"SELECT path FROM image_hashes WHERE sha256=? AND source='archive-date' LIMIT 1",
|
||||
(sha,)).fetchone()
|
||||
if match:
|
||||
results["dup"] += 1
|
||||
dups.append((str(p), match[0]))
|
||||
results["dup_in_bydate"] += 1
|
||||
continue
|
||||
# 3. route target
|
||||
dest = _route_target(p)
|
||||
if dest == p:
|
||||
results["no_target"] += 1
|
||||
@@ -104,21 +111,21 @@ def scan_orphans(conn) -> dict:
|
||||
if dest.exists():
|
||||
results["collision"] += 1
|
||||
continue
|
||||
results["new"] += 1
|
||||
results["approved"] += 1
|
||||
moves.append((str(p), str(dest)))
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
results["errors"] += 1
|
||||
return results, moves, dups
|
||||
return results, moves, skipped
|
||||
|
||||
|
||||
def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int:
|
||||
"""Move files (or report them in dry-run). Returns count."""
|
||||
"""Move files (or report in dry-run). Returns count."""
|
||||
moved = 0
|
||||
for src, dest in moves:
|
||||
d = Path(dest)
|
||||
d.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dry_run:
|
||||
print(f" MOVE {Path(src).name} → {d.parent.name}/")
|
||||
print(f" MOVE {Path(src).name} → {d.parent.parent.name}/{d.parent.name}/")
|
||||
moved += 1
|
||||
continue
|
||||
try:
|
||||
@@ -130,22 +137,20 @@ def apply_moves(moves: list[tuple[str, str]], dry_run: bool = True) -> int:
|
||||
|
||||
|
||||
def merge_orphans(dry_run: bool = True) -> dict:
|
||||
"""Main entry: scan + (dry-run or apply)."""
|
||||
"""Main entry: scan + (dry-run or apply). Only approved photos move."""
|
||||
db.init_db()
|
||||
conn = db.get_db()
|
||||
results, moves, dups = scan_orphans(conn)
|
||||
results, moves, skipped = scan_orphans(conn)
|
||||
conn.close()
|
||||
print(f"scan: {results}")
|
||||
if dups:
|
||||
print(f" first 5 dups already in by_date:")
|
||||
for s, m in dups[:5]:
|
||||
print(f" {Path(s).name} == {m}")
|
||||
print(f"scan: approved={results['approved']} keep_unreviewed={results['keep_unreviewed']} "
|
||||
f"rejected={results['rejected']} dup_in_bydate={results['dup_in_bydate']} "
|
||||
f"collisions={results['collision']} errors={results['errors']}")
|
||||
moved = apply_moves(moves, dry_run=dry_run)
|
||||
results["moves"] = moved
|
||||
if dry_run:
|
||||
print(f"[dry-run] {moved} files would move — nothing touched")
|
||||
print(f"[dry-run] {moved} APPROVED photos would move — nothing touched")
|
||||
else:
|
||||
print(f"[applied] {moved} files moved")
|
||||
print(f"[applied] {moved} approved photos moved")
|
||||
return results
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user