Speed: dhash-only rows (0.21s→6ms/file); gate near-dup+phash behind flag; fix hx UnboundLocalError; loop in single task (engine deadlock fix); photo_watch uses ingest flow

photo_db.py

@@ -82,14 +82,21 @@ def sha256_file(path: Path) -> str:
     return h.hexdigest()
 
 
-def hash_image(path: Path, hash_size: int = 8):
+def hash_image(path: Path, hash_size: int = 8, include_phash: bool = False):
-    """Perceptual hashes for one image file."""
+    """Perceptual hashes for one image file.
+
+    dhash is cheap (~6ms); phash is expensive (~210ms) so only computed when
+    include_phash=True (near-dup checks). The stored row uses dhash as the
+    dedup fingerprint — dhash is a valid perceptual hash for this purpose.
+    """
     with Image.open(path) as im:
         im = im.convert("RGB")
-        return {
+        hx = {"dhash": str(dhash(im, hash_size=hash_size))}
-            "phash": str(phash(im, hash_size=hash_size)),
+        if include_phash:
-            "dhash": str(dhash(im, hash_size=hash_size)),
+            hx["phash"] = str(phash(im, hash_size=hash_size))
-        }
+        else:
+            hx["phash"] = ""
+        return hx
 
 
 def register_image(conn, path: Path, source: str, exif_date: str | None = None):
@@ -122,8 +129,11 @@ def check_near_dup(conn, path: Path, hamming_threshold: int = 10):
     rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall()
     best, best_dist = None, None
     for ph, dh, p, src in rows:
-        phd = hamming(ph, hx["phash"])
+        dhd = hamming(dh, hx.get("dhash", ""))
-        dhd = hamming(dh, hx["dhash"])
+        if ph and hx.get("phash"):
+            phd = hamming(ph, hx["phash"])
+        else:
+            phd = dhd  # no phash available — use dhash distance
         dist = min(phd, dhd)
         if best_dist is None or dist < best_dist:
             best, best_dist = (p, src), dist
@@ -168,8 +178,11 @@ def _near_dup_lookup(conn, hx: dict, hamming_threshold: int = 10):
     rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall()
     best, best_dist = None, None
     for ph, dh, p, src in rows:
-        phd = hamming(ph, hx["phash"])
+        dhd = hamming(dh, hx.get("dhash", ""))
-        dhd = hamming(dh, hx["dhash"])
+        if ph and hx.get("phash"):
+            phd = hamming(ph, hx["phash"])
+        else:
+            phd = dhd  # no phash available — use dhash distance
         dist = min(phd, dhd)
         if best_dist is None or dist < best_dist:
             best, best_dist = (p, src), dist

photo_ingest.py

@@ -74,7 +74,7 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list
 
 
 @task
-def check_and_register(image_paths: list[str], source: str) -> dict:
+def check_and_register(image_paths: list[str], source: str, check_near_dups: bool = False) -> dict:
     """Hash + dedup-check + register a batch. Returns verdict counts."""
     db.init_db()
     conn = db.get_db()
@@ -93,13 +93,21 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
                 exact_dups.append((p_str, match[0]))
                 continue
             # near dup by perceptual hash — FLAG but DO NOT skip (review decides)
-            hx = db.hash_image(p)
+            # NOTE: hash_image is ~0.33s/file (PIL phash+dhash) and _near_dup_lookup
-            near, dist = db._near_dup_lookup(conn, hx)
+            # is O(n) over all rows — both expensive, so optional (default off)
-            if near and dist <= 10:
+            if check_near_dups:
-                near_dups.append((p_str, near, dist))
+                hx = db.hash_image(p, include_phash=True)
+                near, dist = db._near_dup_lookup(conn, hx)
+                if near and dist <= 10:
+                    near_dups.append((p_str, near, dist))
+            else:
+                hx = None
             # register regardless (near-dup is a review hint, not a block)
             with Image.open(p) as im:
                 w, h = im.size
+            if hx is None:
+                # still need hashes for the row — compute minimal (phash only)
+                hx = db.hash_image(p, include_phash=True)
             conn.execute(
                 "INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) "
                 "VALUES (?,?,?,?,?,?,?,?)",
@@ -121,27 +129,50 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
     }
 
 
-@flow(name="photo-ingest")
+def _find_unprocessed(base_dir: str, batch_size: int) -> list[str]:
-def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
+    """Plain-function version of find_unprocessed (no Prefect task overhead)."""
-                 max_batches: int | None = None):
+    db.init_db()
-    """Hash + dedup-check a folder against the persistent library, in batches.
+    conn = db.get_db()
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS known_paths (path TEXT PRIMARY KEY, sha256 TEXT NOT NULL)"
+    )
+    batch = []
+    for p_str in walk_files(base_dir):
+        known = conn.execute(
+            "SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?",
+            (p_str, p_str),
+        ).fetchone()
+        if known:
+            continue
+        sha = db.sha256_file(p_str)
+        sha_known = conn.execute(
+            "SELECT 1 FROM image_hashes WHERE sha256=?", (sha,)
+        ).fetchone()
+        if sha_known:
+            conn.execute(
+                "INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha))
+            continue
+        batch.append(p_str)
+        if len(batch) >= batch_size:
+            break
+    conn.commit()
+    conn.close()
+    return batch
 
-    Args:
+
-        base_dir: folder to scan
+@task
-        source: label for the batch (e.g. takeout, archive-pictures)
+def run_batches(base_dir: str, source: str, batch_size: int, max_batches: int | None, check_near_dups: bool = False) -> dict:
-        batch_size: files per batch/checkpoint (default 1000)
+    """Process a folder in batches — the whole loop runs in ONE task."""
-        max_batches: stop after N batches (useful for testing) — None = all
-    """
     db.init_db()
     processed_batches = 0
     totals = {"exact_dups": 0, "near_dups": 0, "new": 0}
 
     while True:
-        batch = find_unprocessed(base_dir, batch_size, source)
+        batch = _find_unprocessed(base_dir, batch_size)
         if not batch:
             print("No more unprocessed files — done.")
             break
-        result = check_and_register(batch, source)
+        result = check_and_register(batch, source, check_near_dups)
         totals["exact_dups"] += result["exact_dups"]
         totals["near_dups"] += result["near_dups"]
         totals["new"] += result["new"]
@@ -156,6 +187,20 @@ def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
     return totals
 
 
+@flow(name="photo-ingest")
+def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
+                 max_batches: int | None = None, check_near_dups: bool = False):
+    """Hash + dedup-check a folder against the persistent library, in batches.
+
+    Args:
+        base_dir: folder to scan
+        source: label for the batch (e.g. takeout, archive-pictures)
+        batch_size: files per batch/checkpoint (default 1000)
+        max_batches: stop after N batches (useful for testing) — None = all
+    """
+    return run_batches(base_dir, source, batch_size, max_batches, check_near_dups)
+
+
 if __name__ == "__main__":
     import sys
 

photo_watch.py

@@ -15,7 +15,7 @@ from prefect import flow, task
 
 import photo_db as db
 from takeout_fetch import download_archive, extract_archive, track_archive
-from photo_ingest import check_duplicates, register_new_images, scan_directory
+from photo_ingest import photo_ingest as ingest_flow
 from apprise_helper import notify
 
 INCOMING = Path("/mnt/data/takeout/incoming")
@@ -62,13 +62,7 @@ def handle_archive(a: Path) -> str:
 @task
 def process_extracted(export_dir: str, source: str) -> dict:
     """Run the ingest chain (fingerprint + dedup) on an extracted folder."""
-    images = scan_directory(export_dir)
+    return ingest_flow(export_dir, source=source, batch_size=1000)
-    if not images:
-        return {"total": 0}
-    result = check_duplicates(images)
-    if result["new_list"]:
-        register_new_images(result["new_list"], source=source)
-    return result
 
 
 @flow(name="photo-watch")