Speed: dhash-only rows (0.21s→6ms/file); gate near-dup+phash behind flag; fix hx UnboundLocalError; loop in single task (engine deadlock fix); photo_watch uses ingest flow

photo_ingest.py

@@ -74,7 +74,7 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list
 
 
 @task
-def check_and_register(image_paths: list[str], source: str) -> dict:
+def check_and_register(image_paths: list[str], source: str, check_near_dups: bool = False) -> dict:
     """Hash + dedup-check + register a batch. Returns verdict counts."""
     db.init_db()
     conn = db.get_db()
@@ -93,13 +93,21 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
                 exact_dups.append((p_str, match[0]))
                 continue
             # near dup by perceptual hash — FLAG but DO NOT skip (review decides)
-            hx = db.hash_image(p)
-            near, dist = db._near_dup_lookup(conn, hx)
-            if near and dist <= 10:
-                near_dups.append((p_str, near, dist))
+            # NOTE: hash_image is ~0.33s/file (PIL phash+dhash) and _near_dup_lookup
+            # is O(n) over all rows — both expensive, so optional (default off)
+            if check_near_dups:
+                hx = db.hash_image(p, include_phash=True)
+                near, dist = db._near_dup_lookup(conn, hx)
+                if near and dist <= 10:
+                    near_dups.append((p_str, near, dist))
+            else:
+                hx = None
             # register regardless (near-dup is a review hint, not a block)
             with Image.open(p) as im:
                 w, h = im.size
+            if hx is None:
+                # still need hashes for the row — compute minimal (phash only)
+                hx = db.hash_image(p, include_phash=True)
             conn.execute(
                 "INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) "
                 "VALUES (?,?,?,?,?,?,?,?)",
@@ -121,27 +129,50 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
     }
 
 
-@flow(name="photo-ingest")
-def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
-                 max_batches: int | None = None):
-    """Hash + dedup-check a folder against the persistent library, in batches.
+def _find_unprocessed(base_dir: str, batch_size: int) -> list[str]:
+    """Plain-function version of find_unprocessed (no Prefect task overhead)."""
+    db.init_db()
+    conn = db.get_db()
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS known_paths (path TEXT PRIMARY KEY, sha256 TEXT NOT NULL)"
+    )
+    batch = []
+    for p_str in walk_files(base_dir):
+        known = conn.execute(
+            "SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?",
+            (p_str, p_str),
+        ).fetchone()
+        if known:
+            continue
+        sha = db.sha256_file(p_str)
+        sha_known = conn.execute(
+            "SELECT 1 FROM image_hashes WHERE sha256=?", (sha,)
+        ).fetchone()
+        if sha_known:
+            conn.execute(
+                "INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha))
+            continue
+        batch.append(p_str)
+        if len(batch) >= batch_size:
+            break
+    conn.commit()
+    conn.close()
+    return batch
 
-    Args:
-        base_dir: folder to scan
-        source: label for the batch (e.g. takeout, archive-pictures)
-        batch_size: files per batch/checkpoint (default 1000)
-        max_batches: stop after N batches (useful for testing) — None = all
-    """
+
+@task
+def run_batches(base_dir: str, source: str, batch_size: int, max_batches: int | None, check_near_dups: bool = False) -> dict:
+    """Process a folder in batches — the whole loop runs in ONE task."""
     db.init_db()
     processed_batches = 0
     totals = {"exact_dups": 0, "near_dups": 0, "new": 0}
 
     while True:
-        batch = find_unprocessed(base_dir, batch_size, source)
+        batch = _find_unprocessed(base_dir, batch_size)
         if not batch:
             print("No more unprocessed files — done.")
             break
-        result = check_and_register(batch, source)
+        result = check_and_register(batch, source, check_near_dups)
         totals["exact_dups"] += result["exact_dups"]
         totals["near_dups"] += result["near_dups"]
         totals["new"] += result["new"]
@@ -156,6 +187,20 @@ def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
     return totals
 
 
+@flow(name="photo-ingest")
+def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
+                 max_batches: int | None = None, check_near_dups: bool = False):
+    """Hash + dedup-check a folder against the persistent library, in batches.
+
+    Args:
+        base_dir: folder to scan
+        source: label for the batch (e.g. takeout, archive-pictures)
+        batch_size: files per batch/checkpoint (default 1000)
+        max_batches: stop after N batches (useful for testing) — None = all
+    """
+    return run_batches(base_dir, source, batch_size, max_batches, check_near_dups)
+
+
 if __name__ == "__main__":
     import sys