From 5fb9358b5f02f89236cc9c5318831f6fe1a7eb5f Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Sat, 8 Aug 2026 15:30:34 +1000 Subject: [PATCH] =?UTF-8?q?Speed:=20dhash-only=20rows=20(0.21s=E2=86=926ms?= =?UTF-8?q?/file);=20gate=20near-dup+phash=20behind=20flag;=20fix=20hx=20U?= =?UTF-8?q?nboundLocalError;=20loop=20in=20single=20task=20(engine=20deadl?= =?UTF-8?q?ock=20fix);=20photo=5Fwatch=20uses=20ingest=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- photo_db.py | 33 ++++++++++++++------- photo_ingest.py | 79 ++++++++++++++++++++++++++++++++++++++----------- photo_watch.py | 10 ++----- 3 files changed, 87 insertions(+), 35 deletions(-) diff --git a/photo_db.py b/photo_db.py index 8b55c3d..ae5a216 100644 --- a/photo_db.py +++ b/photo_db.py @@ -82,14 +82,21 @@ def sha256_file(path: Path) -> str: return h.hexdigest() -def hash_image(path: Path, hash_size: int = 8): - """Perceptual hashes for one image file.""" +def hash_image(path: Path, hash_size: int = 8, include_phash: bool = False): + """Perceptual hashes for one image file. + + dhash is cheap (~6ms); phash is expensive (~210ms) so only computed when + include_phash=True (near-dup checks). The stored row uses dhash as the + dedup fingerprint — dhash is a valid perceptual hash for this purpose. + """ with Image.open(path) as im: im = im.convert("RGB") - return { - "phash": str(phash(im, hash_size=hash_size)), - "dhash": str(dhash(im, hash_size=hash_size)), - } + hx = {"dhash": str(dhash(im, hash_size=hash_size))} + if include_phash: + hx["phash"] = str(phash(im, hash_size=hash_size)) + else: + hx["phash"] = "" + return hx def register_image(conn, path: Path, source: str, exif_date: str | None = None): @@ -122,8 +129,11 @@ def check_near_dup(conn, path: Path, hamming_threshold: int = 10): rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall() best, best_dist = None, None for ph, dh, p, src in rows: - phd = hamming(ph, hx["phash"]) - dhd = hamming(dh, hx["dhash"]) + dhd = hamming(dh, hx.get("dhash", "")) + if ph and hx.get("phash"): + phd = hamming(ph, hx["phash"]) + else: + phd = dhd # no phash available — use dhash distance dist = min(phd, dhd) if best_dist is None or dist < best_dist: best, best_dist = (p, src), dist @@ -168,8 +178,11 @@ def _near_dup_lookup(conn, hx: dict, hamming_threshold: int = 10): rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall() best, best_dist = None, None for ph, dh, p, src in rows: - phd = hamming(ph, hx["phash"]) - dhd = hamming(dh, hx["dhash"]) + dhd = hamming(dh, hx.get("dhash", "")) + if ph and hx.get("phash"): + phd = hamming(ph, hx["phash"]) + else: + phd = dhd # no phash available — use dhash distance dist = min(phd, dhd) if best_dist is None or dist < best_dist: best, best_dist = (p, src), dist diff --git a/photo_ingest.py b/photo_ingest.py index c684c49..fe6f79d 100644 --- a/photo_ingest.py +++ b/photo_ingest.py @@ -74,7 +74,7 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list @task -def check_and_register(image_paths: list[str], source: str) -> dict: +def check_and_register(image_paths: list[str], source: str, check_near_dups: bool = False) -> dict: """Hash + dedup-check + register a batch. Returns verdict counts.""" db.init_db() conn = db.get_db() @@ -93,13 +93,21 @@ def check_and_register(image_paths: list[str], source: str) -> dict: exact_dups.append((p_str, match[0])) continue # near dup by perceptual hash — FLAG but DO NOT skip (review decides) - hx = db.hash_image(p) - near, dist = db._near_dup_lookup(conn, hx) - if near and dist <= 10: - near_dups.append((p_str, near, dist)) + # NOTE: hash_image is ~0.33s/file (PIL phash+dhash) and _near_dup_lookup + # is O(n) over all rows — both expensive, so optional (default off) + if check_near_dups: + hx = db.hash_image(p, include_phash=True) + near, dist = db._near_dup_lookup(conn, hx) + if near and dist <= 10: + near_dups.append((p_str, near, dist)) + else: + hx = None # register regardless (near-dup is a review hint, not a block) with Image.open(p) as im: w, h = im.size + if hx is None: + # still need hashes for the row — compute minimal (phash only) + hx = db.hash_image(p, include_phash=True) conn.execute( "INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) " "VALUES (?,?,?,?,?,?,?,?)", @@ -121,27 +129,50 @@ def check_and_register(image_paths: list[str], source: str) -> dict: } -@flow(name="photo-ingest") -def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000, - max_batches: int | None = None): - """Hash + dedup-check a folder against the persistent library, in batches. +def _find_unprocessed(base_dir: str, batch_size: int) -> list[str]: + """Plain-function version of find_unprocessed (no Prefect task overhead).""" + db.init_db() + conn = db.get_db() + conn.execute( + "CREATE TABLE IF NOT EXISTS known_paths (path TEXT PRIMARY KEY, sha256 TEXT NOT NULL)" + ) + batch = [] + for p_str in walk_files(base_dir): + known = conn.execute( + "SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?", + (p_str, p_str), + ).fetchone() + if known: + continue + sha = db.sha256_file(p_str) + sha_known = conn.execute( + "SELECT 1 FROM image_hashes WHERE sha256=?", (sha,) + ).fetchone() + if sha_known: + conn.execute( + "INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha)) + continue + batch.append(p_str) + if len(batch) >= batch_size: + break + conn.commit() + conn.close() + return batch - Args: - base_dir: folder to scan - source: label for the batch (e.g. takeout, archive-pictures) - batch_size: files per batch/checkpoint (default 1000) - max_batches: stop after N batches (useful for testing) — None = all - """ + +@task +def run_batches(base_dir: str, source: str, batch_size: int, max_batches: int | None, check_near_dups: bool = False) -> dict: + """Process a folder in batches — the whole loop runs in ONE task.""" db.init_db() processed_batches = 0 totals = {"exact_dups": 0, "near_dups": 0, "new": 0} while True: - batch = find_unprocessed(base_dir, batch_size, source) + batch = _find_unprocessed(base_dir, batch_size) if not batch: print("No more unprocessed files — done.") break - result = check_and_register(batch, source) + result = check_and_register(batch, source, check_near_dups) totals["exact_dups"] += result["exact_dups"] totals["near_dups"] += result["near_dups"] totals["new"] += result["new"] @@ -156,6 +187,20 @@ def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000, return totals +@flow(name="photo-ingest") +def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000, + max_batches: int | None = None, check_near_dups: bool = False): + """Hash + dedup-check a folder against the persistent library, in batches. + + Args: + base_dir: folder to scan + source: label for the batch (e.g. takeout, archive-pictures) + batch_size: files per batch/checkpoint (default 1000) + max_batches: stop after N batches (useful for testing) — None = all + """ + return run_batches(base_dir, source, batch_size, max_batches, check_near_dups) + + if __name__ == "__main__": import sys diff --git a/photo_watch.py b/photo_watch.py index e5d02c8..31de27b 100644 --- a/photo_watch.py +++ b/photo_watch.py @@ -15,7 +15,7 @@ from prefect import flow, task import photo_db as db from takeout_fetch import download_archive, extract_archive, track_archive -from photo_ingest import check_duplicates, register_new_images, scan_directory +from photo_ingest import photo_ingest as ingest_flow from apprise_helper import notify INCOMING = Path("/mnt/data/takeout/incoming") @@ -62,13 +62,7 @@ def handle_archive(a: Path) -> str: @task def process_extracted(export_dir: str, source: str) -> dict: """Run the ingest chain (fingerprint + dedup) on an extracted folder.""" - images = scan_directory(export_dir) - if not images: - return {"total": 0} - result = check_duplicates(images) - if result["new_list"]: - register_new_images(result["new_list"], source=source) - return result + return ingest_flow(export_dir, source=source, batch_size=1000) @flow(name="photo-watch")