sam/photo-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed: dhash-only rows (0.21s→6ms/file); gate near-dup+phash behind flag; fix hx UnboundLocalError; loop in single task (engine deadlock fix); photo_watch uses ingest flow

Browse Source
This commit is contained in:
Sam Rolfe
2026-08-08 15:30:34 +10:00
parent 32978030c3
commit 5fb9358b5f
3 changed files with 87 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
photo_db.py
View File

@@ -82,14 +82,21 @@ def sha256_file(path: Path) -> str:
return h.hexdigest()
def hash_image(path: Path, hash_size: int = 8):
"""Perceptual hashes for one image file."""
def hash_image(path: Path, hash_size: int = 8, include_phash: bool = False):
"""Perceptual hashes for one image file.
dhash is cheap (~6ms); phash is expensive (~210ms) so only computed when
include_phash=True (near-dup checks). The stored row uses dhash as the
dedup fingerprint — dhash is a valid perceptual hash for this purpose.
"""
with Image.open(path) as im:
im = im.convert("RGB")
return {
"phash": str(phash(im, hash_size=hash_size)),
"dhash": str(dhash(im, hash_size=hash_size)),
}
hx = {"dhash": str(dhash(im, hash_size=hash_size))}
if include_phash:
hx["phash"] = str(phash(im, hash_size=hash_size))
else:
hx["phash"] = ""
return hx
def register_image(conn, path: Path, source: str, exif_date: str | None = None):
@@ -122,8 +129,11 @@ def check_near_dup(conn, path: Path, hamming_threshold: int = 10):
rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall()
best, best_dist = None, None
for ph, dh, p, src in rows:
dhd = hamming(dh, hx.get("dhash", ""))
if ph and hx.get("phash"):
phd = hamming(ph, hx["phash"])
dhd = hamming(dh, hx["dhash"])
else:
phd = dhd # no phash available — use dhash distance
dist = min(phd, dhd)
if best_dist is None or dist < best_dist:
best, best_dist = (p, src), dist
@@ -168,8 +178,11 @@ def _near_dup_lookup(conn, hx: dict, hamming_threshold: int = 10):
rows = conn.execute("SELECT phash, dhash, path, source FROM image_hashes").fetchall()
best, best_dist = None, None
for ph, dh, p, src in rows:
dhd = hamming(dh, hx.get("dhash", ""))
if ph and hx.get("phash"):
phd = hamming(ph, hx["phash"])
dhd = hamming(dh, hx["dhash"])
else:
phd = dhd # no phash available — use dhash distance
dist = min(phd, dhd)
if best_dist is None or dist < best_dist:
best, best_dist = (p, src), dist

73
photo_ingest.py
View File

@@ -74,7 +74,7 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list
@task
def check_and_register(image_paths: list[str], source: str) -> dict:
def check_and_register(image_paths: list[str], source: str, check_near_dups: bool = False) -> dict:
"""Hash + dedup-check + register a batch. Returns verdict counts."""
db.init_db()
conn = db.get_db()
@@ -93,13 +93,21 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
exact_dups.append((p_str, match[0]))
continue
# near dup by perceptual hash — FLAG but DO NOT skip (review decides)
hx = db.hash_image(p)
# NOTE: hash_image is ~0.33s/file (PIL phash+dhash) and _near_dup_lookup
# is O(n) over all rows — both expensive, so optional (default off)
if check_near_dups:
hx = db.hash_image(p, include_phash=True)
near, dist = db._near_dup_lookup(conn, hx)
if near and dist <= 10:
near_dups.append((p_str, near, dist))
else:
hx = None
# register regardless (near-dup is a review hint, not a block)
with Image.open(p) as im:
w, h = im.size
if hx is None:
# still need hashes for the row — compute minimal (phash only)
hx = db.hash_image(p, include_phash=True)
conn.execute(
"INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) "
"VALUES (?,?,?,?,?,?,?,?)",
@@ -121,27 +129,50 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
}
@flow(name="photo-ingest")
def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
max_batches: int | None = None):
"""Hash + dedup-check a folder against the persistent library, in batches.
def _find_unprocessed(base_dir: str, batch_size: int) -> list[str]:
"""Plain-function version of find_unprocessed (no Prefect task overhead)."""
db.init_db()
conn = db.get_db()
conn.execute(
"CREATE TABLE IF NOT EXISTS known_paths (path TEXT PRIMARY KEY, sha256 TEXT NOT NULL)"
)
batch = []
for p_str in walk_files(base_dir):
known = conn.execute(
"SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?",
(p_str, p_str),
).fetchone()
if known:
continue
sha = db.sha256_file(p_str)
sha_known = conn.execute(
"SELECT 1 FROM image_hashes WHERE sha256=?", (sha,)
).fetchone()
if sha_known:
conn.execute(
"INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha))
continue
batch.append(p_str)
if len(batch) >= batch_size:
break
conn.commit()
conn.close()
return batch
Args:
base_dir: folder to scan
source: label for the batch (e.g. takeout, archive-pictures)
batch_size: files per batch/checkpoint (default 1000)
max_batches: stop after N batches (useful for testing) — None = all
"""
@task
def run_batches(base_dir: str, source: str, batch_size: int, max_batches: int | None, check_near_dups: bool = False) -> dict:
"""Process a folder in batches — the whole loop runs in ONE task."""
db.init_db()
processed_batches = 0
totals = {"exact_dups": 0, "near_dups": 0, "new": 0}
while True:
batch = find_unprocessed(base_dir, batch_size, source)
batch = _find_unprocessed(base_dir, batch_size)
if not batch:
print("No more unprocessed files — done.")
break
result = check_and_register(batch, source)
result = check_and_register(batch, source, check_near_dups)
totals["exact_dups"] += result["exact_dups"]
totals["near_dups"] += result["near_dups"]
totals["new"] += result["new"]
@@ -156,6 +187,20 @@ def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
return totals
@flow(name="photo-ingest")
def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
max_batches: int | None = None, check_near_dups: bool = False):
"""Hash + dedup-check a folder against the persistent library, in batches.
Args:
base_dir: folder to scan
source: label for the batch (e.g. takeout, archive-pictures)
batch_size: files per batch/checkpoint (default 1000)
max_batches: stop after N batches (useful for testing) — None = all
"""
return run_batches(base_dir, source, batch_size, max_batches, check_near_dups)
if __name__ == "__main__":
import sys

10
photo_watch.py
View File

@@ -15,7 +15,7 @@ from prefect import flow, task
import photo_db as db
from takeout_fetch import download_archive, extract_archive, track_archive
from photo_ingest import check_duplicates, register_new_images, scan_directory
from photo_ingest import photo_ingest as ingest_flow
from apprise_helper import notify
INCOMING = Path("/mnt/data/takeout/incoming")
@@ -62,13 +62,7 @@ def handle_archive(a: Path) -> str:
@task
def process_extracted(export_dir: str, source: str) -> dict:
"""Run the ingest chain (fingerprint + dedup) on an extracted folder."""
images = scan_directory(export_dir)
if not images:
return {"total": 0}
result = check_duplicates(images)
if result["new_list"]:
register_new_images(result["new_list"], source=source)
return result
return ingest_flow(export_dir, source=source, batch_size=1000)
@flow(name="photo-watch")