Add 10s per-file timeout to scan_once (prevents hung file blocking large scans)

This commit is contained in:
2026-08-10 06:01:51 +10:00
parent d2a58caca6
commit 1edafbfac8

View File

@@ -12,6 +12,8 @@ Performance: dhash-only rows (~6ms/file); phash only when check_near_dups.
"""
from pathlib import Path
import signal
import time as _time
from prefect import flow, task
from PIL import Image
@@ -49,15 +51,23 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
pending = []
t0 = time.time()
def _timeout_handler(signum, frame):
raise TimeoutError(f"file took >{FILE_TIMEOUT}s")
FILE_TIMEOUT = 10 # seconds per file — guards against hung/corrupt files
signal.signal(signal.SIGALRM, _timeout_handler)
for p_str in walk_files(base_dir):
p = Path(p_str)
try:
signal.alarm(FILE_TIMEOUT)
known = conn.execute(
"SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?",
(p_str, p_str),
).fetchone()
if known:
totals["known"] += 1
signal.alarm(0)
continue
sha = db.sha256_file(p_str)
@@ -68,6 +78,7 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
conn.execute(
"INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha))
totals["exact_dups"] += 1
signal.alarm(0)
continue
hx = db.hash_image(p)
@@ -76,18 +87,24 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
pending.append((sha, hx["phash"], hx["dhash"], p.stat().st_size,
w, h, str(p), source))
totals["new"] += 1
signal.alarm(0)
if len(pending) >= batch_size:
conn.executemany(
"INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) "
"VALUES (?,?,?,?,?,?,?,?)", pending)
conn.commit()
print(f" checkpoint: {totals['new']} new ({time.time()-t0:.0f}s)", flush=True)
print(f" checkpoint: {totals['new']} new ({_time.time()-t0:.0f}s)", flush=True)
pending = []
except TimeoutError as e:
totals["skipped"] += 1
print(f" TIMEOUT {p.name}: {e}", flush=True)
except Exception as e:
totals["skipped"] += 1
print(f" SKIP {p.name}: {type(e).__name__}: {e}", flush=True)
finally:
signal.alarm(0)
if pending:
conn.executemany(