Add 10s per-file timeout to scan_once (prevents hung file blocking large scans)
This commit is contained in:
@@ -12,6 +12,8 @@ Performance: dhash-only rows (~6ms/file); phash only when check_near_dups.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time as _time
|
||||
|
||||
from prefect import flow, task
|
||||
from PIL import Image
|
||||
@@ -49,15 +51,23 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
|
||||
pending = []
|
||||
t0 = time.time()
|
||||
|
||||
def _timeout_handler(signum, frame):
|
||||
raise TimeoutError(f"file took >{FILE_TIMEOUT}s")
|
||||
|
||||
FILE_TIMEOUT = 10 # seconds per file — guards against hung/corrupt files
|
||||
signal.signal(signal.SIGALRM, _timeout_handler)
|
||||
|
||||
for p_str in walk_files(base_dir):
|
||||
p = Path(p_str)
|
||||
try:
|
||||
signal.alarm(FILE_TIMEOUT)
|
||||
known = conn.execute(
|
||||
"SELECT 1 FROM image_hashes WHERE path=? UNION SELECT 1 FROM known_paths WHERE path=?",
|
||||
(p_str, p_str),
|
||||
).fetchone()
|
||||
if known:
|
||||
totals["known"] += 1
|
||||
signal.alarm(0)
|
||||
continue
|
||||
|
||||
sha = db.sha256_file(p_str)
|
||||
@@ -68,6 +78,7 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO known_paths (path, sha256) VALUES (?,?)", (p_str, sha))
|
||||
totals["exact_dups"] += 1
|
||||
signal.alarm(0)
|
||||
continue
|
||||
|
||||
hx = db.hash_image(p)
|
||||
@@ -76,18 +87,24 @@ def scan_once(base_dir: str, source: str, batch_size: int, check_near_dups: bool
|
||||
pending.append((sha, hx["phash"], hx["dhash"], p.stat().st_size,
|
||||
w, h, str(p), source))
|
||||
totals["new"] += 1
|
||||
signal.alarm(0)
|
||||
|
||||
if len(pending) >= batch_size:
|
||||
conn.executemany(
|
||||
"INSERT INTO image_hashes (sha256, phash, dhash, file_size, width, height, path, source) "
|
||||
"VALUES (?,?,?,?,?,?,?,?)", pending)
|
||||
conn.commit()
|
||||
print(f" checkpoint: {totals['new']} new ({time.time()-t0:.0f}s)", flush=True)
|
||||
print(f" checkpoint: {totals['new']} new ({_time.time()-t0:.0f}s)", flush=True)
|
||||
pending = []
|
||||
|
||||
except TimeoutError as e:
|
||||
totals["skipped"] += 1
|
||||
print(f" TIMEOUT {p.name}: {e}", flush=True)
|
||||
except Exception as e:
|
||||
totals["skipped"] += 1
|
||||
print(f" SKIP {p.name}: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
signal.alarm(0)
|
||||
|
||||
if pending:
|
||||
conn.executemany(
|
||||
|
||||
Reference in New Issue
Block a user