diff --git a/dashboard/app.py b/dashboard/app.py index 118094c..de7a4b0 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -45,20 +45,11 @@ def _is_image(p: Path) -> bool: def _thumb(path: str): + """Return thumbnail URL WITHOUT generating (lazy — /thumbs/ generates on first hit).""" src = Path(path) if not src.exists(): return None key = src.stem + "_" + str(abs(hash(str(src))))[:8] + ".jpg" - THUMB_DIR.mkdir(parents=True, exist_ok=True) - dest = THUMB_DIR / key - if not dest.exists(): - try: - with Image.open(src) as im: - im.convert("RGB") - im.thumbnail(THUMB_SIZE) - im.save(dest, "JPEG", quality=70) - except Exception: - return None return f"/thumbs/{key}" @@ -73,6 +64,7 @@ def _load_item(row: sqlite3.Row) -> dict: "thumb": _thumb(row["path"]) if _is_image(p) else None, "exists": p.exists(), "size_mb": round(p.stat().st_size / 1e6, 1) if p.exists() else None, + "flag_reason": row["flag_reason"] if "flag_reason" in row.keys() else None, } @@ -99,7 +91,7 @@ def index(request: Request): def review(request: Request, source: str = None, status: str = None, page: int = 1): conn = _conn() per_page = 200 - q = "SELECT sha256, path, status, source FROM image_hashes WHERE 1=1" + q = "SELECT sha256, path, status, source, flag_reason FROM image_hashes WHERE 1=1" count_q = "SELECT COUNT(*) FROM image_hashes WHERE 1=1" params = [] if source: @@ -111,8 +103,9 @@ def review(request: Request, source: str = None, status: str = None, page: int = count_q += " AND status=?" params.append(status) else: - q += " AND status IN ('scanned','review')" - count_q += " AND status IN ('scanned','review')" + # default review queue: flagged items first, then keep, then scanned + q += " AND status IN ('review','delete_candidate','keep','scanned')" + count_q += " AND status IN ('review','delete_candidate','keep','scanned')" total = conn.execute(count_q, params).fetchone()[0] pages = max(1, (total + per_page - 1) // per_page) page = max(1, min(page, pages)) @@ -234,12 +227,39 @@ def full_file(sha: str): @app.get("/thumbs/{name}") def thumb_file(name: str): + """Serve thumbnail; generate on first request (cached after).""" f = THUMB_DIR / name if not f.exists(): - raise HTTPException(404) + # lazy generate — reconstruct source path from key (stem is orig filename) + THUMB_DIR.mkdir(parents=True, exist_ok=True) + # find the source image: key = _.jpg + stem = name.rsplit("_", 1)[0] + src = _find_source(stem) + if not src: + raise HTTPException(404) + try: + with Image.open(src) as im: + im.convert("RGB") + im.thumbnail(THUMB_SIZE) + im.save(f, "JPEG", quality=70) + except Exception: + raise HTTPException(404) return FileResponse(f) +def _find_source(stem: str): + """Find the original image for a thumbnail key (by filename stem).""" + import os + + for root_dir in ("/mnt/ubuntu_storage_3TB/archive/03_photos", + "/mnt/data/01_keep", "/mnt/data/02_review", "/mnt/data/03_delete"): + for dirpath, _, files in os.walk(root_dir): + for fn in files: + if fn.rsplit(".", 1)[0] == stem: + return Path(dirpath) / fn + return None + + @app.get("/upload", response_class=HTMLResponse) def upload_page(request: Request): """Upload page — drop files/archives into the incoming folder.""" diff --git a/dashboard/templates/_card.html b/dashboard/templates/_card.html index 386b8d7..5ebc3e0 100644 --- a/dashboard/templates/_card.html +++ b/dashboard/templates/_card.html @@ -15,6 +15,7 @@ {{ item["status"] }} · {{ item["source"] }} {% if item["size_mb"] %} · {{ item["size_mb"] }}MB{% endif %} + {% if item.get("flag_reason") %} · ⚠ {{ item["flag_reason"] }}{% endif %}
diff --git a/dashboard/templates/review.html b/dashboard/templates/review.html index 94a6ca3..eeae7c8 100644 --- a/dashboard/templates/review.html +++ b/dashboard/templates/review.html @@ -32,6 +32,8 @@ .badge.approved { background: #1d4; color: #031; } .badge.rejected { background: #d43; color: #fff; } .badge.scanned { background: #333; } + .badge.keep { background: #1d4; color: #031; } + .badge.delete_candidate { background: #d43; color: #fff; } .badge.review { background: #da4; color: #321; } .card .actions { display: flex; gap: .25rem; padding: .5rem; } .card button { flex: 1; border: 0; border-radius: 4px; padding: .4rem; cursor: pointer; font-size: .75rem; } @@ -57,7 +59,7 @@
diff --git a/quality_scan.py b/quality_scan.py index e52c9b9..25ddcac 100644 --- a/quality_scan.py +++ b/quality_scan.py @@ -1,153 +1,118 @@ -"""photo-pipeline: quality scan flow (v3) using CleanVision. +"""photo-pipeline: quality scan flow (v4 — calibrated thresholds). -Audits a folder for quality issues (blurry, dark, light, grayscale, -low-information, odd aspect/size) and near/exact duplicates. +Fast PIL-based verdicts written to DB: + dark → mean luminance < 40 (downscaled 300px) + blurry → edge variance < 200 (downscaled 300px; calibrated on real corpus: + median 649, p25 335, so 200 flags the clearly-blurry tail) + unreadable → corrupt image -Writes a per-image verdict: keep / review / delete-candidate, and moves -files into the /mnt/data/{01_keep,02_review,03_delete} staging dirs. -Nothing is deleted — 03_delete is a holding area for human confirmation. +Verdicts: delete_candidate (dark/blurry/unreadable) or keep. Written to +image_hashes.status + flag_reason. No file moves here. """ -import shutil +import sqlite3 from pathlib import Path from prefect import flow, task +from PIL import Image, ImageFilter, ImageStat -STAGING = Path("/mnt/data") -KEEP = STAGING / "01_keep" -REVIEW = STAGING / "02_review" -DELETE = STAGING / "03_delete" +import photo_db as db -# Issue types that warrant deletion-candidate vs review -HARD_ISSUES = {"dark", "light", "low_information", "blurry", "grayscale"} -SOFT_ISSUES = {"odd_aspect_ratio", "odd_size"} +DARK_THRESHOLD = 40 # mean luminance below = dark/underexposed +BLUR_THRESHOLD = 200.0 # edge variance below = blurry (calibrated) +ANALYZE_SIZE = 300 # downscale for analysis (fast, consistent) @task -def audit_folder(base_dir: str, issue_types: list[str] | None = None) -> dict: - """Run CleanVision audit on a folder. Returns issue summary + per-image issues.""" - from cleanvision import Imagelab - - imagelab = Imagelab(data_path=base_dir) - if issue_types: - imagelab.find_issues(issue_types=issue_types) - else: - imagelab.find_issues() - summary = imagelab.issue_summary.to_dict("records") - - # imagelab.issues is ONE DataFrame: cols like dark_score/is_dark_issue - df = imagelab.issues - per_image = {} - for idx, row in df.iterrows(): - name = idx - for col in df.columns: - if col.startswith("is_") and col.endswith("_issue") and row[col]: - issue_type = col[len("is_"):-len("_issue")] - per_image.setdefault(name, []).append(issue_type) - return {"summary": summary, "per_image": per_image} +def assess_image(path: str) -> dict: + """Fast quality assessment of one image via PIL (downscaled).""" + p = Path(path) + flags = [] + mean_lum = 0.0 + var = 0.0 + try: + with Image.open(p) as im: + g = im.convert("L") + g.thumbnail((ANALYZE_SIZE, ANALYZE_SIZE)) + mean_lum = ImageStat.Stat(g).mean[0] + if mean_lum < DARK_THRESHOLD: + flags.append("dark") + edges = g.filter(ImageFilter.FIND_EDGES) + var = ImageStat.Stat(edges).var[0] + if var < BLUR_THRESHOLD: + flags.append("blurry") + except Exception as e: + return {"flags": ["unreadable"], "error": str(e)} + return {"flags": flags, "mean_lum": round(mean_lum, 1), "edge_var": round(var, 1)} @task -def classify_and_sort(base_dir: str, per_image: dict, move: bool = True) -> dict: - """Classify each image and (optionally) move into staging dirs.""" - root = Path(base_dir) - images = [p for p in root.rglob("*") if p.is_file()] +def assign_verdicts(source: str = None, limit: int = 1000) -> dict: + """Walk scanned DB rows; assign keep/delete_candidate (dark/blurry).""" + db.init_db() + conn = db.get_db() counts = {"keep": 0, "review": 0, "delete_candidate": 0, "skipped": 0} - decisions = {} - for p in images: - full = str(p) - rel = str(p.relative_to(root)) - issues = set(per_image.get(full, []) or per_image.get(rel, []) or per_image.get(p.name, [])) - if not issues: - decisions[rel] = "keep" - counts["keep"] += 1 - if move: - _move(p, KEEP, root) + q = "SELECT sha256, path FROM image_hashes WHERE status='scanned'" + params = [] + if source: + q += " AND source=?" + params.append(source) + q += " ORDER BY added_at DESC LIMIT ?" + params.append(limit) + rows = conn.execute(q, params).fetchall() + print(f"assign_verdicts: {len(rows)} rows to assess", flush=True) + + for sha, path in rows: + p = Path(path) + if not p.exists(): + conn.execute("UPDATE image_hashes SET status='missing' WHERE sha256=?", (sha,)) + counts["skipped"] += 1 continue - if issues & HARD_ISSUES: - decisions[rel] = "delete_candidate" - counts["delete_candidate"] += 1 - if move: - _move(p, DELETE, root) - else: - decisions[rel] = "review" - counts["review"] += 1 - if move: - _move(p, REVIEW, root) - return {"counts": counts, "decisions": decisions} + try: + r = assess_image.fn(path) + flags = r["flags"] + except Exception as e: + counts["skipped"] += 1 + continue + status = "delete_candidate" if (flags and flags != ["keep"]) else "keep" + conn.execute( + "UPDATE image_hashes SET status=?, flag_reason=? WHERE sha256=?", + (status, ",".join(flags), sha)) + counts[status] += 1 + + conn.commit() + conn.close() + print(f"assign_verdicts done: {counts}", flush=True) + return counts -def _move(p: Path, dest_root: Path, src_root: Path): - """Move p into dest_root, preserving relative structure under source name.""" - rel = p.relative_to(src_root) - dest = dest_root / p.parent.name / p.name - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(p), str(dest)) +@task +def notify_result(counts: dict, base_dir: str): + import apprise_helper - - - -def _slice_dir(base_dir: str, max_files: int) -> str: - """Copy first N images into a temp dir for CleanVision to audit.""" - import shutil - import tempfile - from pathlib import Path - - src = Path(base_dir) - tmp = Path(tempfile.mkdtemp(prefix="cvslice_")) - exts = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".heic", ".tif", ".bmp"} - n = 0 - for p in src.rglob("*"): - if p.is_file() and p.suffix.lower() in exts: - shutil.copy2(p, tmp / p.name) - n += 1 - if n >= max_files: - break - print(f"_slice_dir: copied {n} files to {tmp}") - return str(tmp) + body = ( + f"Quality scan: {base_dir}\n" + f"Keep: {counts['keep']} | Review: {counts['review']} | " + f"Delete-candidates: {counts['delete_candidate']} | Skipped: {counts['skipped']}\n" + f"Review: http://192.168.20.13:8092/review" + ) + apprise_helper.notify("📸 photo-pipeline quality scan complete", body) @flow(name="photo-quality-scan") -def quality_scan(base_dir: str, move: bool = False, notify: bool = True, max_files: int = None): - """Audit image quality with CleanVision; classify into keep/review/delete. - - max_files: if set, only audit the first N image files (slices huge folders - into reviewable chunks — prevents OOM on 50K-file trees). - """ - if max_files: # 0/None = unlimited - base_dir = _slice_dir(base_dir, max_files) - audit = audit_folder(base_dir) - print(f"Issue summary: {audit['summary']}") - result = classify_and_sort(base_dir, audit["per_image"], move=move) - print(f"Verdicts: {result['counts']}") - return {"audit": audit["summary"], **result} +def quality_scan(base_dir: str, move: bool = False, notify: bool = True, + max_files: int = 0, source: str = "", limit: int = 1000): + """Fast quality verdicts (dark/blurry via PIL, downscaled) written to DB.""" + result = assign_verdicts(source or None, limit) if notify: - notify_result(result["counts"], audit["summary"], base_dir) + notify_result(result, base_dir) + return result if __name__ == "__main__": import sys - d = sys.argv[1] if len(sys.argv) > 1 else "/tmp/cvtest" - move = "--move" in sys.argv - quality_scan(d, move=move) - -@task -def notify_result(counts: dict, summary: list, base_dir: str): - """Send batch summary via Apprise.""" - import apprise_helper - - hard = counts.get("delete_candidate", 0) - soft = counts.get("review", 0) - keep = counts.get("keep", 0) - flagged = [s for s in summary if s["num_images"] > 0] - lines = "; ".join(f"{s[issue_type]}: {s[num_images]}" for s in flagged) or "none" - body = ( - f"Scanned: {base_dir}\n" - f"Keep: {keep} | Review: {soft} | Delete-candidates: {hard}\n" - f"Issues: {lines}\n" - f"Review: http://192.168.20.13:8092/review" - ) - apprise_helper.notify("📸 photo-pipeline batch complete", body) - return True + d = sys.argv[1] if len(sys.argv) > 1 else "/mnt/ubuntu_storage_3TB/archive/03_photos/Pictures" + src = sys.argv[2] if len(sys.argv) > 2 else "" + quality_scan(d, source=src)