314 lines
10 KiB
Python
314 lines
10 KiB
Python
"""photo-pipeline dashboard — FastAPI + htmx review UI.
|
|
|
|
Reads photo_pipeline.db + staging dirs on .13. Serves:
|
|
/ — overview: source/batch stats + verdict counts + folder targets
|
|
/review — thumbnail review grid with keep/reject/reset (htmx)
|
|
/file/{sha} — full-size original image
|
|
/thumbs/... — generated thumbnails
|
|
/stats — JSON stats
|
|
|
|
Review actions update the DB ONLY (instant). File moves happen at import time
|
|
(via the immich-import / process-staging flow) — this keeps review responsive
|
|
even for large batches; cross-filesystem moves are slow.
|
|
"""
|
|
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.responses import HTMLResponse, FileResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.templating import Jinja2Templates
|
|
from PIL import Image
|
|
|
|
BASE = Path(__file__).parent
|
|
DB_PATH = BASE.parent / "photo_pipeline.db"
|
|
STAGING = Path("/mnt/data")
|
|
THUMB_DIR = Path("/mnt/data/.thumbs")
|
|
THUMB_SIZE = (320, 320)
|
|
|
|
app = FastAPI(title="photo-pipeline dashboard")
|
|
templates = Jinja2Templates(directory=str(BASE / "templates"))
|
|
app.mount("/static", StaticFiles(directory=str(BASE / "static")), name="static")
|
|
|
|
EXT_IMAGES = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".heic"}
|
|
|
|
|
|
def _conn():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
def _is_image(p: Path) -> bool:
|
|
return p.suffix.lower() in EXT_IMAGES and p.exists()
|
|
|
|
|
|
def _thumb(path: str):
|
|
"""Return thumbnail URL WITHOUT generating (lazy — /thumbs/ generates on first hit)."""
|
|
src = Path(path)
|
|
if not src.exists():
|
|
return None
|
|
key = src.stem + "_" + str(abs(hash(str(src))))[:8] + ".jpg"
|
|
return f"/thumbs/{key}"
|
|
|
|
|
|
def _load_item(row: sqlite3.Row) -> dict:
|
|
p = Path(row["path"])
|
|
return {
|
|
"sha256": row["sha256"],
|
|
"path": row["path"],
|
|
"name": p.name,
|
|
"status": row["status"],
|
|
"source": row["source"],
|
|
"thumb": _thumb(row["path"]) if _is_image(p) else None,
|
|
"exists": p.exists(),
|
|
"size_mb": round(p.stat().st_size / 1e6, 1) if p.exists() else None,
|
|
"flag_reason": row["flag_reason"] if "flag_reason" in row.keys() else None,
|
|
}
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
def index(request: Request):
|
|
conn = _conn()
|
|
rows = conn.execute(
|
|
"""SELECT source, status, COUNT(*) as n FROM image_hashes
|
|
GROUP BY source, status ORDER BY source, status"""
|
|
).fetchall()
|
|
total = conn.execute("SELECT COUNT(*) FROM image_hashes").fetchone()[0]
|
|
conn.close()
|
|
folders = {}
|
|
for name in ("01_keep", "02_review", "03_delete"):
|
|
d = STAGING / name
|
|
folders[name] = sum(1 for _ in d.rglob("*") if _.is_file()) if d.exists() else 0
|
|
return templates.TemplateResponse(
|
|
request, "index.html",
|
|
{"rows": rows, "total": total, "folders": folders},
|
|
)
|
|
|
|
|
|
@app.get("/review", response_class=HTMLResponse)
|
|
def review(request: Request, source: str = None, status: str = None, page: int = 1):
|
|
conn = _conn()
|
|
per_page = 200
|
|
q = "SELECT sha256, path, status, source, flag_reason FROM image_hashes WHERE 1=1"
|
|
count_q = "SELECT COUNT(*) FROM image_hashes WHERE 1=1"
|
|
params = []
|
|
if source:
|
|
q += " AND source=?"
|
|
count_q += " AND source=?"
|
|
params.append(source)
|
|
if status:
|
|
q += " AND status=?"
|
|
count_q += " AND status=?"
|
|
params.append(status)
|
|
else:
|
|
# default review queue: flagged items first, then keep, then scanned
|
|
q += " AND status IN ('review','delete_candidate','keep','scanned')"
|
|
count_q += " AND status IN ('review','delete_candidate','keep','scanned')"
|
|
total = conn.execute(count_q, params).fetchone()[0]
|
|
pages = max(1, (total + per_page - 1) // per_page)
|
|
page = max(1, min(page, pages))
|
|
q += " ORDER BY added_at DESC LIMIT ? OFFSET ?"
|
|
rows = conn.execute(q, params + [per_page, (page - 1) * per_page]).fetchall()
|
|
conn.close()
|
|
items = [_load_item(r) for r in rows]
|
|
return templates.TemplateResponse(
|
|
request, "review.html",
|
|
{"items": items, "source": source, "status": status,
|
|
"page": page, "pages": pages, "total": total},
|
|
)
|
|
|
|
|
|
@app.get("/stats")
|
|
def stats():
|
|
import json
|
|
|
|
conn = _conn()
|
|
total = conn.execute("SELECT COUNT(*) FROM image_hashes").fetchone()[0]
|
|
by_status = dict(conn.execute(
|
|
"SELECT status, COUNT(*) FROM image_hashes GROUP BY status").fetchall())
|
|
by_source = dict(conn.execute(
|
|
"SELECT source, COUNT(*) FROM image_hashes GROUP BY source").fetchall())
|
|
conn.close()
|
|
return json.dumps({"total": total, "by_status": by_status, "by_source": by_source})
|
|
|
|
|
|
@app.get("/pipeline", response_class=HTMLResponse)
|
|
def pipeline(request: Request):
|
|
"""Pipeline status: recent Prefect flow runs + recent review actions."""
|
|
import sys
|
|
if str(BASE.parent) not in sys.path:
|
|
sys.path.insert(0, str(BASE.parent))
|
|
import status_helper
|
|
|
|
runs = status_helper.get_flow_runs(limit=15)
|
|
actions = status_helper.get_recent_actions(limit=20)
|
|
return templates.TemplateResponse(
|
|
request, "pipeline.html",
|
|
{"runs": runs, "actions": actions},
|
|
)
|
|
|
|
|
|
def _set_status(sha: str, status: str, request: Request) -> HTMLResponse:
|
|
"""Update DB status ONLY — instant. File moves happen at import time."""
|
|
conn = _conn()
|
|
row = conn.execute("SELECT * FROM image_hashes WHERE sha256=?", (sha,)).fetchone()
|
|
if not row:
|
|
conn.close()
|
|
raise HTTPException(404)
|
|
conn.execute(
|
|
"UPDATE image_hashes SET status=?, reviewed_at=datetime('now') WHERE sha256=?",
|
|
(status, sha))
|
|
conn.commit()
|
|
row = conn.execute("SELECT * FROM image_hashes WHERE sha256=?", (sha,)).fetchone()
|
|
conn.close()
|
|
return templates.TemplateResponse(
|
|
request, "_card.html", {"item": _load_item(row)}
|
|
)
|
|
|
|
|
|
@app.post("/review/{sha}/approve")
|
|
def approve(sha: str, request: Request):
|
|
return _set_status(sha, "approved", request)
|
|
|
|
|
|
@app.post("/review/{sha}/reject")
|
|
def reject(sha: str, request: Request):
|
|
return _set_status(sha, "rejected", request)
|
|
|
|
|
|
@app.post("/review/{sha}/reset")
|
|
def reset_status(sha: str, request: Request):
|
|
return _set_status(sha, "scanned", request)
|
|
|
|
|
|
@app.post("/review/bulk")
|
|
async def bulk_action(request: Request):
|
|
"""Bulk status update — DB only, instant. Body: {"action", "shas": []}"""
|
|
import json
|
|
|
|
body = json.loads(await request.body())
|
|
action = body.get("action")
|
|
shas = body.get("shas", [])
|
|
if action not in ("approve", "reject", "reset"):
|
|
raise HTTPException(400, "action must be approve/reject/reset")
|
|
status_map = {"approve": "approved", "reject": "rejected", "reset": "scanned"}
|
|
status = status_map[action]
|
|
|
|
conn = _conn()
|
|
updated = 0
|
|
missing = 0
|
|
for sha in shas:
|
|
cur = conn.execute(
|
|
"UPDATE image_hashes SET status=?, reviewed_at=datetime('now') WHERE sha256=?",
|
|
(status, sha))
|
|
if cur.rowcount:
|
|
updated += 1
|
|
else:
|
|
missing += 1
|
|
conn.commit()
|
|
conn.close()
|
|
return {"action": action, "updated": updated, "missing": missing, "total": len(shas)}
|
|
|
|
|
|
@app.get("/file/{sha}")
|
|
def full_file(sha: str):
|
|
conn = _conn()
|
|
row = conn.execute("SELECT path FROM image_hashes WHERE sha256=?", (sha,)).fetchone()
|
|
conn.close()
|
|
if not row:
|
|
raise HTTPException(404)
|
|
p = Path(row["path"])
|
|
if not p.exists():
|
|
raise HTTPException(404)
|
|
return FileResponse(p)
|
|
|
|
|
|
@app.get("/thumbs/{name}")
|
|
def thumb_file(name: str):
|
|
"""Serve thumbnail; generate on first request (cached after)."""
|
|
f = THUMB_DIR / name
|
|
if not f.exists():
|
|
# lazy generate — reconstruct source path from key (stem is orig filename)
|
|
THUMB_DIR.mkdir(parents=True, exist_ok=True)
|
|
# find the source image: key = <stem>_<hash8>.jpg
|
|
stem = name.rsplit("_", 1)[0]
|
|
src = _find_source(stem)
|
|
if not src:
|
|
raise HTTPException(404)
|
|
try:
|
|
with Image.open(src) as im:
|
|
im.convert("RGB")
|
|
im.thumbnail(THUMB_SIZE)
|
|
im.save(f, "JPEG", quality=70)
|
|
except Exception:
|
|
raise HTTPException(404)
|
|
return FileResponse(f)
|
|
|
|
|
|
def _find_source(stem: str):
|
|
"""Find the original image for a thumbnail key (by filename stem)."""
|
|
import os
|
|
|
|
for root_dir in ("/mnt/ubuntu_storage_3TB/archive/03_photos",
|
|
"/mnt/data/01_keep", "/mnt/data/02_review", "/mnt/data/03_delete"):
|
|
for dirpath, _, files in os.walk(root_dir):
|
|
for fn in files:
|
|
if fn.rsplit(".", 1)[0] == stem:
|
|
return Path(dirpath) / fn
|
|
return None
|
|
|
|
|
|
@app.get("/upload", response_class=HTMLResponse)
|
|
def upload_page(request: Request):
|
|
"""Upload page — drop files/archives into the incoming folder."""
|
|
return templates.TemplateResponse(request, "upload.html", {})
|
|
|
|
|
|
@app.get("/howto", response_class=HTMLResponse)
|
|
def howto(request: Request):
|
|
"""How-to documentation page."""
|
|
return templates.TemplateResponse(request, "howto.html", {})
|
|
|
|
|
|
@app.post("/upload")
|
|
async def upload(request: Request):
|
|
"""Receive uploaded files → save to /mnt/data/takeout/incoming/."""
|
|
import uuid
|
|
|
|
from starlette.datastructures import UploadFile
|
|
|
|
form = await request.form()
|
|
incoming = STAGING / "takeout" / "incoming"
|
|
incoming.mkdir(parents=True, exist_ok=True)
|
|
saved = []
|
|
for field in form.values():
|
|
if isinstance(field, UploadFile) and field.filename:
|
|
# sanitize: keep name but avoid path traversal
|
|
name = Path(field.filename).name
|
|
dest = incoming / f"{uuid.uuid4().hex[:8]}_{name}"
|
|
with open(dest, "wb") as f:
|
|
while chunk := await field.read(1024 * 1024):
|
|
f.write(chunk)
|
|
saved.append(dest.name)
|
|
# notify
|
|
try:
|
|
import sys
|
|
if str(BASE.parent) not in sys.path:
|
|
sys.path.insert(0, str(BASE.parent))
|
|
import apprise_helper
|
|
apprise_helper.notify(
|
|
"📥 photo-pipeline: upload received",
|
|
f"{len(saved)} file(s) saved to incoming. Watch flow will process them.",
|
|
)
|
|
except Exception:
|
|
pass
|
|
return {"saved": len(saved), "files": saved}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
|
|
uvicorn.run(app, host="0.0.0.0", port=8092)
|