From 39b5a393345befe788e1baaf4cf61ba655296d32 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Sat, 8 Aug 2026 13:55:40 +1000 Subject: [PATCH] Fix infinite loop in find_unprocessed: check sha256 content (not just path), record dup paths; Optional[int] for max_batches --- dashboard/app.py | 6 ++ dashboard/templates/howto.html | 132 +++++++++++++++++++++++++++++++++ photo_ingest.py | 20 +++-- 3 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 dashboard/templates/howto.html diff --git a/dashboard/app.py b/dashboard/app.py index 2238eac..118094c 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -246,6 +246,12 @@ def upload_page(request: Request): return templates.TemplateResponse(request, "upload.html", {}) +@app.get("/howto", response_class=HTMLResponse) +def howto(request: Request): + """How-to documentation page.""" + return templates.TemplateResponse(request, "howto.html", {}) + + @app.post("/upload") async def upload(request: Request): """Receive uploaded files → save to /mnt/data/takeout/incoming/.""" diff --git a/dashboard/templates/howto.html b/dashboard/templates/howto.html new file mode 100644 index 0000000..6083ef2 --- /dev/null +++ b/dashboard/templates/howto.html @@ -0,0 +1,132 @@ + + + + + + How To — photo-pipeline + + + +
+

📸 photo-pipeline

+ Overview + Review queue + Pipeline + Upload + How To +
+
+ +

What this system does

+

An automated photo ingestion pipeline: it downloads Google Takeout exports, +fingerprints every image (deduplication), audits quality (blurry/dark/etc), +and imports approved photos into your Immich library — with +notifications and a review dashboard at every step.

+ +
Core safety rule: nothing is ever deleted +automatically. Rejected photos go to a holding folder (03_delete) and +stay there until you confirm.
+ +

The pipeline at a glance

+ + + + + + + + + + + +
StepFlowWhat happens
1. Export— (manual)You create a Google Takeout export
2. Feedupload / scpDrop manifest or archives into incoming/
3. Fetchtakeout-fetchDownloads archives (resumable), tracks them, extracts
4. Ingestphoto-ingestHashes every file, flags exact + near duplicates
5. Qualityphoto-quality-scanCleanVision flags blurry/dark/odd images
6. NotifyAppriseYou get a message with counts + dashboard link
7. Reviewthis dashboardYou Keep or Reject images in the review queue
8. Stageprocess-stagingApproved → 01_keep, Rejected → 03_delete
9. Importimmich-importApproved photos uploaded to Immich
+ +

How to add photos — 4 ways

+ +

A. From Google Takeout (bulk)

+
    +
  1. Go to takeout.google.com and create an export (Photos, batched by year so each archive is <10GB). Google has no API for this — it's the one manual step.
    2. +
  2. When it's ready, copy the download URLs into a text file (one per line) — call it urls.txt.
    3. +
  3. Drop it in incoming/ — via the Upload page, or from a terminal: scp urls.txt sam@100.114.62.46:/mnt/data/takeout/incoming/ (Tailscale IP).
    4. +
  4. The watch flow picks it up within 15 minutes and runs the whole pipeline. You get an Apprise notification with results.
    5. +
+ +

B. An already-downloaded archive

+

If you already have Takeout archives (.zip/.tgz), drop them in incoming/ the same way. The pipeline extracts and processes them.

+ +

C. A folder already on .13 (like your archive)

+

For folders already on the server (e.g. archive/03_photos), run the ingest flow directly — it scans in batches (1000 at a time) with checkpoints, so it's safe to interrupt:

+
export PREFECT_API_URL=http://localhost:4200/api
+~/photo-pipeline/.venv/bin/prefect deployment run "photo-ingest/ingest" \
+  --param base_dir=/mnt/ubuntu_storage_3TB/archive/03_photos/Pictures \
+  --param source=archive-pictures --param batch_size=1000
+ +

D. From your phone

+

Install the Immich app on your phone and enable auto-backup — photos upload straight to your Immich library, bypassing Google entirely. This is the recommended phone path.

+ +

How to review

+
    +
  1. Open the Review queue. It shows 200 images per page, newest first.
    2. +
  2. Click a photo to see it full-size (opens in a new tab).
    3. +
  3. Keep = approved for Immich. Reject = moves to the delete-holding folder. ↺ = undo back to unscanned.
    4. +
  4. Use the checkbox + Select all + Bulk Keep/Reject for large batches.
    5. +
  5. Filter by status or source with the controls above the grid.
    6. +
  6. Track progress on the Pipeline page (auto-refreshes every 30s).
    7. +
+
Remember: review decisions are instant DB updates. +Files physically move only when the process-staging flow runs. Nothing is +deleted without your explicit confirmation.
+ +

How to check on work

+ + +

Common operations

+

Run a flow manually

+
export PREFECT_API_URL=http://localhost:4200/api
+~/photo-pipeline/.venv/bin/prefect deployment run "photo-watch/watch"
+

Replace photo-watch/watch with any deployment name: +takeout-fetch/fetch, photo-ingest/ingest, +photo-quality-scan/quality, process-staging/staging, +immich-import/import.

+ +

Restart the dashboard

+
systemctl --user restart photo-dashboard.service
+ +

See recent actions

+

Open the Pipeline page — the lower table shows your last 20 review decisions.

+ +

How it's built (for the curious)

+

Prefect 3 orchestrates everything on photo-pool; a SQLite database +(photo_pipeline.db) is the source of truth for every image's hash, +path and status. imagehash detects duplicates (exact sha256 + perceptual phash/dhash), +CleanVision audits quality, FastAPI+htmx powers this dashboard, and Apprise sends +notifications. Everything runs on your .13 server as Docker + systemd services. +Code lives in the photo-pipeline repo on your Gitea.

+ +
+ + diff --git a/photo_ingest.py b/photo_ingest.py index ce8e6c3..9d15528 100644 --- a/photo_ingest.py +++ b/photo_ingest.py @@ -42,20 +42,28 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list conn = db.get_db() batch = [] for p_str in walk_files(base_dir): - # skip if already registered for this source (or any source) - row = conn.execute( - "SELECT 1 FROM image_hashes WHERE sha256=?", - (db.sha256_file(p_str),), - ).fetchone() if False else None # cheap check: path already known? known = conn.execute( "SELECT 1 FROM image_hashes WHERE path=?", (p_str,) ).fetchone() if known: continue + # content check: sha256 already registered (catches same photo at other paths) + sha = db.sha256_file(p_str) + sha_known = conn.execute( + "SELECT 1 FROM image_hashes WHERE sha256=?", (sha,) + ).fetchone() + if sha_known: + # record this path too, so we don't re-hash it every loop + conn.execute( + "INSERT OR IGNORE INTO image_hashes (sha256, phash, dhash, file_size, path, source) " + "SELECT sha256, phash, dhash, file_size, ?, source FROM image_hashes WHERE sha256=?", + (p_str, sha)) + continue batch.append(p_str) if len(batch) >= batch_size: break + conn.commit() conn.close() print(f"find_unprocessed: {len(batch)} new files (batch_size={batch_size})") return batch @@ -112,7 +120,7 @@ def check_and_register(image_paths: list[str], source: str) -> dict: @flow(name="photo-ingest") def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000, - max_batches: int = None): + max_batches: int | None = None): """Hash + dedup-check a folder against the persistent library, in batches. Args: