sam/photo-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix infinite loop in find_unprocessed: check sha256 content (not just path), record dup paths; Optional[int] for max_batches

Browse Source
This commit is contained in:
Sam Rolfe
2026-08-08 13:55:40 +10:00
parent 0d20c29f35
commit 39b5a39334
3 changed files with 152 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dashboard/app.py
View File

@@ -246,6 +246,12 @@ def upload_page(request: Request):
return templates.TemplateResponse(request, "upload.html", {}) return templates.TemplateResponse(request, "upload.html", {})
@app.get("/howto", response_class=HTMLResponse)
def howto(request: Request):
"""How-to documentation page."""
return templates.TemplateResponse(request, "howto.html", {})
@app.post("/upload") @app.post("/upload")
async def upload(request: Request): async def upload(request: Request):
"""Receive uploaded files → save to /mnt/data/takeout/incoming/.""" """Receive uploaded files → save to /mnt/data/takeout/incoming/."""

132
dashboard/templates/howto.html Normal file
View File

@@ -0,0 +1,132 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>How To — photo-pipeline</title>
<style>
:root { color-scheme: dark; }
body { font-family: system-ui, sans-serif; margin: 0; background: #111; color: #eee; line-height: 1.6; }
header { padding: 1rem 1.5rem; border-bottom: 1px solid #333; display: flex; gap: 1.5rem; align-items: baseline; }
header h1 { font-size: 1.2rem; margin: 0; }
header a { color: #6cf; text-decoration: none; }
main { padding: 1.5rem 2rem; max-width: 900px; margin: 0 auto; }
h2 { border-bottom: 1px solid #333; padding-bottom: .4rem; margin-top: 2.5rem; }
h3 { color: #6cf; margin-top: 1.5rem; }
code { background: #222; padding: .1rem .4rem; border-radius: 4px; font-size: .9em; }
pre { background: #1a1a1a; border: 1px solid #333; border-radius: 8px; padding: 1rem; overflow-x: auto; }
.steps { counter-reset: step; list-style: none; padding: 0; }
.steps li { counter-increment: step; margin-bottom: 1.2rem; padding-left: 2.5rem; position: relative; }
.steps li::before { content: counter(step); position: absolute; left: 0; top: .1rem; width: 1.6rem; height: 1.6rem; background: #1d4; color: #031; border-radius: 50%; display: flex; align-items: center; justify-content: center; font-weight: 700; font-size: .85rem; }
table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
th, td { text-align: left; padding: .5rem .75rem; border-bottom: 1px solid #222; }
th { color: #999; font-size: .8rem; text-transform: uppercase; }
.warn { background: #2a1a1a; border: 1px solid #d43; border-left: 4px solid #d43; border-radius: 8px; padding: .8rem 1rem; margin: 1rem 0; }
.good { background: #16221a; border: 1px solid #1d4; border-left: 4px solid #1d4; border-radius: 8px; padding: .8rem 1rem; margin: 1rem 0; }
a { color: #6cf; }
</style>
</head>
<body>
<header>
<h1>📸 photo-pipeline</h1>
<a href="/">Overview</a>
<a href="/review">Review queue</a>
<a href="/pipeline">Pipeline</a>
<a href="/upload">Upload</a>
<a href="/howto">How To</a>
</header>
<main>
<h2>What this system does</h2>
<p>An automated photo ingestion pipeline: it downloads Google Takeout exports,
fingerprints every image (deduplication), audits quality (blurry/dark/etc),
and imports approved photos into your <strong>Immich</strong> library — with
notifications and a review dashboard at every step.</p>
<div class="good"><strong>Core safety rule:</strong> nothing is ever deleted
automatically. Rejected photos go to a holding folder (<code>03_delete</code>) and
stay there until <em>you</em> confirm.</div>
<h2>The pipeline at a glance</h2>
<table>
<tr><th>Step</th><th>Flow</th><th>What happens</th></tr>
<tr><td>1. Export</td><td>— (manual)</td><td>You create a Google Takeout export</td></tr>
<tr><td>2. Feed</td><td>upload / scp</td><td>Drop manifest or archives into <code>incoming/</code></td></tr>
<tr><td>3. Fetch</td><td>takeout-fetch</td><td>Downloads archives (resumable), tracks them, extracts</td></tr>
<tr><td>4. Ingest</td><td>photo-ingest</td><td>Hashes every file, flags exact + near duplicates</td></tr>
<tr><td>5. Quality</td><td>photo-quality-scan</td><td>CleanVision flags blurry/dark/odd images</td></tr>
<tr><td>6. Notify</td><td>Apprise</td><td>You get a message with counts + dashboard link</td></tr>
<tr><td>7. Review</td><td>this dashboard</td><td>You Keep or Reject images in the review queue</td></tr>
<tr><td>8. Stage</td><td>process-staging</td><td>Approved → 01_keep, Rejected → 03_delete</td></tr>
<tr><td>9. Import</td><td>immich-import</td><td>Approved photos uploaded to Immich</td></tr>
</table>
<h2>How to add photos — 4 ways</h2>
<h3>A. From Google Takeout (bulk)</h3>
<ol class="steps">
<li>Go to <a href="https://takeout.google.com" target="_blank">takeout.google.com</a> and create an export (Photos, batched by year so each archive is &lt;10GB). Google has no API for this — it's the one manual step.</li>
<li>When it's ready, copy the download URLs into a text file (one per line) — call it <code>urls.txt</code>.</li>
<li>Drop it in <code>incoming/</code> — via the <a href="/upload">Upload page</a>, or from a terminal: <code>scp urls.txt sam@100.114.62.46:/mnt/data/takeout/incoming/</code> (Tailscale IP).</li>
<li>The <strong>watch flow</strong> picks it up within 15 minutes and runs the whole pipeline. You get an Apprise notification with results.</li>
</ol>
<h3>B. An already-downloaded archive</h3>
<p>If you already have Takeout archives (<code>.zip</code>/<code>.tgz</code>), drop them in <code>incoming/</code> the same way. The pipeline extracts and processes them.</p>
<h3>C. A folder already on .13 (like your archive)</h3>
<p>For folders already on the server (e.g. <code>archive/03_photos</code>), run the ingest flow directly — it scans in batches (1000 at a time) with checkpoints, so it's safe to interrupt:</p>
<pre>export PREFECT_API_URL=http://localhost:4200/api
~/photo-pipeline/.venv/bin/prefect deployment run "photo-ingest/ingest" \
--param base_dir=/mnt/ubuntu_storage_3TB/archive/03_photos/Pictures \
--param source=archive-pictures --param batch_size=1000</pre>
<h3>D. From your phone</h3>
<p>Install the <strong>Immich app</strong> on your phone and enable auto-backup — photos upload straight to your Immich library, bypassing Google entirely. This is the recommended phone path.</p>
<h2>How to review</h2>
<ol class="steps">
<li>Open the <a href="/review">Review queue</a>. It shows 200 images per page, newest first.</li>
<li>Click a photo to see it full-size (opens in a new tab).</li>
<li><strong>Keep</strong> = approved for Immich. <strong>Reject</strong> = moves to the delete-holding folder. <strong>↺</strong> = undo back to unscanned.</li>
<li>Use the checkbox + <strong>Select all</strong> + <strong>Bulk Keep/Reject</strong> for large batches.</li>
<li>Filter by status or source with the controls above the grid.</li>
<li>Track progress on the <a href="/pipeline">Pipeline page</a> (auto-refreshes every 30s).</li>
</ol>
<div class="warn"><strong>Remember:</strong> review decisions are instant DB updates.
Files physically move only when the <code>process-staging</code> flow runs. Nothing is
deleted without your explicit confirmation.</div>
<h2>How to check on work</h2>
<ul>
<li><strong>This dashboard</strong> — overview counts, review, pipeline status.</li>
<li><strong>Prefect UI</strong> — <a href="https://prefect.home.lab" target="_blank">prefect.home.lab</a> — every flow run, logs, retries, schedules. (username/password: your Caddy basic-auth)</li>
<li><strong>Immich</strong> — <a href="https://immich.home.lab" target="_blank">immich.home.lab</a> — the photo library itself.</li>
</ul>
<h2>Common operations</h2>
<h3>Run a flow manually</h3>
<pre>export PREFECT_API_URL=http://localhost:4200/api
~/photo-pipeline/.venv/bin/prefect deployment run "photo-watch/watch"</pre>
<p>Replace <code>photo-watch/watch</code> with any deployment name:
<code>takeout-fetch/fetch</code>, <code>photo-ingest/ingest</code>,
<code>photo-quality-scan/quality</code>, <code>process-staging/staging</code>,
<code>immich-import/import</code>.</p>
<h3>Restart the dashboard</h3>
<pre>systemctl --user restart photo-dashboard.service</pre>
<h3>See recent actions</h3>
<p>Open the <a href="/pipeline">Pipeline page</a> — the lower table shows your last 20 review decisions.</p>
<h2>How it's built (for the curious)</h2>
<p>Prefect 3 orchestrates everything on <code>photo-pool</code>; a SQLite database
(<code>photo_pipeline.db</code>) is the source of truth for every image's hash,
path and status. imagehash detects duplicates (exact sha256 + perceptual phash/dhash),
CleanVision audits quality, FastAPI+htmx powers this dashboard, and Apprise sends
notifications. Everything runs on your .13 server as Docker + systemd services.
Code lives in the <code>photo-pipeline</code> repo on your Gitea.</p>
</main>
</body>
</html>

20
photo_ingest.py
View File

@@ -42,20 +42,28 @@ def find_unprocessed(base_dir: str, batch_size: int, source: str = None) -> list
conn = db.get_db() conn = db.get_db()
batch = [] batch = []
for p_str in walk_files(base_dir): for p_str in walk_files(base_dir):
# skip if already registered for this source (or any source)
row = conn.execute(
"SELECT 1 FROM image_hashes WHERE sha256=?",
(db.sha256_file(p_str),),
).fetchone() if False else None
# cheap check: path already known? # cheap check: path already known?
known = conn.execute( known = conn.execute(
"SELECT 1 FROM image_hashes WHERE path=?", (p_str,) "SELECT 1 FROM image_hashes WHERE path=?", (p_str,)
).fetchone() ).fetchone()
if known: if known:
continue continue
# content check: sha256 already registered (catches same photo at other paths)
sha = db.sha256_file(p_str)
sha_known = conn.execute(
"SELECT 1 FROM image_hashes WHERE sha256=?", (sha,)
).fetchone()
if sha_known:
# record this path too, so we don't re-hash it every loop
conn.execute(
"INSERT OR IGNORE INTO image_hashes (sha256, phash, dhash, file_size, path, source) "
"SELECT sha256, phash, dhash, file_size, ?, source FROM image_hashes WHERE sha256=?",
(p_str, sha))
continue
batch.append(p_str) batch.append(p_str)
if len(batch) >= batch_size: if len(batch) >= batch_size:
break break
conn.commit()
conn.close() conn.close()
print(f"find_unprocessed: {len(batch)} new files (batch_size={batch_size})") print(f"find_unprocessed: {len(batch)} new files (batch_size={batch_size})")
return batch return batch
@@ -112,7 +120,7 @@ def check_and_register(image_paths: list[str], source: str) -> dict:
@flow(name="photo-ingest") @flow(name="photo-ingest")
def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000, def photo_ingest(base_dir: str, source: str = "takeout", batch_size: int = 1000,
max_batches: int = None): max_batches: int | None = None):
"""Hash + dedup-check a folder against the persistent library, in batches. """Hash + dedup-check a folder against the persistent library, in batches.
Args: Args: