Pagination (200/page), batched+checkpointed ingest (batch_size, resume-safe), quality max_files slicing, upload page; Caddy routes for prefect/photo-filter

dashboard/app.py

@@ -96,25 +96,34 @@ def index(request: Request):
 
 
 @app.get("/review", response_class=HTMLResponse)
-def review(request: Request, source: str = None, status: str = None):
+def review(request: Request, source: str = None, status: str = None, page: int = 1):
     conn = _conn()
+    per_page = 200
     q = "SELECT sha256, path, status, source FROM image_hashes WHERE 1=1"
+    count_q = "SELECT COUNT(*) FROM image_hashes WHERE 1=1"
     params = []
     if source:
         q += " AND source=?"
+        count_q += " AND source=?"
         params.append(source)
     if status:
         q += " AND status=?"
+        count_q += " AND status=?"
         params.append(status)
     else:
         q += " AND status IN ('scanned','review')"
-    q += " ORDER BY added_at DESC LIMIT 300"
-    rows = conn.execute(q, params).fetchall()
+        count_q += " AND status IN ('scanned','review')"
+    total = conn.execute(count_q, params).fetchone()[0]
+    pages = max(1, (total + per_page - 1) // per_page)
+    page = max(1, min(page, pages))
+    q += " ORDER BY added_at DESC LIMIT ? OFFSET ?"
+    rows = conn.execute(q, params + [per_page, (page - 1) * per_page]).fetchall()
     conn.close()
     items = [_load_item(r) for r in rows]
     return templates.TemplateResponse(
         request, "review.html",
-        {"items": items, "source": source, "status": status},
+        {"items": items, "source": source, "status": status,
+         "page": page, "pages": pages, "total": total},
     )
 
 
@@ -231,6 +240,47 @@ def thumb_file(name: str):
     return FileResponse(f)
 
 
+@app.get("/upload", response_class=HTMLResponse)
+def upload_page(request: Request):
+    """Upload page — drop files/archives into the incoming folder."""
+    return templates.TemplateResponse(request, "upload.html", {})
+
+
+@app.post("/upload")
+async def upload(request: Request):
+    """Receive uploaded files → save to /mnt/data/takeout/incoming/."""
+    import uuid
+
+    from starlette.datastructures import UploadFile
+
+    form = await request.form()
+    incoming = STAGING / "takeout" / "incoming"
+    incoming.mkdir(parents=True, exist_ok=True)
+    saved = []
+    for field in form.values():
+        if isinstance(field, UploadFile) and field.filename:
+            # sanitize: keep name but avoid path traversal
+            name = Path(field.filename).name
+            dest = incoming / f"{uuid.uuid4().hex[:8]}_{name}"
+            with open(dest, "wb") as f:
+                while chunk := await field.read(1024 * 1024):
+                    f.write(chunk)
+            saved.append(dest.name)
+    # notify
+    try:
+        import sys
+        if str(BASE.parent) not in sys.path:
+            sys.path.insert(0, str(BASE.parent))
+        import apprise_helper
+        apprise_helper.notify(
+            "📥 photo-pipeline: upload received",
+            f"{len(saved)} file(s) saved to incoming. Watch flow will process them.",
+        )
+    except Exception:
+        pass
+    return {"saved": len(saved), "files": saved}
+
+
 if __name__ == "__main__":
     import uvicorn
 

dashboard/templates/review.html

@@ -41,6 +41,9 @@
     .none { color: #666; font-style: italic; padding: 2rem; text-align: center; }
     .selall { display: inline-flex; align-items: center; gap: .35rem; background: #222; border: 1px solid #444; border-radius: 6px; padding: .4rem .8rem; cursor: pointer; }
     .toast { position: fixed; bottom: 1.2rem; right: 1.2rem; background: #1d4; color: #031; padding: .7rem 1rem; border-radius: 8px; display: none; z-index: 50; font-size: .9rem; box-shadow: 0 2px 12px rgba(0,0,0,.5); }
+    .pager { display: flex; gap: 1rem; align-items: center; justify-content: center; padding: 1.5rem; }
+    .pager a { color: #6cf; text-decoration: none; padding: .4rem .8rem; background: #222; border: 1px solid #444; border-radius: 6px; }
+    .pager span { color: #999; }
   </style>
 </head>
 <body>
@@ -72,7 +75,7 @@
   <button class="btn-reset" id="bulk-reset">↺ Reset</button>
 </div>
 <div class="toast" id="toast"></div>
-<p class="count">{{ items|length }} images</p>
+<p class="count">{{ total }} images · page {{ page }}/{{ pages }}</p>
 <main>
   <div class="grid" id="review-grid">
     {% for item in items %}
@@ -81,6 +84,17 @@
     <p class="none">No images match the filter.</p>
     {% endfor %}
   </div>
+  {% if pages > 1 %}
+  <div class="pager">
+    {% if page > 1 %}
+    <a href="/review?page={{ page-1 }}{% if source %}&source={{ source }}{% endif %}{% if status %}&status={{ status }}{% endif %}">← Prev</a>
+    {% endif %}
+    <span>page {{ page }} / {{ pages }}</span>
+    {% if page < pages %}
+    <a href="/review?page={{ page+1 }}{% if source %}&source={{ source }}{% endif %}{% if status %}&status={{ status }}{% endif %}">Next →</a>
+    {% endif %}
+  </div>
+  {% endif %}
 </main>
 <script>
   // Selection is DERIVED from DOM checkboxes — single source of truth.
@@ -158,7 +172,11 @@
   function updateCount() {
     const el = document.querySelector('.count');
     const n = document.querySelectorAll('.card').length;
-    if (el) el.textContent = n + ' images';
+    if (el && el.textContent.includes('images')) {
+      // preserve the "· page X/Y" part, update the leading card count
+      const m = el.textContent.match(/page (\d+)\/(\d+)/);
+      el.textContent = n + ' images' + (m ? ' · page ' + m[1] + '/' + m[2] : '');
+    }
   }
 
   function showToast(msg, isErr) {

dashboard/templates/upload.html

@@ -0,0 +1,80 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Upload — photo-pipeline</title>
+  <style>
+    :root { color-scheme: dark; }
+    body { font-family: system-ui, sans-serif; margin: 0; background: #111; color: #eee; }
+    header { padding: 1rem 1.5rem; border-bottom: 1px solid #333; display: flex; gap: 1.5rem; align-items: baseline; }
+    header h1 { font-size: 1.2rem; margin: 0; }
+    header a { color: #6cf; text-decoration: none; }
+    main { padding: 1.5rem; max-width: 700px; margin: 0 auto; }
+    .drop { border: 2px dashed #444; border-radius: 12px; padding: 3rem 2rem; text-align: center; color: #888; }
+    .drop.dragover { border-color: #6cf; background: #16222a; }
+    input[type=file] { margin: 1rem 0; }
+    button { background: #1d4; color: #031; border: 0; border-radius: 8px; padding: .6rem 1.2rem; font-size: 1rem; cursor: pointer; font-weight: 600; }
+    .hint { color: #666; font-size: .85rem; margin-top: 1.5rem; }
+    .hint code { background: #222; padding: .1rem .4rem; border-radius: 4px; }
+  </style>
+</head>
+<body>
+<header>
+  <h1>📸 photo-pipeline</h1>
+  <a href="/">Overview</a>
+  <a href="/review">Review queue</a>
+  <a href="/pipeline">Pipeline</a>
+  <a href="/upload">Upload</a>
+</header>
+<main>
+  <h2>Upload to pipeline</h2>
+  <p>Drop Takeout archives (<code>.zip</code>/<code>.tgz</code>) or manifest files here.
+  They land in <code>/mnt/data/takeout/incoming/</code> — the watch flow picks them up
+  within 15 minutes and runs the pipeline automatically.</p>
+
+  <form id="upform" method="post" action="/upload" enctype="multipart/form-data">
+    <div class="drop" id="drop">
+      <p>Drag &amp; drop files here, or click to browse</p>
+      <input type="file" id="fileinput" name="files" multiple>
+    </div>
+    <p><button type="submit">Upload</button></p>
+    <p id="status" style="color:#6cf;"></p>
+  </form>
+
+  <div class="hint">
+    <p>What can go here:</p>
+    <ul>
+      <li><code>*.txt</code> — a manifest of Takeout download URLs (one per line). The pipeline downloads them itself.</li>
+      <li><code>*.zip</code> / <code>*.tgz</code> — an already-downloaded Takeout archive.</li>
+    </ul>
+  </div>
+</main>
+<script>
+  const drop = document.getElementById('drop');
+  const input = document.getElementById('fileinput');
+  const status = document.getElementById('status');
+
+  drop.addEventListener('dragover', e => { e.preventDefault(); drop.classList.add('dragover'); });
+  drop.addEventListener('dragleave', () => drop.classList.remove('dragover'));
+  drop.addEventListener('drop', e => {
+    e.preventDefault();
+    drop.classList.remove('dragover');
+    input.files = e.dataTransfer.files;
+    status.textContent = input.files.length + ' file(s) selected';
+  });
+  input.addEventListener('change', () => {
+    status.textContent = input.files.length + ' file(s) selected';
+  });
+
+  document.getElementById('upform').addEventListener('submit', async e => {
+    e.preventDefault();
+    status.textContent = 'Uploading…';
+    const fd = new FormData(document.getElementById('upform'));
+    const resp = await fetch('/upload', { method: 'POST', body: fd });
+    const data = await resp.json();
+    status.textContent = `Done: ${data.saved} file(s) saved to incoming. Watch flow will process them.`;
+  });
+</script>
+</body>
+</html>