diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index a585eb4..08c368f 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "300 areas/370 Family Home/basketball training.md", - "mode": "preview", + "file": "200 projects/240 Web Sites/Kontra Day.md", + "mode": "source", "source": false }, "icon": "lucide-file", - "title": "basketball training" + "title": "Kontra Day" } } ] @@ -183,7 +183,7 @@ "state": { "type": "file-properties", "state": { - "file": "300 areas/370 Family Home/basketball training.md" + "file": "200 projects/240 Web Sites/Kontra Day.md" }, "icon": "lucide-info", "title": "File properties" @@ -209,8 +209,15 @@ "templater-obsidian:Templater": false } }, - "active": "7066c9273c6fb936", + "active": "8f0c8ae5063438f6", "lastOpenFiles": [ + "200 projects/240 Web Sites/Where Woof.md", + "200 projects/Tools Software WebUI/Family Home Lab.md", + "200 projects/Tools Software WebUI/Google Cloud Images Immich.md", + "200 projects/Tools Software WebUI/Photo Consolidation Plan.md", + "200 projects/Tools Software WebUI/Paseo Pi GUI Tool.md", + "200 projects/240 Web Sites/Kontra Day.md", + "300 areas/370 Family Home/basketball training.md", "000 daily/Daily Notes and To Do.md", "000 daily/Update Install.md", "300 areas/350 AI/AI Design.md", @@ -240,13 +247,6 @@ "300 areas/360 Dev-Ops Network Computers/Backup System β€” Borg, Kopia & Restic.md", "300 areas/360 Dev-Ops Network Computers/Pi Neovim Coding Harness.md", "300 areas/360 Dev-Ops Network Computers/Dev-Ops Tooling β€” Mermaid, Archify, Vikunja & Outline.md", - "300 areas/360 Dev-Ops Network Computers/Home Network Map Overview.md", - "300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md", - "300 areas/370 Family Home/basketball training.md", - "200 projects/220 Web Host Migration/Project Ops β€” Build Plan (Vikunja+Outline+Obsidian).md", - "300 areas/350 AI/Tools to try with AI.md", - "400 resources/470 Holidays Travel/Ski Stuf.md", - "300 areas/380 Social Politics Economics Issues/Chinese Non Silicon Micro Chip.md", "Untitled.canvas" ] } \ No newline at end of file diff --git a/200 projects/240 Web Sites/Kontra Day.md b/200 projects/240 Web Sites/Kontra Day.md new file mode 100644 index 0000000..9ebd52d --- /dev/null +++ b/200 projects/240 Web Sites/Kontra Day.md @@ -0,0 +1,13 @@ +--- +created: 2026-09-09 11:54 +modified: 2026-09-09 11:54 +type: note +tags: + - website + - politics + - portfolio +aliases: [] +--- +# [[Kontra Day]] + + diff --git a/200 projects/Tools Software WebUI/Google Cloud Images Immich.md b/200 projects/Tools Software WebUI/Google Cloud Images Immich.md new file mode 100644 index 0000000..5554bf6 --- /dev/null +++ b/200 projects/Tools Software WebUI/Google Cloud Images Immich.md @@ -0,0 +1,159 @@ +--- +created: 2026-09-09 11:57 +modified: 2026-09-09 +type: note +tags: + - tool + - tools + - photos + - ai + - immich + - pipeline + - google + - prefect +aliases: [photo-pipeline, google-photos-migration] +--- +# [[Google Cloud Images Immich]] + +> **Photo Pipeline β€” migrate Google Photos to a private archive, serve via Immich, back up offsite.** +> A Prefect-orchestrated pipeline that downloads Google Takeout exports, fingerprints + dedups every image, applies quality analysis (blurry/dark), flows through a human review dashboard, merges approved photos into a date-sorted master archive (`by_date`), imports them into Immich, and protects everything with layered backups (Borg, Kopia, AWS S3). + +--- + +## πŸ—ΊοΈ The pipeline (Mermaid) + +```mermaid +flowchart LR + TO[Google Takeout
134GB / 14 zips] -->|drop manifest| W[photo-watch
every 15 min] + W -->|download + extract| I[photo-ingest
sha256 + dhash dedup] + I -->|fingerprints| Q[quality-scan
PIL blurry/dark] + Q -->|verdicts| R[Review dashboard
photo-filter.home.lab] + R -->|approved| M[merge_orphans
EXIF date routing] + M --> B[(by_date master
35K files)] + B -->|incremental import| IM[Immich
.35 library] + R -->|rejected| T[(trash β†’ purge)] + B -.->|offsite| S3[AWS S3
Glacier lifecycle] +``` + +**Explore the interactive map:** https://maps.lab.audasmedia.com.au/google_cloud_and_images/docs/photo-pipeline-map.html + +--- + +## What it is + +A **self-hosted Google Photos replacement pipeline**. Photos leave Google via Takeout, are verified + deduplicated + quality-checked locally, reviewed by a human, then land in a **date-sorted master archive** (the single source of truth) and are served through **Immich** (a self-hosted photo server). The goal is full independence from Google + a robust local + offsite backup story. + +### Why it matters +- Content-based dedup by **sha256** β€” re-importing never doubles photos +- Quality **verdicts are deterministic** (PIL image math, not ML) β€” reliable at 35K+ scale +- **Human gate** β€” nothing merges until a person approves it +- **Incremental import** β€” only new photos go to Immich (no 376GB re-uploads) +- **Safe cleanup** β€” rejected β†’ trash (recoverable) β†’ purge only on confirmation +- **PHOTOS-TO-DELETE album** β€” remove unwanted photos from Immich + by_date in one action + +--- + +## How to use it + +### Reviewing photos (main daily task) +1. Open **https://photo-filter.home.lab** (basic-auth: `sam`) +2. Filter: `?source=takeout&status=keep` (new Google photos) or `?status=delete_candidate` (blurry/dark) +3. **Bulk Keep** (approve) or **Bulk Reject** β€” decisions save instantly to the DB +4. Approved photos are ready for merge; rejected sit in trash (recoverable) + +### Pipeline actions (dashboard buttons) +Go to the **"Pipeline actions"** page in the dashboard: + +| Button | What it does | Safe? | +|---|---|---| +| πŸ” Dry-run merge | Preview what approved photos WOULD move | βœ… read-only | +| β–Ά Merge approved β†’ by_date | Move approved files (EXIF-date routed) | Confirmation required | +| πŸ“€ Import by_date β†’ Immich | Upload only NEW photos (dedup-safe) | βœ… incremental | +| πŸ—‘ Rejected preview / trash / purge | Move rejected β†’ trash β†’ permanent delete | Trash recoverable | +| πŸ”„ Check PHOTOS-TO-DELETE album | See what's in the removal album | βœ… read-only | +| πŸ—‘ Remove album photos | Trash by_date copies + delete from Immich | Confirmation required | + +### Removing unwanted photos (Immich side) +1. In **Immich**, move photos you don't want into the **PHOTOS-TO-DELETE** album +2. Dashboard β†’ Pipeline β†’ **Check PHOTOS-TO-DELETE** +3. **Remove album photos** β†’ by_date copies go to trash (recoverable), Immich copies are deleted + +### Adding NEW Google Takeout in future +1. Download zips to `incoming/` (or drop a manifest of URLs) +2. `photo-watch` auto-processes every 15 min β†’ extract β†’ ingest β†’ quality β†’ review +3. Review β†’ merge β†’ import β†’ **then** you can delete from Google + +--- + +## πŸ› οΈTools used (resume showcase) + +| Layer | Technology | Role | +|---|---|---| +| **Orchestration** | **Prefect 3** | Flows, scheduled deploys, worker lifecycle, crash recovery | +| **Backend** | Python, **FastAPI**, SQLite | Review dashboard, dedup, quality scan, incremental import | +| **Image analysis** | **PIL/Pillow** | Blurry/dark verdicts (luminance + edge variance at 300px) | +| **Frontend** | htmx, Jinja2, JS | Dashboard with live polling, batch actions, confirm dialogs | +| **Data stores** | SQLite (pipeline), Postgres (Immich) | Fingerprints + assets/albums | +| **Import** | Immich server-bundled CLI | Checksum-dedup'd upload | +| **Backup** | Borg, Kopia, AWS S3 (+ rclone/aws cli) | Local chains + offsite | +| **Diagrams** | Mermaid, Archify (published to maps.lab) | Architecture visualization | +| **Ops/infra** | NixOS, Docker, Caddy, Gitea, Vikunja, Outline | Full self-hosted stack | + +**Demonstrated strengths:** content-addressed dedup, incremental import design, data-safety discipline (backup-verify-before-merge), image-quality analysis at scale, full-stack dashboard with background jobs, cross-machine SSH automation, backup failure detection + repair. + +--- + +## Where things are + +| What | Where | +|---|---| +| **Master archive (`by_date`)** | `.13:/mnt/ubuntu_storage_3TB/archive/03_photos/by_date` β€” 35K files, YYYY/MM-Mon | +| **Pipeline code** | Gitea: `ssh://gitea.lab.audasmedia.com.au:2222/sam/photo-pipeline.git` | +| **Review dashboard** | https://photo-filter.home.lab (`.13:8092`) β€” basic-auth | +| **Prefect UI** | https://prefect.home.lab (`.13:4200`) | +| **Immich** | `http://192.168.20.35:2283` β€” self-hosted photo server | +| **Interactive map** | https://maps.lab.audasmedia.com.au/google_cloud_and_images/docs/photo-pipeline-map.html | +| **Fingerprint DB** | `.13:/mnt/data/photo_pipeline.db` (SQLite) | +| **Takeout staging** | `.13:/mnt/data/takeout/` β€” transient downloads/extract | +| **Removed photos trash** | `.13:/mnt/data/takeout/removed/` + `rejected-trash/` β€” recoverable | +| **Vikunja (tasks)** | Project **"Photo Pipeline & Archive"** β€” kanban of remaining work | +| **Outline (docs)** | Collection **"Photo Pipeline & Archive"** β€” Overview / Architecture / Operations / Storage | + +--- + +## Current status (Sep 2026) + +| Item | State | +|---|---| +| Google Takeout download | βœ… Complete (134GB / 14 zips, sha256-verified) | +| Extract + ingest | βœ… 12.3K new photos fingerprinted, dedup-skipped 225 | +| Quality scan | βœ… Verdicts applied (PIL) | +| **Review** | πŸ”„ **2,246 approved Β· 8,877 keep to review Β· 1,203 rejected** | +| Merge + import to Immich | βœ… Built + tested (incremental, dedup-safe) | +| Rejected cleanup | βœ… Trash β†’ purge flow working | +| PHOTOS-TO-DELETE removal | βœ… Working (album β†’ remove from Immich + by_date) | +| S3 offsite backup | ⏳ Pending seed (by_date + archive/rest β‰ˆ 712GB) | +| Delete from Google (final) | ⏳ After local verification complete | + +--- + +## βš™οΈ Config notes (for future maintenance) + +- **SSH .13 β†’ .35** uses key: `-i ~/.ssh/id_ed25519_rsync` +- **Dashboard service** (on .13): `systemctl --user restart photo-dashboard` (users must **refresh browser** after code changes β€” stale JS causes "buttons do nothing") +- **Prefect worker**: `systemctl --user status prefect-worker` (runs photo-pool) +- **Backup job**: `borgbackup-job-archive-to-23` (daily 06:00, .13 β†’ .23) β€” covers `archive/` **and** `~/photo-pipeline` (DB) +- The **3TB archive drive is USB** β€” got unplugged once (Aug 21-26 2026 β†’ backup failures); replugging resolved. Remount if `/mnt/ubuntu_storage_3TB` missing: the disk is `sde1` (UUID `037a542c-…`), `sudo mount /mnt/ubuntu_storage_3TB` +- **Immich API key** lives at `~/.ssh/../photo-pipeline/.immich-key` (0600) β€” needs user.read + asset.read + asset.upload; skip server.about scope (not granted) +- **382 HEIC files** skip the PIL quality scan (no HEIF plugin) β€” safe on disk, handle with pillow-heif later +- **Known gap**: phone-only photos uploaded directly to Immich are NOT in by_date and NOT backed up β€” decision deferred +- Gitea repo pushes use SSH port **2222**; .13 SSH can be intermittently flaky β€” retry after 30s if a push times out + +--- + +## Resources +- Interactive map: https://maps.lab.audasmedia.com.au/google_cloud_and_images/docs/photo-pipeline-map.html +- [[Photo Project Master Overview]] β€” deep-dive human reference (storage map, backup S3 plan, tools table) +- [[Backup Architecture β€” Offsite to AWS S3]] β€” the S3 offsite plan +- [[Filesystem Drive Map]] β€” disk layout +- Vikunja: project **Photo Pipeline & Archive** Β· Outline: collection **Photo Pipeline & Archive** \ No newline at end of file diff --git a/200 projects/Tools Software WebUI/Paseo Pi GUI Tool.md b/200 projects/Tools Software WebUI/Paseo Pi GUI Tool.md new file mode 100644 index 0000000..c919b54 --- /dev/null +++ b/200 projects/Tools Software WebUI/Paseo Pi GUI Tool.md @@ -0,0 +1,16 @@ +--- +created: 2026-09-09 11:55 +modified: 2026-09-09 11:55 +type: note +tags: + - ai + - ai-agents + - ai-resume + - pi + - tool + - tools +aliases: [] +--- +# [[Paseo Pi GUI Tool]] + +