diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index d72f5b0..7741d2e 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "200 projects/240 Web Sites/Kontra Day.md", + "file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Kontra Day" + "title": "FAMILY S3 Storage Integration & Blueprint" } } ] @@ -183,7 +183,7 @@ "state": { "type": "file-properties", "state": { - "file": "200 projects/240 Web Sites/Kontra Day.md" + "file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md" }, "icon": "lucide-info", "title": "File properties" @@ -211,8 +211,8 @@ }, "active": "8f0c8ae5063438f6", "lastOpenFiles": [ - "200 projects/220 Web Host Migration/000 — Migration Overview.md", "200 projects/240 Web Sites/Kontra Day.md", + "200 projects/220 Web Host Migration/000 — Migration Overview.md", "200 projects/240 Web Sites/Where Woof.md", "200 projects/Tools Software WebUI/Family Home Lab.md", "200 projects/Tools Software WebUI/Google Cloud Images Immich.md", diff --git a/000 daily/FAMILY S3 Storage Integration & Blueprint.md b/000 daily/FAMILY S3 Storage Integration & Blueprint.md deleted file mode 100644 index c76ed16..0000000 --- a/000 daily/FAMILY S3 Storage Integration & Blueprint.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -created: 2026-08-24 09:39 -modified: 2026-08-24 09:39 -type: note -tags: - - homelab - - storage - - s3 - - garage - - family -aliases: [] ---- -# [[FAMILY S3 Storage Integration & Blueprint]] - -> **FINAL PLAN (approved 2026-08 session).** Implementation tracked in `/home/sam/home_network/custom_tools/family_home_lab/plan.md`. Supersedes the generic blueprint below. - -## Final architecture - -| Item | Decision | -|---|---| -| Engine | Fresh **Garage v1.x** container inside the family-home-lab Docker Compose stack | -| Host | `.13` nixos-desktop (always-on server) | -| Compose location | `/home/sam/Docker/Containers/family-home-lab/` | -| Data location | `/mnt/data/family-home-lab/garage-data/` + `/mnt/data/family-home-lab/garage-meta/` — **new subdirs only**. The Takeout landing zone (`01_keep`, `02_review`, `03_delete`, `takeout`, `.thumbs`) is never touched or reformatted | -| Ports | `3900` (S3 API), `3902` (web) — old test instance `garage-garage-1` retired first to free these | -| Region | `homelab` | -| Buckets | `sam`, `jo`, `harry`, `finn`, `shared-media` | -| Access | Portal (FastAPI) uses boto3, S3v4 signatures, endpoint `.13:3900` | -| Backups | `/mnt/data/family-home-lab/` added to .13's existing Borg job → rsync→.35→.23 chain | - -## Migration steps - -1. Sanity-check old test instance data: `sudo du -sh /var/lib/docker/volumes/garage_garage_data/_data/*` -2. Stop & remove old `garage-garage-1` (compose at `/home/sam/deployment/garage/`) -3. Deploy new Garage in family-home-lab stack with data dirs on `/mnt/data` -4. Create buckets + per-user S3 access keys via `garage` admin CLI -5. Verify portal upload/download round-trip -6. Add path to Borg backup sources - -## Consumers - -- **Family console portal** (`console.home.lab`) — uploads/downloads per-user buckets -- Media tools (Photopea/video/audio containers) — bind-mount or rclone-mount bucket paths -- DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]) — optional workspace persistence in user buckets - ---- - -## Original generic blueprint (reference only) - -This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based), replacing legacy volume mappings with decoupled web APIs to unify storage across network nodes. - -### Garage engine config (final form) - -```toml -metadata_dir = "/var/lib/garage/meta" # → mapped to /mnt/data/family-home-lab/garage-meta -data_dir = "/var/lib/garage/data" # → mapped to /mnt/data/family-home-lab/garage-data -db_engine = "sqlite" -replication_factor = 1 -rpc_bind_addr = "[::]:3901" - -[s3_api] -s3_region = "homelab" -api_bind_addr = "[::]:3900" - -[s3_web] -bind_addr = "[::]:3902" -root_domain = "web.s3.home.lab" -``` - -### Caddy routing (.35) - -```caddy -s3.home.lab { - reverse_proxy 192.168.20.13:3900 -} -``` - -### Full original compose example - -
expand - -```yaml -version: '3.8' -services: - studio-s3: - image: dxflrs/garage:v1.0.0 - container_name: studio_garage_s3 - restart: unless-stopped - ports: - - "3900:3900" - - "3901:3901" - volumes: - - ./garage/garage.toml:/etc/garage.toml - - /mnt/data/family-home-lab/garage-meta:/var/lib/garage/meta - - /mnt/data/family-home-lab/garage-data:/var/lib/garage/data - environment: - - GARAGE_RPC_SECRET= -``` - -
diff --git a/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md b/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md deleted file mode 100644 index a933cdf..0000000 --- a/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md +++ /dev/null @@ -1,127 +0,0 @@ ---- -created: 2026-08-21 20:08 -modified: 2026-08-24 09:39 -type: note -tags: - - ai - - family - - homelab -aliases: [] ---- -# [[# DeepSeek Harness (dsh) Home Lab Setup]] - -> **IMPLEMENTATION BRIEF for the dsh agent (updated by family-home-lab parent project).** -> The dsh agent builds these instances. The **parent console** (FastAPI portal at `console.lab.audasmedia.com.au`) is built separately and provides login, session routing and tool launching. Read this whole note before deploying. - -## 0. Context — what the parent console provides (do NOT rebuild) - -The family-home-lab project (`/home/sam/home_network/custom_tools/family_home_lab/plan.md` on .27) deploys on `.13`: - -- A FastAPI + Jinja2 + HTMX console at **`console.lab.audasmedia.com.au`** (port `8500` on .13) with per-user login (**Sam, Jo, Harry, Finn**) — bcrypt password auth, signed HTTP-only session cookies. -- The console links to each user's dsh instance after they log in. -- RabbitMQ + Celery workers, PostgreSQL+pgvector, Garage S3 (see [[FAMILY S3 Storage Integration & Blueprint]]). - -**dsh must NOT implement its own cross-user account system** — one user per container instance is correct; the console decides which instance a user sees. - -## 1. Network environment (verified live) - -| Item | Value | -|---|---| -| Subnet | `192.168.20.0/24`, gateway `.1` | -| dsh host | **`.13` nixos-desktop** (`sam@192.168.20.13`), NixOS, Docker host, 15.5 GB RAM | -| Reverse proxy | **Caddy on `.35`** (`sam@192.168.20.35`) — already proxies `*.home.lab`. Add entries there; do NOT run a second Caddy in compose | -| Local DNS | Pi-hole on `.13` — add local records → `.35` | -| LLM routing | **OmniRoute already running at `.13:20129`** (API) / `:20128` (dashboard). Point dsh providers at `http://192.168.20.13:20129/v1` instead of external APIs | -| Open WebUI | Being retired — dsh replaces it as the chat layer | - -### Port allocation (avoid collisions — verified in use on .13) - -In-use: 53, 1883, 3001, 3002, 4000, 5000, 5432-5434, 5678, 8079-8082, 8088, 8090-8091, 8787, 9001, 9010-9011, 9090-9091, 10200, 20128-20129, plus new family-home-lab stack starting at **8500**. - -**dsh instances:** use `3081` (sam), `3082` (jo), `3083` (harry), `3084` (finn). - -## 2. System architecture - -Subdomains (not subpaths) to prevent WebSocket failures: - -``` - [ Caddy on .35 ] - │ - ┌──────────┬────────┼─────────┬──────────────┐ - ▼ ▼ ▼ ▼ ▼ -dsh-sam dsh-jo dsh-harry dsh-finn console.lab.audasmedia.com.au -:3081 :3082 :3083 :3084 (.13:8500) -``` - -## 3. Docker Compose (deploy inside `/home/sam/Docker/Containers/dsh/`) - -Match existing conventions: PUID/PGID 1000, restart unless-stopped. - -```yaml -services: - dsh-sam: - image: node:20-slim - container_name: dsh-sam - command: npx @deepseek-ai/dsh web --port 3080 - ports: - - "3081:3080" - volumes: - - /mnt/data/family-home-lab/dsh/sam/workspace:/workspace - - /mnt/data/family-home-lab/dsh/sam/config:/root/.config - restart: unless-stopped - # repeat for jo(:3082), harry(:3084→3083), finn(:3084) -``` - -Data lives under `/mnt/data/family-home-lab/dsh//` so it's covered by the Borg backup job. - -## 4. Caddy entries (on .35, `/etc/caddy/Caddyfile` or its compose config) - -```caddy -dsh-sam.lab.audasmedia.com.au { - reverse_proxy 192.168.20.13:3081 -} -dsh-jo.lab.audasmedia.com.au { - reverse_proxy 192.168.20.13:3082 -} -dsh-harry.lab.audasmedia.com.au { - reverse_proxy 192.168.20.13:3083 -} -dsh-finn.lab.audasmedia.com.au { - reverse_proxy 192.168.20.13:3084 -} -``` - -**Iframe requirement (from parent console):** tools are embedded inline in the console where possible. Ensure Caddy does not add `X-Frame-Options: DENY` and that dsh sets no restrictive CSP frame-ancestors. If dsh sets headers itself, allow framing from `console.lab.audasmedia.com.au`. - -**Session handoff (to coordinate with console):** the console links out with a short-lived one-time token (`?token=...`) so users don't re-enter credentials; dsh should accept and exchange it, or at minimum tolerate being launched from the console. Coordinate final design with the parent project. - -## 5. Hardening & customizing tools (`dsh.config.yaml`) - -Providers point at local OmniRoute: - -```yaml -providers: - openai-compatible: - baseUrl: "http://192.168.20.13:20129/v1" - apiKey: "" - -plugins: - - name: dsh-plugin-file-editor - - name: dsh-plugin-translation - - name: dsh-plugin-ocr - -# STRICTLY EXCLUDED for kids' instances: -# - dsh-plugin-shell / persistent-bash -``` - -Per-user restrictions: Sam full plugins; Jo/Harry/Finn no shell/bash plugins. - -## 6. Deployment checklist - -1. `mkdir -p /mnt/data/family-home-lab/dsh/{sam,jo,harry,finn}/{workspace,config}` -2. Compose up from `/home/sam/Docker/Containers/dsh/` -3. ~~Pi-hole records~~ not needed — public DNS `*.lab.audasmedia.com.au` already resolves to `.35` -4. Add Caddyfile entries on .35, reload -5. Configure OmniRoute provider keys per instance -6. Verify WebSocket connectivity through Caddy -7. Notify parent project when URLs are live so console tool cards can link them diff --git a/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md b/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md deleted file mode 100644 index ef0972e..0000000 --- a/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md +++ /dev/null @@ -1,266 +0,0 @@ ---- -created: 2026-08-21 21:37 -modified: 2026-08-21 21:37 -type: note -tags: - - ai - - family - - tools -aliases: [] ---- -# [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint]] - -> **Build status:** see [[FAMILY Home Lab Console — Build Status]] (portal written repo-local, deployment pending). -> -> **SUPERSEDED by final plan** in `/home/sam/home_network/custom_tools/family_home_lab/plan.md` (approved). Key changes from this blueprint: host = `.13`; reuse existing Caddy on `.35` + Pi-hole DNS (no Caddy container); RabbitMQ deployed fresh (verified none exists); Garage S3 data on `/mnt/data/family-home-lab/` (see [[FAMILY S3 Storage Integration & Blueprint]]); compose at `/home/sam/Docker/Containers/family-home-lab/`; portal port `8500`; frontend FastAPI+HTMX; Open WebUI retired in favour of DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]); users Sam/Jo/Harry/Finn with username+password auth. Original blueprint below for reference. - -# Enterprise Homelab AI Multimedia Suite: Systems Architecture & Blueprint - -This document outlines the deployment strategy for a self-hosted, custom-built multimedia and AI workspace. It features individual family login profiles, an asynchronous RabbitMQ queue, and persistent user understanding via PostgreSQL with pgvector. - -## 1. Directory Structure (Managed via Pi) - -```text -/home/user/ai-studio/ -├── docker-compose.yml -├── caddy/ -│ └── Caddyfile -├── gateway-app/ # Custom FastAPI Web Application -│ ├── main.py # Async Web Controller -│ ├── database.py # PostgreSQL Connection Layer -│ ├── tasks.py # Background Worker Tasks (Celery) -│ ├── templates/ # HTMX / Frontend Layouts -│ └── static/ -└── storage/ - ├── shared_media/ # Central Media Asset Pool - ├── mum_workspace/ - └── son_workspace/ -``` - -## 2. Core Production Infrastructure Stack (`docker-compose.yml`) - -```yaml -version: '3.8' - -services: - # 1. NETWORK PROXY - caddy: - image: caddy:2-alpine - container_name: network_proxy - restart: unless-stopped - ports: - - "80:80" - - "443:443" - volumes: - - ./caddy/Caddyfile:/etc/caddy/Caddyfile - - caddy_data:/data - - caddy_config:/config - network_mode: host - - # 2. THE MASTER ENTRY PORTAL (Custom Gateway UI) - studio-portal: - image: python:3.11-slim - container_name: studio_portal_app - restart: unless-stopped - working_dir: /app - command: > - sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart && - uvicorn main:app --host 0.0.0.0 --port 8000" - ports: - - "8000:8000" - volumes: - - ./gateway-app:/app - - ./storage:/app/storage - depends_on: - - studio-db - - studio-rabbitmq - - # 3. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ) - media-worker: - image: python:3.11-slim - container_name: async_media_worker - restart: unless-stopped - working_dir: /app - command: celery -A tasks worker --loglevel=info - volumes: - - ./gateway-app:/app - - ./storage:/app/storage - - /var/run/docker.sock:/var/run/docker.sock # Safe container orchestration loop - depends_on: - - studio-rabbitmq - - # 4. INDUSTRIAL MESSAGE BROKER (RabbitMQ) - studio-rabbitmq: - image: rabbitmq:3-management-alpine - container_name: studio_message_broker - restart: unless-stopped - ports: - - "5672:5672" # RabbitMQ message port - - "15672:15672" # Management Web UI dashboard - environment: - - RABBITMQ_DEFAULT_USER=studio_broker - - RABBITMQ_DEFAULT_PASS=broker_secure_pass - - # 5. ENTERPRISE COGNITIVE DATABASE - studio-db: - image: pgvector/pgvector:pg16 # PostgreSQL natively equipped with AI Vector support - container_name: studio_cognitive_db - restart: unless-stopped - environment: - - POSTGRES_USER=studio_admin - - POSTGRES_PASSWORD=studio_secure_pass - - POSTGRES_DB=studio_memory_matrix - volumes: - - postgres_data:/var/lib/postgresql/data - - # 6. OMNIROUTE GATEWAY - omniroute: - image: omniroute/gateway:latest - container_name: omniroute_gateway - restart: unless-stopped - ports: - - "20128:20128" - environment: - - OPENAI_API_KEY=your_secure_cloud_key - - GEMINI_API_KEY=your_secure_cloud_key - volumes: - - ./omniroute/config:/app/config - - # 7. MULTIMEDIA STUDIO CONTAINERS - photopea: - image: shtse8/photopea:1.0 - container_name: photopea_studio - ports: - - "8487:8887" - - kdenlive-studio: - image: lscr.io/linuxserver/kdenlive:latest # HTML5 Streamed Pro Video Studio - container_name: pro_video_studio - ports: - - "8081:3000" - environment: - - PUID=1000 - - PGID=1000 - volumes: - - ./storage/shared_media:/config - - # AUDIO STUDIO A: Browser-Streamed Audacity - audacity-studio: - image: lscr.io/linuxserver/audacity:latest - container_name: pro_audio_audacity - ports: - - "3000:3000" - environment: - - PUID=1000 - - PGID=1000 - volumes: - - ./storage/shared_media:/config - - # AUDIO STUDIO B: Browser-Streamed Pro DAW (Zrythm) - zrythm-studio: - image: lscr.io/linuxserver/zrythm:latest # Full multi-track timeline automation DAW - container_name: pro_audio_zrythm - ports: - - "3001:3000" - environment: - - PUID=1000 - - PGID=1000 - volumes: - - ./storage/shared_media:/config - -volumes: - postgres_data: - caddy_data: - caddy_config: -``` - -## 3. Cognitive Engine: Relational Vector Schema (`database.py`) - -```python -import psycopg2 - -def init_db(): - conn = psycopg2.connect("host=studio-db dbname=studio_memory_matrix user=studio_admin password=studio_secure_pass") - cur = conn.cursor() - - # Enable the Vector extension explicitly - cur.execute("CREATE EXTENSION IF NOT EXISTS vector;") - - # User Profile table - cur.execute(""" - CREATE TABLE IF NOT EXISTS user_profiles ( - user_id VARCHAR PRIMARY KEY, - preference_matrix JSONB, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - """) - - # Semantic Memory Table with Vector Embeddings - cur.execute(""" - CREATE TABLE IF NOT EXISTS user_memories ( - memory_id SERIAL PRIMARY KEY, - user_id VARCHAR, - summary TEXT, - embedding vector(1536), -- Standard cloud embedding dimensions - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - """) - conn.commit() - cur.close() - conn.close() -``` - -## 4. Asynchronous Task Routing Backend (`tasks.py`) - -```python -from celery import Celery -import subprocess - -# Configured to use RabbitMQ as the robust message broker -celery_app = Celery('studio_tasks', broker='amqp://studio_broker:broker_secure_pass@studio-rabbitmq:5672//') - -@celery_app.task -def process_video_rotation(input_file, output_file): - """ - Executes a fast, headless FFmpeg run inside RabbitMQ queue context. - Prevents the main FastAPI frontend from lagging during large transfers. - """ - subprocess.run([ - "docker", "run", "--rm", - "-v", "/home/user/ai-studio/storage/shared_media:/media", - "jrottenberg/ffmpeg", - "-i", f"/media/{input_file}", - "-vf", "transpose=1", - f"/media/{output_file}" - ]) -``` - -## 5. Network Routing Configuration (`Caddyfile`) - -```caddy -# Core Application Portal Entry Point -ds.home.lab { - reverse_proxy 127.0.0.1:8000 -} - -# Image Design Lab -photo.home.lab { - reverse_proxy 127.0.0.1:8487 -} - -# Pro Video Editor Layout -video.home.lab { - reverse_proxy 127.0.0.1:8081 -} - -# Pro Audio Station A (Audacity Wrapper) -audacity.home.lab { - reverse_proxy 127.0.0.1:3000 -} - -# Pro Audio Station B (Zrythm DAW Studio) -zrythm.home.lab { - reverse_proxy 127.0.0.1:3001 -} -``` - diff --git a/000 daily/FAMILY Home Lab Console — Build Status.md b/000 daily/FAMILY Home Lab Console — Build Status.md deleted file mode 100644 index c26bfd6..0000000 --- a/000 daily/FAMILY Home Lab Console — Build Status.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -created: 2026-08-24 -modified: 2026-08-24 -type: note -tags: - - family - - home-lab - - portal - - build -aliases: - - Home Lab Console Build ---- -# Home Lab Console — Build Status - -Status of the [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint|Home Lab]] console portal build. Authoritative plan: `/home/sam/home_network/custom_tools/family_home_lab/plan.md` (FINAL v3, approved). Design: `DESIGN.md` (Notion language — canvas `#f6f5f4`, primary `#0075de`, indigo login `#213183`). - -## Where it runs -- **Host:** `.13` (192.168.20.13, nixos-desktop, Docker host) -- **Compose:** `/home/sam/Docker/Containers/family-home-lab/` (ported from repo) -- **Proxy/DNS:** existing Caddy on `.35`; public zone `*.lab.audasmedia.com.au` → router → `.35` → `.13` -- **Portal:** FastAPI + Jinja2 + HTMX, port `8500`, `console.lab.audasmedia.com.au` - -## Repo-local code (written 2026-08-24, not yet deployed) -> **Update 2026-08-24 (deploy):** core stack **deployed and live** at https://console.lab.audasmedia.com.au (login → dashboard verified over public HTTPS). Garage up with buckets `sam, jo-media, harry, finn, shared-media`; portal key RW on all. Borg backup edit to `backup.nix` pending (needs sudo). -> -> (Original, below, now historical.) -- `portal/` — `main.py` (routes, first-run admin seed), `auth.py` (bcrypt + signed HTTP-only session cookies), `database.py` (SQLAlchemy async + pgvector/pg16), `tasks.py` (Celery + RabbitMQ), `tools.py` (per-user dsh URL), `config.py`, `Dockerfile`, `requirements.txt` -- `portal/templates/` (9 pages) + `portal/static/tokens.css` + `app.css` -- `docker-compose.yml` (postgres / redis / rabbitmq / garage / portal / worker; tool containers behind `profile: tools`), `garage/garage.toml`, `.env.example`, per-tool `README`s (image/video/audio/ai/docs/chat) - -## Verified -- All modules compile; all 9 templates render; per-user sections + dsh link resolve; `docker compose config -q` valid. - -## Deployment checklist (pending) -1. Local smoke test of login → dashboard flow. -2. Sync repo → `.13` at `/home/sam/Docker/Containers/family-home-lab/`; `docker compose up -d`. -3. Generate Garage tokens + create buckets `sam jo harry finn shared-media` + access keys (runbook in `deploy/DEPLOYMENT.md`). -4. Apply `deploy/caddy/Caddyfile.snippet` on `.35` and reload. -5. Add `/mnt/data/family-home-lab/` to `.13` Borg backup sources. -6. Enable tool containers (`--profile tools`) once images verified (plan §4). - -## Deployment checklist (further) -1. ✅ Core stack up on `.13`; console URL live. -2. ✅ Garage buckets + portal access key provisioned (`jo` → `jo-media` per S3 name rules). -3. ✅ Caddy `console` + `photo`/`video`/`audio` rules applied & reloaded on `.35`; certs obtained. -4. ✅ **Media tools live**: Photopea (photo), KdenLive (video), Audacity (audio) — public URLs verified. Images: shtse8/photopea:1.0, lscr.io/linuxserver/kdenlive, lscr.io/linuxserver/audacity. Host ports video **8083** / audio **8084** (moved off airflow-8081 & NixOS-3000). -5. ⏳ Add `/mnt/data/family-home-lab/` to `.13` `backup.nix` (`services.borgbackup.jobs."local-to-storage".paths`) — needs sudo. -6. ⏳ DeepSeek Harness instances (dsh agent) — console already routes `dsh-{user}.lab.audasmedia.com.au`. - -## Related -- [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint]] -- [[FAMILY S3 Storage Integration & Blueprint]] -- [[FAMILY DeepSeek Harness (dsh) Home Lab Setup]] \ No newline at end of file diff --git a/500 archive/520 Projects/Family Home Lab Planning Archive.md b/500 archive/520 Projects/Family Home Lab Planning Archive.md new file mode 100644 index 0000000..65fde47 --- /dev/null +++ b/500 archive/520 Projects/Family Home Lab Planning Archive.md @@ -0,0 +1,572 @@ +--- +created: 2026-09-09 +modified: 2026-09-09 +type: note +tags: + - family + - home-lab + - planning + - archive +aliases: + - Family Home Lab Planning +--- +# Family Home Lab Planning Archive + +> **Archive.** Original planning/build notes for the Family Home Lab, merged from the four +> `000 daily` FAMILY files (2026-08). **Superseded by** [[FAMILY CONSOLE]] (human overview) and +> "Family Home Lab" in `200 projects/Tools Software WebUI/`, plus the Outline + Vikunja project. Kept for provenance. + +--- +###### FILE: FAMILY Home Lab Console — Build Status.md ###### +--- +created: 2026-08-24 +modified: 2026-08-24 +type: note +tags: + - family + - home-lab + - portal + - build +aliases: + - Home Lab Console Build +--- +# Home Lab Console — Build Status + +Status of the [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint|Home Lab]] console portal build. Authoritative plan: `/home/sam/home_network/custom_tools/family_home_lab/plan.md` (FINAL v3, approved). Design: `DESIGN.md` (Notion language — canvas `#f6f5f4`, primary `#0075de`, indigo login `#213183`). + +## Where it runs +- **Host:** `.13` (192.168.20.13, nixos-desktop, Docker host) +- **Compose:** `/home/sam/Docker/Containers/family-home-lab/` (ported from repo) +- **Proxy/DNS:** existing Caddy on `.35`; public zone `*.lab.audasmedia.com.au` → router → `.35` → `.13` +- **Portal:** FastAPI + Jinja2 + HTMX, port `8500`, `console.lab.audasmedia.com.au` + +## Repo-local code (written 2026-08-24, not yet deployed) +> **Update 2026-08-24 (deploy):** core stack **deployed and live** at https://console.lab.audasmedia.com.au (login → dashboard verified over public HTTPS). Garage up with buckets `sam, jo-media, harry, finn, shared-media`; portal key RW on all. Borg backup edit to `backup.nix` pending (needs sudo). +> +> (Original, below, now historical.) +- `portal/` — `main.py` (routes, first-run admin seed), `auth.py` (bcrypt + signed HTTP-only session cookies), `database.py` (SQLAlchemy async + pgvector/pg16), `tasks.py` (Celery + RabbitMQ), `tools.py` (per-user dsh URL), `config.py`, `Dockerfile`, `requirements.txt` +- `portal/templates/` (9 pages) + `portal/static/tokens.css` + `app.css` +- `docker-compose.yml` (postgres / redis / rabbitmq / garage / portal / worker; tool containers behind `profile: tools`), `garage/garage.toml`, `.env.example`, per-tool `README`s (image/video/audio/ai/docs/chat) + +## Verified +- All modules compile; all 9 templates render; per-user sections + dsh link resolve; `docker compose config -q` valid. + +## Deployment checklist (pending) +1. Local smoke test of login → dashboard flow. +2. Sync repo → `.13` at `/home/sam/Docker/Containers/family-home-lab/`; `docker compose up -d`. +3. Generate Garage tokens + create buckets `sam jo harry finn shared-media` + access keys (runbook in `deploy/DEPLOYMENT.md`). +4. Apply `deploy/caddy/Caddyfile.snippet` on `.35` and reload. +5. Add `/mnt/data/family-home-lab/` to `.13` Borg backup sources. +6. Enable tool containers (`--profile tools`) once images verified (plan §4). + +## Deployment checklist (further) +1. ✅ Core stack up on `.13`; console URL live. +2. ✅ Garage buckets + portal access key provisioned (`jo` → `jo-media` per S3 name rules). +3. ✅ Caddy `console` + `photo`/`video`/`audio` rules applied & reloaded on `.35`; certs obtained. +4. ✅ **Media tools live**: Photopea (photo), KdenLive (video), Audacity (audio) — public URLs verified. Images: shtse8/photopea:1.0, lscr.io/linuxserver/kdenlive, lscr.io/linuxserver/audacity. Host ports video **8083** / audio **8084** (moved off airflow-8081 & NixOS-3000). +5. ⏳ Add `/mnt/data/family-home-lab/` to `.13` `backup.nix` (`services.borgbackup.jobs."local-to-storage".paths`) — needs sudo. +6. ⏳ DeepSeek Harness instances (dsh agent) — console already routes `dsh-{user}.lab.audasmedia.com.au`. + +## Related +- [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint]] +- [[FAMILY S3 Storage Integration & Blueprint]] +- [[FAMILY DeepSeek Harness (dsh) Home Lab Setup]] +###### FILE: FAMILY DeepSeek Harness (dsh) Home Lab Setup.md ###### +--- +created: 2026-08-21 20:08 +modified: 2026-08-24 09:39 +type: note +tags: + - ai + - family + - homelab +aliases: [] +--- +# [[# DeepSeek Harness (dsh) Home Lab Setup]] + +> **IMPLEMENTATION BRIEF for the dsh agent (updated by family-home-lab parent project).** +> The dsh agent builds these instances. The **parent console** (FastAPI portal at `console.lab.audasmedia.com.au`) is built separately and provides login, session routing and tool launching. Read this whole note before deploying. + +## 0. Context — what the parent console provides (do NOT rebuild) + +The family-home-lab project (`/home/sam/home_network/custom_tools/family_home_lab/plan.md` on .27) deploys on `.13`: + +- A FastAPI + Jinja2 + HTMX console at **`console.lab.audasmedia.com.au`** (port `8500` on .13) with per-user login (**Sam, Jo, Harry, Finn**) — bcrypt password auth, signed HTTP-only session cookies. +- The console links to each user's dsh instance after they log in. +- RabbitMQ + Celery workers, PostgreSQL+pgvector, Garage S3 (see [[FAMILY S3 Storage Integration & Blueprint]]). + +**dsh must NOT implement its own cross-user account system** — one user per container instance is correct; the console decides which instance a user sees. + +## 1. Network environment (verified live) + +| Item | Value | +|---|---| +| Subnet | `192.168.20.0/24`, gateway `.1` | +| dsh host | **`.13` nixos-desktop** (`sam@192.168.20.13`), NixOS, Docker host, 15.5 GB RAM | +| Reverse proxy | **Caddy on `.35`** (`sam@192.168.20.35`) — already proxies `*.home.lab`. Add entries there; do NOT run a second Caddy in compose | +| Local DNS | Pi-hole on `.13` — add local records → `.35` | +| LLM routing | **OmniRoute already running at `.13:20129`** (API) / `:20128` (dashboard). Point dsh providers at `http://192.168.20.13:20129/v1` instead of external APIs | +| Open WebUI | Being retired — dsh replaces it as the chat layer | + +### Port allocation (avoid collisions — verified in use on .13) + +In-use: 53, 1883, 3001, 3002, 4000, 5000, 5432-5434, 5678, 8079-8082, 8088, 8090-8091, 8787, 9001, 9010-9011, 9090-9091, 10200, 20128-20129, plus new family-home-lab stack starting at **8500**. + +**dsh instances:** use `3081` (sam), `3082` (jo), `3083` (harry), `3084` (finn). + +## 2. System architecture + +Subdomains (not subpaths) to prevent WebSocket failures: + +``` + [ Caddy on .35 ] + │ + ┌──────────┬────────┼─────────┬──────────────┐ + ▼ ▼ ▼ ▼ ▼ +dsh-sam dsh-jo dsh-harry dsh-finn console.lab.audasmedia.com.au +:3081 :3082 :3083 :3084 (.13:8500) +``` + +## 3. Docker Compose (deploy inside `/home/sam/Docker/Containers/dsh/`) + +Match existing conventions: PUID/PGID 1000, restart unless-stopped. + +```yaml +services: + dsh-sam: + image: node:20-slim + container_name: dsh-sam + command: npx @deepseek-ai/dsh web --port 3080 + ports: + - "3081:3080" + volumes: + - /mnt/data/family-home-lab/dsh/sam/workspace:/workspace + - /mnt/data/family-home-lab/dsh/sam/config:/root/.config + restart: unless-stopped + # repeat for jo(:3082), harry(:3084→3083), finn(:3084) +``` + +Data lives under `/mnt/data/family-home-lab/dsh//` so it's covered by the Borg backup job. + +## 4. Caddy entries (on .35, `/etc/caddy/Caddyfile` or its compose config) + +```caddy +dsh-sam.lab.audasmedia.com.au { + reverse_proxy 192.168.20.13:3081 +} +dsh-jo.lab.audasmedia.com.au { + reverse_proxy 192.168.20.13:3082 +} +dsh-harry.lab.audasmedia.com.au { + reverse_proxy 192.168.20.13:3083 +} +dsh-finn.lab.audasmedia.com.au { + reverse_proxy 192.168.20.13:3084 +} +``` + +**Iframe requirement (from parent console):** tools are embedded inline in the console where possible. Ensure Caddy does not add `X-Frame-Options: DENY` and that dsh sets no restrictive CSP frame-ancestors. If dsh sets headers itself, allow framing from `console.lab.audasmedia.com.au`. + +**Session handoff (to coordinate with console):** the console links out with a short-lived one-time token (`?token=...`) so users don't re-enter credentials; dsh should accept and exchange it, or at minimum tolerate being launched from the console. Coordinate final design with the parent project. + +## 5. Hardening & customizing tools (`dsh.config.yaml`) + +Providers point at local OmniRoute: + +```yaml +providers: + openai-compatible: + baseUrl: "http://192.168.20.13:20129/v1" + apiKey: "" + +plugins: + - name: dsh-plugin-file-editor + - name: dsh-plugin-translation + - name: dsh-plugin-ocr + +# STRICTLY EXCLUDED for kids' instances: +# - dsh-plugin-shell / persistent-bash +``` + +Per-user restrictions: Sam full plugins; Jo/Harry/Finn no shell/bash plugins. + +## 6. Deployment checklist + +1. `mkdir -p /mnt/data/family-home-lab/dsh/{sam,jo,harry,finn}/{workspace,config}` +2. Compose up from `/home/sam/Docker/Containers/dsh/` +3. ~~Pi-hole records~~ not needed — public DNS `*.lab.audasmedia.com.au` already resolves to `.35` +4. Add Caddyfile entries on .35, reload +5. Configure OmniRoute provider keys per instance +6. Verify WebSocket connectivity through Caddy +7. Notify parent project when URLs are live so console tool cards can link them + +###### FILE: FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md ###### +--- +created: 2026-08-21 21:37 +modified: 2026-08-21 21:37 +type: note +tags: + - ai + - family + - tools +aliases: [] +--- +# [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint]] + +> **Build status:** see [[FAMILY Home Lab Console — Build Status]] (portal written repo-local, deployment pending). +> +> **SUPERSEDED by final plan** in `/home/sam/home_network/custom_tools/family_home_lab/plan.md` (approved). Key changes from this blueprint: host = `.13`; reuse existing Caddy on `.35` + Pi-hole DNS (no Caddy container); RabbitMQ deployed fresh (verified none exists); Garage S3 data on `/mnt/data/family-home-lab/` (see [[FAMILY S3 Storage Integration & Blueprint]]); compose at `/home/sam/Docker/Containers/family-home-lab/`; portal port `8500`; frontend FastAPI+HTMX; Open WebUI retired in favour of DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]); users Sam/Jo/Harry/Finn with username+password auth. Original blueprint below for reference. + +# Enterprise Homelab AI Multimedia Suite: Systems Architecture & Blueprint + +This document outlines the deployment strategy for a self-hosted, custom-built multimedia and AI workspace. It features individual family login profiles, an asynchronous RabbitMQ queue, and persistent user understanding via PostgreSQL with pgvector. + +## 1. Directory Structure (Managed via Pi) + +```text +/home/user/ai-studio/ +├── docker-compose.yml +├── caddy/ +│ └── Caddyfile +├── gateway-app/ # Custom FastAPI Web Application +│ ├── main.py # Async Web Controller +│ ├── database.py # PostgreSQL Connection Layer +│ ├── tasks.py # Background Worker Tasks (Celery) +│ ├── templates/ # HTMX / Frontend Layouts +│ └── static/ +└── storage/ + ├── shared_media/ # Central Media Asset Pool + ├── mum_workspace/ + └── son_workspace/ +``` + +## 2. Core Production Infrastructure Stack (`docker-compose.yml`) + +```yaml +version: '3.8' + +services: + # 1. NETWORK PROXY + caddy: + image: caddy:2-alpine + container_name: network_proxy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + network_mode: host + + # 2. THE MASTER ENTRY PORTAL (Custom Gateway UI) + studio-portal: + image: python:3.11-slim + container_name: studio_portal_app + restart: unless-stopped + working_dir: /app + command: > + sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart && + uvicorn main:app --host 0.0.0.0 --port 8000" + ports: + - "8000:8000" + volumes: + - ./gateway-app:/app + - ./storage:/app/storage + depends_on: + - studio-db + - studio-rabbitmq + + # 3. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ) + media-worker: + image: python:3.11-slim + container_name: async_media_worker + restart: unless-stopped + working_dir: /app + command: celery -A tasks worker --loglevel=info + volumes: + - ./gateway-app:/app + - ./storage:/app/storage + - /var/run/docker.sock:/var/run/docker.sock # Safe container orchestration loop + depends_on: + - studio-rabbitmq + + # 4. INDUSTRIAL MESSAGE BROKER (RabbitMQ) + studio-rabbitmq: + image: rabbitmq:3-management-alpine + container_name: studio_message_broker + restart: unless-stopped + ports: + - "5672:5672" # RabbitMQ message port + - "15672:15672" # Management Web UI dashboard + environment: + - RABBITMQ_DEFAULT_USER=studio_broker + - RABBITMQ_DEFAULT_PASS=broker_secure_pass + + # 5. ENTERPRISE COGNITIVE DATABASE + studio-db: + image: pgvector/pgvector:pg16 # PostgreSQL natively equipped with AI Vector support + container_name: studio_cognitive_db + restart: unless-stopped + environment: + - POSTGRES_USER=studio_admin + - POSTGRES_PASSWORD=studio_secure_pass + - POSTGRES_DB=studio_memory_matrix + volumes: + - postgres_data:/var/lib/postgresql/data + + # 6. OMNIROUTE GATEWAY + omniroute: + image: omniroute/gateway:latest + container_name: omniroute_gateway + restart: unless-stopped + ports: + - "20128:20128" + environment: + - OPENAI_API_KEY=your_secure_cloud_key + - GEMINI_API_KEY=your_secure_cloud_key + volumes: + - ./omniroute/config:/app/config + + # 7. MULTIMEDIA STUDIO CONTAINERS + photopea: + image: shtse8/photopea:1.0 + container_name: photopea_studio + ports: + - "8487:8887" + + kdenlive-studio: + image: lscr.io/linuxserver/kdenlive:latest # HTML5 Streamed Pro Video Studio + container_name: pro_video_studio + ports: + - "8081:3000" + environment: + - PUID=1000 + - PGID=1000 + volumes: + - ./storage/shared_media:/config + + # AUDIO STUDIO A: Browser-Streamed Audacity + audacity-studio: + image: lscr.io/linuxserver/audacity:latest + container_name: pro_audio_audacity + ports: + - "3000:3000" + environment: + - PUID=1000 + - PGID=1000 + volumes: + - ./storage/shared_media:/config + + # AUDIO STUDIO B: Browser-Streamed Pro DAW (Zrythm) + zrythm-studio: + image: lscr.io/linuxserver/zrythm:latest # Full multi-track timeline automation DAW + container_name: pro_audio_zrythm + ports: + - "3001:3000" + environment: + - PUID=1000 + - PGID=1000 + volumes: + - ./storage/shared_media:/config + +volumes: + postgres_data: + caddy_data: + caddy_config: +``` + +## 3. Cognitive Engine: Relational Vector Schema (`database.py`) + +```python +import psycopg2 + +def init_db(): + conn = psycopg2.connect("host=studio-db dbname=studio_memory_matrix user=studio_admin password=studio_secure_pass") + cur = conn.cursor() + + # Enable the Vector extension explicitly + cur.execute("CREATE EXTENSION IF NOT EXISTS vector;") + + # User Profile table + cur.execute(""" + CREATE TABLE IF NOT EXISTS user_profiles ( + user_id VARCHAR PRIMARY KEY, + preference_matrix JSONB, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """) + + # Semantic Memory Table with Vector Embeddings + cur.execute(""" + CREATE TABLE IF NOT EXISTS user_memories ( + memory_id SERIAL PRIMARY KEY, + user_id VARCHAR, + summary TEXT, + embedding vector(1536), -- Standard cloud embedding dimensions + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """) + conn.commit() + cur.close() + conn.close() +``` + +## 4. Asynchronous Task Routing Backend (`tasks.py`) + +```python +from celery import Celery +import subprocess + +# Configured to use RabbitMQ as the robust message broker +celery_app = Celery('studio_tasks', broker='amqp://studio_broker:broker_secure_pass@studio-rabbitmq:5672//') + +@celery_app.task +def process_video_rotation(input_file, output_file): + """ + Executes a fast, headless FFmpeg run inside RabbitMQ queue context. + Prevents the main FastAPI frontend from lagging during large transfers. + """ + subprocess.run([ + "docker", "run", "--rm", + "-v", "/home/user/ai-studio/storage/shared_media:/media", + "jrottenberg/ffmpeg", + "-i", f"/media/{input_file}", + "-vf", "transpose=1", + f"/media/{output_file}" + ]) +``` + +## 5. Network Routing Configuration (`Caddyfile`) + +```caddy +# Core Application Portal Entry Point +ds.home.lab { + reverse_proxy 127.0.0.1:8000 +} + +# Image Design Lab +photo.home.lab { + reverse_proxy 127.0.0.1:8487 +} + +# Pro Video Editor Layout +video.home.lab { + reverse_proxy 127.0.0.1:8081 +} + +# Pro Audio Station A (Audacity Wrapper) +audacity.home.lab { + reverse_proxy 127.0.0.1:3000 +} + +# Pro Audio Station B (Zrythm DAW Studio) +zrythm.home.lab { + reverse_proxy 127.0.0.1:3001 +} +``` + + +###### FILE: FAMILY S3 Storage Integration & Blueprint.md ###### +--- +created: 2026-08-24 09:39 +modified: 2026-08-24 09:39 +type: note +tags: + - homelab + - storage + - s3 + - garage + - family +aliases: [] +--- +# [[FAMILY S3 Storage Integration & Blueprint]] + +> **FINAL PLAN (approved 2026-08 session).** Implementation tracked in `/home/sam/home_network/custom_tools/family_home_lab/plan.md`. Supersedes the generic blueprint below. + +## Final architecture + +| Item | Decision | +|---|---| +| Engine | Fresh **Garage v1.x** container inside the family-home-lab Docker Compose stack | +| Host | `.13` nixos-desktop (always-on server) | +| Compose location | `/home/sam/Docker/Containers/family-home-lab/` | +| Data location | `/mnt/data/family-home-lab/garage-data/` + `/mnt/data/family-home-lab/garage-meta/` — **new subdirs only**. The Takeout landing zone (`01_keep`, `02_review`, `03_delete`, `takeout`, `.thumbs`) is never touched or reformatted | +| Ports | `3900` (S3 API), `3902` (web) — old test instance `garage-garage-1` retired first to free these | +| Region | `homelab` | +| Buckets | `sam`, `jo`, `harry`, `finn`, `shared-media` | +| Access | Portal (FastAPI) uses boto3, S3v4 signatures, endpoint `.13:3900` | +| Backups | `/mnt/data/family-home-lab/` added to .13's existing Borg job → rsync→.35→.23 chain | + +## Migration steps + +1. Sanity-check old test instance data: `sudo du -sh /var/lib/docker/volumes/garage_garage_data/_data/*` +2. Stop & remove old `garage-garage-1` (compose at `/home/sam/deployment/garage/`) +3. Deploy new Garage in family-home-lab stack with data dirs on `/mnt/data` +4. Create buckets + per-user S3 access keys via `garage` admin CLI +5. Verify portal upload/download round-trip +6. Add path to Borg backup sources + +## Consumers + +- **Family console portal** (`console.home.lab`) — uploads/downloads per-user buckets +- Media tools (Photopea/video/audio containers) — bind-mount or rclone-mount bucket paths +- DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]) — optional workspace persistence in user buckets + +--- + +## Original generic blueprint (reference only) + +This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based), replacing legacy volume mappings with decoupled web APIs to unify storage across network nodes. + +### Garage engine config (final form) + +```toml +metadata_dir = "/var/lib/garage/meta" # → mapped to /mnt/data/family-home-lab/garage-meta +data_dir = "/var/lib/garage/data" # → mapped to /mnt/data/family-home-lab/garage-data +db_engine = "sqlite" +replication_factor = 1 +rpc_bind_addr = "[::]:3901" + +[s3_api] +s3_region = "homelab" +api_bind_addr = "[::]:3900" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = "web.s3.home.lab" +``` + +### Caddy routing (.35) + +```caddy +s3.home.lab { + reverse_proxy 192.168.20.13:3900 +} +``` + +### Full original compose example + +
expand + +```yaml +version: '3.8' +services: + studio-s3: + image: dxflrs/garage:v1.0.0 + container_name: studio_garage_s3 + restart: unless-stopped + ports: + - "3900:3900" + - "3901:3901" + volumes: + - ./garage/garage.toml:/etc/garage.toml + - /mnt/data/family-home-lab/garage-meta:/var/lib/garage/meta + - /mnt/data/family-home-lab/garage-data:/var/lib/garage/data + environment: + - GARAGE_RPC_SECRET= +``` + +
+ diff --git a/kontra_content b/kontra_content deleted file mode 120000 index bb49dd8..0000000 --- a/kontra_content +++ /dev/null @@ -1 +0,0 @@ -/home/sam/home_network/web_sites/kontra_day/content \ No newline at end of file