From 95ecbd233cd1e302f6272961d5cd8fe99c9cd93c Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Mon, 24 Aug 2026 13:54:08 +1000 Subject: [PATCH] sam-4screen-desktop 2026-8-24:13:54:8 --- ...ILY S3 Storage Integration & Blueprint.md | 297 ++++-------------- ...Y DeepSeek Harness (dsh) Home Lab Setup.md | 135 ++++---- ...Suite. Systems Architecture & Blueprint.md | 2 + 3 files changed, 135 insertions(+), 299 deletions(-) diff --git a/000 daily/FAMILY S3 Storage Integration & Blueprint.md b/000 daily/FAMILY S3 Storage Integration & Blueprint.md index e8d1ff2..c76ed16 100644 --- a/000 daily/FAMILY S3 Storage Integration & Blueprint.md +++ b/000 daily/FAMILY S3 Storage Integration & Blueprint.md @@ -2,280 +2,99 @@ created: 2026-08-24 09:39 modified: 2026-08-24 09:39 type: note -tags: - - ai - - amazon - - s3 - - aws - - family - - tools -aliases: [] ---- -# [[FAMILY S3 Storage Integration & Blueprint]] - ---- -created: 2026-08-24 09:38 -modified: 2026-08-24 09:38 -type: note tags: - homelab - storage - s3 - garage + - family aliases: [] --- # [[FAMILY S3 Storage Integration & Blueprint]] -# FAMILY Enterprise Homelab AI Multimedia Suite: S3 Storage Integration & Blueprint +> **FINAL PLAN (approved 2026-08 session).** Implementation tracked in `/home/sam/home_network/custom_tools/family_home_lab/plan.md`. Supersedes the generic blueprint below. -This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based). This integration replaces legacy volume mappings with decoupled web APIs to unify storage across multiple network nodes. +## Final architecture -## 1. Updated Directory Structure (Managed via Pi) +| Item | Decision | +|---|---| +| Engine | Fresh **Garage v1.x** container inside the family-home-lab Docker Compose stack | +| Host | `.13` nixos-desktop (always-on server) | +| Compose location | `/home/sam/Docker/Containers/family-home-lab/` | +| Data location | `/mnt/data/family-home-lab/garage-data/` + `/mnt/data/family-home-lab/garage-meta/` — **new subdirs only**. The Takeout landing zone (`01_keep`, `02_review`, `03_delete`, `takeout`, `.thumbs`) is never touched or reformatted | +| Ports | `3900` (S3 API), `3902` (web) — old test instance `garage-garage-1` retired first to free these | +| Region | `homelab` | +| Buckets | `sam`, `jo`, `harry`, `finn`, `shared-media` | +| Access | Portal (FastAPI) uses boto3, S3v4 signatures, endpoint `.13:3900` | +| Backups | `/mnt/data/family-home-lab/` added to .13's existing Borg job → rsync→.35→.23 chain | -```text -/home/user/ai-studio/ -├── docker-compose.yml -├── caddy/ -│ └── Caddyfile -├── garage/ -│ └── garage.toml # Garage Engine Configuration -├── gateway-app/ # Custom FastAPI Web Application -│ ├── main.py # Modified Async Web Controller (Boto3 Native) -│ ├── database.py # PostgreSQL Connection Layer (pgvector) -│ ├── tasks.py # Background Worker Tasks (Celery) -│ └── ... -└── storage/ - ├── garage_meta/ # Fast metadata tracking blocks - └── shared_media/ # Physical encrypted object storage pools -``` +## Migration steps -## 2. Integrated Production Stack Layout (`docker-compose.yml`) +1. Sanity-check old test instance data: `sudo du -sh /var/lib/docker/volumes/garage_garage_data/_data/*` +2. Stop & remove old `garage-garage-1` (compose at `/home/sam/deployment/garage/`) +3. Deploy new Garage in family-home-lab stack with data dirs on `/mnt/data` +4. Create buckets + per-user S3 access keys via `garage` admin CLI +5. Verify portal upload/download round-trip +6. Add path to Borg backup sources -```yaml -version: '3.8' +## Consumers -services: - # 1. NETWORK PROXY - caddy: - image: caddy:2-alpine - container_name: network_proxy - restart: unless-stopped - ports: - - "80:80" - - "443:443" - volumes: - - ./caddy/Caddyfile:/etc/caddy/Caddyfile - - caddy_data:/data - - caddy_config:/config - network_mode: host +- **Family console portal** (`console.home.lab`) — uploads/downloads per-user buckets +- Media tools (Photopea/video/audio containers) — bind-mount or rclone-mount bucket paths +- DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]) — optional workspace persistence in user buckets - # 2. PURE OPEN-SOURCE LOCAL S3 ENGINE (Rust-powered) - studio-s3: - image: dxflrs/garage:v1.0.0 - container_name: studio_garage_s3 - restart: unless-stopped - ports: - - "3900:3900" # S3 Web API Engine Endpoint - - "3901:3901" # Administrative Cluster RPC Port - volumes: - - ./garage/garage.toml:/etc/garage.toml - - ./storage/garage_meta:/var/lib/garage/meta - - ./storage/shared_media:/var/lib/garage/data - environment: - - GK_rpc_secret=7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00 - command: /usr/local/bin/garage server +--- - # 3. THE MASTER ENTRY PORTAL (Custom Gateway UI with Boto3 S3 Integration) - studio-portal: - image: python:3.11-slim - container_name: studio_portal_app - restart: unless-stopped - working_dir: /app - command: > - sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart boto3 && - uvicorn main:app --host 0.0.0.0 --port 8000" - ports: - - "8000:8000" - volumes: - - ./gateway-app:/app - depends_on: - - studio-db - - studio-rabbitmq - - studio-s3 +## Original generic blueprint (reference only) - # 4. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ) - media-worker: - image: python:3.11-slim - container_name: async_media_worker - restart: unless-stopped - working_dir: /app - command: celery -A tasks worker --loglevel=info - volumes: - - ./gateway-app:/app - - /var/run/docker.sock:/var/run/docker.sock - depends_on: - - studio-rabbitmq - - studio-s3 +This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based), replacing legacy volume mappings with decoupled web APIs to unify storage across network nodes. - # 5. INDUSTRIAL MESSAGE BROKER (RabbitMQ) - studio-rabbitmq: - image: rabbitmq:3-management-alpine - container_name: studio_message_broker - restart: unless-stopped - ports: - - "5672:5672" - - "15672:15672" - environment: - - RABBITMQ_DEFAULT_USER=studio_broker - - RABBITMQ_DEFAULT_PASS=broker_secure_pass - - # 6. ENTERPRISE COGNITIVE DATABASE - studio-db: - image: pgvector/pgvector:pg16 - container_name: studio_cognitive_db - restart: unless-stopped - environment: - - POSTGRES_USER=studio_admin - - POSTGRES_PASSWORD=studio_secure_pass - - POSTGRES_DB=studio_memory_matrix - volumes: - - postgres_data:/var/lib/postgresql/data - - # 7. OMNIROUTE GATEWAY - omniroute: - image: omniroute/gateway:latest - container_name: omniroute_gateway - restart: unless-stopped - ports: - - "20128:20128" - environment: - - OPENAI_API_KEY=your_secure_cloud_key - - GEMINI_API_KEY=your_secure_cloud_key - volumes: - - ./omniroute/config:/app/config - - # 8. MULTIMEDIA STUDIO CONTAINERS (Configured to mount S3 blocks or talk directly via API) - photopea: - image: shtse8/photopea:1.0 - container_name: photopea_studio - ports: - - "8487:8887" - - kdenlive-studio: - image: lscr.io/linuxserver/kdenlive:latest - container_name: pro_video_studio - ports: - - "8081:3000" - environment: - - PUID=1000 - - PGID=1000 - - audacity-studio: - image: lscr.io/linuxserver/audacity:latest - container_name: pro_audio_audacity - ports: - - "3000:3000" - environment: - - PUID=1000 - - PGID=1000 - - zrythm-studio: - image: lscr.io/linuxserver/zrythm:latest - container_name: pro_audio_zrythm - ports: - - "3001:3000" - environment: - - PUID=1000 - - PGID=1000 - -volumes: - postgres_data: - caddy_data: - caddy_config: -``` - -## 3. Dedicated Storage Engine Configuration (`garage.toml`) +### Garage engine config (final form) ```toml -metadata_dir = "/var/lib/garage/meta" -data_dir = "/var/lib/garage/data" +metadata_dir = "/var/lib/garage/meta" # → mapped to /mnt/data/family-home-lab/garage-meta +data_dir = "/var/lib/garage/data" # → mapped to /mnt/data/family-home-lab/garage-data db_engine = "sqlite" - +replication_factor = 1 rpc_bind_addr = "[::]:3901" -rpc_secret = "7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00" [s3_api] -api_bind_addr = "[::]:3900" s3_region = "homelab" -root_domain = "s3.home.lab" +api_bind_addr = "[::]:3900" [s3_web] bind_addr = "[::]:3902" root_domain = "web.s3.home.lab" ``` -## 4. FastAPI Gateway S3 Client Integration Snippet (`main.py`) - -```python -import os -import boto3 -from botocore.client import Config -from fastapi import FastAPI, UploadFile, File - -app = FastAPI() - -# Point S3 Client to the local Garage Engine Container -s3_client = boto3.client( - 's3', - endpoint_url='http://studio-s3:3900', - aws_access_key_id=os.getenv("GARAGE_ACCESS_KEY", "studio_key_id"), - aws_secret_access_key=os.getenv("GARAGE_SECRET_KEY", "studio_secret_key"), - config=Config(signature_version='s3v4'), - region_name='homelab' -) - -@app.post("/upload/{workspace_bucket}") -async def upload_to_family_vault(workspace_bucket: str, file: UploadFile = File(...)): - """Streams uploads from dashboard direct to isolated Garage family buckets""" - s3_client.upload_fileobj( - file.file, - workspace_bucket, - file.filename - ) - return {"status": "success", "uri": f"s3://{workspace_bucket}/{file.filename}"} -``` - -## 5. Network Routing Configuration Update (`Caddyfile`) +### Caddy routing (.35) ```caddy -# Core Application Portal Entry Point -ds.home.lab { - reverse_proxy 127.0.0.1:8000 -} - -# Image Design Lab -photo.home.lab { - reverse_proxy 127.0.0.1:8487 -} - -# Pro Video Editor Layout -video.home.lab { - reverse_proxy 127.0.0.1:8081 -} - -# Pro Audio Station A (Audacity Wrapper) -audacity.home.lab { - reverse_proxy 127.0.0.1:3000 -} - -# Pro Audio Station B (Zrythm DAW Studio) -zrythm.home.lab { - reverse_proxy 127.0.0.1:3001 -} - -# Garage S3 Native Web API Endpoint s3.home.lab { - reverse_proxy 127.0.0.1:3900 + reverse_proxy 192.168.20.13:3900 } ``` +### Full original compose example + +
expand + +```yaml +version: '3.8' +services: + studio-s3: + image: dxflrs/garage:v1.0.0 + container_name: studio_garage_s3 + restart: unless-stopped + ports: + - "3900:3900" + - "3901:3901" + volumes: + - ./garage/garage.toml:/etc/garage.toml + - /mnt/data/family-home-lab/garage-meta:/var/lib/garage/meta + - /mnt/data/family-home-lab/garage-data:/var/lib/garage/data + environment: + - GARAGE_RPC_SECRET= +``` + +
diff --git a/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md b/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md index 0d5a633..2bfab62 100644 --- a/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md +++ b/000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md @@ -1,112 +1,127 @@ --- created: 2026-08-21 20:08 -modified: 2026-08-21 20:08 +modified: 2026-08-24 09:39 type: note tags: - ai - family - - tools + - homelab aliases: [] --- # [[# DeepSeek Harness (dsh) Home Lab Setup]] +> **IMPLEMENTATION BRIEF for the dsh agent (updated by family-home-lab parent project).** +> The dsh agent builds these instances. The **parent console** (FastAPI portal at `console.home.lab`) is built separately and provides login, session routing and tool launching. Read this whole note before deploying. -A brief overview for deploying isolated, tool-restricted DeepSeek Harness instances using Docker, Caddy, and subdomains. +## 0. Context — what the parent console provides (do NOT rebuild) -## 1. System Architecture +The family-home-lab project (`/home/sam/home_network/custom_tools/family_home_lab/plan.md` on .27) deploys on `.13`: -Instead of subfolders, use subdomains to prevent WebSocket connection failures. Each family member gets an isolated container, distinct workspace volume, and restricted plugin configuration. +- A FastAPI + Jinja2 + HTMX console at **`console.home.lab`** (port `8500` on .13) with per-user login (**Sam, Jo, Harry, Finn**) — bcrypt password auth, signed HTTP-only session cookies. +- The console links to each user's dsh instance after they log in. +- RabbitMQ + Celery workers, PostgreSQL+pgvector, Garage S3 (see [[FAMILY S3 Storage Integration & Blueprint]]). + +**dsh must NOT implement its own cross-user account system** — one user per container instance is correct; the console decides which instance a user sees. + +## 1. Network environment (verified live) + +| Item | Value | +|---|---| +| Subnet | `192.168.20.0/24`, gateway `.1` | +| dsh host | **`.13` nixos-desktop** (`sam@192.168.20.13`), NixOS, Docker host, 15.5 GB RAM | +| Reverse proxy | **Caddy on `.35`** (`sam@192.168.20.35`) — already proxies `*.home.lab`. Add entries there; do NOT run a second Caddy in compose | +| Local DNS | Pi-hole on `.13` — add local records → `.35` | +| LLM routing | **OmniRoute already running at `.13:20129`** (API) / `:20128` (dashboard). Point dsh providers at `http://192.168.20.13:20129/v1` instead of external APIs | +| Open WebUI | Being retired — dsh replaces it as the chat layer | + +### Port allocation (avoid collisions — verified in use on .13) + +In-use: 53, 1883, 3001, 3002, 4000, 5000, 5432-5434, 5678, 8079-8082, 8088, 8090-8091, 8787, 9001, 9010-9011, 9090-9091, 10200, 20128-20129, plus new family-home-lab stack starting at **8500**. + +**dsh instances:** use `3081` (sam), `3082` (jo), `3083` (harry), `3084` (finn). + +## 2. System architecture + +Subdomains (not subpaths) to prevent WebSocket failures: ``` - [ Internet / Home Network ] - │ - [ Caddy Proxy ] - ┌─────────────────────┼─────────────────────┐ - ▼ ▼ ▼ -[ mum.ds.home.lab ] [ dad.ds.home.lab ] [ son.ds.home.lab ] - (Port 3081) (Port 3082) (Port 3083) - Container: dsh-mum Container: dsh-dad Container: dsh-son - No Bash Plugin No Bash Plugin Full Coder Plugins + [ Caddy on .35 ] + │ + ┌──────────┬────────┼─────────┬──────────────┐ + ▼ ▼ ▼ ▼ ▼ +dsh-sam dsh-jo dsh-harry dsh-finn console.home.lab +:3081 :3082 :3083 :3084 (.13:8500) ``` ---- +## 3. Docker Compose (deploy inside `/home/sam/Docker/Containers/dsh/`) -## 2. Docker Compose Configuration (`docker-compose.yml`) - -This configuration isolates the file system environments. Adjust host directories to match your local system paths. +Match existing conventions: PUID/PGID 1000, restart unless-stopped. ```yaml -version: '3.8' - services: - dsh-mum: + dsh-sam: image: node:20-slim - container_name: dsh-mum + container_name: dsh-sam command: npx @deepseek-ai/dsh web --port 3080 ports: - "3081:3080" volumes: - - /home/user/dsh/mum/workspace:/workspace - - /home/user/dsh/mum/config:/root/.config - restart: unless-stopped - - dsh-dad: - image: node:20-slim - container_name: dsh-dad - command: npx @deepseek-ai/dsh web --port 3080 - ports: - - "3082:3080" - volumes: - - /home/user/dsh/dad/workspace:/workspace - - /home/user/dsh/dad/config:/root/.config + - /mnt/data/family-home-lab/dsh/sam/workspace:/workspace + - /mnt/data/family-home-lab/dsh/sam/config:/root/.config restart: unless-stopped + # repeat for jo(:3082), harry(:3084→3083), finn(:3084) ``` ---- +Data lives under `/mnt/data/family-home-lab/dsh//` so it's covered by the Borg backup job. -## 3. Caddy Reverse Proxy Configuration (`Caddyfile`) - -Map local subdomains cleanly to prevent container path routing errors. +## 4. Caddy entries (on .35, `/etc/caddy/Caddyfile` or its compose config) ```caddy -mum.ds.home.lab { - reverse_proxy 127.0.0.1:3081 +dsh-sam.home.lab { + reverse_proxy 192.168.20.13:3081 } - -dad.ds.home.lab { - reverse_proxy 127.0.0.1:3082 +dsh-jo.home.lab { + reverse_proxy 192.168.20.13:3082 +} +dsh-harry.home.lab { + reverse_proxy 192.168.20.13:3083 +} +dsh-finn.home.lab { + reverse_proxy 192.168.20.13:3084 } ``` ---- +**Iframe requirement (from parent console):** tools are embedded inline in the console where possible. Ensure Caddy does not add `X-Frame-Options: DENY` and that dsh sets no restrictive CSP frame-ancestors. If dsh sets headers itself, allow framing from `console.home.lab`. -## 4. Hardening & Customizing Tools (`dsh.config.yaml`) +**Session handoff (to coordinate with console):** the console links out with a short-lived one-time token (`?token=...`) so users don't re-enter credentials; dsh should accept and exchange it, or at minimum tolerate being launched from the console. Coordinate final design with the parent project. -To remove terminal or bash access, drop the shell execution plugins from the runtime profile. Place this file inside the user's config volume directory. +## 5. Hardening & customizing tools (`dsh.config.yaml`) + +Providers point at local OmniRoute: ```yaml -# Target OmniRoute or external endpoints providers: openai-compatible: - baseUrl: "https://api.omniroute.example/v1" # Replace with your OmniRoute URL - apiKey: "your-omniroute-api-key" + baseUrl: "http://192.168.20.13:20129/v1" + apiKey: "" -# Explicitly register safe household plugins plugins: - name: dsh-plugin-file-editor - name: dsh-plugin-translation - name: dsh-plugin-ocr -# STRICTLY EXCLUDED (Do not list these to block terminal access): -# - dsh-plugin-shell -# - persistent-bash +# STRICTLY EXCLUDED for kids' instances: +# - dsh-plugin-shell / persistent-bash ``` ---- +Per-user restrictions: Sam full plugins; Jo/Harry/Finn no shell/bash plugins. -## 5. Deployment Commands - -1. **Create directories:** `mkdir -p /home/user/dsh/{mum,dad}/{workspace,config}` -2. **Launch containers:** `docker compose up -d` -3. **Reload proxy:** `caddy reload` +## 6. Deployment checklist +1. `mkdir -p /mnt/data/family-home-lab/dsh/{sam,jo,harry,finn}/{workspace,config}` +2. Compose up from `/home/sam/Docker/Containers/dsh/` +3. Add Pi-hole DNS records: `dsh-{sam,jo,harry,finn}.home.lab` → `.35` +4. Add Caddyfile entries on .35, reload +5. Configure OmniRoute provider keys per instance +6. Verify WebSocket connectivity through Caddy +7. Notify parent project when URLs are live so console tool cards can link them diff --git a/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md b/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md index beea355..dba3f6a 100644 --- a/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md +++ b/000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md @@ -10,6 +10,8 @@ aliases: [] --- # [[FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint]] +> **SUPERSEDED by final plan** in `/home/sam/home_network/custom_tools/family_home_lab/plan.md` (approved). Key changes from this blueprint: host = `.13`; reuse existing Caddy on `.35` + Pi-hole DNS (no Caddy container); RabbitMQ deployed fresh (verified none exists); Garage S3 data on `/mnt/data/family-home-lab/` (see [[FAMILY S3 Storage Integration & Blueprint]]); compose at `/home/sam/Docker/Containers/family-home-lab/`; portal port `8500`; frontend FastAPI+HTMX; Open WebUI retired in favour of DeepSeek Harness instances ([[FAMILY DeepSeek Harness (dsh) Home Lab Setup]]); users Sam/Jo/Harry/Finn with username+password auth. Original blueprint below for reference. + # Enterprise Homelab AI Multimedia Suite: Systems Architecture & Blueprint This document outlines the deployment strategy for a self-hosted, custom-built multimedia and AI workspace. It features individual family login profiles, an asynchronous RabbitMQ queue, and persistent user understanding via PostgreSQL with pgvector.