From 37bc8af9020f9bb0c9eebab9480cc962c3ef38f2 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Mon, 24 Aug 2026 09:54:08 +1000 Subject: [PATCH] sam-4screen-desktop 2026-8-24:9:54:8 --- .obsidian/workspace.json | 12 +- ...ILY S3 Storage Integration & Blueprint.md | 281 ++++++++++++++++++ 2 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 000 daily/FAMILY S3 Storage Integration & Blueprint.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index ff3aa00..9776b8e 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md", + "file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint" + "title": "FAMILY S3 Storage Integration & Blueprint" } } ] @@ -183,7 +183,7 @@ "state": { "type": "file-properties", "state": { - "file": "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md" + "file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md" }, "icon": "lucide-info", "title": "File properties" @@ -211,12 +211,13 @@ }, "active": "b0724d9a8e78110f", "lastOpenFiles": [ + "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md", + "000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md", + "000 daily/FAMILY S3 Storage Integration & Blueprint.md", "200 projects/220 Web Host Migration/AWS Cloud Emulator Floci.md", "200 projects/220 Web Host Migration/Local Web Host.md", "400 resources/450 Recipes/Easy Yogurt flat bread.md", "400 resources/450 Recipes/Chilli Sauce Hack.md", - "000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md", - "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md", "000 daily/basketball training.md", "000 daily/Daily Notes and To Do.md", "200 projects/210 AI Resume/AI Resume - Content Organization Plan.md", @@ -236,7 +237,6 @@ "200 projects/210 AI Resume/Websites pages for AI resume.md", "200 projects/210 AI Resume/Resume Ideas.md", "200 projects/210 AI Resume/Local Hybrid Vector + Graph RAG Setup.md", - "200 projects/260 Build Create Make/Add to shopping list.md", "000 daily/ThinkPad Recommendations", "500 archive/510 Daily", "300 areas/305 Ideas Businesses", diff --git a/000 daily/FAMILY S3 Storage Integration & Blueprint.md b/000 daily/FAMILY S3 Storage Integration & Blueprint.md new file mode 100644 index 0000000..e8d1ff2 --- /dev/null +++ b/000 daily/FAMILY S3 Storage Integration & Blueprint.md @@ -0,0 +1,281 @@ +--- +created: 2026-08-24 09:39 +modified: 2026-08-24 09:39 +type: note +tags: + - ai + - amazon + - s3 + - aws + - family + - tools +aliases: [] +--- +# [[FAMILY S3 Storage Integration & Blueprint]] + +--- +created: 2026-08-24 09:38 +modified: 2026-08-24 09:38 +type: note +tags: + - homelab + - storage + - s3 + - garage +aliases: [] +--- +# [[FAMILY S3 Storage Integration & Blueprint]] + +# FAMILY Enterprise Homelab AI Multimedia Suite: S3 Storage Integration & Blueprint + +This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based). This integration replaces legacy volume mappings with decoupled web APIs to unify storage across multiple network nodes. + +## 1. Updated Directory Structure (Managed via Pi) + +```text +/home/user/ai-studio/ +├── docker-compose.yml +├── caddy/ +│ └── Caddyfile +├── garage/ +│ └── garage.toml # Garage Engine Configuration +├── gateway-app/ # Custom FastAPI Web Application +│ ├── main.py # Modified Async Web Controller (Boto3 Native) +│ ├── database.py # PostgreSQL Connection Layer (pgvector) +│ ├── tasks.py # Background Worker Tasks (Celery) +│ └── ... +└── storage/ + ├── garage_meta/ # Fast metadata tracking blocks + └── shared_media/ # Physical encrypted object storage pools +``` + +## 2. Integrated Production Stack Layout (`docker-compose.yml`) + +```yaml +version: '3.8' + +services: + # 1. NETWORK PROXY + caddy: + image: caddy:2-alpine + container_name: network_proxy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + network_mode: host + + # 2. PURE OPEN-SOURCE LOCAL S3 ENGINE (Rust-powered) + studio-s3: + image: dxflrs/garage:v1.0.0 + container_name: studio_garage_s3 + restart: unless-stopped + ports: + - "3900:3900" # S3 Web API Engine Endpoint + - "3901:3901" # Administrative Cluster RPC Port + volumes: + - ./garage/garage.toml:/etc/garage.toml + - ./storage/garage_meta:/var/lib/garage/meta + - ./storage/shared_media:/var/lib/garage/data + environment: + - GK_rpc_secret=7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00 + command: /usr/local/bin/garage server + + # 3. THE MASTER ENTRY PORTAL (Custom Gateway UI with Boto3 S3 Integration) + studio-portal: + image: python:3.11-slim + container_name: studio_portal_app + restart: unless-stopped + working_dir: /app + command: > + sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart boto3 && + uvicorn main:app --host 0.0.0.0 --port 8000" + ports: + - "8000:8000" + volumes: + - ./gateway-app:/app + depends_on: + - studio-db + - studio-rabbitmq + - studio-s3 + + # 4. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ) + media-worker: + image: python:3.11-slim + container_name: async_media_worker + restart: unless-stopped + working_dir: /app + command: celery -A tasks worker --loglevel=info + volumes: + - ./gateway-app:/app + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - studio-rabbitmq + - studio-s3 + + # 5. INDUSTRIAL MESSAGE BROKER (RabbitMQ) + studio-rabbitmq: + image: rabbitmq:3-management-alpine + container_name: studio_message_broker + restart: unless-stopped + ports: + - "5672:5672" + - "15672:15672" + environment: + - RABBITMQ_DEFAULT_USER=studio_broker + - RABBITMQ_DEFAULT_PASS=broker_secure_pass + + # 6. ENTERPRISE COGNITIVE DATABASE + studio-db: + image: pgvector/pgvector:pg16 + container_name: studio_cognitive_db + restart: unless-stopped + environment: + - POSTGRES_USER=studio_admin + - POSTGRES_PASSWORD=studio_secure_pass + - POSTGRES_DB=studio_memory_matrix + volumes: + - postgres_data:/var/lib/postgresql/data + + # 7. OMNIROUTE GATEWAY + omniroute: + image: omniroute/gateway:latest + container_name: omniroute_gateway + restart: unless-stopped + ports: + - "20128:20128" + environment: + - OPENAI_API_KEY=your_secure_cloud_key + - GEMINI_API_KEY=your_secure_cloud_key + volumes: + - ./omniroute/config:/app/config + + # 8. MULTIMEDIA STUDIO CONTAINERS (Configured to mount S3 blocks or talk directly via API) + photopea: + image: shtse8/photopea:1.0 + container_name: photopea_studio + ports: + - "8487:8887" + + kdenlive-studio: + image: lscr.io/linuxserver/kdenlive:latest + container_name: pro_video_studio + ports: + - "8081:3000" + environment: + - PUID=1000 + - PGID=1000 + + audacity-studio: + image: lscr.io/linuxserver/audacity:latest + container_name: pro_audio_audacity + ports: + - "3000:3000" + environment: + - PUID=1000 + - PGID=1000 + + zrythm-studio: + image: lscr.io/linuxserver/zrythm:latest + container_name: pro_audio_zrythm + ports: + - "3001:3000" + environment: + - PUID=1000 + - PGID=1000 + +volumes: + postgres_data: + caddy_data: + caddy_config: +``` + +## 3. Dedicated Storage Engine Configuration (`garage.toml`) + +```toml +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +rpc_bind_addr = "[::]:3901" +rpc_secret = "7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00" + +[s3_api] +api_bind_addr = "[::]:3900" +s3_region = "homelab" +root_domain = "s3.home.lab" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = "web.s3.home.lab" +``` + +## 4. FastAPI Gateway S3 Client Integration Snippet (`main.py`) + +```python +import os +import boto3 +from botocore.client import Config +from fastapi import FastAPI, UploadFile, File + +app = FastAPI() + +# Point S3 Client to the local Garage Engine Container +s3_client = boto3.client( + 's3', + endpoint_url='http://studio-s3:3900', + aws_access_key_id=os.getenv("GARAGE_ACCESS_KEY", "studio_key_id"), + aws_secret_access_key=os.getenv("GARAGE_SECRET_KEY", "studio_secret_key"), + config=Config(signature_version='s3v4'), + region_name='homelab' +) + +@app.post("/upload/{workspace_bucket}") +async def upload_to_family_vault(workspace_bucket: str, file: UploadFile = File(...)): + """Streams uploads from dashboard direct to isolated Garage family buckets""" + s3_client.upload_fileobj( + file.file, + workspace_bucket, + file.filename + ) + return {"status": "success", "uri": f"s3://{workspace_bucket}/{file.filename}"} +``` + +## 5. Network Routing Configuration Update (`Caddyfile`) + +```caddy +# Core Application Portal Entry Point +ds.home.lab { + reverse_proxy 127.0.0.1:8000 +} + +# Image Design Lab +photo.home.lab { + reverse_proxy 127.0.0.1:8487 +} + +# Pro Video Editor Layout +video.home.lab { + reverse_proxy 127.0.0.1:8081 +} + +# Pro Audio Station A (Audacity Wrapper) +audacity.home.lab { + reverse_proxy 127.0.0.1:3000 +} + +# Pro Audio Station B (Zrythm DAW Studio) +zrythm.home.lab { + reverse_proxy 127.0.0.1:3001 +} + +# Garage S3 Native Web API Endpoint +s3.home.lab { + reverse_proxy 127.0.0.1:3900 +} +``` +