282 lines
7.1 KiB
Markdown
282 lines
7.1 KiB
Markdown
---
|
|
created: 2026-08-24 09:39
|
|
modified: 2026-08-24 09:39
|
|
type: note
|
|
tags:
|
|
- ai
|
|
- amazon
|
|
- s3
|
|
- aws
|
|
- family
|
|
- tools
|
|
aliases: []
|
|
---
|
|
# [[FAMILY S3 Storage Integration & Blueprint]]
|
|
|
|
---
|
|
created: 2026-08-24 09:38
|
|
modified: 2026-08-24 09:38
|
|
type: note
|
|
tags:
|
|
- homelab
|
|
- storage
|
|
- s3
|
|
- garage
|
|
aliases: []
|
|
---
|
|
# [[FAMILY S3 Storage Integration & Blueprint]]
|
|
|
|
# FAMILY Enterprise Homelab AI Multimedia Suite: S3 Storage Integration & Blueprint
|
|
|
|
This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based). This integration replaces legacy volume mappings with decoupled web APIs to unify storage across multiple network nodes.
|
|
|
|
## 1. Updated Directory Structure (Managed via Pi)
|
|
|
|
```text
|
|
/home/user/ai-studio/
|
|
├── docker-compose.yml
|
|
├── caddy/
|
|
│ └── Caddyfile
|
|
├── garage/
|
|
│ └── garage.toml # Garage Engine Configuration
|
|
├── gateway-app/ # Custom FastAPI Web Application
|
|
│ ├── main.py # Modified Async Web Controller (Boto3 Native)
|
|
│ ├── database.py # PostgreSQL Connection Layer (pgvector)
|
|
│ ├── tasks.py # Background Worker Tasks (Celery)
|
|
│ └── ...
|
|
└── storage/
|
|
├── garage_meta/ # Fast metadata tracking blocks
|
|
└── shared_media/ # Physical encrypted object storage pools
|
|
```
|
|
|
|
## 2. Integrated Production Stack Layout (`docker-compose.yml`)
|
|
|
|
```yaml
|
|
version: '3.8'
|
|
|
|
services:
|
|
# 1. NETWORK PROXY
|
|
caddy:
|
|
image: caddy:2-alpine
|
|
container_name: network_proxy
|
|
restart: unless-stopped
|
|
ports:
|
|
- "80:80"
|
|
- "443:443"
|
|
volumes:
|
|
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
|
|
- caddy_data:/data
|
|
- caddy_config:/config
|
|
network_mode: host
|
|
|
|
# 2. PURE OPEN-SOURCE LOCAL S3 ENGINE (Rust-powered)
|
|
studio-s3:
|
|
image: dxflrs/garage:v1.0.0
|
|
container_name: studio_garage_s3
|
|
restart: unless-stopped
|
|
ports:
|
|
- "3900:3900" # S3 Web API Engine Endpoint
|
|
- "3901:3901" # Administrative Cluster RPC Port
|
|
volumes:
|
|
- ./garage/garage.toml:/etc/garage.toml
|
|
- ./storage/garage_meta:/var/lib/garage/meta
|
|
- ./storage/shared_media:/var/lib/garage/data
|
|
environment:
|
|
- GK_rpc_secret=7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00
|
|
command: /usr/local/bin/garage server
|
|
|
|
# 3. THE MASTER ENTRY PORTAL (Custom Gateway UI with Boto3 S3 Integration)
|
|
studio-portal:
|
|
image: python:3.11-slim
|
|
container_name: studio_portal_app
|
|
restart: unless-stopped
|
|
working_dir: /app
|
|
command: >
|
|
sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart boto3 &&
|
|
uvicorn main:app --host 0.0.0.0 --port 8000"
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./gateway-app:/app
|
|
depends_on:
|
|
- studio-db
|
|
- studio-rabbitmq
|
|
- studio-s3
|
|
|
|
# 4. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ)
|
|
media-worker:
|
|
image: python:3.11-slim
|
|
container_name: async_media_worker
|
|
restart: unless-stopped
|
|
working_dir: /app
|
|
command: celery -A tasks worker --loglevel=info
|
|
volumes:
|
|
- ./gateway-app:/app
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
depends_on:
|
|
- studio-rabbitmq
|
|
- studio-s3
|
|
|
|
# 5. INDUSTRIAL MESSAGE BROKER (RabbitMQ)
|
|
studio-rabbitmq:
|
|
image: rabbitmq:3-management-alpine
|
|
container_name: studio_message_broker
|
|
restart: unless-stopped
|
|
ports:
|
|
- "5672:5672"
|
|
- "15672:15672"
|
|
environment:
|
|
- RABBITMQ_DEFAULT_USER=studio_broker
|
|
- RABBITMQ_DEFAULT_PASS=broker_secure_pass
|
|
|
|
# 6. ENTERPRISE COGNITIVE DATABASE
|
|
studio-db:
|
|
image: pgvector/pgvector:pg16
|
|
container_name: studio_cognitive_db
|
|
restart: unless-stopped
|
|
environment:
|
|
- POSTGRES_USER=studio_admin
|
|
- POSTGRES_PASSWORD=studio_secure_pass
|
|
- POSTGRES_DB=studio_memory_matrix
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
|
|
# 7. OMNIROUTE GATEWAY
|
|
omniroute:
|
|
image: omniroute/gateway:latest
|
|
container_name: omniroute_gateway
|
|
restart: unless-stopped
|
|
ports:
|
|
- "20128:20128"
|
|
environment:
|
|
- OPENAI_API_KEY=your_secure_cloud_key
|
|
- GEMINI_API_KEY=your_secure_cloud_key
|
|
volumes:
|
|
- ./omniroute/config:/app/config
|
|
|
|
# 8. MULTIMEDIA STUDIO CONTAINERS (Configured to mount S3 blocks or talk directly via API)
|
|
photopea:
|
|
image: shtse8/photopea:1.0
|
|
container_name: photopea_studio
|
|
ports:
|
|
- "8487:8887"
|
|
|
|
kdenlive-studio:
|
|
image: lscr.io/linuxserver/kdenlive:latest
|
|
container_name: pro_video_studio
|
|
ports:
|
|
- "8081:3000"
|
|
environment:
|
|
- PUID=1000
|
|
- PGID=1000
|
|
|
|
audacity-studio:
|
|
image: lscr.io/linuxserver/audacity:latest
|
|
container_name: pro_audio_audacity
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
- PUID=1000
|
|
- PGID=1000
|
|
|
|
zrythm-studio:
|
|
image: lscr.io/linuxserver/zrythm:latest
|
|
container_name: pro_audio_zrythm
|
|
ports:
|
|
- "3001:3000"
|
|
environment:
|
|
- PUID=1000
|
|
- PGID=1000
|
|
|
|
volumes:
|
|
postgres_data:
|
|
caddy_data:
|
|
caddy_config:
|
|
```
|
|
|
|
## 3. Dedicated Storage Engine Configuration (`garage.toml`)
|
|
|
|
```toml
|
|
metadata_dir = "/var/lib/garage/meta"
|
|
data_dir = "/var/lib/garage/data"
|
|
db_engine = "sqlite"
|
|
|
|
rpc_bind_addr = "[::]:3901"
|
|
rpc_secret = "7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00"
|
|
|
|
[s3_api]
|
|
api_bind_addr = "[::]:3900"
|
|
s3_region = "homelab"
|
|
root_domain = "s3.home.lab"
|
|
|
|
[s3_web]
|
|
bind_addr = "[::]:3902"
|
|
root_domain = "web.s3.home.lab"
|
|
```
|
|
|
|
## 4. FastAPI Gateway S3 Client Integration Snippet (`main.py`)
|
|
|
|
```python
|
|
import os
|
|
import boto3
|
|
from botocore.client import Config
|
|
from fastapi import FastAPI, UploadFile, File
|
|
|
|
app = FastAPI()
|
|
|
|
# Point S3 Client to the local Garage Engine Container
|
|
s3_client = boto3.client(
|
|
's3',
|
|
endpoint_url='http://studio-s3:3900',
|
|
aws_access_key_id=os.getenv("GARAGE_ACCESS_KEY", "studio_key_id"),
|
|
aws_secret_access_key=os.getenv("GARAGE_SECRET_KEY", "studio_secret_key"),
|
|
config=Config(signature_version='s3v4'),
|
|
region_name='homelab'
|
|
)
|
|
|
|
@app.post("/upload/{workspace_bucket}")
|
|
async def upload_to_family_vault(workspace_bucket: str, file: UploadFile = File(...)):
|
|
"""Streams uploads from dashboard direct to isolated Garage family buckets"""
|
|
s3_client.upload_fileobj(
|
|
file.file,
|
|
workspace_bucket,
|
|
file.filename
|
|
)
|
|
return {"status": "success", "uri": f"s3://{workspace_bucket}/{file.filename}"}
|
|
```
|
|
|
|
## 5. Network Routing Configuration Update (`Caddyfile`)
|
|
|
|
```caddy
|
|
# Core Application Portal Entry Point
|
|
ds.home.lab {
|
|
reverse_proxy 127.0.0.1:8000
|
|
}
|
|
|
|
# Image Design Lab
|
|
photo.home.lab {
|
|
reverse_proxy 127.0.0.1:8487
|
|
}
|
|
|
|
# Pro Video Editor Layout
|
|
video.home.lab {
|
|
reverse_proxy 127.0.0.1:8081
|
|
}
|
|
|
|
# Pro Audio Station A (Audacity Wrapper)
|
|
audacity.home.lab {
|
|
reverse_proxy 127.0.0.1:3000
|
|
}
|
|
|
|
# Pro Audio Station B (Zrythm DAW Studio)
|
|
zrythm.home.lab {
|
|
reverse_proxy 127.0.0.1:3001
|
|
}
|
|
|
|
# Garage S3 Native Web API Endpoint
|
|
s3.home.lab {
|
|
reverse_proxy 127.0.0.1:3900
|
|
}
|
|
```
|
|
|