sam/obsidian-vault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sam-4screen-desktop 2026-8-24:9:54:8

Browse Source
This commit is contained in:
Sam Rolfe
2026-08-24 09:54:08 +10:00
parent 9dbbb4c926
commit 37bc8af902
2 changed files with 287 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.obsidian/workspace.json vendored
View File

@@ -13,12 +13,12 @@
"state": {
"type": "markdown",
"state": {
"file": "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md",
"file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md",
"mode": "source",
"source": false
},
"icon": "lucide-file",
"title": "FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint"
"title": "FAMILY S3 Storage Integration & Blueprint"
}
}
]
@@ -183,7 +183,7 @@
"state": {
"type": "file-properties",
"state": {
"file": "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md"
"file": "000 daily/FAMILY S3 Storage Integration & Blueprint.md"
},
"icon": "lucide-info",
"title": "File properties"
@@ -211,12 +211,13 @@
},
"active": "b0724d9a8e78110f",
"lastOpenFiles": [
"000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md",
"000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md",
"000 daily/FAMILY S3 Storage Integration & Blueprint.md",
"200 projects/220 Web Host Migration/AWS Cloud Emulator Floci.md",
"200 projects/220 Web Host Migration/Local Web Host.md",
"400 resources/450 Recipes/Easy Yogurt flat bread.md",
"400 resources/450 Recipes/Chilli Sauce Hack.md",
"000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md",
"000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md",
"000 daily/basketball training.md",
"000 daily/Daily Notes and To Do.md",
"200 projects/210 AI Resume/AI Resume - Content Organization Plan.md",
@@ -236,7 +237,6 @@
"200 projects/210 AI Resume/Websites pages for AI resume.md",
"200 projects/210 AI Resume/Resume Ideas.md",
"200 projects/210 AI Resume/Local Hybrid Vector + Graph RAG Setup.md",
"200 projects/260 Build Create Make/Add to shopping list.md",
"000 daily/ThinkPad Recommendations",
"500 archive/510 Daily",
"300 areas/305 Ideas Businesses",

281
000 daily/FAMILY S3 Storage Integration & Blueprint.md Normal file
View File

@@ -0,0 +1,281 @@
---
created: 2026-08-24 09:39
modified: 2026-08-24 09:39
type: note
tags:
- ai
- amazon
- s3
- aws
- family
- tools
aliases: []
---
# [[FAMILY S3 Storage Integration & Blueprint]]
---
created: 2026-08-24 09:38
modified: 2026-08-24 09:38
type: note
tags:
- homelab
- storage
- s3
- garage
aliases: []
---
# [[FAMILY S3 Storage Integration & Blueprint]]
# FAMILY Enterprise Homelab AI Multimedia Suite: S3 Storage Integration & Blueprint
This document details the architectural expansion of the self-hosted AI and multimedia workspace. It introduces a pure open-source, local S3 object storage tier using **Garage** (Rust-based). This integration replaces legacy volume mappings with decoupled web APIs to unify storage across multiple network nodes.
## 1. Updated Directory Structure (Managed via Pi)
```text
/home/user/ai-studio/
├── docker-compose.yml
├── caddy/
│ └── Caddyfile
├── garage/
│ └── garage.toml # Garage Engine Configuration
├── gateway-app/ # Custom FastAPI Web Application
│ ├── main.py # Modified Async Web Controller (Boto3 Native)
│ ├── database.py # PostgreSQL Connection Layer (pgvector)
│ ├── tasks.py # Background Worker Tasks (Celery)
│ └── ...
└── storage/
├── garage_meta/ # Fast metadata tracking blocks
└── shared_media/ # Physical encrypted object storage pools
```
## 2. Integrated Production Stack Layout (`docker-compose.yml`)
```yaml
version: '3.8'
services:
# 1. NETWORK PROXY
caddy:
image: caddy:2-alpine
container_name: network_proxy
restart: unless-stopped
ports:
- "80:80"
- "443:443"
volumes:
- ./caddy/Caddyfile:/etc/caddy/Caddyfile
- caddy_data:/data
- caddy_config:/config
network_mode: host
# 2. PURE OPEN-SOURCE LOCAL S3 ENGINE (Rust-powered)
studio-s3:
image: dxflrs/garage:v1.0.0
container_name: studio_garage_s3
restart: unless-stopped
ports:
- "3900:3900" # S3 Web API Engine Endpoint
- "3901:3901" # Administrative Cluster RPC Port
volumes:
- ./garage/garage.toml:/etc/garage.toml
- ./storage/garage_meta:/var/lib/garage/meta
- ./storage/shared_media:/var/lib/garage/data
environment:
- GK_rpc_secret=7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00
command: /usr/local/bin/garage server
# 3. THE MASTER ENTRY PORTAL (Custom Gateway UI with Boto3 S3 Integration)
studio-portal:
image: python:3.11-slim
container_name: studio_portal_app
restart: unless-stopped
working_dir: /app
command: >
sh -c "pip install fastapi uvicorn psycopg2-binary celery jinja2 python-multipart boto3 &&
uvicorn main:app --host 0.0.0.0 --port 8000"
ports:
- "8000:8000"
volumes:
- ./gateway-app:/app
depends_on:
- studio-db
- studio-rabbitmq
- studio-s3
# 4. ASYNCHRONOUS MEDIA WORKER (Fueled by RabbitMQ)
media-worker:
image: python:3.11-slim
container_name: async_media_worker
restart: unless-stopped
working_dir: /app
command: celery -A tasks worker --loglevel=info
volumes:
- ./gateway-app:/app
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
- studio-rabbitmq
- studio-s3
# 5. INDUSTRIAL MESSAGE BROKER (RabbitMQ)
studio-rabbitmq:
image: rabbitmq:3-management-alpine
container_name: studio_message_broker
restart: unless-stopped
ports:
- "5672:5672"
- "15672:15672"
environment:
- RABBITMQ_DEFAULT_USER=studio_broker
- RABBITMQ_DEFAULT_PASS=broker_secure_pass
# 6. ENTERPRISE COGNITIVE DATABASE
studio-db:
image: pgvector/pgvector:pg16
container_name: studio_cognitive_db
restart: unless-stopped
environment:
- POSTGRES_USER=studio_admin
- POSTGRES_PASSWORD=studio_secure_pass
- POSTGRES_DB=studio_memory_matrix
volumes:
- postgres_data:/var/lib/postgresql/data
# 7. OMNIROUTE GATEWAY
omniroute:
image: omniroute/gateway:latest
container_name: omniroute_gateway
restart: unless-stopped
ports:
- "20128:20128"
environment:
- OPENAI_API_KEY=your_secure_cloud_key
- GEMINI_API_KEY=your_secure_cloud_key
volumes:
- ./omniroute/config:/app/config
# 8. MULTIMEDIA STUDIO CONTAINERS (Configured to mount S3 blocks or talk directly via API)
photopea:
image: shtse8/photopea:1.0
container_name: photopea_studio
ports:
- "8487:8887"
kdenlive-studio:
image: lscr.io/linuxserver/kdenlive:latest
container_name: pro_video_studio
ports:
- "8081:3000"
environment:
- PUID=1000
- PGID=1000
audacity-studio:
image: lscr.io/linuxserver/audacity:latest
container_name: pro_audio_audacity
ports:
- "3000:3000"
environment:
- PUID=1000
- PGID=1000
zrythm-studio:
image: lscr.io/linuxserver/zrythm:latest
container_name: pro_audio_zrythm
ports:
- "3001:3000"
environment:
- PUID=1000
- PGID=1000
volumes:
postgres_data:
caddy_data:
caddy_config:
```
## 3. Dedicated Storage Engine Configuration (`garage.toml`)
```toml
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
rpc_bind_addr = "[::]:3901"
rpc_secret = "7b61f201d4a8e632b9c70811e54a3900112233445566778899aabbccddeeff00"
[s3_api]
api_bind_addr = "[::]:3900"
s3_region = "homelab"
root_domain = "s3.home.lab"
[s3_web]
bind_addr = "[::]:3902"
root_domain = "web.s3.home.lab"
```
## 4. FastAPI Gateway S3 Client Integration Snippet (`main.py`)
```python
import os
import boto3
from botocore.client import Config
from fastapi import FastAPI, UploadFile, File
app = FastAPI()
# Point S3 Client to the local Garage Engine Container
s3_client = boto3.client(
's3',
endpoint_url='http://studio-s3:3900',
aws_access_key_id=os.getenv("GARAGE_ACCESS_KEY", "studio_key_id"),
aws_secret_access_key=os.getenv("GARAGE_SECRET_KEY", "studio_secret_key"),
config=Config(signature_version='s3v4'),
region_name='homelab'
)
@app.post("/upload/{workspace_bucket}")
async def upload_to_family_vault(workspace_bucket: str, file: UploadFile = File(...)):
"""Streams uploads from dashboard direct to isolated Garage family buckets"""
s3_client.upload_fileobj(
file.file,
workspace_bucket,
file.filename
)
return {"status": "success", "uri": f"s3://{workspace_bucket}/{file.filename}"}
```
## 5. Network Routing Configuration Update (`Caddyfile`)
```caddy
# Core Application Portal Entry Point
ds.home.lab {
reverse_proxy 127.0.0.1:8000
}
# Image Design Lab
photo.home.lab {
reverse_proxy 127.0.0.1:8487
}
# Pro Video Editor Layout
video.home.lab {
reverse_proxy 127.0.0.1:8081
}
# Pro Audio Station A (Audacity Wrapper)
audacity.home.lab {
reverse_proxy 127.0.0.1:3000
}
# Pro Audio Station B (Zrythm DAW Studio)
zrythm.home.lab {
reverse_proxy 127.0.0.1:3001
}
# Garage S3 Native Web API Endpoint
s3.home.lab {
reverse_proxy 127.0.0.1:3900
}
```