From f3248ff071fb32c55aa235ae1e4080eab12148e1 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Fri, 9 Jan 2026 10:28:44 +1100 Subject: [PATCH] add read me --- README.md | 56 +++++++++++++++++++ app.py | 40 ++++++++++++++ docker-compose.yml | 31 +++++++++++ mqtt_audio_bridge.py | 124 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 README.md create mode 100644 app.py create mode 100644 docker-compose.yml create mode 100644 mqtt_audio_bridge.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..11ae3c4 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# Voice Assistant Bridge (Docker Backend) + +This repository contains the backend infrastructure for the local Voice Assistant system. It acts as the middleware between the ESP32 audio stream and Home Assistant. + +## 🏗️ Architecture + +```mermaid +graph LR + ESP32[ESP32 Hardware] -->|MQTT Audio Stream| Bridge + Bridge[Python Bridge] -->|HTTP Request| Whisper[Faster-Whisper Container] + Whisper -->|Text| Bridge + Bridge -->|MQTT Text Command| HA[Home Assistant] + Bridge -->|MQTT Status| ESP32 +``` + +## 🧩 Components + +### 1. `docker-compose.yml` +Orchestrates two containers: +* **`voice-bridge`**: The logic handler. Listens to MQTT audio, buffers it, detects Wake Words via `OpenWakeWord`, and handles routing. +* **`whisper-api`**: A lightweight Flask API wrapping `faster-whisper` for Speech-to-Text conversion. + +### 2. `mqtt_audio_bridge.py` +The main Python script running inside the `voice-bridge` container. +* **Input:** Listens to raw PCM audio on `voice/audio_stream` (Broker .13). +* **Processing:** + * Uses **OpenWakeWord** to detect "Hey Jarvis". + * Buffers audio and sends to Whisper API. + * Safety mechanisms (Memory limits, Log rotation). +* **Output:** Publishes transcribed text to `homeassistant/voice/text` (Broker .30). + +### 3. `app.py` +The Whisper API endpoint. +* **Model:** `small.en` (Optimized for CPU usage with `int8` quantization). +* **Language:** Locked to English to prevent hallucinations on static/silence. + +## 🚀 Deployment + +1. **Requirements:** Docker & Docker Compose. +2. **Configuration:** Update IP addresses in `mqtt_audio_bridge.py` for your MQTT brokers. +3. **Run:** + ```bash + docker compose up -d + ``` + +## 🔧 Debugging + +**View Logs:** +```bash +docker compose logs -f voice_bridge +``` + +**Restart Stack:** +```bash +docker compose restart +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000..ccc44fd --- /dev/null +++ b/app.py @@ -0,0 +1,40 @@ +from flask import Flask, request, jsonify +from faster_whisper import WhisperModel +import os +import tempfile + +app = Flask(__name__) + +# Load Model +model_size = "small.en" +model = WhisperModel(model_size, device="cpu", compute_type="int8") + +@app.route('/transcribe', methods=['POST']) +def transcribe_audio(): + if 'audio' not in request.files: + return jsonify({"error": "No audio file provided"}), 400 + + audio_file = request.files['audio'] + temp_file_path = None # Initialize variable + + # Save file + with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_audio: + audio_file.save(tmp_audio.name) + temp_file_path = tmp_audio.name + + try: + # Now we can safely use temp_file_path + segments, info = model.transcribe(temp_file_path, beam_size=5, language="en") + transcript = " ".join([segment.text for segment in segments]) + print(f"Transcript: {transcript}") + return jsonify({"transcript": transcript}) + except Exception as e: + print(f"Error: {e}") + return jsonify({"error": str(e)}), 500 + finally: + # Cleanup + if temp_file_path and os.path.exists(temp_file_path): + os.remove(temp_file_path) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fdfa197 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +services: + whisper-api: + image: python:3.9-slim + container_name: voice_whisper + restart: unless-stopped + working_dir: /app + volumes: + - ./app.py:/app/app.py + command: > + sh -c "pip install flask faster-whisper && python app.py" + ports: + - "5000:5000" + voice-bridge: + image: python:3.9-slim + container_name: voice_bridge + restart: unless-stopped + deploy: + resources: + limits: + memory: 500M + depends_on: + - whisper-api + working_dir: /app + volumes: + - ./mqtt_audio_bridge.py:/app/mqtt_audio_bridge.py + environment: + - XDG_CACHE_HOME=/root/.cache + command: > + sh -c "pip install paho-mqtt requests 'numpy<2' openwakeword tflite-runtime && + python -c 'import openwakeword; openwakeword.utils.download_models()' && + python mqtt_audio_bridge.py" diff --git a/mqtt_audio_bridge.py b/mqtt_audio_bridge.py new file mode 100644 index 0000000..66f8162 --- /dev/null +++ b/mqtt_audio_bridge.py @@ -0,0 +1,124 @@ +import paho.mqtt.client as mqtt +import requests +import wave +import io +import numpy as np +import openwakeword +from openwakeword.model import Model +import collections +import threading + +# --- CONFIG --- +# 1. VOICE SERVER (Audio IO) +VOICE_BROKER = "192.168.20.13" +VOICE_TOPIC_AUDIO = "voice/audio_stream" +VOICE_TOPIC_STATUS = "voice/status" + +# 2. HOME ASSISTANT (Text Output) +HA_BROKER = "192.168.20.30" +HA_TOPIC_TEXT = "homeassistant/voice/text" + +MQTT_USER = "MQTT_USER_REDACTED" +MQTT_PASS = "MQTT_PASS_REDACTED" + +WHISPER_URL = "http://whisper-api:5000/transcribe" +WAKE_WORD = "hey_jarvis" + +# --- STATE --- +BUFFER_SIZE = 1280 * 4 +audio_buffer = collections.deque(maxlen=BUFFER_SIZE) +command_buffer = bytearray() +is_command_mode = False + +print("Loading Models...") +owwModel = Model() + +# --- HELPER FUNCTIONS --- +def transcribe_and_send(raw_data): + wav_io = io.BytesIO() + with wave.open(wav_io, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(raw_data) + wav_io.seek(0) + + try: + files = {'audio': ('cmd.wav', wav_io, 'audio/wav')} + res = requests.post(WHISPER_URL, files=files) + if res.status_code == 200: + text = res.json().get("transcript", "").strip() + print(f"TEXT: {text}") + if text: + # PUBLISH TO HOME ASSISTANT BROKER + ha_client.publish(HA_TOPIC_TEXT, text) + + # FEEDBACK TO ESP32 (Via Voice Broker) + voice_client.publish(VOICE_TOPIC_STATUS, "OK") + except Exception as e: + print(f"API Error: {e}") + +# --- CALLBACKS --- +def on_voice_message(client, userdata, msg): + global is_command_mode, command_buffer + + if msg.topic == VOICE_TOPIC_AUDIO: + payload = msg.payload + + # 1. COMMAND MODE (Recording for Whisper) + if is_command_mode: + # Safety Check: Prevent Memory Overflow (cap at ~3MB) + if len(command_buffer) > 3000000: + print("Error: Buffer overflow. Resetting.") + is_command_mode = False + command_buffer = bytearray() + owwModel.reset() + return + + # Append the audio chunk + command_buffer.extend(payload) + return + + # 2. LISTENING MODE (Wake Word Detection) + # Convert bytes to numpy int16 for OpenWakeWord + audio_int16 = np.frombuffer(payload, dtype=np.int16) + + # Feed the model + prediction = owwModel.predict(audio_int16) + + if prediction[WAKE_WORD] > 0.5: + print(f"--- WAKE WORD DETECTED ---") + is_command_mode = True + command_buffer = bytearray() + client.publish(VOICE_TOPIC_STATUS, "WAKE") + + elif msg.topic == "voice/status" and msg.payload.decode() == "processing": + # ESP32 stopped recording + if is_command_mode: + print("Processing...") + transcribe_and_send(command_buffer) + is_command_mode = False + command_buffer = bytearray() + owwModel.reset() +# --- MAIN --- +# Client 1: Voice Server (The Listener) +voice_client = mqtt.Client(client_id="Bridge_Voice") +voice_client.username_pw_set(MQTT_USER, MQTT_PASS) +voice_client.on_message = on_voice_message +voice_client.connect(VOICE_BROKER, 1883) +voice_client.subscribe([(VOICE_TOPIC_AUDIO, 0), ("voice/status", 0)]) + +# Client 2: Home Assistant (The Sender) +ha_client = mqtt.Client(client_id="Bridge_HA") +ha_client.username_pw_set(MQTT_USER, MQTT_PASS) +ha_client.connect(HA_BROKER, 1883) + +print("Bridge Running (Dual Broker)...") + +# Run loops in threads or blocking +# Since loop_forever blocks, we run HA loop in background and Voice loop in main +ha_thread = threading.Thread(target=ha_client.loop_forever) +ha_thread.daemon = True +ha_thread.start() + +voice_client.loop_forever()