sam/voice_bridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync live code; move MQTT credentials out of source; untrack venv/

Browse Source 
- mqtt_audio_bridge.py: MQTT_USER / MQTT_PASS are read from the environment,
  falling back to ~/.config/environment.d/10-secrets.conf (mode 600). The
  environment.d trap means a fresh SSH shell needs the file fallback.
- docker-compose.yml: pass MQTT_USER / MQTT_PASS through; keeps the live
  network_mode / Dockerfile topology.
- start_bridge.sh: launcher that sources the secrets file before compose up.
- app.py: commits the live initial_prompt that was uncommitted.
- .gitignore: add venv/ (was missing), __pycache__, *.bak*, *.wav.
- git rm -r --cached venv/: 13,413 tracked venv files removed from the index.

Credential note: MQTT_PASS was hardcoded in edaf914. origin/main was later
force-rewritten to redact it, but the value was exposed and must still be
rotated. Removing venv/ from the index does not shrink the repository: the
objects stay in history until a history rewrite and GC.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Sam Rolfe
2026-10-07 18:10:51 +11:00
parent f3248ff071
commit de9b06a3d1
7 changed files with 226 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored Normal file
View File

@@ -0,0 +1,7 @@
__pycache__/
*.pyc
# Backups must never be committed: they can contain the hardcoded MQTT password.
*.bak
*.bak-*
*.wav
venv/

7
Dockerfile Normal file
View File

@@ -0,0 +1,7 @@
FROM python:3.9-slim
WORKDIR /app
# Install dependencies once during the build process
RUN pip install flask faster-whisper paho-mqtt requests 'numpy<2' openwakeword tflite-runtime
# Pre-download models so they are ready at boot
RUN python -c 'import openwakeword; openwakeword.utils.download_models()'

7
Dockerfile.bridge Normal file
View File

@@ -0,0 +1,7 @@
FROM python:3.9-slim
WORKDIR /app
RUN pip install paho-mqtt requests 'numpy<2' openwakeword tflite-runtime
RUN python -c 'import openwakeword; openwakeword.utils.download_models()'
COPY mqtt_audio_bridge.py /app/
ENV PYTHONUNBUFFERED=1
CMD ["python", "-u", "mqtt_audio_bridge.py"]

12
app.py
View File

@@ -24,7 +24,17 @@ def transcribe_audio():
try:
# Now we can safely use temp_file_path
segments, info = model.transcribe(temp_file_path, beam_size=5, language="en")
initial_prompt = (
"Jervis, feed the fish, turn on the lights, turn off the lights, "
"shopping list, add to the shopping list, calendar, weather, time, "
"remind me, heater, watering the plants, play music, stop"
)
segments, info = model.transcribe(
temp_file_path,
beam_size=5,
language="en",
initial_prompt=initial_prompt,
)
transcript = " ".join([segment.text for segment in segments])
print(f"Transcript: {transcript}")
return jsonify({"transcript": transcript})

25
docker-compose.yml
View File

@@ -1,31 +1,34 @@
services:
whisper-api:
image: python:3.9-slim
build: .
container_name: voice_whisper
restart: unless-stopped
working_dir: /app
volumes:
- ./app.py:/app/app.py
command: >
sh -c "pip install flask faster-whisper && python app.py"
command: python app.py
ports:
- "5000:5000"
voice-bridge:
image: python:3.9-slim
build:
context: .
dockerfile: Dockerfile.bridge
image: voice_bridge:local
container_name: voice_bridge
restart: unless-stopped
network_mode: "host"
deploy:
resources:
limits:
memory: 500M
depends_on:
- whisper-api
working_dir: /app
volumes:
- ./mqtt_audio_bridge.py:/app/mqtt_audio_bridge.py
environment:
- XDG_CACHE_HOME=/root/.cache
command: >
sh -c "pip install paho-mqtt requests 'numpy<2' openwakeword tflite-runtime &&
python -c 'import openwakeword; openwakeword.utils.download_models()' &&
python mqtt_audio_bridge.py"
# Passed through from the invoking shell. start_bridge.sh sources
# ~/.config/environment.d/10-secrets.conf first, because environment.d
# does not reach a fresh SSH shell (AGENTS.md section 4.4).
# No secret values live in this file or in git.
- MQTT_USER
- MQTT_PASS

164
mqtt_audio_bridge.py
View File

@@ -2,12 +2,41 @@ import paho.mqtt.client as mqtt
import requests
import wave
import io
import os
import numpy as np
import openwakeword
from openwakeword.model import Model
import collections
import threading
def _load_secret(name):
"""Resolve a secret from the environment, then from the systemd secrets file.
environment.d reaches the systemd user manager, not a fresh SSH shell
(AGENTS.md section 4.4), so the file fallback is required. Secret values are
never logged.
"""
value = os.environ.get(name)
if value:
return value
path = os.path.expanduser(
os.environ.get("SECRETS_FILE", "~/.config/environment.d/10-secrets.conf")
)
try:
with open(path) as handle:
for raw in handle:
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, val = line.partition("=")
if key.strip() == name:
return val.strip().strip('"').strip("'")
except OSError:
pass
return None
# --- CONFIG ---
# 1. VOICE SERVER (Audio IO)
VOICE_BROKER = "192.168.20.13"
@@ -18,20 +47,35 @@ VOICE_TOPIC_STATUS = "voice/status"
HA_BROKER = "192.168.20.30"
HA_TOPIC_TEXT = "homeassistant/voice/text"
MQTT_USER = "MQTT_USER_REDACTED"
MQTT_PASS = "MQTT_PASS_REDACTED"
# MQTT credentials come from the environment or
# ~/.config/environment.d/10-secrets.conf (mode 600). They were previously
# hardcoded in this file and that commit is in Gitea; the password must be
# rotated. See the project brief section 1.6 / 1.9.
MQTT_USER = _load_secret("MQTT_USER")
MQTT_PASS = _load_secret("MQTT_PASS")
if not MQTT_USER or not MQTT_PASS:
raise SystemExit(
"MQTT_USER and MQTT_PASS must be set in the environment or in "
"~/.config/environment.d/10-secrets.conf"
)
WHISPER_URL = "http://whisper-api:5000/transcribe"
WHISPER_URL = "http://127.0.0.1:5000/transcribe"
WAKE_WORD = "hey_jarvis"
# --- OpenWakeWord expects 128-sample frames (16kHz, 0.008s per frame) ---
# We accumulate at least MIN_SAMPLES before calling predict()
MIN_SAMPLES = 128 # Minimum samples for a valid predict call
BUFFER_SIZE = 1280 * 8 # ~10 seconds of 16kHz int16 audio (20KB)
# --- STATE ---
BUFFER_SIZE = 1280 * 4
audio_buffer = collections.deque(maxlen=BUFFER_SIZE)
audio_buffer = collections.deque(maxlen=BUFFER_SIZE)
command_buffer = bytearray()
is_command_mode = False
leftover_bytes = bytearray() # Accumulates partial/odd-byte chunks from MQTT
print("Loading Models...")
print("Loading OpenWakeWord model...")
owwModel = Model()
print("Model loaded. Ready.")
# --- HELPER FUNCTIONS ---
def transcribe_and_send(raw_data):
@@ -48,77 +92,123 @@ def transcribe_and_send(raw_data):
res = requests.post(WHISPER_URL, files=files)
if res.status_code == 200:
text = res.json().get("transcript", "").strip()
print(f"TEXT: {text}")
# --- known mishears (homophones whisper gets wrong) ---
CORRECTIONS = {
"fade the fish": "feed the fish",
"fade fish": "feed the fish",
"feed fish": "feed the fish",
}
_t = text.strip().lower()
if _t in CORRECTIONS:
text = CORRECTIONS[_t]
print(f"TEXT (corrected): {text}")
else:
print(f"TEXT: {text}")
if text:
# PUBLISH TO HOME ASSISTANT BROKER
ha_client.publish(HA_TOPIC_TEXT, text)
# FEEDBACK TO ESP32 (Via Voice Broker)
voice_client.publish(VOICE_TOPIC_STATUS, "OK")
voice_client.publish(VOICE_TOPIC_STATUS, "OK")
except Exception as e:
print(f"API Error: {e}")
def safe_predict(samples):
"""Run OpenWakeWord predict with length validation."""
if len(samples) < MIN_SAMPLES:
return None
# Truncate to nearest multiple of MIN_SAMPLES for model compatibility
valid_len = (len(samples) // MIN_SAMPLES) * MIN_SAMPLES
if valid_len == 0:
return None
try:
return owwModel.predict(samples[:valid_len])
except Exception as e:
print(f"Predict error (skipping chunk): {e}")
return None
# --- CALLBACKS ---
def on_voice_message(client, userdata, msg):
global is_command_mode, command_buffer
global is_command_mode, command_buffer, leftover_bytes
if msg.topic == VOICE_TOPIC_AUDIO:
payload = msg.payload
# 1. COMMAND MODE (Recording for Whisper)
if is_command_mode:
# Safety Check: Prevent Memory Overflow (cap at ~3MB)
if len(command_buffer) > 3000000:
print("Error: Buffer overflow. Resetting.")
is_command_mode = False
command_buffer = bytearray()
owwModel.reset()
leftover_bytes = bytearray()
return
# Append the audio chunk
command_buffer.extend(payload)
return
# 2. LISTENING MODE (Wake Word Detection)
# Convert bytes to numpy int16 for OpenWakeWord
audio_int16 = np.frombuffer(payload, dtype=np.int16)
# Feed the model
prediction = owwModel.predict(audio_int16)
if prediction[WAKE_WORD] > 0.5:
print(f"--- WAKE WORD DETECTED ---")
# Prepend any leftover bytes from a previous truncated chunk
if leftover_bytes:
payload = leftover_bytes + payload
leftover_bytes = bytearray()
# Discard odd byte if payload length is not a multiple of 2 (int16)
if len(payload) % 2 != 0:
leftover_bytes = bytearray([payload[-1]])
payload = payload[:-1]
if len(payload) < 2:
return # Not enough data yet
try:
audio_int16 = np.frombuffer(payload, dtype=np.int16)
except Exception as e:
print(f"Buffer conversion error: {e}")
leftover_bytes = bytearray()
return
if len(audio_int16) < MIN_SAMPLES:
# Save for next chunk
leftover_bytes = payload
return
prediction = safe_predict(audio_int16)
if prediction is None:
return
if WAKE_WORD in prediction and prediction[WAKE_WORD] > 0.5:
print(f"--- WAKE WORD DETECTED ({WAKE_WORD}) ---")
is_command_mode = True
command_buffer = bytearray()
client.publish(VOICE_TOPIC_STATUS, "WAKE")
# Save leftover samples that weren't used in prediction
valid_len = (len(audio_int16) // MIN_SAMPLES) * MIN_SAMPLES
if valid_len < len(audio_int16):
command_buffer.extend(audio_int16[valid_len:].tobytes())
voice_client.publish(VOICE_TOPIC_STATUS, "WAKE")
elif msg.topic == "voice/status" and msg.payload.decode() == "processing":
# ESP32 stopped recording
if is_command_mode:
print("Processing...")
print("Processing command audio...")
transcribe_and_send(command_buffer)
is_command_mode = False
command_buffer = bytearray()
owwModel.reset()
leftover_bytes = bytearray()
# --- MAIN ---
# Client 1: Voice Server (The Listener)
voice_client = mqtt.Client(client_id="Bridge_Voice")
voice_client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION1, client_id="Bridge_Voice")
voice_client.username_pw_set(MQTT_USER, MQTT_PASS)
voice_client.on_message = on_voice_message
voice_client.connect(VOICE_BROKER, 1883)
voice_client.connect(VOICE_BROKER, 1883, 60)
voice_client.subscribe([(VOICE_TOPIC_AUDIO, 0), ("voice/status", 0)])
# Client 2: Home Assistant (The Sender)
ha_client = mqtt.Client(client_id="Bridge_HA")
ha_client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION1, client_id="Bridge_HA")
ha_client.username_pw_set(MQTT_USER, MQTT_PASS)
ha_client.connect(HA_BROKER, 1883)
ha_client.connect(HA_BROKER, 1883, 60)
print("Bridge Running (Dual Broker)...")
# Run loops in threads or blocking
# Since loop_forever blocks, we run HA loop in background and Voice loop in main
ha_thread = threading.Thread(target=ha_client.loop_forever)
ha_thread.daemon = True
ha_thread = threading.Thread(target=ha_client.loop_forever, daemon=True)
ha_thread.start()
voice_client.loop_forever()

53
start_bridge.sh Executable file
View File

@@ -0,0 +1,53 @@
#!/usr/bin/env bash
#
# start_bridge.sh — (re)start voice_bridge with MQTT credentials from the
# environment, not from source.
#
# environment.d does NOT reach a fresh SSH shell (AGENTS.md section 4.4), so
# this script sources the mode-600 secrets file itself. Only MQTT_USER and
# MQTT_PASS are passed to the container; no secret values are written to git.
#
# Usage:
# ./start_bridge.sh # rebuild if needed and (re)create voice_bridge
# ./start_bridge.sh --status # show container state and recent logs
# ./start_bridge.sh --logs # follow the bridge log
#
# Rollback: the previous image is kept as voice_bridge:pre-credential-hygiene.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SECRETS="${SECRETS_FILE:-$HOME/.config/environment.d/10-secrets.conf}"
load_secrets() {
if [ -f "$SECRETS" ]; then
# shellcheck disable=SC1090
set -a; . "$SECRETS"; set +a
else
echo "start_bridge.sh: $SECRETS not found" >&2
exit 1
fi
if [ -z "${MQTT_USER:-}" ] || [ -z "${MQTT_PASS:-}" ]; then
echo "start_bridge.sh: MQTT_USER / MQTT_PASS missing from $SECRETS" >&2
exit 1
fi
}
case "${1:-}" in
--status)
docker ps --filter name=voice_bridge --format '{{.Names}}\t{{.Image}}\t{{.Status}}'
docker logs --tail 20 voice_bridge 2>&1 || true
exit 0
;;
--logs)
exec docker logs -f voice_bridge
;;
"")
load_secrets
exec docker compose -f "$HERE/docker-compose.yml" up -d --build voice-bridge
;;
*)
echo "start_bridge.sh: unknown option $1" >&2
exit 2
;;
esac