From de9b06a3d1f19590d425d6777ad57936528258e4 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Wed, 7 Oct 2026 18:10:51 +1100 Subject: [PATCH] Sync live code; move MQTT credentials out of source; untrack venv/ - mqtt_audio_bridge.py: MQTT_USER / MQTT_PASS are read from the environment, falling back to ~/.config/environment.d/10-secrets.conf (mode 600). The environment.d trap means a fresh SSH shell needs the file fallback. - docker-compose.yml: pass MQTT_USER / MQTT_PASS through; keeps the live network_mode / Dockerfile topology. - start_bridge.sh: launcher that sources the secrets file before compose up. - app.py: commits the live initial_prompt that was uncommitted. - .gitignore: add venv/ (was missing), __pycache__, *.bak*, *.wav. - git rm -r --cached venv/: 13,413 tracked venv files removed from the index. Credential note: MQTT_PASS was hardcoded in edaf914. origin/main was later force-rewritten to redact it, but the value was exposed and must still be rotated. Removing venv/ from the index does not shrink the repository: the objects stay in history until a history rewrite and GC. Co-Authored-By: Paperclip --- .gitignore | 7 ++ Dockerfile | 7 ++ Dockerfile.bridge | 7 ++ app.py | 12 +++- docker-compose.yml | 25 ++++--- mqtt_audio_bridge.py | 164 +++++++++++++++++++++++++++++++++---------- start_bridge.sh | 53 ++++++++++++++ 7 files changed, 226 insertions(+), 49 deletions(-) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Dockerfile.bridge create mode 100755 start_bridge.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..934fda3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +# Backups must never be committed: they can contain the hardcoded MQTT password. +*.bak +*.bak-* +*.wav +venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..836700f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.9-slim +WORKDIR /app +# Install dependencies once during the build process +RUN pip install flask faster-whisper paho-mqtt requests 'numpy<2' openwakeword tflite-runtime +# Pre-download models so they are ready at boot +RUN python -c 'import openwakeword; openwakeword.utils.download_models()' + diff --git a/Dockerfile.bridge b/Dockerfile.bridge new file mode 100644 index 0000000..0786580 --- /dev/null +++ b/Dockerfile.bridge @@ -0,0 +1,7 @@ +FROM python:3.9-slim +WORKDIR /app +RUN pip install paho-mqtt requests 'numpy<2' openwakeword tflite-runtime +RUN python -c 'import openwakeword; openwakeword.utils.download_models()' +COPY mqtt_audio_bridge.py /app/ +ENV PYTHONUNBUFFERED=1 +CMD ["python", "-u", "mqtt_audio_bridge.py"] diff --git a/app.py b/app.py index ccc44fd..b4604e8 100644 --- a/app.py +++ b/app.py @@ -24,7 +24,17 @@ def transcribe_audio(): try: # Now we can safely use temp_file_path - segments, info = model.transcribe(temp_file_path, beam_size=5, language="en") + initial_prompt = ( + "Jervis, feed the fish, turn on the lights, turn off the lights, " + "shopping list, add to the shopping list, calendar, weather, time, " + "remind me, heater, watering the plants, play music, stop" + ) + segments, info = model.transcribe( + temp_file_path, + beam_size=5, + language="en", + initial_prompt=initial_prompt, + ) transcript = " ".join([segment.text for segment in segments]) print(f"Transcript: {transcript}") return jsonify({"transcript": transcript}) diff --git a/docker-compose.yml b/docker-compose.yml index fdfa197..4321739 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,31 +1,34 @@ services: whisper-api: - image: python:3.9-slim + build: . container_name: voice_whisper restart: unless-stopped working_dir: /app volumes: - ./app.py:/app/app.py - command: > - sh -c "pip install flask faster-whisper && python app.py" + command: python app.py ports: - "5000:5000" + voice-bridge: - image: python:3.9-slim + build: + context: . + dockerfile: Dockerfile.bridge + image: voice_bridge:local container_name: voice_bridge restart: unless-stopped + network_mode: "host" deploy: resources: limits: memory: 500M depends_on: - whisper-api - working_dir: /app - volumes: - - ./mqtt_audio_bridge.py:/app/mqtt_audio_bridge.py environment: - XDG_CACHE_HOME=/root/.cache - command: > - sh -c "pip install paho-mqtt requests 'numpy<2' openwakeword tflite-runtime && - python -c 'import openwakeword; openwakeword.utils.download_models()' && - python mqtt_audio_bridge.py" + # Passed through from the invoking shell. start_bridge.sh sources + # ~/.config/environment.d/10-secrets.conf first, because environment.d + # does not reach a fresh SSH shell (AGENTS.md section 4.4). + # No secret values live in this file or in git. + - MQTT_USER + - MQTT_PASS diff --git a/mqtt_audio_bridge.py b/mqtt_audio_bridge.py index 66f8162..357e95d 100644 --- a/mqtt_audio_bridge.py +++ b/mqtt_audio_bridge.py @@ -2,12 +2,41 @@ import paho.mqtt.client as mqtt import requests import wave import io +import os import numpy as np import openwakeword from openwakeword.model import Model import collections import threading + +def _load_secret(name): + """Resolve a secret from the environment, then from the systemd secrets file. + + environment.d reaches the systemd user manager, not a fresh SSH shell + (AGENTS.md section 4.4), so the file fallback is required. Secret values are + never logged. + """ + value = os.environ.get(name) + if value: + return value + path = os.path.expanduser( + os.environ.get("SECRETS_FILE", "~/.config/environment.d/10-secrets.conf") + ) + try: + with open(path) as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + if key.strip() == name: + return val.strip().strip('"').strip("'") + except OSError: + pass + return None + + # --- CONFIG --- # 1. VOICE SERVER (Audio IO) VOICE_BROKER = "192.168.20.13" @@ -18,20 +47,35 @@ VOICE_TOPIC_STATUS = "voice/status" HA_BROKER = "192.168.20.30" HA_TOPIC_TEXT = "homeassistant/voice/text" -MQTT_USER = "MQTT_USER_REDACTED" -MQTT_PASS = "MQTT_PASS_REDACTED" +# MQTT credentials come from the environment or +# ~/.config/environment.d/10-secrets.conf (mode 600). They were previously +# hardcoded in this file and that commit is in Gitea; the password must be +# rotated. See the project brief section 1.6 / 1.9. +MQTT_USER = _load_secret("MQTT_USER") +MQTT_PASS = _load_secret("MQTT_PASS") +if not MQTT_USER or not MQTT_PASS: + raise SystemExit( + "MQTT_USER and MQTT_PASS must be set in the environment or in " + "~/.config/environment.d/10-secrets.conf" + ) -WHISPER_URL = "http://whisper-api:5000/transcribe" +WHISPER_URL = "http://127.0.0.1:5000/transcribe" WAKE_WORD = "hey_jarvis" +# --- OpenWakeWord expects 128-sample frames (16kHz, 0.008s per frame) --- +# We accumulate at least MIN_SAMPLES before calling predict() +MIN_SAMPLES = 128 # Minimum samples for a valid predict call +BUFFER_SIZE = 1280 * 8 # ~10 seconds of 16kHz int16 audio (20KB) + # --- STATE --- -BUFFER_SIZE = 1280 * 4 -audio_buffer = collections.deque(maxlen=BUFFER_SIZE) +audio_buffer = collections.deque(maxlen=BUFFER_SIZE) command_buffer = bytearray() is_command_mode = False +leftover_bytes = bytearray() # Accumulates partial/odd-byte chunks from MQTT -print("Loading Models...") +print("Loading OpenWakeWord model...") owwModel = Model() +print("Model loaded. Ready.") # --- HELPER FUNCTIONS --- def transcribe_and_send(raw_data): @@ -48,77 +92,123 @@ def transcribe_and_send(raw_data): res = requests.post(WHISPER_URL, files=files) if res.status_code == 200: text = res.json().get("transcript", "").strip() - print(f"TEXT: {text}") + # --- known mishears (homophones whisper gets wrong) --- + CORRECTIONS = { + "fade the fish": "feed the fish", + "fade fish": "feed the fish", + "feed fish": "feed the fish", + } + _t = text.strip().lower() + if _t in CORRECTIONS: + text = CORRECTIONS[_t] + print(f"TEXT (corrected): {text}") + else: + print(f"TEXT: {text}") if text: - # PUBLISH TO HOME ASSISTANT BROKER ha_client.publish(HA_TOPIC_TEXT, text) - - # FEEDBACK TO ESP32 (Via Voice Broker) - voice_client.publish(VOICE_TOPIC_STATUS, "OK") + voice_client.publish(VOICE_TOPIC_STATUS, "OK") except Exception as e: print(f"API Error: {e}") + +def safe_predict(samples): + """Run OpenWakeWord predict with length validation.""" + if len(samples) < MIN_SAMPLES: + return None + # Truncate to nearest multiple of MIN_SAMPLES for model compatibility + valid_len = (len(samples) // MIN_SAMPLES) * MIN_SAMPLES + if valid_len == 0: + return None + try: + return owwModel.predict(samples[:valid_len]) + except Exception as e: + print(f"Predict error (skipping chunk): {e}") + return None + + # --- CALLBACKS --- def on_voice_message(client, userdata, msg): - global is_command_mode, command_buffer + global is_command_mode, command_buffer, leftover_bytes if msg.topic == VOICE_TOPIC_AUDIO: payload = msg.payload - + # 1. COMMAND MODE (Recording for Whisper) if is_command_mode: - # Safety Check: Prevent Memory Overflow (cap at ~3MB) if len(command_buffer) > 3000000: print("Error: Buffer overflow. Resetting.") is_command_mode = False command_buffer = bytearray() owwModel.reset() + leftover_bytes = bytearray() return - - # Append the audio chunk command_buffer.extend(payload) return # 2. LISTENING MODE (Wake Word Detection) - # Convert bytes to numpy int16 for OpenWakeWord - audio_int16 = np.frombuffer(payload, dtype=np.int16) - - # Feed the model - prediction = owwModel.predict(audio_int16) - - if prediction[WAKE_WORD] > 0.5: - print(f"--- WAKE WORD DETECTED ---") + # Prepend any leftover bytes from a previous truncated chunk + if leftover_bytes: + payload = leftover_bytes + payload + leftover_bytes = bytearray() + + # Discard odd byte if payload length is not a multiple of 2 (int16) + if len(payload) % 2 != 0: + leftover_bytes = bytearray([payload[-1]]) + payload = payload[:-1] + + if len(payload) < 2: + return # Not enough data yet + + try: + audio_int16 = np.frombuffer(payload, dtype=np.int16) + except Exception as e: + print(f"Buffer conversion error: {e}") + leftover_bytes = bytearray() + return + + if len(audio_int16) < MIN_SAMPLES: + # Save for next chunk + leftover_bytes = payload + return + + prediction = safe_predict(audio_int16) + if prediction is None: + return + + if WAKE_WORD in prediction and prediction[WAKE_WORD] > 0.5: + print(f"--- WAKE WORD DETECTED ({WAKE_WORD}) ---") is_command_mode = True command_buffer = bytearray() - client.publish(VOICE_TOPIC_STATUS, "WAKE") + # Save leftover samples that weren't used in prediction + valid_len = (len(audio_int16) // MIN_SAMPLES) * MIN_SAMPLES + if valid_len < len(audio_int16): + command_buffer.extend(audio_int16[valid_len:].tobytes()) + voice_client.publish(VOICE_TOPIC_STATUS, "WAKE") elif msg.topic == "voice/status" and msg.payload.decode() == "processing": - # ESP32 stopped recording if is_command_mode: - print("Processing...") + print("Processing command audio...") transcribe_and_send(command_buffer) is_command_mode = False command_buffer = bytearray() owwModel.reset() + leftover_bytes = bytearray() + + # --- MAIN --- -# Client 1: Voice Server (The Listener) -voice_client = mqtt.Client(client_id="Bridge_Voice") +voice_client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION1, client_id="Bridge_Voice") voice_client.username_pw_set(MQTT_USER, MQTT_PASS) voice_client.on_message = on_voice_message -voice_client.connect(VOICE_BROKER, 1883) +voice_client.connect(VOICE_BROKER, 1883, 60) voice_client.subscribe([(VOICE_TOPIC_AUDIO, 0), ("voice/status", 0)]) -# Client 2: Home Assistant (The Sender) -ha_client = mqtt.Client(client_id="Bridge_HA") +ha_client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION1, client_id="Bridge_HA") ha_client.username_pw_set(MQTT_USER, MQTT_PASS) -ha_client.connect(HA_BROKER, 1883) +ha_client.connect(HA_BROKER, 1883, 60) print("Bridge Running (Dual Broker)...") -# Run loops in threads or blocking -# Since loop_forever blocks, we run HA loop in background and Voice loop in main -ha_thread = threading.Thread(target=ha_client.loop_forever) -ha_thread.daemon = True +ha_thread = threading.Thread(target=ha_client.loop_forever, daemon=True) ha_thread.start() voice_client.loop_forever() diff --git a/start_bridge.sh b/start_bridge.sh new file mode 100755 index 0000000..de7c998 --- /dev/null +++ b/start_bridge.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# start_bridge.sh — (re)start voice_bridge with MQTT credentials from the +# environment, not from source. +# +# environment.d does NOT reach a fresh SSH shell (AGENTS.md section 4.4), so +# this script sources the mode-600 secrets file itself. Only MQTT_USER and +# MQTT_PASS are passed to the container; no secret values are written to git. +# +# Usage: +# ./start_bridge.sh # rebuild if needed and (re)create voice_bridge +# ./start_bridge.sh --status # show container state and recent logs +# ./start_bridge.sh --logs # follow the bridge log +# +# Rollback: the previous image is kept as voice_bridge:pre-credential-hygiene. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SECRETS="${SECRETS_FILE:-$HOME/.config/environment.d/10-secrets.conf}" + +load_secrets() { + if [ -f "$SECRETS" ]; then + # shellcheck disable=SC1090 + set -a; . "$SECRETS"; set +a + else + echo "start_bridge.sh: $SECRETS not found" >&2 + exit 1 + fi + if [ -z "${MQTT_USER:-}" ] || [ -z "${MQTT_PASS:-}" ]; then + echo "start_bridge.sh: MQTT_USER / MQTT_PASS missing from $SECRETS" >&2 + exit 1 + fi +} + +case "${1:-}" in + --status) + docker ps --filter name=voice_bridge --format '{{.Names}}\t{{.Image}}\t{{.Status}}' + docker logs --tail 20 voice_bridge 2>&1 || true + exit 0 + ;; + --logs) + exec docker logs -f voice_bridge + ;; + "") + load_secrets + exec docker compose -f "$HERE/docker-compose.yml" up -d --build voice-bridge + ;; + *) + echo "start_bridge.sh: unknown option $1" >&2 + exit 2 + ;; +esac