From e1bb4a558ff8adfa0a2b62f728ddfabb5a2ea125 Mon Sep 17 00:00:00 2001 From: Sam Rolfe Date: Sun, 6 Sep 2026 08:24:08 +1000 Subject: [PATCH] sam-4screen-desktop 2026-9-6:8:24:8 --- .obsidian/workspace.json | 15 ++-- .../Home Voice Assistant System.md | 74 +++++++++++++------ 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 79bd425..ceab8a6 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,12 +13,12 @@ "state": { "type": "markdown", "state": { - "file": "000 daily/basketball training.md", + "file": "300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "basketball training" + "title": "Home Voice Assistant System" } } ] @@ -55,7 +55,7 @@ "state": { "type": "search", "state": { - "query": "basketb", + "query": "Jervi", "matchingCase": false, "explainSearch": false, "collapseAll": false, @@ -76,8 +76,7 @@ "title": "Bookmarks" } } - ], - "currentTab": 1 + ] } ], "direction": "horizontal", @@ -184,7 +183,7 @@ "state": { "type": "file-properties", "state": { - "file": "000 daily/basketball training.md" + "file": "300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md" }, "icon": "lucide-info", "title": "File properties" @@ -210,8 +209,9 @@ "templater-obsidian:Templater": false } }, - "active": "8018aa52f0591df9", + "active": "7066c9273c6fb936", "lastOpenFiles": [ + "000 daily/basketball training.md", "200 projects/220 Web Host Migration/000 — Migration Overview.md", "200 projects/220 Web Host Migration/Garage Web UI — handoff.md", "200 projects/220 Web Host Migration/Project Ops — Build Plan (Vikunja+Outline+Obsidian).md", @@ -237,7 +237,6 @@ "000 daily/FAMILY S3 Storage Integration & Blueprint.md", "000 daily/FAMILY DeepSeek Harness (dsh) Home Lab Setup.md", "000 daily/FAMILY Enterprise Homelab AI Multimedia Suite. Systems Architecture & Blueprint.md", - "200 projects/220 Web Host Migration/Backup Architecture — Offsite to AWS S3.md", "000 daily/ThinkPad Recommendations", "500 archive/510 Daily", "300 areas/305 Ideas Businesses", diff --git a/300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md b/300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md index cad1283..6efccba 100644 --- a/300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md +++ b/300 areas/360 Dev-Ops Network Computers/Home Voice Assistant System.md @@ -1,11 +1,12 @@ --- created: 2026-08-30 -modified: 2026-08-30 +modified: 2026-09-05 type: note tags: - voice - system - ai + - project aliases: - Jervis - Voice Assistant @@ -13,34 +14,61 @@ aliases: # Home Voice Assistant System (Jervis) -> **Source of truth:** Gitea repo `sam/voice-assistant` → `https://gitea.lab.audasmedia.com.au/sam/voice-assistant` -> Docs: `docs/overview.md`, `docs/agent-api.md`, `docs/clients.md` (in that repo). +> **Human-readable overview of the local voice-assistant project.** +> - **AI/project tracking:** Vikunja → *Jervis Voice Assistant (Whisper ESP HA Flow)* (project id 5) +> - **AI documentation (for bots):** Outline → *Voice Assistant (Whisper ESP HA)* collection → [Project Overview](/doc/project-overview-genev9IHFT) +> - **API/docs (Gitea):** `sam/voice-assistant` → overview / agent-api / clients +> - **Tasks/progress:** see the Vikunja project; docs live in Outline; this note is the human map. -## TL;DR -Local, private voice assistant. "Hey Jervis" → natural language → one LLM brain (voice-agent on `.13:8501`) -decides intent → calls tools (time/weather/shopping/HA control/reminders/character voices) → speaks via piper → Snapcast. +## What it is (TL;DR) +A private, local voice assistant. Say **"Hey Jervis"** (ESP32 wake word) then a natural-language request — a local **LLM agent** decides intent and calls tools: it controls Home Assistant, manages shopping/tasks/calendar, plays music, answers questions, and speaks back through the house speakers (piper → Snapcast). -**Command it from anywhere:** `POST http://192.168.20.13:8501/voice {"text":"add milk to the shopping list"}` (reply is spoken). +**Command it from anywhere:** `POST http://192.168.20.13:8501/voice {"text": "add milk to the shopping list"}` — the reply is spoken. -## Components (host:port) +## Architecture +``` +ESP32 (wake word, mic) → MQTT voice/audio_stream + → voice_bridge + whisper (STT → text) + → Jervis agent (FastAPI :8501, LLM tool-calling) + ├─ Home Assistant (lights, timers, calendar, HA todo lists) + ├─ OmniRoute LLM (voice-fast: OpenRouter → opencode-go → deepseek flash) + ├─ Mopidy → Spotify → Snapcast (music) + ├─ Apprise (family WhatsApp/Telegram/ntfy/email) + ├─ Vikunja (project tasks) + └─ reply → piper → Snapcast speakers +``` + +## Components (host : port) | Component | Host | Port | Role | |---|---|---|---| -| ESP32-S3 (wake word) | — | — | mic, LED | -| voice_bridge / voice_whisper | .13 | docker/:5000 | STT | -| **voice-agent (LLM brain)** | .13 | **:8501** | tool-calling | -| OmniRoute | .13 | :20129 | LLM gateway | -| piper_tts | .13 | :10200 | TTS | -| Snapcast | .13 | :1780 | audio | -| Home Assistant | .30 | :8123 | control | -| KitchenOwl | .35 | owl.lab | shopping | -| mosquitto | .13/.30 | :1883 | MQTT | +| ESP32-S3 (voice_assistant.ino) | devices | — | wake word "Hey Jervis", mic, LED | +| voice_bridge / voice_whisper | .13 | docker / :5000 | speech → text (faster-whisper) | +| **Jervis agent** | .13 | **:8501** | LLM tool-calling brain (systemd) | +| OmniRoute | .13 | :20129 | LLM gateway (voice-fast route) | +| Home Assistant | .30 | :8123 | control, timers, calendar, todo lists | +| Mopidy + Spotify | .13 | :6600 / librespot | music | +| piper_tts | .13 | :10200 | TTS → Snapcast | +| Snapcast | .13 | :1780 | multi-room audio | +| Apprise | .35 | :8210 | notifications | +| Vikunja | .35 | vikunja.lab | project tasks/stages (tracker) | +| Outline | .13 | :3000 | project documentation (AI) | -See the repo `docs/overview.md` for the full diagram + `docs/agent-api.md` for the API contract + `docs/clients.md` for type/web/Android. +## Status (2026-09-05) +- **Voice working end-to-end**: ESP32 wake word → whisper → agent → piper → Snapcast (fixed docker piper, HA→.13 SSH auth, resampler, MQTT client-id). +- **HA integration**: lights/scenes via Assist; shopping lists via HA todo (`add_shopping_item`); timers + spoken reminders (HA `timer` → automation → piper); calendar alerts spoken 30 min before (HA automation); add-calendar tool. +- **Music**: play/pause/skip/volume via Mopidy → Spotify → Snapcast (credentials.json device-auth). +- **Q&A + web search**: Tavily-backed `web_search`; fast LLM route `voice-fast` (~2–3s). +- **Family messaging**: Apprise (sam/finn/harry/joanna; WhatsApp/email/Telegram/ntfy). +- **AI tooling**: `project-ops` skill gives every pi agent Vikunja + Outline access (keys in `10-secrets.conf`). -## Status (2026-08-30) -- Voice-over-Snapcast restored (piper docker + HA→.13 SSH auth fixed); full-length speech (pure-python resampler). -- Agent persistent as NixOS systemd service; tools verified: time, weather, shopping (KitchenOwl), HA control, reminders, feed-fish, character voices (Trump etc.). -- **Music working**: `play_music` → Mopidy → Spotify → Snapcast (gst-plugin-spotify + device-auth credentials.json; Radiohead/Pink Floyd verified). Fast LLM route `voice-fast` (OpenRouter→opencode-go→deepseek flash) cuts response latency to ~2-3s. -- Next: pause/stop music tools, Apprise messaging (sam/jo/harry/finn), calendar (N8N), extended TTS voices. See [[secrets_fixes]] for credential issues. +## Tools & systems (employment/skills) +FastAPI · Python · Docker · MQTT · Home Assistant (REST/Assist/automations) · Whisper · piper TTS · Snapcast · Mopidy / librespot / Spotify · OmniRoute LLM routing · Tavily web search · Apprise · Vikunja & Outline APIs · NixOS (systemd services, flakes) · git-filter-repo (secret hygiene) · ESP32 (Arduino/ESP-IDF). + +## Key configs & links +- **Vikunja project**: `https://vikunja.lab.audasmedia.com.au` → *Jervis Voice Assistant (Whisper ESP HA Flow)* +- **Outline collection**: `https://outline.lab.audasmedia.com.au` → *Voice Assistant (Whisper ESP HA)* +- **Gitea**: `sam/voice-assistant` (overview / agent-api / clients docs) +- **Repo layout / plan**: `~/chats/homeassistant` (plan.md, AGENT.md); agent at `/home/sam/voice-agent/` +- **Secrets**: `~/.config/environment.d/10-secrets.conf` on .27/.13/.51 (VIKUNJA_TOKEN, OL_API_KEY, etc.) — see also the `secrets_fixes` note. See also: [[Docker Containers]], [[Home Network Map Overview]]. \ No newline at end of file