sam/voice_assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Led yellow on response, returns to standby if no response.

Browse Source
This commit is contained in:
sam rolfe
2025-12-08 13:30:40 +11:00
parent 48a7888ac2
commit fd465bc04c
8 changed files with 26495 additions and 26505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2344
build/compile_commands.json
View File

File diff suppressed because it is too large Load Diff

165
build/sketch/voice_assistant.ino.cpp
View File

@ -15,7 +15,7 @@ const char* MQTT_USER = "mqtt-user";
const char* MQTT_PASS = "sam4jo";
const char* TOPIC_AUDIO = "voice/audio_stream";
const char* TOPIC_ACK = "voice/status";
const char* TOPIC_STATUS = "voice/status";
// --- HARDWARE ---
#define I2S_SCK 4
@ -30,12 +30,19 @@ const char* TOPIC_ACK = "voice/status";
const int BLOCK_SIZE = 512;
int16_t sBuffer[BLOCK_SIZE];
// --- OBJECTS ---
WiFiClient espClient;
PubSubClient mqtt(espClient);
Freenove_ESP32_WS2812 strip(NUM_LEDS, LED_PIN, 0, TYPE_GRB);
enum DeviceState { STATE_IDLE, STATE_RECORDING, STATE_TRANSMITTING, STATE_ACK_RECEIVED };
// --- STATES ---
enum DeviceState {
STATE_IDLE,
STATE_STREAMING, // Noise detected (Green)
STATE_LISTENING, // Wake Word Confirmed (Blue)
STATE_PROCESSING, // Sending to Server (Red)
STATE_SUCCESS, // Done (Yellow)
STATE_ERROR // Timeout (Blink Red)
};
DeviceState currentState = STATE_IDLE;
unsigned long stateTimer = 0;
@ -58,106 +65,93 @@ const i2s_pin_config_t pin_config = {
.data_in_num = I2S_SD
};
#line 59 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
// --- LED HELPER ---
#line 67 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void setRingColor(uint8_t r, uint8_t g, uint8_t b);
#line 64 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void updateLEDs();
#line 79 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
#line 73 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void mqttCallback(char* topic, byte* payload, unsigned int length);
#line 84 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
#line 95 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void setup();
#line 132 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
#line 120 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void loop();
#line 59 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
#line 67 "/media/sam/8294CD2994CD2111/Users/Dell/Documents/Arduino/voice_assistant/voice_assistant.ino"
void setRingColor(uint8_t r, uint8_t g, uint8_t b) {
strip.setAllLedsColor(r, g, b);
strip.show();
}
void updateLEDs() {
if (currentState == STATE_ACK_RECEIVED) {
for (int j = 0; j < 255; j += 20) {
for (int i = 0; i < NUM_LEDS; i++) {
byte pos = (i * 256 / NUM_LEDS + j) & 255;
strip.setLedColorData(i, pos, 255-pos, 0); // Simple fast rainbow
}
strip.show();
delay(5);
}
currentState = STATE_IDLE;
setRingColor(0,0,0);
}
}
// --- MQTT HANDLING ---
void mqttCallback(char* topic, byte* payload, unsigned int length) {
Serial.print("MSG Recv: "); Serial.println(topic);
currentState = STATE_ACK_RECEIVED;
char msg[length + 1];
memcpy(msg, payload, length);
msg[length] = '\0';
String message = String(msg);
// Serial.print("Recv: "); Serial.println(message); // Debug
if (message == "WAKE") {
// Server heard "Hey Jarvis" -> Turn BLUE
currentState = STATE_LISTENING;
stateTimer = millis(); // Reset timer to prevent premature timeout
setRingColor(0, 0, 255);
}
else if (message == "OK") {
// Command executed -> Turn YELLOW
currentState = STATE_SUCCESS;
stateTimer = millis();
setRingColor(255, 200, 0); // Yellow/Orange
}
}
void setup() {
delay(3000); // Wait for Serial
delay(2000);
Serial.begin(115200);
Serial.println("\n\n=== BOOTING DEBUG MODE ===");
// 1. LEDs
mqtt.setBufferSize(2048);
strip.begin();
strip.setBrightness(30);
setRingColor(50, 50, 50); // White
Serial.println("1. LEDs Ready");
setRingColor(50, 50, 50); // White Boot
// 2. I2S
if (i2s_driver_install(I2S_PORT, &i2s_config, 0, NULL) != ESP_OK) {
Serial.println("ERR: I2S Driver Fail");
} else {
i2s_driver_install(I2S_PORT, &i2s_config, 0, NULL);
i2s_set_pin(I2S_PORT, &pin_config);
Serial.println("2. I2S Ready");
}
// 3. WiFi
Serial.print("3. WiFi Connecting: ");
Serial.println(WIFI_SSID);
WiFi.begin(WIFI_SSID, WIFI_PASS);
int wifi_timeout = 0;
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
setRingColor(0, 0, 255); delay(100); setRingColor(0,0,0);
wifi_timeout++;
if(wifi_timeout > 20) {
Serial.println("\nERR: WiFi Timeout! Checking Creds...");
wifi_timeout = 0; // Keep trying or reset?
setRingColor(0, 0, 100); delay(100); setRingColor(0,0,0);
}
}
Serial.print("\n WiFi Connected! IP: ");
Serial.println(WiFi.localIP());
// 4. MQTT
mqtt.setServer(MQTT_BROKER, MQTT_PORT);
mqtt.setCallback(mqttCallback);
mqtt.setBufferSize(2048); // Vital for audio
Serial.println("4. MQTT Configured");
setRingColor(0,0,0);
Serial.println("=== SYSTEM READY ===");
setRingColor(0,0,0); // Idle
}
void loop() {
// MQTT Reconnect Handler
if (!mqtt.connected()) {
Serial.print("MQTT Connecting to "); Serial.print(MQTT_BROKER); Serial.print(" ... ");
if (mqtt.connect("ESP32Mic", MQTT_USER, MQTT_PASS)) {
Serial.println("CONNECTED");
mqtt.subscribe(TOPIC_ACK);
setRingColor(0, 255, 0); delay(500); setRingColor(0,0,0); // Flash Green on Connect
} else {
Serial.print("FAIL rc="); Serial.println(mqtt.state());
delay(2000);
return;
mqtt.subscribe(TOPIC_STATUS);
}
}
mqtt.loop();
updateLEDs();
// --- TIMEOUT LOGIC (Fixes the "Lock Up") ---
if (currentState == STATE_PROCESSING && millis() - stateTimer > 6000) {
// We waited 6 seconds for an answer, but got nothing.
// Blink Red 3 times then Reset.
for(int i=0; i<3; i++) {
setRingColor(255, 0, 0); delay(200);
setRingColor(0, 0, 0); delay(200);
}
currentState = STATE_IDLE;
}
// Reset "Success" yellow light after 3 seconds
if (currentState == STATE_SUCCESS && millis() - stateTimer > 3000) {
currentState = STATE_IDLE;
setRingColor(0,0,0);
}
size_t bytesRead;
i2s_read(I2S_PORT, &sBuffer, sizeof(sBuffer), &bytesRead, portMAX_DELAY);
@ -176,37 +170,38 @@ void loop() {
}
int avg = sum / (bytesRead / 2);
// Uncomment to debug sensitivity
// Serial.printf("Vol: %d\n", avg);
// --- AUDIO LOGIC ---
// Trigger Logic
// 1. Detect Noise -> Green
if (currentState == STATE_IDLE && avg > VAD_THRESHOLD) {
Serial.println("TRIGGERED");
currentState = STATE_RECORDING;
currentState = STATE_STREAMING;
stateTimer = millis();
setRingColor(0, 255, 0); // Green
setRingColor(0, 255, 0); // Green (I hear you)
// START OF SENTENCE FIX: Send the trigger buffer immediately
// Send trigger buffer
mqtt.publish(TOPIC_AUDIO, (const uint8_t*)sBuffer, bytesRead);
}
if (currentState == STATE_RECORDING) {
// 2. Stream Audio
if (currentState == STATE_STREAMING || currentState == STATE_LISTENING) {
mqtt.publish(TOPIC_AUDIO, (const uint8_t*)sBuffer, bytesRead);
if (avg > VAD_THRESHOLD) stateTimer = millis();
// Silence Timeout (1.5s)
if (millis() - stateTimer > 1500) {
Serial.println("SILENCE DETECTED");
currentState = STATE_TRANSMITTING;
mqtt.publish("voice/status", "processing");
setRingColor(0, 0, 255); // Blue
}
}
if (currentState == STATE_TRANSMITTING && millis() - stateTimer > 4000) {
Serial.println("TIMEOUT WAIT");
if (currentState == STATE_LISTENING) {
// We were talking to Jarvis, now we wait for reply
currentState = STATE_PROCESSING;
stateTimer = millis(); // Start timeout timer
mqtt.publish(TOPIC_STATUS, "processing");
setRingColor(255, 0, 0); // Red (Thinking/Wait)
} else {
// Just noise, never woke up
currentState = STATE_IDLE;
setRingColor(0,0,0);
setRingColor(0, 0, 0);
}
}
}
}

BIN
build/sketch/voice_assistant.ino.cpp.o
View File

Binary file not shown.

BIN
build/voice_assistant.ino.bin
View File

Binary file not shown.

BIN
build/voice_assistant.ino.elf
View File

Binary file not shown.

50429
build/voice_assistant.ino.map
View File

File diff suppressed because one or more lines are too long

BIN
build/voice_assistant.ino.merged.bin
View File

Binary file not shown.

56
voice_assistant.ino
View File

@ -20,7 +20,7 @@ const char* TOPIC_STATUS = "voice/status";
#define I2S_WS 5
#define I2S_SD 6
#define I2S_PORT I2S_NUM_0
#define LED_PIN 1 // Updated to Pin 1
#define LED_PIN 1
#define NUM_LEDS 12
// --- TUNING ---
@ -35,10 +35,11 @@ Freenove_ESP32_WS2812 strip(NUM_LEDS, LED_PIN, 0, TYPE_GRB);
// --- STATES ---
enum DeviceState {
STATE_IDLE,
STATE_STREAMING, // Sending audio (LED OFF)
STATE_LISTENING, // Wake Word Confirmed (LED BLUE)
STATE_PROCESSING, // Sending to Whisper (LED RED)
STATE_SUCCESS // Done (LED GREEN)
STATE_STREAMING, // Noise detected (Green)
STATE_LISTENING, // Wake Word Confirmed (Blue)
STATE_PROCESSING, // Sending to Server (Red)
STATE_SUCCESS, // Done (Yellow)
STATE_ERROR // Timeout (Blink Red)
};
DeviceState currentState = STATE_IDLE;
unsigned long stateTimer = 0;
@ -72,21 +73,22 @@ void setRingColor(uint8_t r, uint8_t g, uint8_t b) {
void mqttCallback(char* topic, byte* payload, unsigned int length) {
char msg[length + 1];
memcpy(msg, payload, length);
msg[length] = '\0'; // Null terminate
msg[length] = '\0';
String message = String(msg);
Serial.print("Status Recv: "); Serial.println(message);
// Serial.print("Recv: "); Serial.println(message); // Debug
if (message == "WAKE") {
// Server heard "Hey Jarvis"
// Server heard "Hey Jarvis" -> Turn BLUE
currentState = STATE_LISTENING;
setRingColor(0, 0, 255); // BLUE
stateTimer = millis(); // Reset timer to prevent premature timeout
setRingColor(0, 0, 255);
}
else if (message == "OK") {
// Command executed
// Command executed -> Turn YELLOW
currentState = STATE_SUCCESS;
stateTimer = millis();
setRingColor(0, 255, 0); // GREEN
setRingColor(255, 200, 0); // Yellow/Orange
}
}
@ -98,7 +100,7 @@ void setup() {
strip.begin();
strip.setBrightness(30);
setRingColor(50, 50, 50); // White on Boot
setRingColor(50, 50, 50); // White Boot
i2s_driver_install(I2S_PORT, &i2s_config, 0, NULL);
i2s_set_pin(I2S_PORT, &pin_config);
@ -112,7 +114,7 @@ void setup() {
mqtt.setServer(MQTT_BROKER, MQTT_PORT);
mqtt.setCallback(mqttCallback);
setRingColor(0,0,0); // Idle = Off
setRingColor(0,0,0); // Idle
}
void loop() {
@ -123,7 +125,18 @@ void loop() {
}
mqtt.loop();
// Reset Success/Green LED after 3 seconds
// --- TIMEOUT LOGIC (Fixes the "Lock Up") ---
if (currentState == STATE_PROCESSING && millis() - stateTimer > 6000) {
// We waited 6 seconds for an answer, but got nothing.
// Blink Red 3 times then Reset.
for(int i=0; i<3; i++) {
setRingColor(255, 0, 0); delay(200);
setRingColor(0, 0, 0); delay(200);
}
currentState = STATE_IDLE;
}
// Reset "Success" yellow light after 3 seconds
if (currentState == STATE_SUCCESS && millis() - stateTimer > 3000) {
currentState = STATE_IDLE;
setRingColor(0,0,0);
@ -140,27 +153,25 @@ void loop() {
sBuffer[i] = (int16_t)boosted;
}
// Calculate Volume
long sum = 0;
for (int i = 0; i < bytesRead / 2; i++) {
sum += abs(sBuffer[i]);
}
int avg = sum / (bytesRead / 2);
// --- LOGIC ---
// --- AUDIO LOGIC ---
// 1. Detect Sound (Invisible)
// 1. Detect Noise -> Green
if (currentState == STATE_IDLE && avg > VAD_THRESHOLD) {
currentState = STATE_STREAMING;
stateTimer = millis();
// LED stays OFF (We don't know if it's Jarvis yet)
setRingColor(0, 255, 0); // Green (I hear you)
// Send trigger buffer
mqtt.publish(TOPIC_AUDIO, (const uint8_t*)sBuffer, bytesRead);
}
// 2. Stream Audio
// We stream if we are just hearing noise OR if we are actively listening for a command
if (currentState == STATE_STREAMING || currentState == STATE_LISTENING) {
mqtt.publish(TOPIC_AUDIO, (const uint8_t*)sBuffer, bytesRead);
@ -169,12 +180,13 @@ void loop() {
// Silence Timeout (1.5s)
if (millis() - stateTimer > 1500) {
if (currentState == STATE_LISTENING) {
// If we were listening, now we process
// We were talking to Jarvis, now we wait for reply
currentState = STATE_PROCESSING;
stateTimer = millis(); // Start timeout timer
mqtt.publish(TOPIC_STATUS, "processing");
setRingColor(255, 0, 0); // RED (Thinking)
setRingColor(255, 0, 0); // Red (Thinking/Wait)
} else {
// If we were just streaming noise and never heard Jarvis, go back to sleep
// Just noise, never woke up
currentState = STATE_IDLE;
setRingColor(0, 0, 0);
}