Phase 5: hands-free "Computer" wake word (esp-sr WakeNet + AFE VAD)
Say "Computer" -> chime + listening -> speak -> AFE VAD detects you stopped -> auto-sends. No taps. Touch still works as a manual override. - esp-sr 2.4.6 added; wn9_computer_tts model packed into a new "model" flash partition (MODEL_IN_FLASH). App moved to 8M, model 4M. - audio.c: replaced the on-demand capture_task with an AFE pipeline — feed_task is the SOLE mic reader (-> afe->feed); detect_task fetches, watches wakeup_state for the wake word and vad_state for end of speech, and forwards AFE-cleaned audio upstream during an utterance. One mic reader ever. - short rising chime acknowledges the wake audibly. Fixes from adversarial review before trusting it: 1. utterance framing (blocking WS sends) moved OFF the AFE fetch thread onto an app_task event queue (EV_TOUCH/EV_WAKE/EV_SPEECH_END) — a 1.5s send could stall fetch and drop the first ~1.5s of speech. 2. app_task is now the single serializer of start/end -> no TOCTOU double-start (was: two utterance_start on a tap during wake). 3. VAD accounting resets on every streaming (re)start (wake OR tap), not just wake -> a tapped utterance can no longer end instantly on stale silence. 4. chime/reply set s_playing (+DMA tail hold) and detect_task skips the mic while s_playing -> our own audio no longer streams into STT or false-triggers the wake at a playback boundary (no AEC yet). 5. NULL-checked AFE create + feed buffer; tasks only start if AFE is up. Verified on hardware: model loads, AFE inits with the Computer word, boots and connects clean, no crash/wedge. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
#include "esp_log.h"
|
||||
#include "esp_system.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/semphr.h"
|
||||
#include "freertos/task.h"
|
||||
#include "bsp/esp-bsp.h"
|
||||
@@ -23,59 +24,80 @@
|
||||
|
||||
static const char *TAG = "desklock";
|
||||
|
||||
static SemaphoreHandle_t s_touch_sem;
|
||||
static bool s_talking;
|
||||
/* All utterance framing (blocking WS sends, chime, face) runs on app_task, never
|
||||
* on the LVGL touch callback nor the AFE detect/fetch thread. The wake word, VAD,
|
||||
* and touch just POST an event; app_task is the single serializer of start/end,
|
||||
* which removes both the fetch-thread stall and the double-start race. */
|
||||
typedef enum { EV_TOUCH, EV_WAKE, EV_SPEECH_END } app_event_t;
|
||||
static QueueHandle_t s_events;
|
||||
|
||||
/* Called from the LVGL touch callback — MUST be fast and non-blocking. */
|
||||
void app_on_touch(void)
|
||||
static void post_event(app_event_t ev)
|
||||
{
|
||||
if (s_touch_sem != NULL) {
|
||||
xSemaphoreGive(s_touch_sem);
|
||||
if (s_events != NULL) {
|
||||
xQueueSend(s_events, &ev, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/* Called from the WS event task when the link drops: forget any half-utterance
|
||||
* so the next tap starts a fresh one instead of hitting the "stop" branch. */
|
||||
void app_on_disconnect(void)
|
||||
{
|
||||
s_talking = false;
|
||||
}
|
||||
/* --- these three are the fast, non-blocking hooks called from other tasks --- */
|
||||
void app_on_touch(void) { post_event(EV_TOUCH); } /* LVGL touch callback */
|
||||
void app_on_wake(void) { post_event(EV_WAKE); } /* AFE detect_task: wake word */
|
||||
void app_on_speech_end(void) { post_event(EV_SPEECH_END); } /* AFE detect_task: VAD silence */
|
||||
|
||||
/* Runs in app_task (normal context): blocking sends here don't stall rendering. */
|
||||
static void handle_touch(void)
|
||||
/* --- the actual work, all on app_task --- */
|
||||
static void start_utterance(void)
|
||||
{
|
||||
face_activity();
|
||||
|
||||
if (!gw_connected()) {
|
||||
s_talking = false; /* link gone; nothing to talk to */
|
||||
if (!gw_connected() || audio_is_playing() || audio_capture_active()
|
||||
|| face_get() == FACE_EFFORT) {
|
||||
return;
|
||||
}
|
||||
if (!s_talking) {
|
||||
if (audio_is_playing() || face_get() == FACE_EFFORT) {
|
||||
return; /* barge-in is phase 4 */
|
||||
}
|
||||
s_talking = true;
|
||||
gw_send_text("{\"type\":\"utterance_start\"}");
|
||||
audio_capture_start();
|
||||
face_set(FACE_LISTENING);
|
||||
ESP_LOGI(TAG, "listening…");
|
||||
} else {
|
||||
s_talking = false;
|
||||
audio_capture_stop();
|
||||
gw_send_text("{\"type\":\"utterance_end\"}");
|
||||
face_set(FACE_PENSIVE);
|
||||
ESP_LOGI(TAG, "utterance sent");
|
||||
face_activity();
|
||||
audio_play_chime(); /* audible "I heard you" */
|
||||
gw_send_text("{\"type\":\"utterance_start\"}"); /* must precede any audio frames */
|
||||
audio_capture_start(); /* now detect_task forwards audio */
|
||||
face_set(FACE_LISTENING);
|
||||
ESP_LOGI(TAG, "listening…");
|
||||
}
|
||||
|
||||
static void end_utterance(void)
|
||||
{
|
||||
if (!audio_capture_active()) {
|
||||
return;
|
||||
}
|
||||
audio_capture_stop();
|
||||
gw_send_text("{\"type\":\"utterance_end\"}");
|
||||
face_set(FACE_PENSIVE);
|
||||
ESP_LOGI(TAG, "utterance sent");
|
||||
}
|
||||
|
||||
/* WS link dropped: abandon any half-utterance so the next wake starts fresh. */
|
||||
void app_on_disconnect(void)
|
||||
{
|
||||
audio_capture_stop();
|
||||
}
|
||||
|
||||
static void app_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
app_event_t ev;
|
||||
for (;;) {
|
||||
if (xSemaphoreTake(s_touch_sem, portMAX_DELAY) == pdTRUE) {
|
||||
vTaskDelay(pdMS_TO_TICKS(60)); /* debounce */
|
||||
xSemaphoreTake(s_touch_sem, 0); /* coalesce repeats during debounce */
|
||||
handle_touch();
|
||||
if (xQueueReceive(s_events, &ev, portMAX_DELAY) != pdTRUE) {
|
||||
continue;
|
||||
}
|
||||
if (ev == EV_TOUCH) {
|
||||
vTaskDelay(pdMS_TO_TICKS(60)); /* debounce */
|
||||
xQueueReset(s_events); /* coalesce a bouncy tap burst */
|
||||
face_activity();
|
||||
if (!gw_connected()) {
|
||||
audio_capture_stop();
|
||||
} else if (!audio_capture_active()) {
|
||||
start_utterance(); /* tap = manual wake */
|
||||
} else {
|
||||
end_utterance(); /* tap = manual end */
|
||||
}
|
||||
} else if (ev == EV_WAKE) {
|
||||
start_utterance();
|
||||
} else if (ev == EV_SPEECH_END) {
|
||||
end_utterance();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,7 +140,7 @@ void app_main(void)
|
||||
{
|
||||
ESP_LOGI(TAG, "DeskLock starting");
|
||||
|
||||
s_touch_sem = xSemaphoreCreateBinary();
|
||||
s_events = xQueueCreate(8, sizeof(app_event_t));
|
||||
|
||||
bsp_display_start();
|
||||
bsp_display_backlight_on();
|
||||
|
||||
Reference in New Issue
Block a user