From b462dff0824d08a3edfc4e876bb697932edbb73c Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 15 Jul 2026 10:52:41 +0200 Subject: [PATCH] Phase 5: hands-free "Computer" wake word (esp-sr WakeNet + AFE VAD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Say "Computer" -> chime + listening -> speak -> AFE VAD detects you stopped -> auto-sends. No taps. Touch still works as a manual override. - esp-sr 2.4.6 added; wn9_computer_tts model packed into a new "model" flash partition (MODEL_IN_FLASH). App moved to 8M, model 4M. - audio.c: replaced the on-demand capture_task with an AFE pipeline — feed_task is the SOLE mic reader (-> afe->feed); detect_task fetches, watches wakeup_state for the wake word and vad_state for end of speech, and forwards AFE-cleaned audio upstream during an utterance. One mic reader ever. - short rising chime acknowledges the wake audibly. Fixes from adversarial review before trusting it: 1. utterance framing (blocking WS sends) moved OFF the AFE fetch thread onto an app_task event queue (EV_TOUCH/EV_WAKE/EV_SPEECH_END) — a 1.5s send could stall fetch and drop the first ~1.5s of speech. 2. app_task is now the single serializer of start/end -> no TOCTOU double-start (was: two utterance_start on a tap during wake). 3. VAD accounting resets on every streaming (re)start (wake OR tap), not just wake -> a tapped utterance can no longer end instantly on stale silence. 4. chime/reply set s_playing (+DMA tail hold) and detect_task skips the mic while s_playing -> our own audio no longer streams into STT or false-triggers the wake at a playback boundary (no AEC yet). 5. NULL-checked AFE create + feed buffer; tasks only start if AFE is up. Verified on hardware: model loads, AFE inits with the Computer word, boots and connects clean, no crash/wedge. Co-Authored-By: Claude Fable 5 --- firmware/main/CMakeLists.txt | 2 +- firmware/main/audio.c | 178 ++++++++++++++++++++++++++++---- firmware/main/desklock.h | 6 +- firmware/main/desklock_main.c | 98 +++++++++++------- firmware/main/idf_component.yml | 1 + firmware/partitions.csv | 1 + firmware/sdkconfig.defaults | 4 + 7 files changed, 228 insertions(+), 62 deletions(-) diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 35bebc1..3081cc9 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -12,7 +12,7 @@ idf_component_register( "fonts/font_rage_64.c" INCLUDE_DIRS "." EMBED_FILES "c6_slave.bin" - PRIV_REQUIRES nvs_flash esp_wifi esp_event esp_netif json esp_timer esp_app_format esp_http_server + PRIV_REQUIRES nvs_flash esp_wifi esp_event esp_netif json esp_timer esp_app_format esp_http_server esp-sr ) target_compile_definitions(${COMPONENT_LIB} PRIVATE LV_LVGL_H_INCLUDE_SIMPLE) diff --git a/firmware/main/audio.c b/firmware/main/audio.c index 8f3bbfc..6325c83 100644 --- a/firmware/main/audio.c +++ b/firmware/main/audio.c @@ -11,10 +11,18 @@ #include "bsp/esp-bsp.h" #include "esp_codec_dev.h" +#include "esp_afe_config.h" +#include "esp_afe_sr_iface.h" +#include "esp_afe_sr_models.h" +#include "esp_wn_models.h" + #include "desklock.h" static const char *TAG = "audio"; +/* end an utterance after this much post-speech silence (AFE VAD) */ +#define VAD_END_SILENCE_MS 800 + #define RATE 16000 #define CAPTURE_CHUNK 3200 /* 100 ms */ #define REPLY_MAX (RATE * 2 * 60) /* 60 s of reply audio */ @@ -31,11 +39,15 @@ static int16_t *s_gong; static uint8_t *s_reply; static volatile size_t s_reply_len; static volatile bool s_playing; -static volatile bool s_capturing; -typedef enum { JOB_GONG, JOB_REPLY } job_t; +/* wake word + utterance (esp-sr AFE) */ +static const esp_afe_sr_iface_t *s_afe; +static esp_afe_sr_data_t *s_afe_data; +static volatile bool s_streaming; /* detect_task streams AFE audio upstream while true */ +static int16_t *s_chime; + +typedef enum { JOB_GONG, JOB_REPLY, JOB_CHIME } job_t; static QueueHandle_t s_jobs; -static TaskHandle_t s_capture_task; /* --- gong: see docs/architecture.md "Sound signature" --- */ @@ -107,40 +119,149 @@ static void synth_gong(void) free(dry); } +/* Short rising two-tone chime played when the wake word fires (you're often + * not looking at the face, so acknowledge audibly). ~180 ms. */ +#define CHIME_SAMPLES (RATE / 5) +static void synth_chime(void) +{ + s_chime = heap_caps_malloc(CHIME_SAMPLES * sizeof(int16_t), MALLOC_CAP_SPIRAM); + if (s_chime == NULL) { + return; + } + float phase = 0.0f; + for (int i = 0; i < CHIME_SAMPLES; i++) { + float freq = (i < CHIME_SAMPLES / 2) ? 660.0f : 990.0f; + phase += 2.0f * (float)M_PI * freq / RATE; + float env = 1.0f - (float)i / CHIME_SAMPLES; + s_chime[i] = (int16_t)(sinf(phase) * 9000.0f * env); + } +} + /* --- playback worker --- */ static void audio_task(void *arg) { (void)arg; synth_gong(); - ESP_LOGI(TAG, "gong ready"); + synth_chime(); + ESP_LOGI(TAG, "gong+chime ready"); job_t job; while (xQueueReceive(s_jobs, &job, portMAX_DELAY) == pdTRUE) { + /* s_playing gates the mic path: while our own speaker is active, the + * detect_task neither wakes nor streams (no AEC), so we don't hear + * ourselves. The tail delay covers the DMA that plays after write() + * returns, preventing a playback-boundary false wake. */ if (job == JOB_GONG && s_gong != NULL) { esp_codec_dev_write(s_spk, s_gong, GONG_SAMPLES * sizeof(int16_t)); + } else if (job == JOB_CHIME && s_chime != NULL) { + s_playing = true; + esp_codec_dev_write(s_spk, s_chime, CHIME_SAMPLES * sizeof(int16_t)); + vTaskDelay(pdMS_TO_TICKS(120)); + s_playing = false; } else if (job == JOB_REPLY) { s_playing = true; esp_codec_dev_write(s_spk, s_reply, s_reply_len); + vTaskDelay(pdMS_TO_TICKS(250)); s_playing = false; app_on_playback_done(); } } } -/* One persistent capture task, created once. It parks on a notification until - * capture is requested, then streams mic -> gateway while s_capturing holds. - * This makes capture single-instance: no create/delete restart race, so a rapid - * re-tap (or a WS-disconnect stop racing an app_task start) can never put two - * readers on the one mic/I2S handle. */ -static void capture_task(void *arg) +/* AFE wake-word + VAD pipeline. Two tasks: + * - feed_task: the SOLE mic reader. Reads mic -> afe->feed(). Always running. + * - detect_task: afe->fetch() -> when ARMED, watch for the wake word; when + * streaming an utterance, forward AFE-cleaned audio upstream and watch VAD + * for end-of-speech. One mic reader ever — no two-readers-on-one-mic hazard. */ + +static bool afe_init(void) +{ + srmodel_list_t *models = esp_srmodel_init("model"); + char *wn = models ? esp_srmodel_filter(models, ESP_WN_PREFIX, NULL) : NULL; + + afe_config_t *cfg = afe_config_init("M", models, AFE_TYPE_SR, AFE_MODE_LOW_COST); + if (cfg == NULL) { + ESP_LOGE(TAG, "afe_config_init failed"); + return false; + } + cfg->aec_init = false; /* no echo canceller in the always-on path (barge-in is later) */ + cfg->se_init = false; /* single logical mic, no beamforming */ + cfg->ns_init = false; + cfg->agc_init = false; + cfg->vad_init = true; /* VAD gives us hands-free end-of-utterance */ + cfg->vad_mode = VAD_MODE_1; + cfg->vad_min_speech_ms = 128; + cfg->vad_min_noise_ms = 600; + cfg->wakenet_init = (wn != NULL); + cfg->wakenet_model_name = wn; + cfg->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; + + s_afe = esp_afe_handle_from_config(cfg); + s_afe_data = s_afe ? s_afe->create_from_config(cfg) : NULL; + if (s_afe == NULL || s_afe_data == NULL) { + ESP_LOGE(TAG, "AFE create failed"); + return false; + } + ESP_LOGI(TAG, "AFE up, wake word = %s", wn ? wn : "NONE"); + return true; +} + +static void feed_task(void *arg) { (void)arg; - uint8_t *chunk = heap_caps_malloc(CAPTURE_CHUNK, MALLOC_CAP_DEFAULT); + int nch = s_afe->get_feed_channel_num(s_afe_data); + int nsamp = s_afe->get_feed_chunksize(s_afe_data); + int bytes = nsamp * nch * sizeof(int16_t); + int16_t *buf = heap_caps_malloc(bytes, MALLOC_CAP_DEFAULT); + if (buf == NULL) { + ESP_LOGE(TAG, "feed buffer alloc failed"); + vTaskDelete(NULL); + } for (;;) { - ulTaskNotifyTake(pdTRUE, portMAX_DELAY); - while (s_capturing) { - if (esp_codec_dev_read(s_mic, chunk, CAPTURE_CHUNK) == ESP_CODEC_DEV_OK) { - gw_send_bin(chunk, CAPTURE_CHUNK); + if (esp_codec_dev_read(s_mic, buf, bytes) == ESP_CODEC_DEV_OK) { + s_afe->feed(s_afe_data, buf); + } + } +} + +static void detect_task(void *arg) +{ + (void)arg; + bool was_streaming = false; + bool had_speech = false; + int silence_ms = 0; + for (;;) { + afe_fetch_result_t *res = s_afe->fetch(s_afe_data); + if (res == NULL || res->ret_value == ESP_FAIL) { + continue; + } + int frame_ms = res->data_size / (int)sizeof(int16_t) / (RATE / 1000); + + /* reset VAD accounting whenever streaming (re)starts — from a wake word + * OR a tap — so a stale silence count can't end the next utterance early */ + if (s_streaming && !was_streaming) { + had_speech = false; + silence_ms = 0; + } + was_streaming = s_streaming; + + if (!s_streaming) { + /* ARMED: listen for the wake word (not while our speaker is active) */ + if (!s_playing && res->wakeup_state == WAKENET_DETECTED) { + app_on_wake(); + } + } else if (!s_playing) { + /* UTTERANCE: forward cleaned audio, end on post-speech silence. + * Skipped while s_playing so the chime/reply doesn't self-stream. */ + gw_send_bin((const uint8_t *)res->data, res->data_size); + if (res->vad_state == VAD_SPEECH) { + had_speech = true; + silence_ms = 0; + } else { + silence_ms += frame_ms; + } + if (had_speech && silence_ms >= VAD_END_SILENCE_MS) { + app_on_speech_end(); } } } @@ -166,28 +287,41 @@ void audio_init(void) s_reply = heap_caps_malloc(REPLY_MAX, MALLOC_CAP_SPIRAM); s_jobs = xQueueCreate(4, sizeof(job_t)); xTaskCreate(audio_task, "audio", 4096, NULL, 5, NULL); - xTaskCreate(capture_task, "capture", 4096, NULL, 6, &s_capture_task); + + if (s_mic != NULL && afe_init()) { + xTaskCreate(feed_task, "afe_feed", 4096, NULL, 6, NULL); + xTaskCreate(detect_task, "afe_detect", 8192, NULL, 5, NULL); + } ESP_LOGI(TAG, "audio up (spk=%d mic=%d)", s_spk != NULL, s_mic != NULL); } +void audio_play_chime(void) +{ + job_t job = JOB_CHIME; + xQueueSend(s_jobs, &job, 0); +} + void audio_play_gong(void) { job_t job = JOB_GONG; xQueueSend(s_jobs, &job, 0); } +/* Streaming = the detect_task forwards AFE audio upstream. Wake word and touch + * both begin it; VAD silence and touch both end it. */ void audio_capture_start(void) { - if (s_mic == NULL || s_capturing || s_capture_task == NULL) { - return; - } - s_capturing = true; - xTaskNotifyGive(s_capture_task); /* wake the persistent task's inner loop */ + s_streaming = true; } void audio_capture_stop(void) { - s_capturing = false; + s_streaming = false; +} + +bool audio_capture_active(void) +{ + return s_streaming; } void audio_playback_begin(void) diff --git a/firmware/main/desklock.h b/firmware/main/desklock.h index 794e648..c850b48 100644 --- a/firmware/main/desklock.h +++ b/firmware/main/desklock.h @@ -26,8 +26,10 @@ void face_status(const char *text); /* bottom status line (diag/boot) */ /* audio.c */ void audio_init(void); void audio_play_gong(void); -void audio_capture_start(void); +void audio_play_chime(void); /* short wake acknowledgement */ +void audio_capture_start(void); /* begin streaming the utterance upstream */ void audio_capture_stop(void); +bool audio_capture_active(void); void audio_playback_begin(void); void audio_playback_feed(const uint8_t *data, size_t len); void audio_playback_end(void); @@ -50,5 +52,7 @@ void c6_ota_start(void); /* desklock_main.c */ void app_on_touch(void); +void app_on_wake(void); /* wake word detected (from audio detect_task) */ +void app_on_speech_end(void); /* AFE VAD detected end of utterance */ void app_on_disconnect(void); void app_on_playback_done(void); diff --git a/firmware/main/desklock_main.c b/firmware/main/desklock_main.c index 71985ae..2e036f5 100644 --- a/firmware/main/desklock_main.c +++ b/firmware/main/desklock_main.c @@ -12,6 +12,7 @@ #include "esp_log.h" #include "esp_system.h" #include "freertos/FreeRTOS.h" +#include "freertos/queue.h" #include "freertos/semphr.h" #include "freertos/task.h" #include "bsp/esp-bsp.h" @@ -23,59 +24,80 @@ static const char *TAG = "desklock"; -static SemaphoreHandle_t s_touch_sem; -static bool s_talking; +/* All utterance framing (blocking WS sends, chime, face) runs on app_task, never + * on the LVGL touch callback nor the AFE detect/fetch thread. The wake word, VAD, + * and touch just POST an event; app_task is the single serializer of start/end, + * which removes both the fetch-thread stall and the double-start race. */ +typedef enum { EV_TOUCH, EV_WAKE, EV_SPEECH_END } app_event_t; +static QueueHandle_t s_events; -/* Called from the LVGL touch callback — MUST be fast and non-blocking. */ -void app_on_touch(void) +static void post_event(app_event_t ev) { - if (s_touch_sem != NULL) { - xSemaphoreGive(s_touch_sem); + if (s_events != NULL) { + xQueueSend(s_events, &ev, 0); } } -/* Called from the WS event task when the link drops: forget any half-utterance - * so the next tap starts a fresh one instead of hitting the "stop" branch. */ -void app_on_disconnect(void) -{ - s_talking = false; -} +/* --- these three are the fast, non-blocking hooks called from other tasks --- */ +void app_on_touch(void) { post_event(EV_TOUCH); } /* LVGL touch callback */ +void app_on_wake(void) { post_event(EV_WAKE); } /* AFE detect_task: wake word */ +void app_on_speech_end(void) { post_event(EV_SPEECH_END); } /* AFE detect_task: VAD silence */ -/* Runs in app_task (normal context): blocking sends here don't stall rendering. */ -static void handle_touch(void) +/* --- the actual work, all on app_task --- */ +static void start_utterance(void) { - face_activity(); - - if (!gw_connected()) { - s_talking = false; /* link gone; nothing to talk to */ + if (!gw_connected() || audio_is_playing() || audio_capture_active() + || face_get() == FACE_EFFORT) { return; } - if (!s_talking) { - if (audio_is_playing() || face_get() == FACE_EFFORT) { - return; /* barge-in is phase 4 */ - } - s_talking = true; - gw_send_text("{\"type\":\"utterance_start\"}"); - audio_capture_start(); - face_set(FACE_LISTENING); - ESP_LOGI(TAG, "listening…"); - } else { - s_talking = false; - audio_capture_stop(); - gw_send_text("{\"type\":\"utterance_end\"}"); - face_set(FACE_PENSIVE); - ESP_LOGI(TAG, "utterance sent"); + face_activity(); + audio_play_chime(); /* audible "I heard you" */ + gw_send_text("{\"type\":\"utterance_start\"}"); /* must precede any audio frames */ + audio_capture_start(); /* now detect_task forwards audio */ + face_set(FACE_LISTENING); + ESP_LOGI(TAG, "listening…"); +} + +static void end_utterance(void) +{ + if (!audio_capture_active()) { + return; } + audio_capture_stop(); + gw_send_text("{\"type\":\"utterance_end\"}"); + face_set(FACE_PENSIVE); + ESP_LOGI(TAG, "utterance sent"); +} + +/* WS link dropped: abandon any half-utterance so the next wake starts fresh. */ +void app_on_disconnect(void) +{ + audio_capture_stop(); } static void app_task(void *arg) { (void)arg; + app_event_t ev; for (;;) { - if (xSemaphoreTake(s_touch_sem, portMAX_DELAY) == pdTRUE) { - vTaskDelay(pdMS_TO_TICKS(60)); /* debounce */ - xSemaphoreTake(s_touch_sem, 0); /* coalesce repeats during debounce */ - handle_touch(); + if (xQueueReceive(s_events, &ev, portMAX_DELAY) != pdTRUE) { + continue; + } + if (ev == EV_TOUCH) { + vTaskDelay(pdMS_TO_TICKS(60)); /* debounce */ + xQueueReset(s_events); /* coalesce a bouncy tap burst */ + face_activity(); + if (!gw_connected()) { + audio_capture_stop(); + } else if (!audio_capture_active()) { + start_utterance(); /* tap = manual wake */ + } else { + end_utterance(); /* tap = manual end */ + } + } else if (ev == EV_WAKE) { + start_utterance(); + } else if (ev == EV_SPEECH_END) { + end_utterance(); } } } @@ -118,7 +140,7 @@ void app_main(void) { ESP_LOGI(TAG, "DeskLock starting"); - s_touch_sem = xSemaphoreCreateBinary(); + s_events = xQueueCreate(8, sizeof(app_event_t)); bsp_display_start(); bsp_display_backlight_on(); diff --git a/firmware/main/idf_component.yml b/firmware/main/idf_component.yml index ac02367..5a9e024 100644 --- a/firmware/main/idf_component.yml +++ b/firmware/main/idf_component.yml @@ -4,3 +4,4 @@ dependencies: espressif/esp_wifi_remote: "*" espressif/esp_hosted: "^2.12" espressif/esp_websocket_client: "~1.3.0" + espressif/esp-sr: "^2.4" diff --git a/firmware/partitions.csv b/firmware/partitions.csv index 79d7a73..533f9f3 100644 --- a/firmware/partitions.csv +++ b/firmware/partitions.csv @@ -2,3 +2,4 @@ nvs, data, nvs, 0x9000, 0x6000 phy_init, data, phy, 0xf000, 0x1000 factory, app, factory, 0x10000, 8M +model, data, spiffs, , 4M, diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults index 805023e..353c15e 100644 --- a/firmware/sdkconfig.defaults +++ b/firmware/sdkconfig.defaults @@ -63,3 +63,7 @@ CONFIG_ESP_HOSTED_SDIO_1_BIT_BUS=y # absorb the mic-audio upstream burst without stalling the SDIO write CONFIG_ESP_HOSTED_SDIO_TX_Q_SIZE=32 CONFIG_ESP_HOSTED_SDIO_RX_Q_SIZE=32 + +# esp-sr wake word: "Computer" (wn9_computer_tts), models in the flash "model" partition +CONFIG_MODEL_IN_FLASH=y +CONFIG_SR_WN_WN9_COMPUTER_TTS=y