Phase 5: hands-free "Computer" wake word (esp-sr WakeNet + AFE VAD)

Say "Computer" -> chime + listening -> speak -> AFE VAD detects you
stopped -> auto-sends. No taps. Touch still works as a manual override.

- esp-sr 2.4.6 added; wn9_computer_tts model packed into a new "model"
  flash partition (MODEL_IN_FLASH). App moved to 8M, model 4M.
- audio.c: replaced the on-demand capture_task with an AFE pipeline —
  feed_task is the SOLE mic reader (-> afe->feed); detect_task fetches,
  watches wakeup_state for the wake word and vad_state for end of
  speech, and forwards AFE-cleaned audio upstream during an utterance.
  One mic reader ever.
- short rising chime acknowledges the wake audibly.

Fixes from adversarial review before trusting it:
1. utterance framing (blocking WS sends) moved OFF the AFE fetch thread
   onto an app_task event queue (EV_TOUCH/EV_WAKE/EV_SPEECH_END) — a
   1.5s send could stall fetch and drop the first ~1.5s of speech.
2. app_task is now the single serializer of start/end -> no TOCTOU
   double-start (was: two utterance_start on a tap during wake).
3. VAD accounting resets on every streaming (re)start (wake OR tap),
   not just wake -> a tapped utterance can no longer end instantly on
   stale silence.
4. chime/reply set s_playing (+DMA tail hold) and detect_task skips the
   mic while s_playing -> our own audio no longer streams into STT or
   false-triggers the wake at a playback boundary (no AEC yet).
5. NULL-checked AFE create + feed buffer; tasks only start if AFE is up.

Verified on hardware: model loads, AFE inits with the Computer word,
boots and connects clean, no crash/wedge.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

firmware/main/CMakeLists.txt

@@ -12,7 +12,7 @@ idf_component_register(
         "fonts/font_rage_64.c"
     INCLUDE_DIRS "."
     EMBED_FILES "c6_slave.bin"
-    PRIV_REQUIRES nvs_flash esp_wifi esp_event esp_netif json esp_timer esp_app_format esp_http_server
+    PRIV_REQUIRES nvs_flash esp_wifi esp_event esp_netif json esp_timer esp_app_format esp_http_server esp-sr
 )
 
 target_compile_definitions(${COMPONENT_LIB} PRIVATE LV_LVGL_H_INCLUDE_SIMPLE)

firmware/main/audio.c

@@ -11,10 +11,18 @@
 #include "bsp/esp-bsp.h"
 #include "esp_codec_dev.h"
 
+#include "esp_afe_config.h"
+#include "esp_afe_sr_iface.h"
+#include "esp_afe_sr_models.h"
+#include "esp_wn_models.h"
+
 #include "desklock.h"
 
 static const char *TAG = "audio";
 
+/* end an utterance after this much post-speech silence (AFE VAD) */
+#define VAD_END_SILENCE_MS 800
+
 #define RATE 16000
 #define CAPTURE_CHUNK 3200 /* 100 ms */
 #define REPLY_MAX (RATE * 2 * 60) /* 60 s of reply audio */
@@ -31,11 +39,15 @@ static int16_t *s_gong;
 static uint8_t *s_reply;
 static volatile size_t s_reply_len;
 static volatile bool s_playing;
-static volatile bool s_capturing;
 
-typedef enum { JOB_GONG, JOB_REPLY } job_t;
+/* wake word + utterance (esp-sr AFE) */
+static const esp_afe_sr_iface_t *s_afe;
+static esp_afe_sr_data_t *s_afe_data;
+static volatile bool s_streaming;   /* detect_task streams AFE audio upstream while true */
+static int16_t *s_chime;
+
+typedef enum { JOB_GONG, JOB_REPLY, JOB_CHIME } job_t;
 static QueueHandle_t s_jobs;
-static TaskHandle_t s_capture_task;
 
 /* --- gong: see docs/architecture.md "Sound signature" --- */
 
@@ -107,40 +119,149 @@ static void synth_gong(void)
     free(dry);
 }
 
+/* Short rising two-tone chime played when the wake word fires (you're often
+ * not looking at the face, so acknowledge audibly). ~180 ms. */
+#define CHIME_SAMPLES (RATE / 5)
+static void synth_chime(void)
+{
+    s_chime = heap_caps_malloc(CHIME_SAMPLES * sizeof(int16_t), MALLOC_CAP_SPIRAM);
+    if (s_chime == NULL) {
+        return;
+    }
+    float phase = 0.0f;
+    for (int i = 0; i < CHIME_SAMPLES; i++) {
+        float freq = (i < CHIME_SAMPLES / 2) ? 660.0f : 990.0f;
+        phase += 2.0f * (float)M_PI * freq / RATE;
+        float env = 1.0f - (float)i / CHIME_SAMPLES;
+        s_chime[i] = (int16_t)(sinf(phase) * 9000.0f * env);
+    }
+}
+
 /* --- playback worker --- */
 
 static void audio_task(void *arg)
 {
     (void)arg;
     synth_gong();
-    ESP_LOGI(TAG, "gong ready");
+    synth_chime();
+    ESP_LOGI(TAG, "gong+chime ready");
     job_t job;
     while (xQueueReceive(s_jobs, &job, portMAX_DELAY) == pdTRUE) {
+        /* s_playing gates the mic path: while our own speaker is active, the
+         * detect_task neither wakes nor streams (no AEC), so we don't hear
+         * ourselves. The tail delay covers the DMA that plays after write()
+         * returns, preventing a playback-boundary false wake. */
         if (job == JOB_GONG && s_gong != NULL) {
             esp_codec_dev_write(s_spk, s_gong, GONG_SAMPLES * sizeof(int16_t));
+        } else if (job == JOB_CHIME && s_chime != NULL) {
+            s_playing = true;
+            esp_codec_dev_write(s_spk, s_chime, CHIME_SAMPLES * sizeof(int16_t));
+            vTaskDelay(pdMS_TO_TICKS(120));
+            s_playing = false;
         } else if (job == JOB_REPLY) {
             s_playing = true;
             esp_codec_dev_write(s_spk, s_reply, s_reply_len);
+            vTaskDelay(pdMS_TO_TICKS(250));
             s_playing = false;
             app_on_playback_done();
         }
     }
 }
 
-/* One persistent capture task, created once. It parks on a notification until
- * capture is requested, then streams mic -> gateway while s_capturing holds.
- * This makes capture single-instance: no create/delete restart race, so a rapid
- * re-tap (or a WS-disconnect stop racing an app_task start) can never put two
- * readers on the one mic/I2S handle. */
-static void capture_task(void *arg)
+/* AFE wake-word + VAD pipeline. Two tasks:
+ *  - feed_task: the SOLE mic reader. Reads mic -> afe->feed(). Always running.
+ *  - detect_task: afe->fetch() -> when ARMED, watch for the wake word; when
+ *    streaming an utterance, forward AFE-cleaned audio upstream and watch VAD
+ *    for end-of-speech. One mic reader ever — no two-readers-on-one-mic hazard. */
+
+static bool afe_init(void)
+{
+    srmodel_list_t *models = esp_srmodel_init("model");
+    char *wn = models ? esp_srmodel_filter(models, ESP_WN_PREFIX, NULL) : NULL;
+
+    afe_config_t *cfg = afe_config_init("M", models, AFE_TYPE_SR, AFE_MODE_LOW_COST);
+    if (cfg == NULL) {
+        ESP_LOGE(TAG, "afe_config_init failed");
+        return false;
+    }
+    cfg->aec_init = false;   /* no echo canceller in the always-on path (barge-in is later) */
+    cfg->se_init = false;    /* single logical mic, no beamforming */
+    cfg->ns_init = false;
+    cfg->agc_init = false;
+    cfg->vad_init = true;    /* VAD gives us hands-free end-of-utterance */
+    cfg->vad_mode = VAD_MODE_1;
+    cfg->vad_min_speech_ms = 128;
+    cfg->vad_min_noise_ms = 600;
+    cfg->wakenet_init = (wn != NULL);
+    cfg->wakenet_model_name = wn;
+    cfg->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
+
+    s_afe = esp_afe_handle_from_config(cfg);
+    s_afe_data = s_afe ? s_afe->create_from_config(cfg) : NULL;
+    if (s_afe == NULL || s_afe_data == NULL) {
+        ESP_LOGE(TAG, "AFE create failed");
+        return false;
+    }
+    ESP_LOGI(TAG, "AFE up, wake word = %s", wn ? wn : "NONE");
+    return true;
+}
+
+static void feed_task(void *arg)
 {
     (void)arg;
-    uint8_t *chunk = heap_caps_malloc(CAPTURE_CHUNK, MALLOC_CAP_DEFAULT);
+    int nch = s_afe->get_feed_channel_num(s_afe_data);
+    int nsamp = s_afe->get_feed_chunksize(s_afe_data);
+    int bytes = nsamp * nch * sizeof(int16_t);
+    int16_t *buf = heap_caps_malloc(bytes, MALLOC_CAP_DEFAULT);
+    if (buf == NULL) {
+        ESP_LOGE(TAG, "feed buffer alloc failed");
+        vTaskDelete(NULL);
+    }
     for (;;) {
-        ulTaskNotifyTake(pdTRUE, portMAX_DELAY);
-        while (s_capturing) {
-            if (esp_codec_dev_read(s_mic, chunk, CAPTURE_CHUNK) == ESP_CODEC_DEV_OK) {
-                gw_send_bin(chunk, CAPTURE_CHUNK);
+        if (esp_codec_dev_read(s_mic, buf, bytes) == ESP_CODEC_DEV_OK) {
+            s_afe->feed(s_afe_data, buf);
+        }
+    }
+}
+
+static void detect_task(void *arg)
+{
+    (void)arg;
+    bool was_streaming = false;
+    bool had_speech = false;
+    int silence_ms = 0;
+    for (;;) {
+        afe_fetch_result_t *res = s_afe->fetch(s_afe_data);
+        if (res == NULL || res->ret_value == ESP_FAIL) {
+            continue;
+        }
+        int frame_ms = res->data_size / (int)sizeof(int16_t) / (RATE / 1000);
+
+        /* reset VAD accounting whenever streaming (re)starts — from a wake word
+         * OR a tap — so a stale silence count can't end the next utterance early */
+        if (s_streaming && !was_streaming) {
+            had_speech = false;
+            silence_ms = 0;
+        }
+        was_streaming = s_streaming;
+
+        if (!s_streaming) {
+            /* ARMED: listen for the wake word (not while our speaker is active) */
+            if (!s_playing && res->wakeup_state == WAKENET_DETECTED) {
+                app_on_wake();
+            }
+        } else if (!s_playing) {
+            /* UTTERANCE: forward cleaned audio, end on post-speech silence.
+             * Skipped while s_playing so the chime/reply doesn't self-stream. */
+            gw_send_bin((const uint8_t *)res->data, res->data_size);
+            if (res->vad_state == VAD_SPEECH) {
+                had_speech = true;
+                silence_ms = 0;
+            } else {
+                silence_ms += frame_ms;
+            }
+            if (had_speech && silence_ms >= VAD_END_SILENCE_MS) {
+                app_on_speech_end();
             }
         }
     }
@@ -166,28 +287,41 @@ void audio_init(void)
     s_reply = heap_caps_malloc(REPLY_MAX, MALLOC_CAP_SPIRAM);
     s_jobs = xQueueCreate(4, sizeof(job_t));
     xTaskCreate(audio_task, "audio", 4096, NULL, 5, NULL);
-    xTaskCreate(capture_task, "capture", 4096, NULL, 6, &s_capture_task);
+
+    if (s_mic != NULL && afe_init()) {
+        xTaskCreate(feed_task, "afe_feed", 4096, NULL, 6, NULL);
+        xTaskCreate(detect_task, "afe_detect", 8192, NULL, 5, NULL);
+    }
     ESP_LOGI(TAG, "audio up (spk=%d mic=%d)", s_spk != NULL, s_mic != NULL);
 }
 
+void audio_play_chime(void)
+{
+    job_t job = JOB_CHIME;
+    xQueueSend(s_jobs, &job, 0);
+}
+
 void audio_play_gong(void)
 {
     job_t job = JOB_GONG;
     xQueueSend(s_jobs, &job, 0);
 }
 
+/* Streaming = the detect_task forwards AFE audio upstream. Wake word and touch
+ * both begin it; VAD silence and touch both end it. */
 void audio_capture_start(void)
 {
-    if (s_mic == NULL || s_capturing || s_capture_task == NULL) {
-        return;
-    }
-    s_capturing = true;
-    xTaskNotifyGive(s_capture_task); /* wake the persistent task's inner loop */
+    s_streaming = true;
 }
 
 void audio_capture_stop(void)
 {
-    s_capturing = false;
+    s_streaming = false;
+}
+
+bool audio_capture_active(void)
+{
+    return s_streaming;
 }
 
 void audio_playback_begin(void)

firmware/main/desklock.h

@@ -26,8 +26,10 @@ void face_status(const char *text);   /* bottom status line (diag/boot) */
 /* audio.c */
 void audio_init(void);
 void audio_play_gong(void);
-void audio_capture_start(void);
+void audio_play_chime(void);            /* short wake acknowledgement */
+void audio_capture_start(void);         /* begin streaming the utterance upstream */
 void audio_capture_stop(void);
+bool audio_capture_active(void);
 void audio_playback_begin(void);
 void audio_playback_feed(const uint8_t *data, size_t len);
 void audio_playback_end(void);
@@ -50,5 +52,7 @@ void c6_ota_start(void);
 
 /* desklock_main.c */
 void app_on_touch(void);
+void app_on_wake(void);         /* wake word detected (from audio detect_task) */
+void app_on_speech_end(void);   /* AFE VAD detected end of utterance */
 void app_on_disconnect(void);
 void app_on_playback_done(void);

firmware/main/desklock_main.c

@@ -12,6 +12,7 @@
 #include "esp_log.h"
 #include "esp_system.h"
 #include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
 #include "freertos/semphr.h"
 #include "freertos/task.h"
 #include "bsp/esp-bsp.h"
@@ -23,59 +24,80 @@
 
 static const char *TAG = "desklock";
 
-static SemaphoreHandle_t s_touch_sem;
-static bool s_talking;
+/* All utterance framing (blocking WS sends, chime, face) runs on app_task, never
+ * on the LVGL touch callback nor the AFE detect/fetch thread. The wake word, VAD,
+ * and touch just POST an event; app_task is the single serializer of start/end,
+ * which removes both the fetch-thread stall and the double-start race. */
+typedef enum { EV_TOUCH, EV_WAKE, EV_SPEECH_END } app_event_t;
+static QueueHandle_t s_events;
 
-/* Called from the LVGL touch callback — MUST be fast and non-blocking. */
-void app_on_touch(void)
+static void post_event(app_event_t ev)
 {
-    if (s_touch_sem != NULL) {
-        xSemaphoreGive(s_touch_sem);
+    if (s_events != NULL) {
+        xQueueSend(s_events, &ev, 0);
     }
 }
 
-/* Called from the WS event task when the link drops: forget any half-utterance
- * so the next tap starts a fresh one instead of hitting the "stop" branch. */
-void app_on_disconnect(void)
-{
-    s_talking = false;
-}
+/* --- these three are the fast, non-blocking hooks called from other tasks --- */
+void app_on_touch(void) { post_event(EV_TOUCH); }        /* LVGL touch callback */
+void app_on_wake(void) { post_event(EV_WAKE); }          /* AFE detect_task: wake word */
+void app_on_speech_end(void) { post_event(EV_SPEECH_END); } /* AFE detect_task: VAD silence */
 
-/* Runs in app_task (normal context): blocking sends here don't stall rendering. */
-static void handle_touch(void)
+/* --- the actual work, all on app_task --- */
+static void start_utterance(void)
 {
-    face_activity();
-
-    if (!gw_connected()) {
-        s_talking = false; /* link gone; nothing to talk to */
+    if (!gw_connected() || audio_is_playing() || audio_capture_active()
+            || face_get() == FACE_EFFORT) {
         return;
     }
-    if (!s_talking) {
-        if (audio_is_playing() || face_get() == FACE_EFFORT) {
-            return; /* barge-in is phase 4 */
-        }
-        s_talking = true;
-        gw_send_text("{\"type\":\"utterance_start\"}");
-        audio_capture_start();
-        face_set(FACE_LISTENING);
-        ESP_LOGI(TAG, "listening…");
-    } else {
-        s_talking = false;
-        audio_capture_stop();
-        gw_send_text("{\"type\":\"utterance_end\"}");
-        face_set(FACE_PENSIVE);
-        ESP_LOGI(TAG, "utterance sent");
+    face_activity();
+    audio_play_chime();                              /* audible "I heard you" */
+    gw_send_text("{\"type\":\"utterance_start\"}");   /* must precede any audio frames */
+    audio_capture_start();                           /* now detect_task forwards audio */
+    face_set(FACE_LISTENING);
+    ESP_LOGI(TAG, "listening…");
+}
+
+static void end_utterance(void)
+{
+    if (!audio_capture_active()) {
+        return;
     }
+    audio_capture_stop();
+    gw_send_text("{\"type\":\"utterance_end\"}");
+    face_set(FACE_PENSIVE);
+    ESP_LOGI(TAG, "utterance sent");
+}
+
+/* WS link dropped: abandon any half-utterance so the next wake starts fresh. */
+void app_on_disconnect(void)
+{
+    audio_capture_stop();
 }
 
 static void app_task(void *arg)
 {
     (void)arg;
+    app_event_t ev;
     for (;;) {
-        if (xSemaphoreTake(s_touch_sem, portMAX_DELAY) == pdTRUE) {
-            vTaskDelay(pdMS_TO_TICKS(60));   /* debounce */
-            xSemaphoreTake(s_touch_sem, 0);  /* coalesce repeats during debounce */
-            handle_touch();
+        if (xQueueReceive(s_events, &ev, portMAX_DELAY) != pdTRUE) {
+            continue;
+        }
+        if (ev == EV_TOUCH) {
+            vTaskDelay(pdMS_TO_TICKS(60));       /* debounce */
+            xQueueReset(s_events);               /* coalesce a bouncy tap burst */
+            face_activity();
+            if (!gw_connected()) {
+                audio_capture_stop();
+            } else if (!audio_capture_active()) {
+                start_utterance();               /* tap = manual wake */
+            } else {
+                end_utterance();                 /* tap = manual end */
+            }
+        } else if (ev == EV_WAKE) {
+            start_utterance();
+        } else if (ev == EV_SPEECH_END) {
+            end_utterance();
         }
     }
 }
@@ -118,7 +140,7 @@ void app_main(void)
 {
     ESP_LOGI(TAG, "DeskLock starting");
 
-    s_touch_sem = xSemaphoreCreateBinary();
+    s_events = xQueueCreate(8, sizeof(app_event_t));
 
     bsp_display_start();
     bsp_display_backlight_on();

firmware/main/idf_component.yml

@@ -4,3 +4,4 @@ dependencies:
   espressif/esp_wifi_remote: "*"
   espressif/esp_hosted: "^2.12"
   espressif/esp_websocket_client: "~1.3.0"
+  espressif/esp-sr: "^2.4"

firmware/partitions.csv

@@ -2,3 +2,4 @@
 nvs,      data, nvs,     0x9000,  0x6000
 phy_init, data, phy,     0xf000,  0x1000
 factory,  app,  factory, 0x10000, 8M
+model,    data, spiffs,  ,        4M,

firmware/sdkconfig.defaults

@@ -63,3 +63,7 @@ CONFIG_ESP_HOSTED_SDIO_1_BIT_BUS=y
 # absorb the mic-audio upstream burst without stalling the SDIO write
 CONFIG_ESP_HOSTED_SDIO_TX_Q_SIZE=32
 CONFIG_ESP_HOSTED_SDIO_RX_Q_SIZE=32
+
+# esp-sr wake word: "Computer" (wn9_computer_tts), models in the flash "model" partition
+CONFIG_MODEL_IN_FLASH=y
+CONFIG_SR_WN_WN9_COMPUTER_TTS=y