From 576fd7d237266419fea3dd8eadfb3ba54b451d0c Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 16:58:59 +0200 Subject: [PATCH] Initial scaffold: ESP32-P4 firmware + voice gateway for Tatlock endpoint DeskLock gives the Tatlock butler a face and voice on a Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C round display in the living room. - firmware/: ESP-IDF project targeting esp32p4 with the Waveshare XC BSP - gateway/: FastAPI voice bridge (faster-whisper STT, Tatlock chat, Piper TTS) - docs/architecture.md: component design and device<->gateway WS protocol Co-Authored-By: Claude Fable 5 --- .gitignore | 23 +++++++ AGENTS.md | 88 ++++++++++++++++++++++++ CLAUDE.md | 38 ++++++++++ README.md | 72 +++++++++++++++++++ docs/architecture.md | 86 +++++++++++++++++++++++ firmware/CMakeLists.txt | 4 ++ firmware/main/CMakeLists.txt | 4 ++ firmware/main/desklock_main.c | 30 ++++++++ firmware/main/idf_component.yml | 3 + firmware/sdkconfig.defaults | 14 ++++ gateway/Dockerfile | 10 +++ gateway/Makefile | 25 +++++++ gateway/pyproject.toml | 38 ++++++++++ gateway/src/desklock_gateway/__init__.py | 0 gateway/src/desklock_gateway/config.py | 17 +++++ gateway/src/desklock_gateway/main.py | 77 +++++++++++++++++++++ gateway/src/desklock_gateway/stt.py | 24 +++++++ gateway/src/desklock_gateway/tatlock.py | 36 ++++++++++ gateway/src/desklock_gateway/tts.py | 23 +++++++ gateway/tests/test_health.py | 10 +++ 20 files changed, 622 insertions(+) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 docs/architecture.md create mode 100644 firmware/CMakeLists.txt create mode 100644 firmware/main/CMakeLists.txt create mode 100644 firmware/main/desklock_main.c create mode 100644 firmware/main/idf_component.yml create mode 100644 firmware/sdkconfig.defaults create mode 100644 gateway/Dockerfile create mode 100644 gateway/Makefile create mode 100644 gateway/pyproject.toml create mode 100644 gateway/src/desklock_gateway/__init__.py create mode 100644 gateway/src/desklock_gateway/config.py create mode 100644 gateway/src/desklock_gateway/main.py create mode 100644 gateway/src/desklock_gateway/stt.py create mode 100644 gateway/src/desklock_gateway/tatlock.py create mode 100644 gateway/src/desklock_gateway/tts.py create mode 100644 gateway/tests/test_health.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b996706 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# ESP-IDF +firmware/build/ +firmware/managed_components/ +firmware/sdkconfig +firmware/sdkconfig.old +firmware/dependencies.lock +firmware/secrets.h + +# Python +__pycache__/ +*.pyc +.venv/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +*.egg-info/ +dist/ + +# Editors / OS +.vscode/ +.idea/ +*.swp +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..aa69b77 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,88 @@ +# AGENTS.md + +> Operational protocols and architecture for AI assistants working on DeskLock. +> Read [docs/architecture.md](docs/architecture.md) before making design changes. + +## What this project is + +DeskLock is the living-room visual/audio endpoint for **Tatlock**, the homelab butler +(`/mnt/media/Projects/tatlock`, API at `http://tatlock.schweitz.internal:8000`). Two halves, +one repo: + +- `firmware/` — ESP-IDF (C, LVGL 9) app for the Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C + (3.4" round 800×800 touch display, dual mics + ES7210 AEC, ES8311 codec + speaker). +- `gateway/` — Python FastAPI container on tower-of-joy doing STT (faster-whisper, GPU), + chat (Tatlock `/v1/chat/completions`), and TTS (Piper). Listens on port **8600**. + +The device and gateway speak a WebSocket protocol defined in `docs/architecture.md`. +**That doc is the contract** — update it in the same change as any protocol edit on +either side. + +## Hard rules + +- **Keep the firmware thin.** No STT, no TTS, no conversation logic on the device. + If a feature needs intelligence, it goes in the gateway or in Tatlock itself. +- **Never modify Tatlock from this repo.** It is a separate project with its own repo. + DeskLock consumes its public API only. +- **Secrets** (Wi-Fi credentials, any future API keys) never go in source. Firmware + gets them via a gitignored `firmware/secrets.h` (see AGENTS notes below) or NVS; + the gateway via environment variables (`DESKLOCK_*`). + +## Firmware (`firmware/`) + +- Toolchain: **ESP-IDF ≥ 5.4** (not Arduino, not PlatformIO). Target `esp32p4`. +- BSP: [`waveshare/esp32_p4_wifi6_touch_lcd_xc`](https://components.espressif.com/components/waveshare/esp32_p4_wifi6_touch_lcd_xc) + from the ESP Component Registry (pulled automatically via `main/idf_component.yml`). +- Reference implementations: [waveshareteam/ESP32-P4-WIFI6-Touch-LCD-XC](https://github.com/waveshareteam/ESP32-P4-WIFI6-Touch-LCD-XC) + `examples/esp-idf/` — notably `08_lvgl_demo_v9` (display), `06_I2SCodec` (audio), + `04_wifistation` (Wi-Fi via ESP-Hosted). When wiring a new peripheral, check the + official example first; do not guess pin mappings. +- ⚠️ **Unverified scaffold**: the BSP API calls in `desklock_main.c` and the + `sdkconfig.defaults` values were written before the first successful build. Validate + against the official examples on first bring-up, then delete this warning. + +```bash +# One-time: install ESP-IDF (not yet installed on tower-of-joy) +git clone -b v5.5 --recursive https://github.com/espressif/esp-idf.git ~/esp-idf +~/esp-idf/install.sh esp32p4 + +# Every shell: +source ~/esp-idf/export.sh + +# Build / flash / monitor (device is on USB-C; check `ls /dev/ttyACM*`) +cd firmware +idf.py set-target esp32p4 # once +idf.py build +idf.py -p /dev/ttyACM0 flash monitor # Ctrl+] exits monitor +``` + +Flashing requires the `dialout` group (or run with sudo once and fix the group). If the +device doesn't enumerate, hold BOOT while pressing RESET to enter download mode. + +## Gateway (`gateway/`) + +```bash +cd gateway +make setup # venv + dev deps (no ML models) +make setup-speech # additionally install faster-whisper + piper +make run # uvicorn on :8600 with reload +make test # pytest +make lint # ruff check + format check +make typecheck # mypy +``` + +- Config via `DESKLOCK_*` env vars — see `src/desklock_gateway/config.py` for the schema + and defaults. +- `stt.py` / `tts.py` defer their heavy imports so the app boots without the `speech` + extra — keep it that way so protocol tests stay fast. +- Deployment: Docker image built from `gateway/Dockerfile`, deployed like other + tower-of-joy stacks (see `/mnt/media/Projects/system-admin-toj/containers/`). Register + the service + port in `CONTAINERS.md` when it first deploys. + +## Homelab context + +- This server **is** tower-of-joy; the device, gateway, and Tatlock all share the LAN. +- Use `tatlock.schweitz.internal:8000` (direct, no SSO) — the public + `tatlock.schweitz.net` route sits behind Authentik and is not for machine-to-machine + traffic. +- Git remote: `git.schweitz.net` (Gitea). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..781a3d5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,38 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Claude Code-specific notes for this project. For architecture, hard rules, and full +command reference — see [AGENTS.md](AGENTS.md), and read it before starting work. + +## Quick orientation + +DeskLock = firmware for a Waveshare ESP32-P4 round-display device (`firmware/`, ESP-IDF/C/LVGL) +plus a voice gateway container (`gateway/`, Python/FastAPI, port 8600) that bridges device +audio to the Tatlock butler API. The device↔gateway WebSocket protocol lives in +`docs/architecture.md` and must stay in sync with both implementations. + +## Commands + +```bash +# Firmware (requires `source ~/esp-idf/export.sh` first; IDF ≥ 5.4) +cd firmware && idf.py build +idf.py -p /dev/ttyACM0 flash monitor + +# Gateway +cd gateway && make setup # once +make run # dev server :8600 +make test # pytest; single test: .venv/bin/pytest tests/test_health.py -k healthz +make lint typecheck +``` + +## Gotchas + +- **ESP-IDF is not yet installed on this machine** — install instructions in AGENTS.md. +- The firmware scaffold has never been built; treat BSP calls and sdkconfig as + provisional until first successful `idf.py build` (see warning in AGENTS.md). +- The device flashes over USB-C on this server, but it doesn't currently enumerate + (`/dev/ttyACM*` empty) and this user lacks the `dialout` group — resolve both before + attempting to flash. +- Gateway speech deps are optional extras; `make setup` alone runs the app and tests + without GPU/ML packages. diff --git a/README.md b/README.md new file mode 100644 index 0000000..49ac9b4 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# DeskLock + +A living-room visual/audio endpoint for **Tatlock**, the homelab butler. DeskLock gives +Tatlock a face and a voice on a Waveshare round touch display: you talk to it, it listens, +thinks, and answers — the first line of contact with the butler backend running on +tower-of-joy. + +## Hardware + +**Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C** + +| Component | Details | +|-----------|---------| +| SoC | ESP32-P4NRW32 — dual-core RISC-V @ 400 MHz + LP core | +| Memory | 32 MB PSRAM (in-package), 32 MB NOR flash | +| Display | 3.4" round IPS, 800×800, MIPI-DSI 2-lane, capacitive touch | +| Radio | ESP32-C6-MINI-1 (Wi-Fi 6 + BLE 5) over SDIO via ESP-Hosted | +| Audio in | Dual onboard microphones + ES7210 echo-cancellation ADC | +| Audio out | ES8311 codec, PH2.0 2-pin speaker connector (8Ω 2W recommended) | +| Flashing | USB-C (hold BOOT during reset for download mode) | + +The device is currently connected over USB-C directly to tower-of-joy, so build/flash +happens on this server. + +## Architecture + +``` +┌─────────────────────┐ WebSocket (PCM audio + JSON events) +│ DeskLock device │◄──────────────────────────────────────────┐ +│ (ESP32-P4) │ │ +│ │ ┌──────────────────────────────────┴──┐ +│ • LVGL face │ │ DeskLock Gateway (container) │ +│ • Touch input │ │ on tower-of-joy │ +│ • Mic capture+AEC │ │ │ +│ • TTS playback │ │ • STT: faster-whisper (GPU) │ +└─────────────────────┘ │ • TTS: Piper │ + │ • Chat: Tatlock /v1/chat/completions│ + └──────────────────┬───────────────────┘ + │ HTTP (LAN, :8000) + ┌──────────┴──────────┐ + │ Tatlock (butler) │ + │ tatlock.schweitz. │ + │ internal │ + └─────────────────────┘ +``` + +Tatlock stays a text-only brain. The **gateway** is Tatlock's ears and mouth: it converts +speech to text on the way in and text to speech on the way out, keeping the device firmware +thin (audio transport + face rendering only). + +See [docs/architecture.md](docs/architecture.md) for the full design. + +## Repository layout + +- `firmware/` — ESP-IDF (C, LVGL 9) application for the ESP32-P4 +- `gateway/` — Python FastAPI voice gateway, deployed as a container on tower-of-joy +- `docs/` — architecture and design notes + +## Roadmap + +1. **Bring-up** — ESP-IDF toolchain, build/flash, BSP display + Wi-Fi working +2. **Face** — LVGL face with idle/listening/thinking/speaking states, clock while idle +3. **Voice (touch-to-talk)** — tap to talk → gateway → Tatlock → spoken reply +4. **Wake word** — esp-sr WakeNet on-device, echo cancellation, barge-in +5. **Polish** — Tatlock-initiated notifications, presence, OTA updates + +## References + +- [Waveshare wiki: ESP32-P4-WIFI6-Touch-LCD-3.4C](https://www.waveshare.com/wiki/ESP32-P4-WIFI6-Touch-LCD-3.4C) +- [Official examples repo (waveshareteam/ESP32-P4-WIFI6-Touch-LCD-XC)](https://github.com/waveshareteam/ESP32-P4-WIFI6-Touch-LCD-XC) +- [BSP component: waveshare/esp32_p4_wifi6_touch_lcd_xc](https://components.espressif.com/components/waveshare/esp32_p4_wifi6_touch_lcd_xc) +- Tatlock backend: `/mnt/media/Projects/tatlock` — https://tatlock.schweitz.net diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..aa566b5 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,86 @@ +# DeskLock Architecture + +## Goal + +A always-on, glanceable butler face in the living room. You speak to it; it relays your +words to Tatlock and speaks the reply back, with a face that reflects what it's doing +(idle, listening, thinking, speaking). It is deliberately a *thin* endpoint: all +intelligence lives in Tatlock, all heavy audio processing lives in the gateway. + +## Components + +### 1. Firmware (`firmware/`) — ESP32-P4 + +Responsibilities: + +- **Face rendering** (LVGL 9 on the 800×800 round MIPI-DSI panel via the + `waveshare/esp32_p4_wifi6_touch_lcd_xc` BSP). Face states: + - `idle` — subtle animation + clock (it's a desk clock when nobody's talking to it) + - `listening` — visual feedback that the mic is hot + - `thinking` — Tatlock is working on a reply + - `speaking` — mouth/waveform animation synced to TTS playback +- **Audio capture**: dual mics through the ES7210 (hardware echo cancellation reference + from the playback path), 16 kHz 16-bit mono PCM. +- **Audio playback**: ES8311 codec → speaker. Plays PCM streamed from the gateway. +- **Transport**: a single WebSocket to the gateway carrying binary PCM frames plus JSON + control events (`state`, `transcript`, `reply_text`, errors). Device reconnects with + backoff; face shows a disconnected state when the gateway is unreachable. +- **Interaction**: phase 1 is touch-to-talk (tap the face). Phase 2 adds esp-sr WakeNet + wake word on the P4 so the interaction is hands-free. + +Non-responsibilities: no STT, no TTS, no conversation state. If it's not rendering, +recording, or playing, it doesn't belong in firmware. + +### 2. Gateway (`gateway/`) — container on tower-of-joy + +A FastAPI service bridging device audio to Tatlock text: + +1. Accepts the device WebSocket (`/ws/voice`). +2. Buffers inbound PCM until end-of-utterance (client-signalled in phase 1; VAD later). +3. **STT**: faster-whisper on the RTX 2080 Ti. +4. **Chat**: POSTs the transcript to Tatlock `/v1/chat/completions` + (`http://tatlock.schweitz.internal:8000`, OpenAI-compatible, streaming). Maintains the + conversation id so follow-ups have context. +5. **TTS**: Piper (fast, CPU-friendly, local) synthesizes the reply. +6. Streams reply PCM back to the device along with `reply_text` for on-screen display. + +The gateway is stateless apart from in-flight conversations; it can restart freely. + +### 3. Tatlock — existing backend (`/mnt/media/Projects/tatlock`) + +Untouched by this project. DeskLock consumes its OpenAI-compatible API over the internal +LAN (port 8000, bypassing the Authentik-protected public route). If device auth is needed +later, the gateway holds the credential — never the firmware. + +## WebSocket protocol (device ↔ gateway) + +Binary frames: raw 16 kHz s16le mono PCM (mic upstream, TTS downstream). +Text frames: JSON control messages. + +``` +device → gateway: {"type": "utterance_start"} +device → gateway: +device → gateway: {"type": "utterance_end"} +gateway → device: {"type": "state", "value": "thinking"} +gateway → device: {"type": "transcript", "text": "..."} +gateway → device: {"type": "reply_text", "text": "..."} +gateway → device: {"type": "audio_start", "sample_rate": 16000} +gateway → device: +gateway → device: {"type": "audio_end"} +``` + +Keep this protocol documented here and mirrored in `firmware/` and `gateway/` constants — +it is the one contract between the two halves of the repo. + +## Key decisions & rationale + +- **ESP-IDF native (not Arduino/ESPHome)**: the P4 + MIPI-DSI + esp-sr stack is only + first-class in ESP-IDF; Waveshare recommends it, and the BSP targets it. +- **Gateway owns STT/TTS (not the device)**: the P4 could run small STT models, but + server-side whisper is dramatically better, and Piper voices beat embedded TTS. The + GPU is already there. Wake word is the only speech task that must be on-device. +- **Separate gateway (not extending Tatlock)**: keeps Tatlock's API text-only and clean; + audio concerns (codecs, VAD, streaming) stay at the edge. The gateway is also where a + future second endpoint (kitchen, office) would connect. +- **Monorepo**: the WS protocol couples firmware and gateway; versioning them together + avoids contract drift. diff --git a/firmware/CMakeLists.txt b/firmware/CMakeLists.txt new file mode 100644 index 0000000..41fbca0 --- /dev/null +++ b/firmware/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(desklock) diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt new file mode 100644 index 0000000..84f7f27 --- /dev/null +++ b/firmware/main/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register( + SRCS "desklock_main.c" + INCLUDE_DIRS "." +) diff --git a/firmware/main/desklock_main.c b/firmware/main/desklock_main.c new file mode 100644 index 0000000..c266dd2 --- /dev/null +++ b/firmware/main/desklock_main.c @@ -0,0 +1,30 @@ +/* + * DeskLock — living-room face and voice for the Tatlock butler. + * + * Boot sequence (planned): + * 1. BSP init: display + LVGL, touch, audio codec (ES8311/ES7210) + * 2. Wi-Fi up via ESP32-C6 (ESP-Hosted over SDIO) + * 3. WebSocket connection to the DeskLock gateway + * 4. Face state machine: idle / listening / thinking / speaking + */ + +#include "esp_log.h" +#include "bsp/esp-bsp.h" + +static const char *TAG = "desklock"; + +void app_main(void) +{ + ESP_LOGI(TAG, "DeskLock starting"); + + bsp_display_start(); + bsp_display_backlight_on(); + + /* TODO(bring-up): render placeholder face via LVGL + * TODO(bring-up): Wi-Fi via ESP-Hosted (C6) + * TODO(voice): mic capture (ES7210) -> WebSocket upstream + * TODO(voice): gateway PCM downstream -> ES8311 playback + */ + + ESP_LOGI(TAG, "DeskLock up (display only)"); +} diff --git a/firmware/main/idf_component.yml b/firmware/main/idf_component.yml new file mode 100644 index 0000000..bc788cf --- /dev/null +++ b/firmware/main/idf_component.yml @@ -0,0 +1,3 @@ +dependencies: + idf: ">=5.4" + waveshare/esp32_p4_wifi6_touch_lcd_xc: "^3.0.1" diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults new file mode 100644 index 0000000..f58aa73 --- /dev/null +++ b/firmware/sdkconfig.defaults @@ -0,0 +1,14 @@ +# Target: Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C +CONFIG_IDF_TARGET="esp32p4" + +# 32 MB NOR flash +CONFIG_ESPTOOLPY_FLASHSIZE_32MB=y +CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y + +# 32 MB in-package PSRAM (hex mode) — LVGL frame buffers live here +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_HEX=y +CONFIG_SPIRAM_SPEED_200M=y + +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_FREERTOS_HZ=1000 diff --git a/gateway/Dockerfile b/gateway/Dockerfile new file mode 100644 index 0000000..e006877 --- /dev/null +++ b/gateway/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY pyproject.toml ./ +COPY src ./src +RUN pip install --no-cache-dir ".[speech]" + +EXPOSE 8600 +CMD ["uvicorn", "desklock_gateway.main:app", "--host", "0.0.0.0", "--port", "8600"] diff --git a/gateway/Makefile b/gateway/Makefile new file mode 100644 index 0000000..2c59179 --- /dev/null +++ b/gateway/Makefile @@ -0,0 +1,25 @@ +.PHONY: setup run test lint typecheck clean + +setup: + python3 -m venv .venv + .venv/bin/pip install -e ".[dev]" + +setup-speech: + .venv/bin/pip install -e ".[dev,speech]" + +run: + .venv/bin/uvicorn desklock_gateway.main:app --host 0.0.0.0 --port 8600 --reload + +test: + .venv/bin/pytest + +lint: + .venv/bin/ruff check src tests + .venv/bin/ruff format --check src tests + +typecheck: + .venv/bin/mypy src + +clean: + rm -rf .venv .pytest_cache .ruff_cache .mypy_cache + find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml new file mode 100644 index 0000000..7500093 --- /dev/null +++ b/gateway/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "desklock-gateway" +version = "0.1.0" +description = "Voice gateway bridging the DeskLock device to the Tatlock butler (STT/chat/TTS)" +requires-python = ">=3.11" +dependencies = [ + "fastapi>=0.115", + "uvicorn[standard]>=0.30", + "httpx>=0.27", + "pydantic-settings>=2.4", +] + +[project.optional-dependencies] +speech = [ + "faster-whisper>=1.0", + "piper-tts>=1.2", +] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.24", + "ruff>=0.6", + "mypy>=1.11", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 100 +src = ["src"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/gateway/src/desklock_gateway/__init__.py b/gateway/src/desklock_gateway/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gateway/src/desklock_gateway/config.py b/gateway/src/desklock_gateway/config.py new file mode 100644 index 0000000..996cdf4 --- /dev/null +++ b/gateway/src/desklock_gateway/config.py @@ -0,0 +1,17 @@ +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + """Gateway configuration, overridable via DESKLOCK_* environment variables.""" + + tatlock_base_url: str = "http://tatlock.schweitz.internal:8000" + tatlock_model: str = "Tatlock" + sample_rate: int = 16000 + stt_model: str = "small" + stt_device: str = "cuda" + tts_voice: str = "en_GB-alan-medium" + + model_config = {"env_prefix": "DESKLOCK_"} + + +settings = Settings() diff --git a/gateway/src/desklock_gateway/main.py b/gateway/src/desklock_gateway/main.py new file mode 100644 index 0000000..04c0ed1 --- /dev/null +++ b/gateway/src/desklock_gateway/main.py @@ -0,0 +1,77 @@ +"""DeskLock gateway: WebSocket voice bridge between the device and Tatlock. + +Protocol (see docs/architecture.md — keep in sync): + device -> gateway : {"type": "utterance_start"} + binary PCM + {"type": "utterance_end"} + gateway -> device : state / transcript / reply_text events, then audio_start + PCM + audio_end +""" + +import asyncio +import logging + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect + +from . import stt, tts +from .config import settings +from .tatlock import TatlockClient + +logger = logging.getLogger("desklock.gateway") + +app = FastAPI(title="DeskLock Gateway") + +PCM_CHUNK_BYTES = 4096 + + +@app.get("/healthz") +async def healthz() -> dict[str, str]: + return {"status": "ok"} + + +@app.websocket("/ws/voice") +async def voice(ws: WebSocket) -> None: + await ws.accept() + tatlock = TatlockClient() + pcm = bytearray() + recording = False + try: + while True: + message = await ws.receive() + if "bytes" in message and message["bytes"] is not None: + if recording: + pcm.extend(message["bytes"]) + continue + if "text" not in message or message["text"] is None: + continue + + import json + + event = json.loads(message["text"]) + if event["type"] == "utterance_start": + pcm.clear() + recording = True + elif event["type"] == "utterance_end": + recording = False + await _handle_utterance(ws, tatlock, bytes(pcm)) + except WebSocketDisconnect: + logger.info("device disconnected") + finally: + await tatlock.aclose() + + +async def _handle_utterance(ws: WebSocket, tatlock: TatlockClient, pcm: bytes) -> None: + await ws.send_json({"type": "state", "value": "thinking"}) + + transcript = await asyncio.to_thread(stt.transcribe, pcm, settings.sample_rate) + await ws.send_json({"type": "transcript", "text": transcript}) + if not transcript: + await ws.send_json({"type": "state", "value": "idle"}) + return + + reply = await tatlock.ask(transcript) + await ws.send_json({"type": "reply_text", "text": reply}) + + audio = await asyncio.to_thread(tts.synthesize, reply) + await ws.send_json({"type": "audio_start", "sample_rate": settings.sample_rate}) + for offset in range(0, len(audio), PCM_CHUNK_BYTES): + await ws.send_bytes(audio[offset : offset + PCM_CHUNK_BYTES]) + await ws.send_json({"type": "audio_end"}) + await ws.send_json({"type": "state", "value": "idle"}) diff --git a/gateway/src/desklock_gateway/stt.py b/gateway/src/desklock_gateway/stt.py new file mode 100644 index 0000000..c1168d4 --- /dev/null +++ b/gateway/src/desklock_gateway/stt.py @@ -0,0 +1,24 @@ +"""Speech-to-text: faster-whisper on the tower-of-joy GPU. + +Import of faster_whisper is deferred so the gateway can run (health checks, +protocol tests) without the heavy speech extras installed. +""" + +from .config import settings + +_model = None + + +def transcribe(pcm: bytes, sample_rate: int | None = None) -> str: + """Transcribe raw s16le mono PCM to text.""" + global _model + if _model is None: + from faster_whisper import WhisperModel + + _model = WhisperModel(settings.stt_model, device=settings.stt_device) + + import numpy as np + + audio = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + segments, _info = _model.transcribe(audio, language="en") + return " ".join(segment.text.strip() for segment in segments).strip() diff --git a/gateway/src/desklock_gateway/tatlock.py b/gateway/src/desklock_gateway/tatlock.py new file mode 100644 index 0000000..f7f2395 --- /dev/null +++ b/gateway/src/desklock_gateway/tatlock.py @@ -0,0 +1,36 @@ +"""Client for the Tatlock butler's OpenAI-compatible chat API.""" + +import httpx + +from .config import settings + + +class TatlockClient: + def __init__(self, base_url: str | None = None) -> None: + self._client = httpx.AsyncClient( + base_url=base_url or settings.tatlock_base_url, + timeout=httpx.Timeout(120.0, connect=5.0), + ) + self._history: list[dict[str, str]] = [] + + async def ask(self, text: str) -> str: + """Send one user utterance, return Tatlock's reply, keeping conversation history.""" + self._history.append({"role": "user", "content": text}) + response = await self._client.post( + "/v1/chat/completions", + json={ + "model": settings.tatlock_model, + "messages": self._history, + "stream": False, + }, + ) + response.raise_for_status() + reply = response.json()["choices"][0]["message"]["content"] + self._history.append({"role": "assistant", "content": reply}) + return reply + + def reset(self) -> None: + self._history.clear() + + async def aclose(self) -> None: + await self._client.aclose() diff --git a/gateway/src/desklock_gateway/tts.py b/gateway/src/desklock_gateway/tts.py new file mode 100644 index 0000000..b1f366c --- /dev/null +++ b/gateway/src/desklock_gateway/tts.py @@ -0,0 +1,23 @@ +"""Text-to-speech: Piper, resampled to the device sample rate. + +Import of piper is deferred so the gateway can run without the speech extras. +""" + +from .config import settings + +_voice = None + + +def synthesize(text: str) -> bytes: + """Synthesize text to raw s16le mono PCM at the configured sample rate.""" + global _voice + if _voice is None: + from piper import PiperVoice + + _voice = PiperVoice.load(settings.tts_voice) + + chunks = bytearray() + for chunk in _voice.synthesize_stream_raw(text): + chunks.extend(chunk) + # TODO: resample from the Piper voice's native rate to settings.sample_rate + return bytes(chunks) diff --git a/gateway/tests/test_health.py b/gateway/tests/test_health.py new file mode 100644 index 0000000..5096daa --- /dev/null +++ b/gateway/tests/test_health.py @@ -0,0 +1,10 @@ +from fastapi.testclient import TestClient + +from desklock_gateway.main import app + + +def test_healthz() -> None: + client = TestClient(app) + response = client.get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "ok"}