From f5df5049db85fd147585d4f29678636c9fd9f2dd Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 17:54:30 +0200 Subject: [PATCH] Add rage table-flip state; wire gateway to live Speaches; add CI pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Face: new 'rage' state — 3-frame kaomoji loop (stare, flip the table, put it back) for in-flight request failures; 'error' stays the quiet persistent face for a dead link. Sim + artifact + design doc updated. - Gateway: stt.py/tts.py are now pluggable backends. Default 'speaches' talks OpenAI-format HTTP to the live container on :8601 (faster-whisper-small STT, Kokoro bm_george TTS with 24->16 kHz audioop resample); 'embedded' fallback kept behind the [speech] extra. Verified with a live TTS->STT round trip (warm: STT 0.27s, TTS 1.9s). Docker image is now slim (no CUDA/ML deps). Python pinned to 3.12 (system 3.8 too old, audioop gone in 3.13). - CI: .gitea/workflows/build.yml — lint+test on main pushes; on v* tags test, build gateway image, push to registry, release, and trigger Watchtower (tatlock pattern; needs REGISTRY_USER/REGISTRY_PASSWORD/ WATCHTOWER_TOKEN secrets). Runtime stack in deploy/desklock-gateway.yml. - architecture.md: measured speech latencies, deployed-Speaches status, CI & deployment section. Co-Authored-By: Claude Fable 5 --- .gitea/workflows/build.yml | 71 ++++++++++++++++++++++++++ AGENTS.md | 22 +++++--- deploy/desklock-gateway.yml | 21 ++++++++ docs/architecture.md | 44 ++++++++++++---- gateway/Dockerfile | 4 +- gateway/Makefile | 5 +- gateway/src/desklock_gateway/config.py | 19 +++++-- gateway/src/desklock_gateway/stt.py | 55 ++++++++++++++++---- gateway/src/desklock_gateway/tts.py | 57 ++++++++++++++++++--- sim/face/index.html | 26 +++++++++- 10 files changed, 283 insertions(+), 41 deletions(-) create mode 100644 .gitea/workflows/build.yml create mode 100644 deploy/desklock-gateway.yml diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..32d9945 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,71 @@ +name: Test, Build and Push + +on: + push: + branches: + - main + tags: + - 'v[0-9]*' + +jobs: + test-gateway: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install + run: pip install -e "./gateway[dev]" + + - name: Lint and test + working-directory: gateway + run: | + ruff check src tests + ruff format --check src tests + pytest + + release: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Create Gitea Release + run: | + curl -sf -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \ + "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases" + + build-gateway: + runs-on: ubuntu-latest + needs: test-gateway + if: startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + + - name: Login to Gitea Registry + uses: docker/login-action@v3 + with: + registry: git.schweitz.internal + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: gateway + push: true + provenance: false + sbom: false + tags: | + git.schweitz.internal/jpmschweitzer/desklock-gateway:latest + git.schweitz.internal/jpmschweitzer/desklock-gateway:${{ github.ref_name }} + + - name: Trigger Watchtower update + if: success() + run: | + curl -sf -H "Authorization: Bearer ${{ secrets.WATCHTOWER_TOKEN }}" \ + http://watchtower:8080/v1/update diff --git a/AGENTS.md b/AGENTS.md index 9171c12..e9a1d45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,10 +13,11 @@ one repo: (3.4" round 800×800 touch display, dual mics + ES7210 AEC, ES8311 codec + speaker). - `gateway/` — Python FastAPI container on tower-of-joy orchestrating STT → chat (Tatlock `/v1/chat/completions`) → TTS. Listens on port **8600**. STT/TTS models live - in a shared **Speaches** container (proposed port 8601, OpenAI-format API), not in the - gateway image; `stt.py`/`tts.py` are pluggable backends (`speaches` default, - `embedded` fallback for dev). See docs/architecture.md — the scaffold currently - implements only `embedded`. + in the shared **Speaches** container (live on port 8601, OpenAI-format API), not in + the gateway image; `stt.py`/`tts.py` are pluggable backends (`speaches` default, + `embedded` fallback needing the `[speech]` extra). Gateway runs on **Python 3.12 + exactly** — system python3 on tower-of-joy is 3.8, and `audioop` (used for TTS + resampling) is removed in 3.13. The device and gateway speak a WebSocket protocol defined in `docs/architecture.md`. **That doc is the contract** — update it in the same change as any protocol edit on @@ -87,9 +88,16 @@ make typecheck # mypy and defaults. - `stt.py` / `tts.py` defer their heavy imports so the app boots without the `speech` extra — keep it that way so protocol tests stay fast. -- Deployment: Docker image built from `gateway/Dockerfile`, deployed like other - tower-of-joy stacks (see `/mnt/media/Projects/system-admin-toj/containers/`). Register - the service + port in `CONTAINERS.md` when it first deploys. +- Deployment is CI-driven: pushing a `v*` tag makes Gitea Actions test, build, and push + `desklock-gateway:{latest,tag}` to the registry and trigger Watchtower + (`.gitea/workflows/build.yml`; needs `REGISTRY_USER`/`REGISTRY_PASSWORD`/ + `WATCHTOWER_TOKEN` secrets). Plain pushes to `main` run lint + tests only. The stack + file is `deploy/desklock-gateway.yml` — copy into + `system-admin-toj/containers/stacks/` and register the service + port 8600 in + `CONTAINERS.md` on first deploy. +- Verify speech changes against the live Speaches container with a real round trip + (TTS → STT of a known phrase, expect the transcript back); warm timings to expect: + STT ~0.3 s, TTS ~2 s per sentence. ## Homelab context diff --git a/deploy/desklock-gateway.yml b/deploy/desklock-gateway.yml new file mode 100644 index 0000000..d5333cb --- /dev/null +++ b/deploy/desklock-gateway.yml @@ -0,0 +1,21 @@ +# DeskLock gateway stack. +# To deploy: copy into system-admin-toj/containers/stacks/ (Portainer) and +# register the service + port 8600 in CONTAINERS.md. +# +# Image is built and pushed by .gitea/workflows/build.yml on version tags; +# Watchtower picks up :latest afterwards. +services: + desklock-gateway: + image: git.schweitz.internal/jpmschweitzer/desklock-gateway:latest + container_name: desklock-gateway + restart: unless-stopped + ports: + - "8600:8600" + environment: + # LAN IP of tower-of-joy: *.schweitz.internal resolves to localhost on the + # host, which is wrong inside a container. Switch to http://speaches:8000 + # (and the tatlock service name) if this stack joins their docker networks. + - DESKLOCK_TATLOCK_BASE_URL=http://192.168.86.149:8000 + - DESKLOCK_SPEACHES_BASE_URL=http://192.168.86.149:8601 + labels: + - com.centurylinklabs.watchtower.enable=true diff --git a/docs/architecture.md b/docs/architecture.md index 789097c..b7320e9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -87,9 +87,6 @@ models — the container stays a slim pure-Python image with no CUDA/ML dependen - `embedded` — in-process faster-whisper / Piper. Kept as a fallback so the gateway can run standalone (dev on a laptop, speech container down), at the cost of a fat image. -> **Status note:** the initial scaffold implements only the `embedded` path; the -> backend switch and Speaches client are the next gateway task. - The gateway is stateless apart from in-flight conversations; it can restart freely. ### 3. Speech layer — Speaches (container, GPU) @@ -99,9 +96,15 @@ is a self-hosted, OpenAI-API-compatible speech server: STT via faster-whisper, T Kokoro/Piper, dynamic model load/offload with a TTL, and a `/v1/realtime` WebSocket API we may adopt later for streaming transcription. -- **Deployment**: its own stack in `system-admin-toj/containers/stacks/`, GPU-enabled. - Proposed host port **8601** (verified free; register in `CONTAINERS.md` at deploy). - LAN-only like the Tatlock internal route — do not expose through NPM without auth. +- **Deployed 2026-07-14**: `ghcr.io/speaches-ai/speaches:latest-cuda` on host port + **8601**, with `Systran/faster-whisper-small` (STT) and + `speaches-ai/Kokoro-82M-v1.0-ONNX` (TTS, 24 kHz — the gateway resamples to the + 16 kHz device contract; default voice `bm_george`, en-GB male). LAN-only like the + Tatlock internal route — do not expose through NPM without auth. Register in + `CONTAINERS.md`. +- **Measured** (live round trip through the gateway code, warm): STT ~0.3 s for a + ~3 s utterance; TTS ~1.9 s for a ~3 s sentence. Cold start after model TTL offload + adds ~5–10 s to the first request. - **Why a shared layer instead of models inside the gateway**: one GPU-resident model instance serves the whole homelab. Open WebUI is currently configured with `AUDIO_STT_ENGINE=openai` / `AUDIO_TTS_ENGINE=openai` (OpenAI *cloud*) — pointing its @@ -137,6 +140,7 @@ ported to LVGL. The `STATES` table in the sim defines the contract: | `pensive` | `· ·` | `~` | 7 streams | cycling `...` thought dots | | `effort` | `> <` | `~` | 40 fast streams | **orbit arc on bezel + `[ Ns ]` elapsed counter**, face jitter | | `speaking` | `^ ^` | cycles `o O - O = o` | 14 streams | mouth animates ~150 ms/frame | +| `rage` | — | — | 34 fast streams | 3-frame kaomoji loop through the eyes slot: `(°□°) ┬─┬` → `(╯°□°)╯︵ ┻━┻` → `┬─┬ ノ( º_º ノ)` — flips the table, then composes itself and puts it back | | `error` | `x x` | `-` | none (rain dies) | face dims to 45% | **Wait cues are a hard requirement** (user-stated): Tatlock turns take 10–25 s, so @@ -145,8 +149,9 @@ elapsed-seconds counter, and max rain. Never a bare static face during a wait, a fake progress bars — only honest cues. **Protocol → face mapping**: gateway `state: thinking` → `effort`; transcription and -other short local waits → `pensive`; `listening`/`speaking` map 1:1; WebSocket -disconnected → `error`; otherwise `idle`. +other short local waits → `pensive`; `listening`/`speaking` map 1:1; an in-flight +request failure (STT/Tatlock/TTS error) → `rage` for a few loops, then `idle`; +WebSocket disconnected → `error` (quiet, persistent); otherwise `idle`. **LVGL port notes** (for phase 2): @@ -155,6 +160,7 @@ disconnected → `error`; otherwise `idle`. - Rain: `lv_canvas` (or a pooled label grid) with per-frame fade; orbit arc = `lv_arc`. - Fonts: generate a large monospace glyph font including the katakana subset used in `GLYPHS` via `lv_font_conv`; the built-in `unscii` fonts are too small for 800 px. + The `rage` frames additionally need `╯ ︵ ┻ ━ ┬ ─ ノ ° □ º` in the subset. - The sim's text glow (`text-shadow`) is browser flair — the device renders flat glyphs. ## Latency budget & streaming @@ -164,8 +170,8 @@ GPU-resident benchmarks of 2026-07-14, gemma4:e2b at ~100 tok/s): | Stage | Cost | |-------|------| -| STT (whisper `small`, GPU) | a few hundred ms for a ~5 s utterance | -| TTS (Piper/Kokoro) | faster than realtime | +| STT (Speaches whisper `small`) | ~0.3 s warm (measured) | +| TTS (Speaches Kokoro) | ~1.9 s per ~3 s sentence, warm (measured) | | Tatlock Steward analysis | ~6 s warm | | **Tatlock, full local flow** | **11–25 s end-to-end** (librarian-routed ~20–25 s) | | Tatlock cold start (>2 h idle) | +~8 s (`OLLAMA_KEEP_ALIVE=2h`) | @@ -239,3 +245,21 @@ it is the one contract between the two halves of the repo. gateway is also where a future second endpoint (kitchen, office) would connect. - **Monorepo**: the WS protocol couples firmware and gateway; versioning them together avoids contract drift. + +## CI & deployment + +Gitea Actions (`.gitea/workflows/build.yml`), following the tatlock/tatlock-ui pattern: + +- **Every push to `main`**: lint + tests for the gateway (Python 3.12). +- **Version tags (`v0.1.0`, …)**: tests, then build `gateway/` into + `git.schweitz.internal/jpmschweitzer/desklock-gateway:{latest,tag}`, push to the + Gitea registry, create a release, and trigger Watchtower to roll the running + container. +- Required repo/org secrets: `REGISTRY_USER`, `REGISTRY_PASSWORD`, + `WATCHTOWER_TOKEN` (same trio tatlock uses). +- The runtime stack definition lives in `deploy/desklock-gateway.yml`; copy it into + `system-admin-toj/containers/stacks/` to deploy, and register port 8600 in + `CONTAINERS.md`. + +Firmware is not containerized: it's flashed over USB (`idf.py flash`), with OTA planned +for phase 5. diff --git a/gateway/Dockerfile b/gateway/Dockerfile index e006877..c3fa203 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -4,7 +4,9 @@ WORKDIR /app COPY pyproject.toml ./ COPY src ./src -RUN pip install --no-cache-dir ".[speech]" +# slim by design: STT/TTS models live in the Speaches container, not this image. +# For the embedded fallback backend build with ".[speech]" instead. +RUN pip install --no-cache-dir . EXPOSE 8600 CMD ["uvicorn", "desklock_gateway.main:app", "--host", "0.0.0.0", "--port", "8600"] diff --git a/gateway/Makefile b/gateway/Makefile index 2c59179..4a9bdc5 100644 --- a/gateway/Makefile +++ b/gateway/Makefile @@ -1,7 +1,10 @@ .PHONY: setup run test lint typecheck clean +# audioop pins us below 3.13; system python3 on tower-of-joy is 3.8 +PYTHON ?= python3.12 + setup: - python3 -m venv .venv + $(PYTHON) -m venv .venv .venv/bin/pip install -e ".[dev]" setup-speech: diff --git a/gateway/src/desklock_gateway/config.py b/gateway/src/desklock_gateway/config.py index 996cdf4..166a2d5 100644 --- a/gateway/src/desklock_gateway/config.py +++ b/gateway/src/desklock_gateway/config.py @@ -6,10 +6,23 @@ class Settings(BaseSettings): tatlock_base_url: str = "http://tatlock.schweitz.internal:8000" tatlock_model: str = "Tatlock" + + # PCM rate of the device WebSocket contract (docs/architecture.md) sample_rate: int = 16000 - stt_model: str = "small" - stt_device: str = "cuda" - tts_voice: str = "en_GB-alan-medium" + + # "speaches" (shared speech container) or "embedded" (in-process models) + stt_backend: str = "speaches" + tts_backend: str = "speaches" + + speaches_base_url: str = "http://localhost:8601" + stt_model: str = "Systran/faster-whisper-small" + tts_model: str = "speaches-ai/Kokoro-82M-v1.0-ONNX" + tts_voice: str = "bm_george" + + # embedded fallback only (requires the [speech] extra) + embedded_stt_model: str = "small" + embedded_stt_device: str = "cuda" + embedded_tts_voice: str = "en_GB-alan-medium" model_config = {"env_prefix": "DESKLOCK_"} diff --git a/gateway/src/desklock_gateway/stt.py b/gateway/src/desklock_gateway/stt.py index c1168d4..aaf95be 100644 --- a/gateway/src/desklock_gateway/stt.py +++ b/gateway/src/desklock_gateway/stt.py @@ -1,24 +1,61 @@ -"""Speech-to-text: faster-whisper on the tower-of-joy GPU. +"""Speech-to-text: Speaches over HTTP (default) or embedded faster-whisper. -Import of faster_whisper is deferred so the gateway can run (health checks, -protocol tests) without the heavy speech extras installed. +The embedded import is deferred so the gateway runs without the heavy +[speech] extras installed. """ +import io +import wave + +import httpx + from .config import settings -_model = None +_client: httpx.Client | None = None +_embedded_model = None -def transcribe(pcm: bytes, sample_rate: int | None = None) -> str: +def transcribe(pcm: bytes, sample_rate: int) -> str: """Transcribe raw s16le mono PCM to text.""" - global _model - if _model is None: + if settings.stt_backend == "speaches": + return _transcribe_speaches(pcm, sample_rate) + return _transcribe_embedded(pcm) + + +def _wav_bytes(pcm: bytes, sample_rate: int) -> bytes: + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(sample_rate) + w.writeframes(pcm) + return buf.getvalue() + + +def _transcribe_speaches(pcm: bytes, sample_rate: int) -> str: + global _client + if _client is None: + _client = httpx.Client(base_url=settings.speaches_base_url, timeout=60.0) + response = _client.post( + "/v1/audio/transcriptions", + files={"file": ("utterance.wav", _wav_bytes(pcm, sample_rate), "audio/wav")}, + data={"model": settings.stt_model, "language": "en"}, + ) + response.raise_for_status() + return response.json()["text"].strip() + + +def _transcribe_embedded(pcm: bytes) -> str: + global _embedded_model + if _embedded_model is None: from faster_whisper import WhisperModel - _model = WhisperModel(settings.stt_model, device=settings.stt_device) + _embedded_model = WhisperModel( + settings.embedded_stt_model, device=settings.embedded_stt_device + ) import numpy as np audio = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 - segments, _info = _model.transcribe(audio, language="en") + segments, _info = _embedded_model.transcribe(audio, language="en") return " ".join(segment.text.strip() for segment in segments).strip() diff --git a/gateway/src/desklock_gateway/tts.py b/gateway/src/desklock_gateway/tts.py index b1f366c..72d7465 100644 --- a/gateway/src/desklock_gateway/tts.py +++ b/gateway/src/desklock_gateway/tts.py @@ -1,23 +1,64 @@ -"""Text-to-speech: Piper, resampled to the device sample rate. +"""Text-to-speech: Speaches/Kokoro over HTTP (default) or embedded Piper. -Import of piper is deferred so the gateway can run without the speech extras. +Output is always s16le mono PCM at settings.sample_rate (the device WS contract). +Kokoro synthesizes at 24 kHz, so the speaches path resamples via audioop — +which pins the runtime to Python 3.12 (audioop is removed in 3.13). """ +import audioop +import io +import wave + +import httpx + from .config import settings -_voice = None +_client: httpx.Client | None = None +_embedded_voice = None def synthesize(text: str) -> bytes: """Synthesize text to raw s16le mono PCM at the configured sample rate.""" - global _voice - if _voice is None: + if settings.tts_backend == "speaches": + return _synthesize_speaches(text) + return _synthesize_embedded(text) + + +def _synthesize_speaches(text: str) -> bytes: + global _client + if _client is None: + _client = httpx.Client(base_url=settings.speaches_base_url, timeout=120.0) + response = _client.post( + "/v1/audio/speech", + json={ + "model": settings.tts_model, + "voice": settings.tts_voice, + "input": text, + "response_format": "wav", + }, + ) + response.raise_for_status() + + with wave.open(io.BytesIO(response.content), "rb") as w: + rate, channels, width = w.getframerate(), w.getnchannels(), w.getsampwidth() + frames = w.readframes(w.getnframes()) + if width != 2: + frames = audioop.lin2lin(frames, width, 2) + if channels == 2: + frames = audioop.tomono(frames, 2, 0.5, 0.5) + if rate != settings.sample_rate: + frames, _state = audioop.ratecv(frames, 2, 1, rate, settings.sample_rate, None) + return frames + + +def _synthesize_embedded(text: str) -> bytes: + global _embedded_voice + if _embedded_voice is None: from piper import PiperVoice - _voice = PiperVoice.load(settings.tts_voice) + _embedded_voice = PiperVoice.load(settings.embedded_tts_voice) chunks = bytearray() - for chunk in _voice.synthesize_stream_raw(text): + for chunk in _embedded_voice.synthesize_stream_raw(text): chunks.extend(chunk) - # TODO: resample from the Piper voice's native rate to settings.sample_rate return bytes(chunks) diff --git a/sim/face/index.html b/sim/face/index.html index 18d09a6..f5c4816 100644 --- a/sim/face/index.html +++ b/sim/face/index.html @@ -74,6 +74,9 @@ body[data-state="effort"] #elapsed { display: block; } @keyframes spin { to { transform: rotate(360deg); } } + body[data-state="rage"] #eyes { font-size: 70px; } + body[data-state="rage"] #mouth { display: none; } + @keyframes breathe { 0%,100% { transform: translateY(0); } 50% { transform: translateY(9px); } } @keyframes jitter { 0% { transform: translate(1px,-1px); } 100% { transform: translate(-1px,1px); } } @@ -116,9 +119,13 @@ const STATES = { effort: { eyes: "> <", mouth: "~", blink: false, rain: { streams: 40, speed: 16 } }, speaking: { eyes: "^ ^", mouth: "o", blink: true, talk: true, rain: { streams: 14, speed: 9 } }, + rage: { mouth: "", blink: false, rain: { streams: 34, speed: 20 }, + frames: [ { t: "(°□°) ┬─┬", ms: 900 }, + { t: "(╯°□°)╯︵ ┻━┻", ms: 1300 }, + { t: "┬─┬ ノ( º_º ノ)", ms: 1400 } ] }, error: { eyes: "x x", mouth: "-", blink: false, rain: { streams: 0, speed: 0 } }, }; -const ORDER = ["idle", "listening", "pensive", "effort", "speaking", "error"]; +const ORDER = ["idle", "listening", "pensive", "effort", "speaking", "rage", "error"]; const TALK = ["o", "O", "-", "O", "=", "o"]; const GLYPHS = "アイウエオカキクケコサシスセソタチツテトナニヌネノ0123456789ACEFHKZ$#%*+=<>"; @@ -161,12 +168,27 @@ function frame(now) { /* ---- face behaviour ---- */ function applyFace() { - eyesEl.textContent = st().eyes; + eyesEl.textContent = st().eyes || ""; mouthEl.textContent = st().mouth; thoughtEl.textContent = ""; document.body.dataset.state = cur; document.querySelectorAll("#controls button[data-state]").forEach( (b) => b.classList.toggle("active", b.dataset.state === cur)); + playFrames(); +} + +/* whole-line kaomoji sequences (rage) render through the eyes slot */ +let animGen = 0; +function playFrames() { + const gen = ++animGen; + const frames = st().frames; + if (!frames) return; + let i = 0; + (function step() { + if (gen !== animGen) return; + eyesEl.textContent = frames[i % frames.length].t; + setTimeout(step, frames[i++ % frames.length].ms); + })(); } function setState(name) { cur = name; stateSince = performance.now(); applyFace(); }