feat(gateway): voice command service
Recognize simple device commands in the transcript and act on them without
a Tatlock round-trip. commands.match() maps volume up/down, mute/unmute,
"set volume to N", and "goes to eleven"/max to a "command" message sent
straight to the device; the utterance never reaches the LLM. Matching is
deliberately precise so real requests ("set an alarm for a quarter to
eleven") are not hijacked. Adds the "command" message to the device
protocol in docs/architecture.md.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LKPbR6DY2JygHbyLjxm7Uu
This commit is contained in:
@@ -10,6 +10,9 @@ until the first tagged release.
|
||||
|
||||
### Added
|
||||
|
||||
- Spoken volume commands ("volume up/down", "mute/unmute", "set volume to N",
|
||||
"this one goes to eleven") are handled instantly on the gateway, bypassing the
|
||||
assistant — no waiting on a reply just to change the volume.
|
||||
- Tap the screen to reveal on-screen controls — a microphone button plus volume
|
||||
down/up and a live level bar, using Phosphor icon glyphs. Tapping the dimmed
|
||||
backdrop dismisses them.
|
||||
|
||||
@@ -266,8 +266,17 @@ gateway → device: {"type": "reply_text", "text": "..."}
|
||||
gateway → device: {"type": "audio_start", "sample_rate": 16000}
|
||||
gateway → device: <binary PCM frames> (may arrive sentence-by-sentence; play as a stream)
|
||||
gateway → device: {"type": "audio_end"}
|
||||
|
||||
gateway → device: {"type": "command", "action": "volume_up"} (LLM-bypass; see below)
|
||||
```
|
||||
|
||||
`command` (gateway → device) is an **alternative to the reply path**: when the
|
||||
gateway recognizes a simple device command in the transcript (volume/mute), it
|
||||
sends a `command` instead of calling Tatlock — no `reply_text`/audio — then returns
|
||||
to `idle`. Actions: `volume_up`, `volume_down`, `mute`, `unmute`, and `volume_set`
|
||||
with an extra `"level"` field (0–11, the on-device volume scale). Matched by the
|
||||
gateway's `commands.py`; applied on the device in `gw_client.c` → `face.c`.
|
||||
|
||||
Planned additions (documented before implemented, here first):
|
||||
|
||||
- `reply_delta` (gateway → device): incremental reply text for on-screen streaming while
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Voice command service.
|
||||
|
||||
Intercepts simple device commands from the STT transcript so they bypass the LLM
|
||||
(no 10-25s Tatlock round-trip). `match()` returns a gateway->device ``command``
|
||||
message dict, or ``None`` if the transcript is not a recognized command and should
|
||||
go to Tatlock.
|
||||
|
||||
Matching is deliberately precise, not clever: a false positive hijacks a real
|
||||
request, so we only fire on explicit volume/mute wording. Extend ``match()`` with
|
||||
new rules as the command vocabulary grows.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
VOL_MAX = 11
|
||||
|
||||
_WORD_NUMBERS = {
|
||||
"zero": 0,
|
||||
"one": 1,
|
||||
"two": 2,
|
||||
"three": 3,
|
||||
"four": 4,
|
||||
"five": 5,
|
||||
"six": 6,
|
||||
"seven": 7,
|
||||
"eight": 8,
|
||||
"nine": 9,
|
||||
"ten": 10,
|
||||
"eleven": 11,
|
||||
}
|
||||
|
||||
Command = dict[str, str | int]
|
||||
|
||||
|
||||
def _norm(text: str) -> str:
|
||||
"""Lower-case, drop punctuation, collapse whitespace."""
|
||||
return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", text.lower())).strip()
|
||||
|
||||
|
||||
def _has(t: str, *words: str) -> bool:
|
||||
return any(re.search(rf"\b{w}\b", t) for w in words)
|
||||
|
||||
|
||||
def _to_int(tok: str) -> int | None:
|
||||
if tok.isdigit():
|
||||
return int(tok)
|
||||
return _WORD_NUMBERS.get(tok)
|
||||
|
||||
|
||||
def _cmd(action: str, level: int | None = None) -> Command:
|
||||
msg: Command = {"type": "command", "action": action}
|
||||
if level is not None:
|
||||
msg["level"] = max(0, min(VOL_MAX, level))
|
||||
return msg
|
||||
|
||||
|
||||
def match(transcript: str) -> Command | None:
|
||||
"""Return a ``command`` message if the transcript is a device command, else None."""
|
||||
t = _norm(transcript)
|
||||
if not t:
|
||||
return None
|
||||
|
||||
# mute / unmute — check unmute first ("unmute" contains "mute")
|
||||
if _has(t, "unmute"):
|
||||
return _cmd("unmute")
|
||||
if _has(t, "mute"):
|
||||
return _cmd("mute")
|
||||
|
||||
# explicit "set volume to N" (digit or number word), requires the word "volume"
|
||||
m = re.search(r"\bvolume\s+(?:to\s+|at\s+|is\s+)?(\w+)\b", t)
|
||||
if m:
|
||||
val = _to_int(m.group(1))
|
||||
if val is not None:
|
||||
return _cmd("volume_set", level=val)
|
||||
|
||||
# max / the meme — "this one goes to eleven", "crank it", "max volume"
|
||||
if (
|
||||
re.search(r"\bgoes to eleven\b", t)
|
||||
or _has(t, "crank")
|
||||
or (_has(t, "max", "maximum", "full") and _has(t, "volume", "sound"))
|
||||
):
|
||||
return _cmd("volume_set", level=VOL_MAX)
|
||||
|
||||
# relative up / down — need explicit volume/sound context or an unambiguous verb
|
||||
vol_ctx = _has(t, "volume", "sound")
|
||||
if _has(t, "louder") or (vol_ctx and _has(t, "up")) or re.search(r"\bturn it up\b", t):
|
||||
return _cmd("volume_up")
|
||||
if (
|
||||
_has(t, "quieter", "softer")
|
||||
or (vol_ctx and _has(t, "down"))
|
||||
or re.search(r"\bturn it down\b", t)
|
||||
):
|
||||
return _cmd("volume_down")
|
||||
|
||||
return None
|
||||
@@ -11,7 +11,7 @@ from datetime import datetime, timezone
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
|
||||
from . import stt, tts
|
||||
from . import commands, stt, tts
|
||||
from .config import settings
|
||||
from .tatlock import TatlockClient
|
||||
|
||||
@@ -104,6 +104,13 @@ async def _handle_utterance(ws: WebSocket, tatlock: TatlockClient, pcm: bytes) -
|
||||
await ws.send_json({"type": "state", "value": "idle"})
|
||||
return
|
||||
|
||||
# Simple device commands (volume/mute) bypass the LLM entirely.
|
||||
command = commands.match(transcript)
|
||||
if command is not None:
|
||||
await ws.send_json(command)
|
||||
await ws.send_json({"type": "state", "value": "idle"})
|
||||
return
|
||||
|
||||
reply = await tatlock.ask(transcript)
|
||||
await ws.send_json({"type": "reply_text", "text": reply})
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
from desklock_gateway import commands
|
||||
|
||||
|
||||
def test_mute_and_unmute() -> None:
|
||||
assert commands.match("mute") == {"type": "command", "action": "mute"}
|
||||
assert commands.match("mute the volume") == {"type": "command", "action": "mute"}
|
||||
# "unmute" contains "mute" — must resolve to unmute
|
||||
assert commands.match("unmute") == {"type": "command", "action": "unmute"}
|
||||
|
||||
|
||||
def test_relative_up_down() -> None:
|
||||
assert commands.match("volume up") == {"type": "command", "action": "volume_up"}
|
||||
assert commands.match("louder please") == {"type": "command", "action": "volume_up"}
|
||||
assert commands.match("turn it up") == {"type": "command", "action": "volume_up"}
|
||||
assert commands.match("volume down") == {"type": "command", "action": "volume_down"}
|
||||
assert commands.match("a bit quieter") == {"type": "command", "action": "volume_down"}
|
||||
assert commands.match("turn it down") == {"type": "command", "action": "volume_down"}
|
||||
|
||||
|
||||
def test_set_to_number_digit_and_word() -> None:
|
||||
assert commands.match("set volume to 7") == {
|
||||
"type": "command",
|
||||
"action": "volume_set",
|
||||
"level": 7,
|
||||
}
|
||||
assert commands.match("volume to seven") == {
|
||||
"type": "command",
|
||||
"action": "volume_set",
|
||||
"level": 7,
|
||||
}
|
||||
assert commands.match("volume 3") == {"type": "command", "action": "volume_set", "level": 3}
|
||||
# clamps into 0..11
|
||||
assert commands.match("set volume to 50") == {
|
||||
"type": "command",
|
||||
"action": "volume_set",
|
||||
"level": 11,
|
||||
}
|
||||
|
||||
|
||||
def test_goes_to_eleven() -> None:
|
||||
assert commands.match("this one goes to eleven") == {
|
||||
"type": "command",
|
||||
"action": "volume_set",
|
||||
"level": 11,
|
||||
}
|
||||
assert commands.match("set volume to eleven") == {
|
||||
"type": "command",
|
||||
"action": "volume_set",
|
||||
"level": 11,
|
||||
}
|
||||
assert commands.match("crank it") == {"type": "command", "action": "volume_set", "level": 11}
|
||||
assert commands.match("max volume") == {"type": "command", "action": "volume_set", "level": 11}
|
||||
|
||||
|
||||
def test_non_commands_fall_through_to_llm() -> None:
|
||||
# these must NOT be hijacked from Tatlock
|
||||
assert commands.match("what's the weather tomorrow") is None
|
||||
assert commands.match("set an alarm for a quarter to eleven") is None
|
||||
assert commands.match("turn up the heating in the lounge") is None
|
||||
assert commands.match("") is None
|
||||
assert commands.match("tell me a joke") is None
|
||||
Reference in New Issue
Block a user