diff --git a/.claude/skills/device-screenshot/SKILL.md b/.claude/skills/device-screenshot/SKILL.md new file mode 100644 index 0000000..6c8dcbc --- /dev/null +++ b/.claude/skills/device-screenshot/SKILL.md @@ -0,0 +1,83 @@ +--- +name: device-screenshot +description: > + Capture a screenshot of the live DeskLock ESP32-P4 display over USB and view + it as a PNG. Use whenever you need to SEE what's on the device screen — verify + a face/UI change, check the tap controls overlay, confirm a state (idle, + listening, effort), or debug layout remotely without the user's camera. + Triggers: "screenshot the device", "what's on the screen", "capture the + display", "show me the face", "did the UI change land". +--- + +# DeskLock device screenshot over USB + +Grabs the current LVGL screen off the device and rebuilds it as a PNG on the +host. No camera, no gateway — it rides the USB serial that's already attached for +flashing. + +## How it works (so you can debug it) + +The firmware (`main/face.c` `face_screenshot_dump`, gated by `DESKLOCK_DEVMODE` +in `desklock_main.c`) runs a task that watches the **USB-serial-JTAG RX FIFO** +for a trigger byte, renders the active screen with `lv_snapshot_take_to_draw_buf` +into a PSRAM buffer, 2×-downscales to 400×400, and streams it as **raw binary** +straight to the USB-serial-JTAG TX FIFO — framed by a text header +`###SHOT_BEGIN … bytes=N crc=0x… bin=1###` + N bytes + `###SHOT_END###`. The host +tool `firmware/tools/device_shot.py` sends the trigger, reads N bytes, checks the +CRC (retrying on the rare dropped frame), and writes a PNG. + +Two things this design is deliberately built around, learned the hard way: +- **Don't use `printf`.** The primary console is UART at 115200 baud (~11 KB/s) — + a frame would take ~37 s. Writing straight to the USB FIFO runs at USB speed + (~1 s). That's why the dump uses `usb_serial_jtag_ll_write_txfifo`, not stdout. +- **Opening the port does NOT reset the P4** (unlike esptool), and there's no USB + stdin, so the trigger is a byte the firmware polls for — on-demand, no reboot, + captures whatever state is currently on screen. + +## Prerequisites + +- Firmware flashed with **dev mode ON** — it's OFF by default (production spends + no internal RAM on the watcher and nothing extra runs on the render path). Flash + a dev build with: + ```bash + cd firmware && sg dialout -c "bash -c 'source ~/esp-idf/export.sh && idf.py -DDESKLOCK_DEVMODE=ON -p /dev/ttyACM0 flash'" + ``` + If screenshots return "no frame", the flashed build is production — reflash with + `-DDESKLOCK_DEVMODE=ON`. Back to production: `-DDESKLOCK_DEVMODE=OFF` (the value + sticks in the CMake cache until you flip it). Leave it ON for a UI-dev session + (you're reflashing for UI changes anyway); flip OFF for the final/production flash. +- Device on `/dev/ttyACM0`. The port needs the `dialout` group, so run the tool + under `sg dialout -c '…'` (this login session predates dialout membership). +- Nothing else holding the port (no `idf.py monitor` running) — one owner only. + +## Take a shot + +From the `desklock` repo root (`/mnt/media/Projects/desklock`): + +```bash +# current screen (whatever state the device is in right now) +sg dialout -c "python3 firmware/tools/device_shot.py /tmp/shot.png" + +# force the tap controls overlay in-frame (mic + volume) — for verifying it +# remotely since you can't physically tap +sg dialout -c "python3 firmware/tools/device_shot.py /tmp/shot.png --overlay" +``` + +Then **Read `/tmp/shot.png`** to view it. A clean run prints +`[try 1] 400x400 320000B crc … OK` and takes ~5 s. + +Trigger bytes: `s` = current screen, `o` = force overlay. The tool retries up to +4× on a CRC mismatch (occasional console contention), so a transient bad frame +self-heals. + +## Notes / caveats + +- **RAM / render path:** the watcher costs one ~5 KB internal-RAM task, so it's + behind `DESKLOCK_DEVMODE` (off by default). Internal RAM is the scarce resource + on this board (it caps the rain sprite pool). Never ship a production build with + it on. +- 400×400 is plenty for layout/UI checks. To change resolution, adjust `SHOT_DS` + in `face.c` (the downscale factor) — the host reads the size from the header. +- If you just reflashed, wait ~7 s for boot before the first shot. +- Only the USB console path is used; this does not touch the device↔gateway + WebSocket protocol. diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 5f849b3..2a1d38c 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -17,3 +17,14 @@ idf_component_register( ) target_compile_definitions(${COMPONENT_LIB} PRIVATE LV_LVGL_H_INCLUDE_SIMPLE) + +# Dev-only features (screenshot watcher). OFF in production so nothing extra runs +# on the render path / spends internal RAM. Deploy dev with: +# idf.py -DDESKLOCK_DEVMODE=ON build flash +# and back to production with -DDESKLOCK_DEVMODE=OFF (the value sticks in the +# CMake cache until you flip it). +option(DESKLOCK_DEVMODE "Enable dev-only features (screenshot watcher)" OFF) +if(DESKLOCK_DEVMODE) + target_compile_definitions(${COMPONENT_LIB} PRIVATE DESKLOCK_DEVMODE=1) + message(STATUS "DeskLock: DEVMODE ON — screenshot watcher enabled") +endif() diff --git a/firmware/main/desklock.h b/firmware/main/desklock.h index a4be3d2..30716a1 100644 --- a/firmware/main/desklock.h +++ b/firmware/main/desklock.h @@ -22,6 +22,7 @@ void face_set(face_state_t state); /* safe from any task */ face_state_t face_get(void); void face_activity(void); void face_status(const char *text); /* bottom status line (diag/boot) */ /* reset the power ladder to active */ +void face_screenshot_dump(bool overlay); /* dev: raw RGB565 screen dump over USB */ /* audio.c */ void audio_init(void); diff --git a/firmware/main/desklock_main.c b/firmware/main/desklock_main.c index 9202caa..65eedcf 100644 --- a/firmware/main/desklock_main.c +++ b/firmware/main/desklock_main.c @@ -30,6 +30,16 @@ * 0 = normal app. */ #define FACE_LOADTEST 0 +/* Dev mode: enabled at deploy time with `idf.py -DDESKLOCK_DEVMODE=ON build flash` + * (see main/CMakeLists.txt), OFF by default in production. Currently gates only the + * screenshot watcher — a task that watches the USB-serial-JTAG RX for a trigger + * byte and dumps the current screen (see the device-screenshot skill / + * firmware/tools/device_shot.py). Off by default so production spends no internal + * RAM on it and nothing extra runs on the render path. */ +#ifndef DESKLOCK_DEVMODE +#define DESKLOCK_DEVMODE 0 +#endif + static const char *TAG = "desklock"; /* All utterance framing (blocking WS sends, chime, face) runs on app_task, never @@ -232,6 +242,29 @@ void sdio_tx_selftest_kick(void) } #endif +#if DESKLOCK_DEVMODE +#include "hal/usb_serial_jtag_ll.h" +/* Watch the USB-serial-JTAG RX FIFO (LL reads, no driver install -> no conflict + * with the secondary console's TX) for a trigger byte and dump one frame. */ +static void screenshot_task(void *arg) +{ + (void)arg; + vTaskDelay(pdMS_TO_TICKS(6000)); /* let boot + first render settle */ + ESP_LOGI(TAG, "screenshot: ready — send 's' on USB serial to capture a frame"); + uint8_t rx[16]; + for (;;) { + if (usb_serial_jtag_ll_rxfifo_data_available()) { + int n = usb_serial_jtag_ll_read_rxfifo(rx, sizeof(rx)); + for (int i = 0; i < n; i++) { + if (rx[i] == 's') { face_screenshot_dump(false); break; } /* current screen */ + if (rx[i] == 'o') { face_screenshot_dump(true); break; } /* force overlay */ + } + } + vTaskDelay(pdMS_TO_TICKS(150)); + } +} +#endif + void app_main(void) { ESP_LOGI(TAG, "DeskLock starting"); @@ -287,4 +320,8 @@ void app_main(void) #if FACE_LOADTEST xTaskCreate(loadtest_task, "loadtest", 4096, NULL, 4, NULL); #endif + +#if DESKLOCK_DEVMODE + xTaskCreate(screenshot_task, "shot", 5120, NULL, 4, NULL); +#endif } diff --git a/firmware/main/face.c b/firmware/main/face.c index f6e657f..d077911 100644 --- a/firmware/main/face.c +++ b/firmware/main/face.c @@ -7,8 +7,12 @@ #include #include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" #include "esp_random.h" #include "esp_heap_caps.h" +#include "esp_rom_crc.h" +#include "hal/usb_serial_jtag_ll.h" #include "bsp/esp-bsp.h" #include "lvgl.h" @@ -701,3 +705,69 @@ void face_activity(void) bsp_display_unlock(); } } + +/* --- dev screenshot over USB (see the device-screenshot skill) --------------- */ + +/* Blocking write straight to the USB-serial-JTAG TX FIFO. Bypasses stdout/printf + * — the primary console is UART at 115200 baud (~11 KB/s), far too slow for a + * frame; the USB FIFO runs at USB speed. Waits for FIFO space so nothing drops. */ +static void shot_usb_write(const uint8_t *data, size_t len) +{ + size_t sent = 0; + while (sent < len) { + if (!usb_serial_jtag_ll_txfifo_writable()) { vTaskDelay(1); continue; } + sent += usb_serial_jtag_ll_write_txfifo(data + sent, len - sent); + usb_serial_jtag_ll_txfifo_flush(); + } +} + +/* Render the live screen to an RGB565 buffer, 2x-downscale it, and stream it as + * RAW BINARY straight over the USB-serial-JTAG, framed by a ###SHOT_BEGIN ...### + * text header (carrying byte count + CRC) and ###SHOT_END###, so the host can + * rebuild a PNG. `overlay` forces the tap controls in-frame. Called only from the + * DESKLOCK_DEVMODE screenshot watcher; not part of normal operation. */ +#define SHOT_DS 2 /* downscale factor (2 -> 400x400) */ +void face_screenshot_dump(bool overlay) +{ + /* Snapshot into our OWN PSRAM buffer — the default lv_snapshot_take() allocs + * the 1.28 MB frame from the small LVGL heap and returns NULL. */ + const int W = SCREEN, H = SCREEN; + const uint32_t stride = lv_draw_buf_width_to_stride(W, LV_COLOR_FORMAT_RGB565); + const size_t buf_size = (size_t)stride * H; + uint8_t *mem = heap_caps_malloc(buf_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (!mem) { printf("###SHOT_FAIL mem###\n"); return; } + lv_draw_buf_t dbuf; + lv_draw_buf_init(&dbuf, W, H, LV_COLOR_FORMAT_RGB565, stride, mem, buf_size); + + bsp_display_lock(UINT32_MAX); + if (overlay) { /* force the tap overlay in-frame */ + controls_show(); + lv_timer_pause(F.ctl_timer); /* don't let it auto-hide mid-capture */ + } + lv_result_t rc = lv_snapshot_take_to_draw_buf(lv_screen_active(), LV_COLOR_FORMAT_RGB565, &dbuf); + bsp_display_unlock(); + if (rc != LV_RESULT_OK) { printf("###SHOT_FAIL take=%d###\n", (int)rc); heap_caps_free(mem); return; } + + const int ow = W / SHOT_DS, oh = H / SHOT_DS; + const size_t olen = (size_t)ow * oh * 2; + uint8_t *out = heap_caps_malloc(olen, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (!out) { printf("###SHOT_FAIL malloc###\n"); heap_caps_free(mem); return; } + + for (int y = 0; y < oh; y++) { + const uint16_t *src = (const uint16_t *)(dbuf.data + (size_t)(y * SHOT_DS) * dbuf.header.stride); + uint16_t *dst = (uint16_t *)(out + (size_t)y * ow * 2); + for (int x = 0; x < ow; x++) dst[x] = src[x * SHOT_DS]; + } + heap_caps_free(mem); + + const uint32_t crc = esp_rom_crc32_le(0, out, olen); + char hdr[128]; + int hlen = snprintf(hdr, sizeof(hdr), + "\n###SHOT_BEGIN w=%d h=%d fmt=rgb565le bytes=%u crc=0x%08x bin=1###\n", + ow, oh, (unsigned)olen, (unsigned)crc); + shot_usb_write((const uint8_t *)hdr, hlen); + shot_usb_write(out, olen); /* raw RGB565-LE, exactly `bytes` long */ + static const char end[] = "\n###SHOT_END###\n"; + shot_usb_write((const uint8_t *)end, sizeof(end) - 1); + heap_caps_free(out); +} diff --git a/firmware/sdkconfig.defaults b/firmware/sdkconfig.defaults index 41ceda2..d78b8ae 100644 --- a/firmware/sdkconfig.defaults +++ b/firmware/sdkconfig.defaults @@ -32,6 +32,10 @@ CONFIG_ESP_WIFI_SOFTAP_SUPPORT=y # custom fonts are uncompressed, but enable the decoder as belt-and-braces CONFIG_LV_USE_FONT_COMPRESSED=y +# lv_snapshot_take() — render the live screen to a buffer for the dev +# screenshot-over-USB dump (face_screenshot_dump) +CONFIG_LV_USE_SNAPSHOT=y + # ESP-Hosted board variant + wifi-remote data-path tuning (from factory brookesia config; # without these the RPC control path works but data frames never flow) CONFIG_SLAVE_IDF_TARGET_ESP32C6=y diff --git a/firmware/tools/device_shot.py b/firmware/tools/device_shot.py new file mode 100755 index 0000000..f07c7da --- /dev/null +++ b/firmware/tools/device_shot.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Capture a screenshot from the DeskLock device over USB. + +The firmware (SCREENSHOT_ENABLE) watches the USB-serial-JTAG RX for a trigger +byte and streams the current screen as a text header + ###SHOT_BEGIN w=.. h=.. fmt=rgb565le bytes=N crc=0xXXXX bin=1###\n +followed by exactly N raw RGB565-LE bytes, then \n###SHOT_END###. We send the +trigger, read the payload, verify the CRC, and write a PNG. + +Usage (run in the dialout group, e.g. under `sg dialout -c '...'`): + python3 device_shot.py [out.png] [--overlay] [--port /dev/ttyACM0] + --overlay send 'o' (force the tap controls overlay) instead of 's' (current screen) +""" +import os, sys, termios, select, time, zlib, re + +port = "/dev/ttyACM0" +out = "shot.png" +trig = b"s" +args = sys.argv[1:] +while args: + a = args.pop(0) + if a == "--overlay": trig = b"o" + elif a == "--port": port = args.pop(0) + elif not a.startswith("-"): out = a + else: sys.stderr.write(f"[warn] ignoring {a}\n") + +ATTEMPTS = 4 +PER_TRY = 8.0 +begin_re = re.compile( + rb"###SHOT_BEGIN w=(\d+) h=(\d+) fmt=(\S+) bytes=(\d+) crc=0x([0-9a-fA-F]+) bin=1###\n") + +fd = os.open(port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK) +a = termios.tcgetattr(fd) +a[0] = 0; a[1] = 0; a[3] = 0 # raw: no CR/NL translation, no canon/echo +a[4] = termios.B115200; a[5] = termios.B115200 +termios.tcsetattr(fd, termios.TCSANOW, a) + + +def drain(sec=0.3): + t = time.time() + while time.time() - t < sec: + r, _, _ = select.select([fd], [], [], 0.05) + if r: + try: os.read(fd, 65536) + except BlockingIOError: pass + + +def capture_once(): + drain(0.3) # clear any in-flight tail from a prior dump + buf = bytearray(); hdr = None; t0 = time.time(); last_trig = 0.0 + while time.time() - t0 < PER_TRY: + now = time.time() + if hdr is None and now - last_trig >= 2.0: + try: os.write(fd, trig) + except OSError: pass + last_trig = now + r, _, _ = select.select([fd], [], [], 0.2) + if r: + try: chunk = os.read(fd, 65536) + except BlockingIOError: chunk = b"" + if chunk: buf += chunk + if hdr is None: + m = begin_re.search(buf) + if m: + hdr = (int(m.group(1)), int(m.group(2)), int(m.group(4)), int(m.group(5), 16)) + buf = bytearray(buf[m.end():]) # everything after header = raw payload + if hdr is not None and len(buf) >= hdr[2]: + return (hdr[0], hdr[1], hdr[2], hdr[3], bytes(buf[:hdr[2]])) + return None + + +result = None +for attempt in range(1, ATTEMPTS + 1): + res = capture_once() + if res is None: + sys.stderr.write(f"[try {attempt}] no frame\n"); continue + w, h, nbytes, crc_dev, raw = res + crc_host = zlib.crc32(raw) & 0xffffffff + ok = crc_host == crc_dev + sys.stderr.write(f"[try {attempt}] {w}x{h} {len(raw)}B " + f"crc dev=0x{crc_dev:08x} host=0x{crc_host:08x} {'OK' if ok else 'RETRY'}\n") + if ok: + result = (w, h, raw); break +os.close(fd) + +if result is None: + sys.stderr.write("[fail] no clean frame — SCREENSHOT_ENABLE flashed? device up?\n") + sys.exit(2) + +w, h, raw = result +from PIL import Image +img = Image.new("RGB", (w, h)); px = img.load(); i = 0 +for y in range(h): + for x in range(w): + v = raw[i] | (raw[i+1] << 8); i += 2 + r5 = (v >> 11) & 0x1f; g6 = (v >> 5) & 0x3f; b5 = v & 0x1f + px[x, y] = ((r5 << 3) | (r5 >> 2), (g6 << 2) | (g6 >> 4), (b5 << 3) | (b5 >> 2)) +img.save(out) +sys.stderr.write(f"[ok] wrote {out} ({w}x{h})\n") +print(out)