From c8d019e6301c390f2c3e28886f1a5b253fb7f5f0 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 21 Apr 2026 15:48:07 +0200 Subject: [PATCH] add web WASM Playwright harness for UI driving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tools/ui/ — scripts + Playwright config that let Claude Code (and humans) drive the Flutter WASM build in a real browser. The point is to avoid screenshot round-trips: every interactive widget in clide ships a Semantics wrapper anyway (a11y requirement), so the automation layer just queries the `flt-semantics[aria-label]` DOM. * build.sh — `flutter build web --wasm` from app/ * serve.sh — `python3 -m http.server 4280` in the background. Before binding, kills any stale listener on the port (orphans from earlier failed runs no longer accumulate). * stop.sh — port-based kill; escalates to SIGKILL after 300ms. The pidfile is now advisory — port ownership is the source of truth. * driver.ts — `ClideDriver` class with `byLabel`, `click`, `type`, `readText`, `screenshot`, `dumpSemanticsTree`, and `waitUntilReady` that auto-clicks the `flt-semantics-placeholder` so the semantic tree is populated before queries. * tests/smoke.spec.ts — first driver test; asserts welcome + disconnected labels surface in the Semantics DOM. Co-Authored-By: Claude --- CHANGELOG.md | 1 + tools/ui/.gitignore | 5 ++ tools/ui/README.md | 61 +++++++++++++++++++ tools/ui/build.sh | 6 ++ tools/ui/driver.ts | 109 ++++++++++++++++++++++++++++++++++ tools/ui/package-lock.json | 91 ++++++++++++++++++++++++++++ tools/ui/package.json | 15 +++++ tools/ui/playwright.config.ts | 26 ++++++++ tools/ui/serve.sh | 42 +++++++++++++ tools/ui/stop.sh | 35 +++++++++++ tools/ui/tests/smoke.spec.ts | 23 +++++++ 11 files changed, 414 insertions(+) create mode 100644 tools/ui/.gitignore create mode 100644 tools/ui/README.md create mode 100755 tools/ui/build.sh create mode 100644 tools/ui/driver.ts create mode 100644 tools/ui/package-lock.json create mode 100644 tools/ui/package.json create mode 100644 tools/ui/playwright.config.ts create mode 100755 tools/ui/serve.sh create mode 100755 tools/ui/stop.sh create mode 100644 tools/ui/tests/smoke.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 533ec9b3..1f721e94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ heading, and (b) bumping `project.yaml` `version:` in the same commit. ### Added +- Web WASM harness under `tools/ui/` — Playwright driver so Claude Code (and humans) can drive the Flutter build in a real browser via the Semantics tree. `build.sh` / `serve.sh` / `stop.sh` manage a local `http.server` on `:4280` with port-based reclaim and kill (so orphaned listeners from earlier runs get swept). `driver.ts` exposes `ClideDriver` with `byLabel` / `click` / `type` / `readText` / `screenshot` / `dumpSemanticsTree` / `waitUntilReady` (auto-clicks the `flt-semantics-placeholder` to enable the semantics tree). First Playwright test `smoke.spec.ts` asserts welcome + disconnected labels render in the browser. - Integration tests under `app/integration_test/`, run with the `integration_test` package against the real built app (not an in-memory widget pump). The load-bearing startup gate lives here: `app_starts_test.dart` boots `ClideApp`, waits for the root shell to settle, and asserts the three-column layout + welcome tab + statusbar connection indicator all render. Also covers theme-picker modal open/select/dismiss (`theme_picker_test.dart`) and extension enable/disable lifecycle with contributions mounting/unmounting (`extension_lifecycle_test.dart`). - App-level test suite under `app/test/` — 168 tests across four layers: - **Unit** (`kernel/`, `extension/`) — events bus, settings (scope + YAML round-trip), log, i18n fallback chain matrix, theme resolver + loader + controller, panel registry + arrangement, command registry + keybinding parser + palette filter, extension-manager dep-order / cycle detection / enable-disable, manifest loader, extension scanner. diff --git a/tools/ui/.gitignore b/tools/ui/.gitignore new file mode 100644 index 00000000..d3fede95 --- /dev/null +++ b/tools/ui/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +out/ +test-results/ +playwright-report/ +.serve.pid diff --git a/tools/ui/README.md b/tools/ui/README.md new file mode 100644 index 00000000..5b466a9b --- /dev/null +++ b/tools/ui/README.md @@ -0,0 +1,61 @@ +# clide UI harness + +A Playwright-based driver that drives the Flutter WASM build of clide +through its semantic DOM tree (the same tree screen readers use). +Used by Claude Code to "actually use the app" without screenshot +round-trips, and by CI to catch web-regression bugs. + +## One-time setup + +```bash +cd tools/ui +npm install +npx playwright install chromium +``` + +## Dev loop + +```bash +# From repo root: +make ui-dev # build web + start local server :4280 +cd tools/ui && npx playwright test + +# When done: +make ui-stop # kill the local server +``` + +Or the one-shot smoke: + +```bash +make ui-smoke # build + serve + run smoke + stop +``` + +## Driver surface + +```ts +import { ClideDriver } from '../driver'; + +test('...', async ({ page }) => { + const clide = new ClideDriver(page); + await clide.goto('/'); + + await clide.click('Open project'); + await clide.type('Name', 'My project'); + const text = await clide.readText('disconnected'); + + await clide.screenshot('out/my-state.png'); + const tree = await clide.dumpSemanticsTree(); +}); +``` + +All lookups go through Flutter's semantics tree (`flt-semantics[aria-label]`). +This only works because every interactive widget in clide emits a +`Semantics(label:, hint:, button:)` wrapper — a requirement that's baked +in for screen-reader support and gets enforced by +`test/a11y/semantic_coverage_test.dart`. + +## CI + +`make ui-smoke` is the CI entry — builds the WASM bundle, runs the +harness, cleans up. Enabling Gitea Actions will start running this on +every push. diff --git a/tools/ui/build.sh b/tools/ui/build.sh new file mode 100755 index 00000000..03635309 --- /dev/null +++ b/tools/ui/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Build the Flutter web WASM bundle that the Playwright harness drives. +set -euo pipefail +cd "$(dirname "$0")/../../app" +flutter build web --wasm "$@" +echo "built app/build/web ($(du -sh build/web | cut -f1))" diff --git a/tools/ui/driver.ts b/tools/ui/driver.ts new file mode 100644 index 00000000..165bfa4b --- /dev/null +++ b/tools/ui/driver.ts @@ -0,0 +1,109 @@ +import { expect } from '@playwright/test'; +import type { Page, Locator } from '@playwright/test'; + +/** + * Playwright helpers that drive the Flutter WASM build by querying + * `flt-semantics` DOM elements instead of pixel coordinates. + * + * Requires the app to have `SemanticsBinding.ensureSemantics()` on + * boot (main.dart already does this). Every interactive widget in + * clide emits a `Semantics(label:, hint:, button:)` wrapper, which + * surfaces as a `flt-semantics[aria-label="…"]` element. + */ +export class ClideDriver { + readonly page: Page; + + constructor(page: Page) { + this.page = page; + } + + /** Navigate to the app root and wait until Flutter has finished first-frame. */ + async goto(path: string = '/'): Promise { + await this.page.goto(path); + await this.waitUntilReady(); + } + + /** + * Wait until the Flutter app is past first frame and click the + * accessibility placeholder so the semantics tree is populated. + * Flutter web ships semantics disabled by default; the placeholder + * at the very top-left of the page is the official way to turn them + * on from outside the app. + */ + async waitUntilReady(): Promise { + await this.page.waitForSelector('flt-glass-pane', { + timeout: 30_000, + state: 'attached', + }); + const placeholder = this.page.locator('flt-semantics-placeholder'); + if ((await placeholder.count()) > 0) { + await placeholder.click({ force: true }); + } + await this.page.waitForSelector('flt-semantics[aria-label]', { + timeout: 30_000, + state: 'attached', + }); + } + + /** + * Returns a locator for a Semantics node whose `aria-label` contains + * [label]. Flutter web merges sibling labels into one aria-label + * (newline-separated), so exact match wouldn't work. Substring match + * is usually unique — narrow with `.filter()` if not. + */ + byLabel(label: string): Locator { + const safe = label.replace(/"/g, '\\"'); + return this.page.locator(`flt-semantics[aria-label*="${safe}"]`); + } + + /** Click an element by its semantic label. Asserts it exists + is enabled. */ + async click(label: string): Promise { + const el = this.byLabel(label); + await el.waitFor({ state: 'attached', timeout: 5_000 }); + await el.click(); + } + + /** Type into the element with the given label. */ + async type(label: string, text: string): Promise { + const el = this.byLabel(label); + await el.waitFor({ state: 'attached', timeout: 5_000 }); + await el.fill(text); + } + + /** Read the visible label of an element (useful for state transitions). */ + async readText(label: string): Promise { + const el = this.byLabel(label); + await el.waitFor({ state: 'attached', timeout: 5_000 }); + return (await el.textContent()) ?? ''; + } + + /** Save a full-page PNG to `path`. */ + async screenshot(path: string): Promise { + await this.page.screenshot({ path, fullPage: true }); + } + + /** + * Dump the entire Flutter semantic tree as structured JSON. Useful + * for test-failure diagnosis ("why didn't my label match?") and for + * Claude's own debugging flow. + */ + async dumpSemanticsTree(): Promise { + return this.page.evaluate(() => { + function walk(el: Element): unknown { + const children = Array.from(el.children) + .filter((c) => c.tagName.toLowerCase().startsWith('flt-semantics')) + .map(walk); + return { + tag: el.tagName.toLowerCase(), + label: el.getAttribute('aria-label'), + role: el.getAttribute('role'), + hint: el.getAttribute('aria-describedby'), + selected: el.getAttribute('aria-selected'), + children, + }; + } + const hosts = Array.from(document.querySelectorAll('flt-semantics-host')); + return hosts.map(walk); + }); + } +} diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json new file mode 100644 index 00000000..545c229e --- /dev/null +++ b/tools/ui/package-lock.json @@ -0,0 +1,91 @@ +{ + "name": "clide-ui-harness", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "clide-ui-harness", + "devDependencies": { + "@playwright/test": "1.50.0", + "typescript": "5.6.3" + } + }, + "node_modules/@playwright/test": { + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.50.0.tgz", + "integrity": "sha512-ZGNXbt+d65EGjBORQHuYKj+XhCewlwpnSd/EDuLPZGSiEWmgOJB5RmMCCYGy5aMfTs9wx61RivfDKi8H/hcMvw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.50.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.0.tgz", + "integrity": "sha512-+GinGfGTrd2IfX1TA4N2gNmeIksSb+IAe589ZH+FlmpV3MYTx6+buChGIuDLQwrGNCw2lWibqV50fU510N7S+w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.50.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.0.tgz", + "integrity": "sha512-CXkSSlr4JaZs2tZHI40DsZUN/NIwgaUPsyLuOAaIZp2CyF2sN5MM5NJsyB188lFSSozFxQ5fPT4qM+f0tH/6wQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/tools/ui/package.json b/tools/ui/package.json new file mode 100644 index 00000000..0ce3e2aa --- /dev/null +++ b/tools/ui/package.json @@ -0,0 +1,15 @@ +{ + "name": "clide-ui-harness", + "description": "Playwright harness Claude Code uses to drive the Flutter web WASM build of clide.", + "private": true, + "type": "module", + "scripts": { + "test": "playwright test", + "test:headed": "playwright test --headed", + "install-browsers": "playwright install chromium" + }, + "devDependencies": { + "@playwright/test": "1.50.0", + "typescript": "5.6.3" + } +} diff --git a/tools/ui/playwright.config.ts b/tools/ui/playwright.config.ts new file mode 100644 index 00000000..b4914527 --- /dev/null +++ b/tools/ui/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './tests', + // Single-worker: the web server is local and global; parallel tests + // would race on the shared browser state. + workers: 1, + reporter: 'line', + use: { + baseURL: process.env.CLIDE_UI_URL ?? 'http://localhost:4280', + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + // WASM + CanvasKit + semantics tree runs headless fine; no extra + // launch args needed for our driver surface. + }, + }, + ], + outputDir: 'out', +}); diff --git a/tools/ui/serve.sh b/tools/ui/serve.sh new file mode 100755 index 00000000..d2331f95 --- /dev/null +++ b/tools/ui/serve.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Serve the WASM bundle on http://localhost:$PORT (default 4280) in the +# background. Robust against orphaned servers: before starting, any +# existing listener on the port (from a previous run whose pidfile got +# lost or overwritten) is killed. The pidfile is advisory — `stop.sh` +# uses port-based kill as the source of truth. +set -euo pipefail + +PORT=${CLIDE_UI_PORT:-4280} +HERE="$(cd "$(dirname "$0")" && pwd)" +DIR="$HERE/../../app/build/web" +PID_FILE="$HERE/.serve.pid" + +if [[ ! -d "$DIR" ]]; then + echo "serve: web build not found at $DIR — run tools/ui/build.sh first" >&2 + exit 2 +fi + +# Port-based orphan kill. Handles the case where the recorded pidfile +# was overwritten by an earlier failed call and the previous server +# is still bound to the port. +existing="" +if command -v lsof >/dev/null 2>&1; then + existing=$(lsof -ti:"$PORT" 2>/dev/null || true) +elif command -v fuser >/dev/null 2>&1; then + existing=$(fuser -n tcp "$PORT" 2>/dev/null | tr -d ' /tcp' || true) +fi +if [[ -n "$existing" ]]; then + echo "serve: reclaiming port $PORT from stale listener(s) $existing" + echo "$existing" | xargs -r kill 2>/dev/null || true + sleep 0.2 +fi + +# Flutter web WASM requires Cross-Origin Opener/Embedder headers on the +# server for `SharedArrayBuffer`. A vanilla python http.server misses +# them, so CanvasKit still works but WasmGC-accelerated Skwasm may not. +# For Tier 0 driver-script automation this is fine; revisit when we +# need Skwasm performance parity. +cd "$DIR" +nohup python3 -m http.server "$PORT" >/tmp/clide-ui-serve.log 2>&1 & +echo $! > "$PID_FILE" +echo "serve: http://localhost:$PORT (pid $(cat "$PID_FILE"), log /tmp/clide-ui-serve.log)" diff --git a/tools/ui/stop.sh b/tools/ui/stop.sh new file mode 100755 index 00000000..4856a2cc --- /dev/null +++ b/tools/ui/stop.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Kill the dev web server on $CLIDE_UI_PORT (default 4280). Port-based +# kill so any orphan from a previous `serve.sh` that lost its pidfile +# dies too. +set -euo pipefail + +PORT=${CLIDE_UI_PORT:-4280} +HERE="$(cd "$(dirname "$0")" && pwd)" +PID_FILE="$HERE/.serve.pid" + +pids="" +if command -v lsof >/dev/null 2>&1; then + pids=$(lsof -ti:"$PORT" 2>/dev/null || true) +elif command -v fuser >/dev/null 2>&1; then + pids=$(fuser -n tcp "$PORT" 2>/dev/null | tr -d ' /tcp' || true) +fi + +if [[ -n "$pids" ]]; then + echo "stop: killing $pids (port $PORT)" + echo "$pids" | xargs -r kill 2>/dev/null || true + # Give SIGTERM 300ms to land; escalate to SIGKILL for stragglers. + sleep 0.3 + remaining="" + if command -v lsof >/dev/null 2>&1; then + remaining=$(lsof -ti:"$PORT" 2>/dev/null || true) + fi + if [[ -n "$remaining" ]]; then + echo "stop: escalating to SIGKILL for $remaining" + echo "$remaining" | xargs -r kill -9 2>/dev/null || true + fi +else + echo "stop: no server on port $PORT" +fi + +rm -f "$PID_FILE" diff --git a/tools/ui/tests/smoke.spec.ts b/tools/ui/tests/smoke.spec.ts new file mode 100644 index 00000000..356a5da8 --- /dev/null +++ b/tools/ui/tests/smoke.spec.ts @@ -0,0 +1,23 @@ +import { test, expect } from '@playwright/test'; +import { ClideDriver } from '../driver'; + +/** + * Browser smoke: boot the WASM bundle and verify the Welcome view + + * statusbar indicator + daemon-disconnected label all render. This is + * the "app actually works in a real browser" regression gate — the web + * counterpart of `integration_test/app_starts_test.dart`. + */ +test('clide boots in the browser, welcome + statusbar visible', async ({ page }) => { + const clide = new ClideDriver(page); + await clide.goto('/'); + + // Semantics nodes are attached even when their rendered canvas is + // visually covered by the Flutter glass-pane; assert "attached" for + // the automation contract — we're reading the tree, not asserting + // rendering. + const openProject = clide.byLabel('Open project'); + await expect(openProject).toHaveCount(1); + + const disconnected = clide.byLabel('disconnected'); + await expect(disconnected.first()).toBeAttached(); +});