diff --git a/.github/workflows/brush-diagnostic.yml b/.github/workflows/brush-diagnostic.yml new file mode 100644 index 000000000..4771861b4 --- /dev/null +++ b/.github/workflows/brush-diagnostic.yml @@ -0,0 +1,74 @@ +name: Brush diagnostic (non-authoritative) + +on: + push: + branches: [test/remediation-brush-diagnostic-ubuntu] + +permissions: + contents: read + +jobs: + browser-diagnostic: + name: Chromium application regressions + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS: '1' + ODYSSEUS_TEST_PYTHON: python + CHROMADB_HOST: 127.0.0.1 + CHROMADB_PORT: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + - name: Install FFmpeg and establish system trust + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg ca-certificates + python scripts/check_ca_trust.py --system + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: '0.13.5' + enable-cache: true + cache-dependency-glob: requirements.txt + prune-cache: false + - run: | + uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt + python scripts/check_ca_trust.py --python + uv pip install --system -r requirements.txt + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.23.4' + cache: npm + - run: npm ci --ignore-scripts --no-audit --no-fund + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - run: npx playwright install --with-deps chromium + - name: Compare trace modes and preserve exact threshold + shell: bash + run: | + lscpu | head -30 + python --version + node --version + for mode in retain-on-failure off on; do + for sample in 1 2 3; do + echo "DIAGNOSTIC_MODE=$mode SAMPLE=$sample" + BRUSH_DIAGNOSTIC_MODE="$mode-$sample" npx playwright test --config tests/e2e/playwright.config.js --grep 'long sampled strokes' --trace "$mode" --reporter=line || true + done + done + - name: Run normal prefix through brush case + shell: bash + run: | + BRUSH_DIAGNOSTIC_MODE=normal-prefix npx playwright test --config tests/e2e/playwright.config.js tests/e2e/photo-editor/adjustment-layers.spec.js tests/e2e/photo-editor/app-compare-mobile.spec.js tests/e2e/photo-editor/brush-cursor.spec.js tests/e2e/photo-editor/brush-engine.spec.js --grep-invert 'eyedropper and retouch' --reporter=line || true + - name: Six-second main-thread stall negative control + shell: bash + run: | + if BRUSH_DIAGNOSTIC_STALL=1 BRUSH_DIAGNOSTIC_MODE=stall npx playwright test --config tests/e2e/playwright.config.js --grep 'long sampled strokes' --trace off --reporter=line; then + echo 'ERROR: negative control was not detected' + exit 1 + fi diff --git a/tests/e2e/photo-editor/brush-engine.spec.js b/tests/e2e/photo-editor/brush-engine.spec.js index b51c2137c..6b97b39fd 100644 --- a/tests/e2e/photo-editor/brush-engine.spec.js +++ b/tests/e2e/photo-editor/brush-engine.spec.js @@ -69,6 +69,8 @@ test('long sampled strokes stay responsive and undo atomically', async ({ page, const metrics = browserName === 'chromium' ? await page.context().newCDPSession(page) : null; if (metrics) await metrics.send('Performance.enable'); const taskTime = async () => (await metrics.send('Performance.getMetrics')).metrics.find(item => item.name === 'TaskDuration').value *1000; + const metricSnapshot = async () => Object.fromEntries((await metrics.send('Performance.getMetrics')).metrics.filter(x => ['TaskDuration','ScriptDuration','LayoutDuration','RecalcStyleDuration'].includes(x.name)).map(x => [x.name, x.value * 1000])); + const beforeMetrics = metrics ? await metricSnapshot() : {}; const beforeTask = metrics ? await taskTime() : null; const startedAt = Date.now(); await page.mouse.move(start.x, start.y); @@ -76,7 +78,10 @@ test('long sampled strokes stay responsive and undo atomically', async ({ page, await page.mouse.move(end.x, end.y, { steps: 240 }); await page.mouse.up(); const strokeDuration = Date.now() - startedAt; + if (process.env.BRUSH_DIAGNOSTIC_STALL === '1') await page.evaluate(() => { const until = performance.now() + 6000; while (performance.now() < until) {} }); const mainThreadDuration = metrics ? (await taskTime()) - beforeTask : null; + const afterMetrics = metrics ? await metricSnapshot() : {}; + console.log('BRUSH_DIAGNOSTIC ' + JSON.stringify({mode: process.env.BRUSH_DIAGNOSTIC_MODE, samples: 240, strokeDuration, controlDuration, mainThreadDuration, breakdown: Object.fromEntries(Object.keys(beforeMetrics).map(k => [k, afterMetrics[k] - beforeMetrics[k]]))})); if (metrics) await metrics.detach(); await testInfo.attach('stroke-timing', { body: Buffer.from(JSON.stringify({ samples: 240, strokeDuration, controlDuration, mainThreadDuration,