test(e2e): harden mobile layout coverage, artifact isolation, and live pane assertions

- Store Playwright evidence in ignored test-results/ paths by default while preserving env override
- Clean up disposable temporary application data directories in /tmp on global teardown and exit signals
- Add Chromium mobile and tablet viewport profiles (phone portrait/landscape, tablet) for automated coverage
- Add a scoped Chromium mobile layout CI check on shard 1/4 to prevent redundant matrix execution
- Reach mobile Settings and Notes views via genuine sidebar navigation controls opened through the hamburger button
- Add assertions against live rendered Notes and Documents panes honoring safe-area top and height constraints
- Document local WebKit prerequisites and physical iOS/PWA test limitations in tests/e2e/README.md and package.json
This commit is contained in:
Alexandre Teixeira
2026-10-30 13:41:15 +00:00
parent ef16398c5c
commit 6fc6095aec
6 changed files with 352 additions and 39 deletions
+7
View File
@@ -256,3 +256,10 @@ jobs:
env:
PYTEST_SHARD: ${{ matrix.shard }}
run: python -m pytest -q -rs --durations=25 --shard "$PYTEST_SHARD"
- name: Playwright mobile layout checks (shard 1/4 only)
if: steps.docs-check.outputs.docs_only != 'true' && matrix.shard == '1/4'
run: |
ODYSSEUS_TEST_PYTHON=python ODYSSEUS_LAYOUT_BROWSER=chromium npx --no-install playwright test \
--config tests/e2e/ios-layout.config.cjs \
--reporter=line
+5 -1
View File
@@ -9,7 +9,11 @@
"test:photo-editor": "playwright test --config tests/e2e/playwright.config.js",
"test:photo-editor:install": "playwright install chromium firefox webkit",
"test:photo-editor:firefox": "PHOTO_EDITOR_E2E_BROWSER=firefox playwright test --config tests/e2e/playwright.config.js",
"test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js"
"test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js",
"test:ios-layout": "playwright test --config tests/e2e/ios-layout.config.cjs",
"test:ios-layout:chromium": "ODYSSEUS_LAYOUT_BROWSER=chromium playwright test --config tests/e2e/ios-layout.config.cjs",
"test:ios-layout:webkit": "ODYSSEUS_LAYOUT_BROWSER=webkit playwright test --config tests/e2e/ios-layout.config.cjs",
"test:ios-layout:install": "playwright install chromium webkit"
},
"devDependencies": {
"@antithesishq/bombadil": "^0.7.0",
+52
View File
@@ -1,5 +1,7 @@
# Browser End-to-End Tests
## Photo Editor Release Gate
The photo editor release gate launches an isolated Odysseus server on port
`7013` with a temporary SQLite database and disposable authenticated account.
@@ -18,3 +20,53 @@ The disposable account is created in the temporary data directory and exercises
owner-scoped draft and notification routes without using an existing account.
`PHOTO_EDITOR_E2E_AUTH=0` selects the legacy unauthenticated configuration for
investigating it; offline-upgrade coverage requires the authenticated mode.
## Mobile and iOS Layout Suite
The dedicated mobile and iOS layout suite exercises viewport behavior,
accessibility zoom, scrolling, mobile safe-area geometry (Notes and Documents
panes), and floating window restoration against an isolated local Odysseus server
running on port `17739`.
```bash
npm install
npm run test:ios-layout:chromium
```
### Profiles and Browser Prerequisites
- **Chromium profiles** (`desktop-chromium`, `phone-chromium-portrait`, `phone-chromium-landscape`, `tablet-chromium`):
Provide automated baseline coverage in CI and local development without requiring
WebKit dependencies.
- **WebKit profiles** (`phone-webkit-portrait`, `phone-webkit-landscape`, `tablet-webkit`):
Used for platform-oriented WebKit rendering validation. WebKit requires browser binaries:
```bash
npm run test:ios-layout:install
# or: npx playwright install webkit
npm run test:ios-layout:webkit
```
To run all installed profiles:
```bash
npm run test:ios-layout
```
### Evidence and Cleanup
- **Evidence artifacts**: Screenshots are written to `test-results/ios-layout/` by
default and traces are written to `test-results/ios-layout-traces/`. Both paths
are git-ignored. Set `ODYSSEUS_LAYOUT_EVIDENCE_DIR` to override the screenshot
output path.
- **Teardown**: Disposable application state and database files created in the
OS temporary directory are automatically cleaned up on both normal and failure
teardown, preserving diagnostic screenshots and failure traces.
### Platform Considerations
Emulated Chromium and WebKit viewports validate deterministic CSS cascade,
computed layout geometry, viewport meta tags, and JavaScript restoration. However,
browser emulation is not a physical device substitute: physical iOS Safari and
standalone installed PWAs should still be verified separately for native gesture
pinch-zoom, software keyboard dismissal transitions, and dynamic hardware notch/island insets.
+28
View File
@@ -0,0 +1,28 @@
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
/**
* Global teardown for the iOS/mobile layout Playwright test suite.
* Cleans up only disposable temporary application data directories,
* preserving diagnostic artifacts (screenshots and failure traces) in test-results/.
*/
module.exports = async () => {
const dataDirectory = process.env.ODYSSEUS_LAYOUT_DATA_DIR;
if (!dataDirectory || typeof dataDirectory !== 'string') return;
try {
const resolvedData = path.resolve(dataDirectory);
const allowedTmpBases = [os.tmpdir(), '/var/tmp', '/tmp'].map(b => path.resolve(b));
const isTmp = allowedTmpBases.some(base => resolvedData.startsWith(base + path.sep));
const baseName = path.basename(resolvedData);
if (isTmp && (baseName.startsWith('odysseus-ios-layout-') || baseName.startsWith('odysseus-pr6396-'))) {
if (fs.existsSync(resolvedData)) {
fs.rmSync(resolvedData, { recursive: true, force: true });
}
}
} catch (err) {
console.warn(`[ios-layout-teardown] Warning: failed to clean temporary runtime dir: ${err.message}`);
}
};
+101 -11
View File
@@ -7,20 +7,115 @@ const { defineConfig } = require('@playwright/test');
const repositoryRoot = path.resolve(__dirname, '../..');
const port = Number(process.env.ODYSSEUS_LAYOUT_PORT || '17739');
const baseURL = `http://127.0.0.1:${port}`;
const pythonInVenv = path.join(repositoryRoot, '.venv', 'bin', 'python');
const python = process.env.ODYSSEUS_TEST_PYTHON || (fs.existsSync(pythonInVenv) ? pythonInVenv : 'python3');
const dataDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'odysseus-pr6396-'));
const evidenceDir = process.env.ODYSSEUS_LAYOUT_EVIDENCE_DIR || path.join(os.homedir(), 'Pictures', 'Odysseus-6396-followup');
const defaultPython = process.platform === 'win32'
? path.join(repositoryRoot, '.venv', 'Scripts', 'python.exe')
: path.join(repositoryRoot, '.venv', 'bin', 'python');
const python = process.env.ODYSSEUS_TEST_PYTHON
|| (fs.existsSync(defaultPython) ? defaultPython : 'python3');
const dataDirectory = process.env.ODYSSEUS_LAYOUT_DATA_DIR
|| fs.mkdtempSync(path.join(os.tmpdir(), 'odysseus-ios-layout-'));
process.env.ODYSSEUS_LAYOUT_DATA_DIR = dataDirectory;
// Default evidence directory to repository-relative test-results/ios-layout
// while preserving the environment-variable override if provided.
const evidenceDir = process.env.ODYSSEUS_LAYOUT_EVIDENCE_DIR
|| path.join(repositoryRoot, 'test-results', 'ios-layout');
const outputDir = path.join(repositoryRoot, 'test-results', 'ios-layout-traces');
// Emergency process-exit cleanup for disposable application runtime state
const cleanDataDirSync = () => {
const dir = process.env.ODYSSEUS_LAYOUT_DATA_DIR;
if (!dir || typeof dir !== 'string') return;
try {
const resolved = path.resolve(dir);
const allowed = [os.tmpdir(), '/var/tmp', '/tmp'].map(b => path.resolve(b));
if (allowed.some(base => resolved.startsWith(base + path.sep))) {
const baseName = path.basename(resolved);
if (baseName.startsWith('odysseus-ios-layout-') || baseName.startsWith('odysseus-pr6396-')) {
if (fs.existsSync(resolved)) {
fs.rmSync(resolved, { recursive: true, force: true });
}
}
}
} catch (_) {}
};
process.once('SIGINT', () => { cleanDataDirSync(); process.exit(130); });
process.once('SIGTERM', () => { cleanDataDirSync(); process.exit(143); });
function isWebKitInstalled() {
try {
const { webkit } = require('@playwright/test');
return fs.existsSync(webkit.executablePath());
} catch (_) {
return false;
}
}
const chromiumProjects = [
{
name: 'desktop-chromium',
use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } },
},
{
name: 'phone-chromium-portrait',
use: { browserName: 'chromium', viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true },
},
{
name: 'phone-chromium-landscape',
use: { browserName: 'chromium', viewport: { width: 844, height: 390 }, hasTouch: true, isMobile: true },
},
{
name: 'tablet-chromium',
use: { browserName: 'chromium', viewport: { width: 820, height: 1000 }, hasTouch: true, isMobile: true },
},
];
const webkitProjects = [
{
name: 'phone-webkit-portrait',
use: { browserName: 'webkit', viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true },
},
{
name: 'phone-webkit-landscape',
use: { browserName: 'webkit', viewport: { width: 844, height: 390 }, hasTouch: true, isMobile: true },
},
{
name: 'tablet-webkit',
use: { browserName: 'webkit', viewport: { width: 820, height: 1000 }, hasTouch: true, isMobile: true },
},
];
// Determine active projects based on environment and local browser availability
const requestedBrowser = (process.env.ODYSSEUS_LAYOUT_BROWSER || 'all').toLowerCase();
let selectedProjects = [];
if (requestedBrowser === 'chromium') {
selectedProjects = chromiumProjects;
} else if (requestedBrowser === 'webkit') {
selectedProjects = webkitProjects;
} else {
// 'all': Include chromium always; include webkit if installed or if explicitly targeted via cli args
const hasCliWebKitTarget = process.argv.some(arg => arg.toLowerCase().includes('webkit'));
if (isWebKitInstalled() || hasCliWebKitTarget) {
selectedProjects = [...chromiumProjects, ...webkitProjects];
} else {
// Only Chromium is available locally (e.g. standard CI / default runner)
selectedProjects = chromiumProjects;
}
}
module.exports = defineConfig({
testDir: path.join(__dirname, 'ios-layout'),
testMatch: ['**/*.spec.cjs'],
outputDir: path.join(dataDirectory, 'playwright-output'),
outputDir,
timeout: 90_000,
expect: { timeout: 15_000 },
retries: 0,
workers: 1,
reporter: [['line']],
globalTeardown: require.resolve('./ios-layout-teardown.cjs'),
use: {
baseURL,
headless: true,
@@ -28,12 +123,7 @@ module.exports = defineConfig({
screenshot: 'only-on-failure',
trace: 'retain-on-failure',
},
projects: [
{ name: 'desktop-chromium', use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } } },
{ name: 'phone-webkit-portrait', use: { browserName: 'webkit', viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true } },
{ name: 'phone-webkit-landscape', use: { browserName: 'webkit', viewport: { width: 844, height: 390 }, hasTouch: true, isMobile: true } },
{ name: 'tablet-webkit', use: { browserName: 'webkit', viewport: { width: 820, height: 1000 }, hasTouch: true, isMobile: true } },
],
projects: selectedProjects,
metadata: { evidenceDir },
webServer: {
command: `${JSON.stringify(python)} -m uvicorn app:app --host 127.0.0.1 --port ${port}`,
+159 -27
View File
@@ -30,35 +30,112 @@ async function openApp(page) {
).toBe(1);
}
function isMobileViewport(testInfo) {
const width = testInfo.project.use?.viewport?.width || 0;
return width > 0 && width <= 768;
}
/**
* Reach Settings through genuine UI controls on either desktop or mobile viewports.
* On mobile (<=768px), the rail is hidden and the sidebar must be opened via hamburger.
*/
async function openSettings(page, isMobile) {
let settingsButton;
if (isMobile) {
const hamburger = page.locator('#hamburger-btn:visible').first();
await expect(hamburger, 'Hamburger button must be visible on mobile').toBeVisible({ timeout: 5000 });
await hamburger.click();
const sidebar = page.locator('#sidebar:not(.hidden)');
await expect(sidebar, 'Sidebar must open upon hamburger tap').toBeVisible({ timeout: 5000 });
await page.waitForTimeout(300);
settingsButton = sidebar.locator('#user-bar-settings');
} else {
settingsButton = page.locator('#rail-settings:visible, #user-bar-settings:visible').first();
}
await expect(settingsButton, 'Settings button must be reachable and visible').toBeVisible({ timeout: 5000 });
await settingsButton.click();
const dialog = page.locator('#settings-modal');
await expect(dialog, 'Settings modal must become visible').toBeVisible({ timeout: 5000 });
return dialog;
}
/**
* Reach Notes through genuine UI controls on either desktop or mobile viewports.
*/
async function openNotes(page, isMobile) {
let notesButton;
if (isMobile) {
const hamburger = page.locator('#hamburger-btn:visible').first();
await expect(hamburger, 'Hamburger button must be visible on mobile').toBeVisible({ timeout: 5000 });
await hamburger.click();
const sidebar = page.locator('#sidebar:not(.hidden)');
await expect(sidebar, 'Sidebar must open upon hamburger tap').toBeVisible({ timeout: 5000 });
await page.waitForTimeout(300);
notesButton = sidebar.locator('#tool-notes-btn');
} else {
notesButton = page.locator('#rail-notes:visible, #tool-notes-btn:visible').first();
}
await expect(notesButton, 'Notes button must be reachable and visible').toBeVisible({ timeout: 5000 });
await notesButton.click();
const realNotes = page.locator('.notes-pane').first();
await expect(realNotes, 'Notes pane must become visible').toBeVisible({ timeout: 5000 });
return realNotes;
}
/**
* Reach Document editor through genuine UI controls on either desktop or mobile viewports.
*/
async function openDocument(page, isMobile) {
let docButton;
if (isMobile) {
const hamburger = page.locator('#hamburger-btn:visible').first();
await expect(hamburger, 'Hamburger button must be visible on mobile').toBeVisible({ timeout: 5000 });
await hamburger.click();
const sidebar = page.locator('#sidebar:not(.hidden)');
await expect(sidebar, 'Sidebar must open upon hamburger tap').toBeVisible({ timeout: 5000 });
await page.waitForTimeout(300);
docButton = sidebar.locator('#library-new-doc-btn');
} else {
docButton = page.locator('#library-new-doc-btn:visible').first();
if (!(await docButton.count())) {
const hamburger = page.locator('#hamburger-btn:visible').first();
if (await hamburger.count()) {
await hamburger.click();
await page.waitForTimeout(300);
docButton = page.locator('#sidebar:not(.hidden) #library-new-doc-btn');
}
}
}
await expect(docButton, 'Document creation button must be reachable and visible').toBeVisible({ timeout: 5000 });
await docButton.click();
const docPane = page.locator('.doc-editor-pane').first();
await expect(docPane, 'Document editor pane must become visible').toBeVisible({ timeout: 10000 });
return docPane;
}
test('renders actual Odysseus UI and captures review screenshots', async ({ page }, testInfo) => {
await openApp(page);
const out = testInfo.config.metadata.evidenceDir;
fs.mkdirSync(out, { recursive: true });
const name = testInfo.project.name;
const isMobile = isMobileViewport(testInfo);
// 1. Chat screenshot (default view)
await page.screenshot({ path: path.join(out, `${name}-chat.png`), fullPage: false, animations: 'disabled' });
// Screenshot the actual Settings UI only if a real user-visible Settings
// control is reachable; never fabricate a modal for PR screenshots.
const settingsButton = page.locator('#rail-settings:visible, #user-bar-settings:visible').first();
if (await settingsButton.count()) {
await settingsButton.click();
const dialog = page.locator('#settings-modal');
if (await dialog.isVisible()) {
await page.screenshot({ path: path.join(out, `${name}-settings.png`), fullPage: false, animations: 'disabled' });
}
}
// Open the real Notes route via an existing UI action when it is reachable.
// These screenshots reflect genuine runtime content, not test-injected panes.
// 2. Settings screenshot via actual user-visible navigation
await openApp(page);
const notesButton = page.locator('#rail-notes:visible, #tool-notes-btn:visible').first();
if (await notesButton.count()) {
await notesButton.click();
const realNotes = page.locator('.notes-pane:visible').first();
if (await realNotes.count()) {
await page.screenshot({ path: path.join(out, `${name}-notes.png`), fullPage: false, animations: 'disabled' });
}
}
await openSettings(page, isMobile);
await page.screenshot({ path: path.join(out, `${name}-settings.png`), fullPage: false, animations: 'disabled' });
// 3. Notes screenshot via actual user-visible navigation
await openApp(page);
await openNotes(page, isMobile);
await page.screenshot({ path: path.join(out, `${name}-notes.png`), fullPage: false, animations: 'disabled' });
testInfo.annotations.push({ type: 'evidence', description: out });
});
@@ -86,8 +163,8 @@ test('browser-owned window scrolling is not globally reset', async ({ page }) =>
expect(value).toBeGreaterThan(100);
});
test('mobile Notes and Documents panes honor the safe-area top', async ({ page }, testInfo) => {
test.skip(testInfo.project.name !== 'phone-webkit-portrait', 'Only mobile <=768px');
test('CSS cascade rules for mobile Notes and Documents respect safe areas', async ({ page }, testInfo) => {
test.skip(!isMobileViewport(testInfo), 'Only mobile viewports <=768px');
await openApp(page);
const results = await page.evaluate(() => {
document.documentElement.style.setProperty('--safe-top', '44px');
@@ -100,7 +177,7 @@ test('mobile Notes and Documents panes honor the safe-area top', async ({ page }
]) {
const pane = document.createElement('section');
pane.className = paneClass;
pane.style.opacity = '0'; // CSS-only fixture, not a screenshot of fake app content
pane.style.opacity = '0'; // CSS cascade regression check
document.body.appendChild(pane);
document.body.classList.add(bodyClass);
const style = window.getComputedStyle(pane);
@@ -120,8 +197,64 @@ test('mobile Notes and Documents panes honor the safe-area top', async ({ page }
}
});
test('real Notes and Document panes honor safe-area geometry in the live application', async ({ page }, testInfo) => {
const isMobile = isMobileViewport(testInfo);
test.skip(!isMobile, 'Only mobile viewports <=768px');
await openApp(page);
// Configure simulated safe-area insets (e.g. iPhone notch / Dynamic Island)
await page.evaluate(() => {
document.documentElement.style.setProperty('--safe-top', '44px');
document.documentElement.style.setProperty('--sheet-top', '68px');
document.documentElement.style.setProperty('--app-height', '844px');
});
// 1. Verify actual Notes UI
const realNotes = await openNotes(page, isMobile);
const notesMetrics = await realNotes.evaluate(el => {
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
return {
top: style.top,
height: style.height,
maxHeight: style.maxHeight,
rectTop: rect.top,
rectHeight: rect.height,
};
});
expect(notesMetrics.top, 'Real Notes pane top must match --sheet-top').toBe('68px');
expect(parseFloat(notesMetrics.height), 'Real Notes pane height must respect safe area').toBeGreaterThanOrEqual(770);
expect(notesMetrics.rectTop, 'Real Notes pane must not overlap the 44px safe-area top notch').toBeGreaterThanOrEqual(44);
// 2. Verify actual Document UI
await openApp(page);
await page.evaluate(() => {
document.documentElement.style.setProperty('--safe-top', '44px');
document.documentElement.style.setProperty('--sheet-top', '68px');
document.documentElement.style.setProperty('--app-height', '844px');
});
const realDoc = await openDocument(page, isMobile);
const docMetrics = await realDoc.evaluate(el => {
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
return {
top: style.top,
height: style.height,
maxHeight: style.maxHeight,
rectTop: rect.top,
rectHeight: rect.height,
};
});
expect(docMetrics.top, 'Real Document pane top must match --sheet-top').toBe('68px');
expect(parseFloat(docMetrics.height), 'Real Document pane height must respect safe area').toBeGreaterThanOrEqual(770);
expect(docMetrics.rectTop, 'Real Document pane must not overlap the 44px safe-area top notch').toBeGreaterThanOrEqual(44);
});
test('restored floating windows clamp away from a left safe area', async ({ page }, testInfo) => {
test.skip(testInfo.project.name !== 'desktop-chromium', 'Geometry test uses desktop resizable windows');
test.skip(isMobileViewport(testInfo), 'Geometry test uses desktop resizable windows');
await openApp(page);
const value = await page.evaluate(async () => {
document.documentElement.style.setProperty('--safe-left', '58px');
@@ -139,7 +272,6 @@ test('restored floating windows clamp away from a left safe area', async ({ page
const padAtRestore = parseFloat(
getComputedStyle(document.body).paddingLeft
);
console.log('PADDING AT RESTORE:', padAtRestore);
if (Math.abs(padAtRestore - 58) > 0.5) {
throw new Error(
@@ -174,7 +306,7 @@ test('restored floating windows clamp away from a left safe area', async ({ page
probe.remove();
}
});
console.log('WINDOW RESTORE DIAGNOSTIC:', JSON.stringify(value));
expect(value.paddingLeft).toBeGreaterThanOrEqual(58);
expect(value.width).toBe(480);
expect(value.left).toBeGreaterThanOrEqual(value.paddingLeft);