ci: gate native suites with isolated browser fixtures

.github/scripts/check_test_execution.py

@@ -0,0 +1,17 @@
+"""Reject skipped/empty native CI runs without conflating driver child counts."""
+import argparse
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('report', type=Path)
+parser.add_argument('--minimum', type=int, required=True)
+args = parser.parse_args()
+root = ET.parse(args.report).getroot()
+cases = list(root.iter('testcase'))
+if len(cases) < args.minimum:
+    raise SystemExit(f'Incomplete execution: {len(cases)} cases, expected at least {args.minimum}')
+for case in cases:
+    if any(case.find(tag) is not None for tag in ('skipped', 'failure', 'error')):
+        raise SystemExit(f'Case did not pass: {case.get("classname")}::{case.get("name")}')
+print(f'Verified {len(cases)} executed cases; no skipped, failed or errored cases')

.github/scripts/run_native_tests.mjs

@@ -0,0 +1,46 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { parse } from 'acorn';
+
+const root = fileURLToPath(new URL('../../', import.meta.url));
+const manifest = JSON.parse(fs.readFileSync(path.join(root, 'tests/native-ci-manifest.json'), 'utf8'));
+function files(directory) {
+  return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
+    const target = path.join(directory, entry.name);
+    return entry.isDirectory() ? files(target) : /\.(?:mjs|cjs|js)$/.test(entry.name) ? [target] : [];
+  });
+}
+function importsNativeTest(node) {
+  if (!node || typeof node !== 'object') return false;
+  if (node.type === 'ImportDeclaration' && node.source.value === 'node:test') return true;
+  if (node.type === 'CallExpression' && node.callee.name === 'require' && node.arguments[0]?.value === 'node:test') return true;
+  return Object.values(node).some(value => Array.isArray(value) ? value.some(importsNativeTest) : importsNativeTest(value));
+}
+const discovered = files(path.join(root, 'tests')).filter(file => {
+  const source = fs.readFileSync(file, 'utf8');
+  return source.includes('node:test') && importsNativeTest(parse(source, { ecmaVersion: 'latest', sourceType: 'module' }));
+}).map(file => path.relative(root, file).split(path.sep).join('/')).sort();
+const assigned = Object.values(manifest).flatMap(group => Array.isArray(group) ? group : Object.keys(group));
+if (new Set(assigned).size !== assigned.length || JSON.stringify([...assigned].sort()) !== JSON.stringify(discovered)) {
+  throw new Error('Native test coverage must be exhaustive and disjoint; update tests/native-ci-manifest.json for added, removed or duplicated files');
+}
+for (const wrappers of Object.values(manifest['python-ci'])) {
+  for (const wrapper of wrappers) if (!fs.existsSync(path.join(root, wrapper))) throw new Error(`Missing pytest wrapper: ${wrapper}`);
+}
+if (process.argv.includes('--list')) {
+  console.log(JSON.stringify({ discovered, runners: manifest }, null, 2));
+} else {
+  const reports = path.join(root, 'test-results/native-node');
+  fs.mkdirSync(reports, { recursive: true });
+  const result = spawnSync(process.execPath, ['--test', '--test-concurrency=1',
+    '--test-reporter=spec', '--test-reporter-destination=stdout',
+    '--test-reporter=junit', `--test-reporter-destination=${path.join(reports, 'junit.xml')}`,
+    ...manifest['native-ci']], {
+    cwd: root, stdio: 'inherit',
+    env: { ...process.env, ENDPOINT_URL: 'http://127.0.0.1:1' },
+  });
+  if (result.error) throw result.error;
+  process.exitCode = result.status ?? 1;
+}

.github/workflows/native-test-suites.yml

@@ -0,0 +1,119 @@
+name: Native test suites
+
+on:
+  pull_request:
+  push:
+    branches: [main, dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: native-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  native-node:
+    name: Native Node regressions
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22.23.4'
+          cache: npm
+      - run: npm ci --ignore-scripts --no-audit --no-fund
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+      - run: npx playwright install --with-deps chromium
+      - name: Validate exhaustive disjoint runner mapping and execute omitted suites
+        run: |
+          node .github/scripts/run_native_tests.mjs
+          python .github/scripts/check_test_execution.py test-results/native-node/junit.xml --minimum 43
+      - name: Save native test identities and results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: native-node-results
+          path: test-results/native-node/
+          retention-days: 7
+          if-no-files-found: warn
+
+  browser:
+    name: Chromium application regressions
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS: '1'
+      ODYSSEUS_TEST_PYTHON: python
+      CHROMADB_HOST: 127.0.0.1
+      CHROMADB_PORT: '1'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+      - name: Install FFmpeg and establish system trust
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends ffmpeg ca-certificates
+          python scripts/check_ca_trust.py --system
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          version: '0.13.5'
+          enable-cache: true
+          cache-dependency-glob: requirements.txt
+          prune-cache: false
+      - run: |
+          uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt
+          python scripts/check_ca_trust.py --python
+          uv pip install --system -r requirements.txt
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22.23.4'
+          cache: npm
+      - run: npm ci --ignore-scripts --no-audit --no-fund
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+      - run: npx playwright install --with-deps chromium
+      - name: Run complete Chromium suite against a fresh authenticated app
+        env:
+          PLAYWRIGHT_JUNIT_OUTPUT_FILE: test-results/browser-junit.xml
+        run: |
+          npm run test:photo-editor -- --reporter=line,junit
+          python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 130
+      - name: Save per-test results and sanitized failure diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: chromium-regression-results
+          path: test-results/
+          retention-days: 7
+          if-no-files-found: warn
+
+  authoritative-native-suites:
+    name: Authoritative native suites
+    if: always()
+    needs: [native-node, browser]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Require both complete suites to succeed
+        env:
+          NODE_RESULT: ${{ needs.native-node.result }}
+          BROWSER_RESULT: ${{ needs.browser.result }}
+        run: |
+          test "$NODE_RESULT" = success
+          test "$BROWSER_RESULT" = success