fix(ci): sanitize diagnostics fail-closed and purge malformed trace records

2026-11-01 02:32:21 +01:00 · 2026-10-30 23:13:46 +00:00
parent fd79776964
commit 84c7eabfce
6 changed files with 518 additions and 15 deletions
@@ -242,3 +242,72 @@ def test_multiple_trace_archives_in_directory(tmp_path):
    with zipfile.ZipFile(z2, "r") as z:
        content = json.loads(z.read("0.trace").decode().strip())
        assert content["params"]["value"] == "[REDACTED]"
+
+
+def test_fail_closed_malformed_jsonl_record(tmp_path):
+    """Malformed JSONL lines in .trace archives fail closed and delete the archive."""
+    zp = str(tmp_path / "malformed_jsonl.zip")
+    with zipfile.ZipFile(zp, "w") as z:
+        z.writestr(
+            "0-trace.trace",
+            b'{"type": "action", "params": {"value": "secret123"}}\n'
+            b'this is invalid json {"password": "leak_raw_pass"}\n'
+        )
+    ok = sanitize_trace_archive(zp)
+    assert ok is False
+    assert not os.path.exists(zp)
+    assert not os.path.exists(zp + ".tmp")
+
+
+def test_fail_closed_malformed_jsonl_truncated(tmp_path):
+    """Truncated JSONL records in .network archives fail closed and delete the archive."""
+    zp = str(tmp_path / "truncated_jsonl.zip")
+    with zipfile.ZipFile(zp, "w") as z:
+        z.writestr(
+            "0-trace.network",
+            b'{"type": "request", "request": {"url": "https://example.com/api/auth", "headers": [{"name": "Authorization", "value": "Bearer secret_tok"'
+        )
+    ok = sanitize_trace_archive(zp)
+    assert ok is False
+    assert not os.path.exists(zp)
+    assert not os.path.exists(zp + ".tmp")
+
+
+def test_fail_closed_malformed_sensitive_body(tmp_path):
+    """Malformed sensitive request body in trace archives fails closed and deletes archive."""
+    zp = str(tmp_path / "malformed_sensitive_body.zip")
+    with zipfile.ZipFile(zp, "w") as z:
+        record = {
+            "type": "request",
+            "request": {
+                "url": "https://example.com/api/login",
+                "postData": '{"password": "super_secret_pw", "corrupt_trailing": ',
+            },
+        }
+        z.writestr("0-trace.network", json.dumps(record).encode() + b"\n")
+    ok = sanitize_trace_archive(zp)
+    assert ok is False
+    assert not os.path.exists(zp)
+    assert not os.path.exists(zp + ".tmp")
+
+
+def test_fail_closed_malformed_har_member(tmp_path):
+    """Malformed JSON in .har archives fails closed and deletes the archive."""
+    zp = str(tmp_path / "malformed_har.zip")
+    with zipfile.ZipFile(zp, "w") as z:
+        z.writestr("network.har", b'{"log": {"entries": [ malformed JSON content')
+    ok = sanitize_trace_archive(zp)
+    assert ok is False
+    assert not os.path.exists(zp)
+    assert not os.path.exists(zp + ".tmp")
+
+
+def test_fail_closed_invalid_utf8_jsonl_record(tmp_path):
+    """Non-UTF8 byte sequences in JSONL trace records fail closed and delete the archive."""
+    zp = str(tmp_path / "invalid_utf8.zip")
+    with zipfile.ZipFile(zp, "w") as z:
+        z.writestr("0-trace.trace", b"\xff\xfe\x00\x00 invalid utf8 raw bytes\n")
+    ok = sanitize_trace_archive(zp)
+    assert ok is False
+    assert not os.path.exists(zp)
+    assert not os.path.exists(zp + ".tmp")
@@ -0,0 +1,209 @@
+"""Tests for the fail-closed artifact verification step (.github/scripts/verify_sanitized_artifacts.py).
+
+Validates that:
+1. Safe JUnit XML reporting is preserved across all scenarios.
+2. Abnormal runner termination without verified disposable CI fixture purges all diagnostics.
+3. Valid disposable CI environments sanitize trace archives fail-closed.
+4. Malformed trace archives are deleted fail-closed without corrupting or deleting JUnit XML reports.
+5. In-progress temporary files, HAR files, and database dumps are purged unconditionally.
+6. Top-level exceptions fail closed to purge all non-XML diagnostics.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+VERIFY_SCRIPT = REPO_ROOT / ".github" / "scripts" / "verify_sanitized_artifacts.py"
+
+import importlib.util
+
+spec = importlib.util.spec_from_file_location("verify_sanitized_artifacts", str(VERIFY_SCRIPT))
+verify_module = importlib.util.module_from_spec(spec)
+sys.modules["verify_sanitized_artifacts"] = verify_module
+spec.loader.exec_module(verify_module)
+
+is_valid_disposable_ci_fixture = verify_module.is_valid_disposable_ci_fixture
+verify_and_sanitize_artifacts = verify_module.verify_and_sanitize_artifacts
+
+
+def _setup_disposable_ci(tmp_path: Path) -> dict[str, str]:
+    data_dir = tmp_path / "odysseus-photo-editor-e2e-fixture"
+    data_dir.mkdir(parents=True, exist_ok=True)
+    db_path = data_dir / "app.db"
+    db_path.write_bytes(b"SQLite format 3\x00")
+    marker_file = data_dir / ".disposable-ci-fixture.json"
+    marker_file.write_text(
+        json.dumps({
+            "synthetic": True,
+            "authenticated": True,
+            "database": str(db_path),
+        })
+    )
+    return {
+        "CI": "true",
+        "GITHUB_ACTIONS": "true",
+        "ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS": "1",
+        "PHOTO_EDITOR_E2E_DATA_DIR": str(data_dir),
+        "PHOTO_EDITOR_E2E_DB_PATH": str(db_path),
+        "PHOTO_EDITOR_E2E_AUTH": "1",
+        "ODYSSEUS_TEST_PYTHON": sys.executable,
+    }
+
+
+def test_preserves_valid_junit_xml_cleanly(tmp_path):
+    """Valid JUnit XML files are preserved intact."""
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "browser-junit.xml"
+    junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
+
+    ok = verify_and_sanitize_artifacts(str(results_dir))
+    assert ok is True
+    assert junit.exists()
+    assert "tests=\"137\"" in junit.read_text()
+
+
+def test_fail_closed_abnormal_termination_purges_all_diagnostics(tmp_path, monkeypatch):
+    """Abnormal runner termination without disposable fixture purges all diagnostics, keeping JUnit."""
+    # Simulate CI without disposable fixture
+    monkeypatch.setenv("CI", "true")
+    monkeypatch.setenv("GITHUB_ACTIONS", "true")
+    monkeypatch.delenv("ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS", raising=False)
+    monkeypatch.delenv("PHOTO_EDITOR_E2E_DATA_DIR", raising=False)
+
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "browser-junit.xml"
+    junit.write_text('<testsuites><testsuite name="photo-editor" tests="10" failures="1"/></testsuites>')
+
+    trace = results_dir / "trace.zip"
+    trace.write_bytes(b"PK0304 sensitive trace archive")
+    har = results_dir / "session.har"
+    har.write_text('{"log": {"entries": ["secret"]}}')
+    png = results_dir / "failure.png"
+    png.write_bytes(b"\x89PNG\r\n\x1a\nfake screenshot")
+    tmp = results_dir / "temp.tmp"
+    tmp.write_text("in-progress trace")
+    db = results_dir / "app.db"
+    db.write_bytes(b"SQLite format 3 database")
+
+    ok = verify_and_sanitize_artifacts(str(results_dir))
+    assert ok is True
+
+    # Safe JUnit reporting MUST be preserved
+    assert junit.exists()
+    assert "failures=\"1\"" in junit.read_text()
+
+    # All sensitive diagnostics MUST be purged
+    assert not trace.exists()
+    assert not har.exists()
+    assert not png.exists()
+    assert not tmp.exists()
+    assert not db.exists()
+
+
+def test_sanitizes_trace_archive_when_disposable_ci_verified(tmp_path, monkeypatch):
+    """When running inside a verified disposable CI fixture, trace archives are sanitized fail-closed."""
+    env = _setup_disposable_ci(tmp_path)
+    for k, v in env.items():
+        monkeypatch.setenv(k, v)
+
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "browser-junit.xml"
+    junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
+
+    trace = results_dir / "trace.zip"
+    with zipfile.ZipFile(trace, "w") as z:
+        action = {"type": "action", "name": "fill", "params": {"value": "super_secret_password"}}
+        z.writestr("0-trace.trace", json.dumps(action).encode() + b"\n")
+
+    png = results_dir / "failure.png"
+    png.write_bytes(b"\x89PNG\r\n\x1a\nvalid screenshot payload")
+
+    ok = verify_and_sanitize_artifacts(str(results_dir))
+    assert ok is True
+
+    assert junit.exists()
+    assert trace.exists()
+    assert png.exists()
+
+    with zipfile.ZipFile(trace, "r") as z:
+        content = json.loads(z.read("0-trace.trace").decode().strip())
+        assert content["params"]["value"] == "[REDACTED]"
+
+
+def test_malformed_trace_archive_purged_preserving_junit_xml(tmp_path, monkeypatch):
+    """Malformed trace archive is purged fail-closed without affecting JUnit XML."""
+    env = _setup_disposable_ci(tmp_path)
+    for k, v in env.items():
+        monkeypatch.setenv(k, v)
+
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "browser-junit.xml"
+    junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="1"/></testsuites>')
+
+    # Create trace archive with malformed JSONL line
+    malformed_trace = results_dir / "trace.zip"
+    with zipfile.ZipFile(malformed_trace, "w") as z:
+        z.writestr("0-trace.trace", b'{"type": "action"}\nmalformed not json line\n')
+
+    ok = verify_and_sanitize_artifacts(str(results_dir))
+    assert ok is True
+
+    # JUnit XML is preserved
+    assert junit.exists()
+    # Malformed archive is purged!
+    assert not malformed_trace.exists()
+
+
+def test_unhandled_exception_fails_closed_purges_all_non_xml(tmp_path, monkeypatch):
+    """Unexpected exception during verification triggers emergency fail-closed purge of non-XML files."""
+    env = _setup_disposable_ci(tmp_path)
+    for k, v in env.items():
+        monkeypatch.setenv(k, v)
+
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "browser-junit.xml"
+    junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
+
+    trace = results_dir / "trace.zip"
+    trace.write_bytes(b"PK0304 trace")
+
+    def broken_sanitize(zp):
+        raise RuntimeError("Simulated unexpected crash in sanitizer")
+
+    monkeypatch.setattr(verify_module, "sanitize_trace_archive", broken_sanitize)
+
+    ok = verify_and_sanitize_artifacts(str(results_dir))
+    assert ok is False
+    # Safe JUnit reporting is preserved
+    assert junit.exists()
+    # Sensitive non-XML file is purged fail-closed!
+    assert not trace.exists()
+
+
+def test_cli_invocation_returns_cleanly(tmp_path):
+    """Running the script as a CLI entrypoint succeeds with exit code 0."""
+    results_dir = tmp_path / "test-results"
+    results_dir.mkdir()
+    junit = results_dir / "junit.xml"
+    junit.write_text('<testsuites><testsuite name="native-node" tests="43" failures="0"/></testsuites>')
+
+    res = subprocess.run(
+        [sys.executable, str(VERIFY_SCRIPT), str(results_dir)],
+        capture_output=True,
+        text=True,
+    )
+    assert res.returncode == 0
+    assert junit.exists()