mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-31 02:02:22 +01:00
fix(ci): sanitize diagnostics fail-closed and purge malformed trace records
This commit is contained in:
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed independent verification and sanitization gate for test diagnostics.
|
||||
|
||||
Runs after test suite completion or abnormal runner termination (e.g. failures, timeouts, crashes).
|
||||
Ensures that no unsanitized traces, raw credentials, temporary archives, or unauthorized
|
||||
diagnostics can ever be published as CI artifacts, while preserving safe JUnit XML reporting.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
# Add repository root to Python path to import sanitizer
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
try:
|
||||
from scripts.sanitize_trace_artifacts import sanitize_trace_archive
|
||||
except ImportError:
|
||||
# If import fails, define fail-closed fallback
|
||||
def sanitize_trace_archive(zp: str) -> bool:
|
||||
if os.path.exists(zp):
|
||||
try:
|
||||
os.remove(zp)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
logger = logging.getLogger("verify_sanitized_artifacts")
|
||||
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def is_valid_disposable_ci_fixture() -> bool:
|
||||
"""Verify that execution ran inside an authorized disposable synthetic fixture."""
|
||||
is_ci = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS"))
|
||||
if not is_ci:
|
||||
# Local runs require explicit trace opt-in
|
||||
trace_opt_in = os.environ.get("PLAYWRIGHT_ENABLE_TRACE", "").lower() in ("1", "true")
|
||||
return trace_opt_in
|
||||
|
||||
if os.environ.get("ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS") != "1":
|
||||
return False
|
||||
if os.environ.get("PHOTO_EDITOR_E2E_AUTH") == "0":
|
||||
return False
|
||||
|
||||
data_dir = os.environ.get("PHOTO_EDITOR_E2E_DATA_DIR")
|
||||
if not data_dir:
|
||||
return False
|
||||
resolved = os.path.abspath(data_dir)
|
||||
base_name = os.path.basename(resolved)
|
||||
if not base_name.startswith("odysseus-photo-editor-e2e-"):
|
||||
return False
|
||||
|
||||
allowed_tmp = [os.path.abspath(p) for p in (tempfile.gettempdir(), "/tmp", "/var/tmp")]
|
||||
if not any(resolved.startswith(b + os.sep) or resolved == b for b in allowed_tmp):
|
||||
return False
|
||||
|
||||
db_path = os.environ.get("PHOTO_EDITOR_E2E_DB_PATH")
|
||||
if db_path != os.path.join(resolved, "app.db"):
|
||||
return False
|
||||
|
||||
marker_path = os.path.join(resolved, ".disposable-ci-fixture.json")
|
||||
if not os.path.exists(marker_path):
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(marker_path, "r", encoding="utf-8") as f:
|
||||
fixture = json.load(f)
|
||||
return (
|
||||
fixture.get("synthetic") is True
|
||||
and fixture.get("authenticated") is True
|
||||
and fixture.get("database") == os.path.join(resolved, "app.db")
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def verify_and_sanitize_artifacts(target_dir: str) -> bool:
|
||||
"""Independently verify and sanitize diagnostics fail-closed, preserving safe JUnit XML.
|
||||
|
||||
Returns True if verification succeeded, False if fail-closed purge was required.
|
||||
"""
|
||||
if not os.path.exists(target_dir):
|
||||
logger.info("Target directory %s does not exist; nothing to sanitize.", target_dir)
|
||||
return True
|
||||
|
||||
disposable_ci = is_valid_disposable_ci_fixture()
|
||||
preserved_xmls = 0
|
||||
sanitized_archives = 0
|
||||
purged_files = 0
|
||||
|
||||
try:
|
||||
for root, _, files in os.walk(target_dir):
|
||||
for filename in files:
|
||||
filepath = os.path.join(root, filename)
|
||||
name_lower = filename.lower()
|
||||
|
||||
# 1. Safe JUnit reporting: preserve valid XML reports
|
||||
if name_lower.endswith(".xml"):
|
||||
try:
|
||||
ET.parse(filepath)
|
||||
preserved_xmls += 1
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.warning("Corrupted XML report %s: %s; purging fail-closed", filepath, exc)
|
||||
try:
|
||||
os.remove(filepath)
|
||||
purged_files += 1
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 2. Check if diagnostics are authorized (disposable synthetic fixture)
|
||||
if not disposable_ci:
|
||||
# In CI without disposable fixture, or local without opt-in: purge all diagnostics
|
||||
logger.info("Purging diagnostic artifact %s (disposable CI fixture not verified)", filepath)
|
||||
try:
|
||||
os.remove(filepath)
|
||||
purged_files += 1
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
|
||||
# 3. Disposable CI is authorized: enforce strict sanitization
|
||||
if name_lower.endswith(".zip"):
|
||||
ok = sanitize_trace_archive(filepath)
|
||||
if ok and os.path.exists(filepath):
|
||||
sanitized_archives += 1
|
||||
else:
|
||||
logger.warning("Trace archive %s failed sanitization; purged fail-closed", filepath)
|
||||
if os.path.exists(filepath):
|
||||
try:
|
||||
os.remove(filepath)
|
||||
except OSError:
|
||||
pass
|
||||
purged_files += 1
|
||||
|
||||
elif name_lower.endswith(".png"):
|
||||
# Validate PNG magic bytes to ensure not a secret payload disguised as an image
|
||||
try:
|
||||
with open(filepath, "rb") as pf:
|
||||
header = pf.read(8)
|
||||
if header == PNG_MAGIC:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
logger.warning("Invalid or suspicious screenshot %s; purged fail-closed", filepath)
|
||||
try:
|
||||
os.remove(filepath)
|
||||
purged_files += 1
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
else:
|
||||
# Purge any .tmp, .har, .db, .sqlite, .log or unrecognized files
|
||||
logger.info("Purging unauthorized diagnostic file %s", filepath)
|
||||
try:
|
||||
os.remove(filepath)
|
||||
purged_files += 1
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
logger.info(
|
||||
"Artifact verification complete for %s: preserved %d XML reports, sanitized %d archives, purged %d files.",
|
||||
target_dir,
|
||||
preserved_xmls,
|
||||
sanitized_archives,
|
||||
purged_files,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Unexpected error during artifact verification: %s; purging all non-XML files fail-closed", exc)
|
||||
try:
|
||||
for p in Path(target_dir).rglob("*"):
|
||||
if p.is_file() and not p.name.lower().endswith(".xml"):
|
||||
try:
|
||||
p.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("target_dir", nargs="?", default="test-results", help="Directory containing test results")
|
||||
args = parser.parse_args()
|
||||
|
||||
ok = verify_and_sanitize_artifacts(args.target_dir)
|
||||
# Always exit 0 so upload-artifact can safely publish verified reports
|
||||
sys.exit(0 if ok else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -39,6 +39,10 @@ jobs:
|
||||
run: |
|
||||
node .github/scripts/run_native_tests.mjs
|
||||
python .github/scripts/check_test_execution.py test-results/native-node/junit.xml --minimum 43
|
||||
- name: Verify and sanitize native test results fail-closed
|
||||
if: always()
|
||||
run: |
|
||||
python .github/scripts/verify_sanitized_artifacts.py test-results/native-node/
|
||||
- name: Save native test identities and results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
@@ -95,6 +99,10 @@ jobs:
|
||||
run: |
|
||||
npm run test:photo-editor -- --reporter=line,junit
|
||||
python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 137
|
||||
- name: Verify and sanitize failure diagnostics fail-closed
|
||||
if: always()
|
||||
run: |
|
||||
python .github/scripts/verify_sanitized_artifacts.py test-results/
|
||||
- name: Save per-test results and sanitized failure diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -25,6 +25,10 @@ Default CI trace suppression remains intact. Only this job's explicit opt-in plu
|
||||
a newly generated authenticated synthetic fixture marker permits failure traces.
|
||||
The existing structured sanitizer runs before artifacts are uploaded; invalid
|
||||
ownership, failed sanitization and missing tooling purge sensitive artifacts.
|
||||
Unsupported or malformed sensitive records cause the affected archive to be removed
|
||||
fail-closed. An independent fail-closed verification step (.github/scripts/verify_sanitized_artifacts.py)
|
||||
runs after test suite execution or abnormal runner termination to prevent publishing
|
||||
unsanitized diagnostics while preserving safe JUnit reporting.
|
||||
Never upload the fixture directory, database or browser storage state.
|
||||
|
||||
The stroke budget now excludes measured synthetic event-delivery pacing and also
|
||||
|
||||
@@ -77,13 +77,14 @@ def sanitize_body_text(text: str) -> str:
|
||||
|
||||
# 1. Try JSON
|
||||
trimmed = text.strip()
|
||||
if (trimmed.startswith("{") and trimmed.endswith("}")) or (trimmed.startswith("[") and trimmed.endswith("]")):
|
||||
if trimmed.startswith(("{", "[")):
|
||||
try:
|
||||
parsed = json.loads(trimmed)
|
||||
sanitized = sanitize_structured_obj(parsed)
|
||||
return json.dumps(sanitized, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if SENSITIVE_KEY_PATTERN.search(trimmed) or any(p in trimmed.lower() for p in SENSITIVE_PARAM_NAMES):
|
||||
raise ValueError(f"Malformed sensitive JSON request body: {exc}") from exc
|
||||
|
||||
# 2. Try URL-encoded form data
|
||||
if "=" in trimmed and not trimmed.startswith("<"):
|
||||
@@ -98,8 +99,13 @@ def sanitize_body_text(text: str) -> str:
|
||||
else:
|
||||
out_pairs.append((k, v))
|
||||
return urllib.parse.urlencode(out_pairs)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if SENSITIVE_KEY_PATTERN.search(trimmed):
|
||||
raise ValueError(f"Malformed sensitive form request body: {exc}") from exc
|
||||
|
||||
# If unparsed body text contains sensitive keys or tokens, fail closed
|
||||
if SENSITIVE_KEY_PATTERN.search(trimmed) or any(p in trimmed.lower() for p in SENSITIVE_PARAM_NAMES):
|
||||
raise ValueError("Malformed or unsupported sensitive request body in trace entry")
|
||||
|
||||
return text
|
||||
|
||||
@@ -207,7 +213,11 @@ def sanitize_structured_obj(obj: Any) -> Any:
|
||||
|
||||
|
||||
def sanitize_jsonl_content(data: bytes) -> bytes:
|
||||
"""Sanitize newline-delimited JSON Lines (.trace, .network, .stacks)."""
|
||||
"""Sanitize newline-delimited JSON Lines (.trace, .network, .stacks).
|
||||
|
||||
If any non-empty record cannot be decoded as valid UTF-8 or parsed as valid JSON,
|
||||
or is malformed, raise ValueError fail-closed so the affected archive is permanently removed.
|
||||
"""
|
||||
lines = data.split(b"\n")
|
||||
out_lines = []
|
||||
for line in lines:
|
||||
@@ -216,10 +226,8 @@ def sanitize_jsonl_content(data: bytes) -> bytes:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(line.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
# Not valid JSON, keep as-is
|
||||
out_lines.append(line)
|
||||
continue
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as err:
|
||||
raise ValueError(f"Malformed or unsupported JSONL trace record: {err}")
|
||||
sanitized = sanitize_structured_obj(parsed)
|
||||
out_lines.append(json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
||||
return b"\n".join(out_lines)
|
||||
@@ -275,11 +283,10 @@ def sanitize_trace_archive(zp: str) -> bool:
|
||||
elif filename_lower.endswith((".json", ".har")):
|
||||
try:
|
||||
parsed = json.loads(content.decode("utf-8"))
|
||||
sanitized = sanitize_structured_obj(parsed)
|
||||
zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
||||
except Exception:
|
||||
# If invalid JSON, treat as text line-by-line
|
||||
zout.writestr(item, sanitize_jsonl_content(content))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise ValueError(f"Malformed JSON in archive member {item.filename}: {exc}") from exc
|
||||
sanitized = sanitize_structured_obj(parsed)
|
||||
zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
||||
else:
|
||||
# Unknown binary or text member: exclude to fail closed
|
||||
continue
|
||||
|
||||
@@ -242,3 +242,72 @@ def test_multiple_trace_archives_in_directory(tmp_path):
|
||||
with zipfile.ZipFile(z2, "r") as z:
|
||||
content = json.loads(z.read("0.trace").decode().strip())
|
||||
assert content["params"]["value"] == "[REDACTED]"
|
||||
|
||||
|
||||
def test_fail_closed_malformed_jsonl_record(tmp_path):
|
||||
"""Malformed JSONL lines in .trace archives fail closed and delete the archive."""
|
||||
zp = str(tmp_path / "malformed_jsonl.zip")
|
||||
with zipfile.ZipFile(zp, "w") as z:
|
||||
z.writestr(
|
||||
"0-trace.trace",
|
||||
b'{"type": "action", "params": {"value": "secret123"}}\n'
|
||||
b'this is invalid json {"password": "leak_raw_pass"}\n'
|
||||
)
|
||||
ok = sanitize_trace_archive(zp)
|
||||
assert ok is False
|
||||
assert not os.path.exists(zp)
|
||||
assert not os.path.exists(zp + ".tmp")
|
||||
|
||||
|
||||
def test_fail_closed_malformed_jsonl_truncated(tmp_path):
|
||||
"""Truncated JSONL records in .network archives fail closed and delete the archive."""
|
||||
zp = str(tmp_path / "truncated_jsonl.zip")
|
||||
with zipfile.ZipFile(zp, "w") as z:
|
||||
z.writestr(
|
||||
"0-trace.network",
|
||||
b'{"type": "request", "request": {"url": "https://example.com/api/auth", "headers": [{"name": "Authorization", "value": "Bearer secret_tok"'
|
||||
)
|
||||
ok = sanitize_trace_archive(zp)
|
||||
assert ok is False
|
||||
assert not os.path.exists(zp)
|
||||
assert not os.path.exists(zp + ".tmp")
|
||||
|
||||
|
||||
def test_fail_closed_malformed_sensitive_body(tmp_path):
|
||||
"""Malformed sensitive request body in trace archives fails closed and deletes archive."""
|
||||
zp = str(tmp_path / "malformed_sensitive_body.zip")
|
||||
with zipfile.ZipFile(zp, "w") as z:
|
||||
record = {
|
||||
"type": "request",
|
||||
"request": {
|
||||
"url": "https://example.com/api/login",
|
||||
"postData": '{"password": "super_secret_pw", "corrupt_trailing": ',
|
||||
},
|
||||
}
|
||||
z.writestr("0-trace.network", json.dumps(record).encode() + b"\n")
|
||||
ok = sanitize_trace_archive(zp)
|
||||
assert ok is False
|
||||
assert not os.path.exists(zp)
|
||||
assert not os.path.exists(zp + ".tmp")
|
||||
|
||||
|
||||
def test_fail_closed_malformed_har_member(tmp_path):
|
||||
"""Malformed JSON in .har archives fails closed and deletes the archive."""
|
||||
zp = str(tmp_path / "malformed_har.zip")
|
||||
with zipfile.ZipFile(zp, "w") as z:
|
||||
z.writestr("network.har", b'{"log": {"entries": [ malformed JSON content')
|
||||
ok = sanitize_trace_archive(zp)
|
||||
assert ok is False
|
||||
assert not os.path.exists(zp)
|
||||
assert not os.path.exists(zp + ".tmp")
|
||||
|
||||
|
||||
def test_fail_closed_invalid_utf8_jsonl_record(tmp_path):
|
||||
"""Non-UTF8 byte sequences in JSONL trace records fail closed and delete the archive."""
|
||||
zp = str(tmp_path / "invalid_utf8.zip")
|
||||
with zipfile.ZipFile(zp, "w") as z:
|
||||
z.writestr("0-trace.trace", b"\xff\xfe\x00\x00 invalid utf8 raw bytes\n")
|
||||
ok = sanitize_trace_archive(zp)
|
||||
assert ok is False
|
||||
assert not os.path.exists(zp)
|
||||
assert not os.path.exists(zp + ".tmp")
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Tests for the fail-closed artifact verification step (.github/scripts/verify_sanitized_artifacts.py).
|
||||
|
||||
Validates that:
|
||||
1. Safe JUnit XML reporting is preserved across all scenarios.
|
||||
2. Abnormal runner termination without verified disposable CI fixture purges all diagnostics.
|
||||
3. Valid disposable CI environments sanitize trace archives fail-closed.
|
||||
4. Malformed trace archives are deleted fail-closed without corrupting or deleting JUnit XML reports.
|
||||
5. In-progress temporary files, HAR files, and database dumps are purged unconditionally.
|
||||
6. Top-level exceptions fail closed to purge all non-XML diagnostics.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
VERIFY_SCRIPT = REPO_ROOT / ".github" / "scripts" / "verify_sanitized_artifacts.py"
|
||||
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("verify_sanitized_artifacts", str(VERIFY_SCRIPT))
|
||||
verify_module = importlib.util.module_from_spec(spec)
|
||||
sys.modules["verify_sanitized_artifacts"] = verify_module
|
||||
spec.loader.exec_module(verify_module)
|
||||
|
||||
is_valid_disposable_ci_fixture = verify_module.is_valid_disposable_ci_fixture
|
||||
verify_and_sanitize_artifacts = verify_module.verify_and_sanitize_artifacts
|
||||
|
||||
|
||||
def _setup_disposable_ci(tmp_path: Path) -> dict[str, str]:
|
||||
data_dir = tmp_path / "odysseus-photo-editor-e2e-fixture"
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
db_path = data_dir / "app.db"
|
||||
db_path.write_bytes(b"SQLite format 3\x00")
|
||||
marker_file = data_dir / ".disposable-ci-fixture.json"
|
||||
marker_file.write_text(
|
||||
json.dumps({
|
||||
"synthetic": True,
|
||||
"authenticated": True,
|
||||
"database": str(db_path),
|
||||
})
|
||||
)
|
||||
return {
|
||||
"CI": "true",
|
||||
"GITHUB_ACTIONS": "true",
|
||||
"ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS": "1",
|
||||
"PHOTO_EDITOR_E2E_DATA_DIR": str(data_dir),
|
||||
"PHOTO_EDITOR_E2E_DB_PATH": str(db_path),
|
||||
"PHOTO_EDITOR_E2E_AUTH": "1",
|
||||
"ODYSSEUS_TEST_PYTHON": sys.executable,
|
||||
}
|
||||
|
||||
|
||||
def test_preserves_valid_junit_xml_cleanly(tmp_path):
|
||||
"""Valid JUnit XML files are preserved intact."""
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "browser-junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||
|
||||
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||
assert ok is True
|
||||
assert junit.exists()
|
||||
assert "tests=\"137\"" in junit.read_text()
|
||||
|
||||
|
||||
def test_fail_closed_abnormal_termination_purges_all_diagnostics(tmp_path, monkeypatch):
|
||||
"""Abnormal runner termination without disposable fixture purges all diagnostics, keeping JUnit."""
|
||||
# Simulate CI without disposable fixture
|
||||
monkeypatch.setenv("CI", "true")
|
||||
monkeypatch.setenv("GITHUB_ACTIONS", "true")
|
||||
monkeypatch.delenv("ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS", raising=False)
|
||||
monkeypatch.delenv("PHOTO_EDITOR_E2E_DATA_DIR", raising=False)
|
||||
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "browser-junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="photo-editor" tests="10" failures="1"/></testsuites>')
|
||||
|
||||
trace = results_dir / "trace.zip"
|
||||
trace.write_bytes(b"PK0304 sensitive trace archive")
|
||||
har = results_dir / "session.har"
|
||||
har.write_text('{"log": {"entries": ["secret"]}}')
|
||||
png = results_dir / "failure.png"
|
||||
png.write_bytes(b"\x89PNG\r\n\x1a\nfake screenshot")
|
||||
tmp = results_dir / "temp.tmp"
|
||||
tmp.write_text("in-progress trace")
|
||||
db = results_dir / "app.db"
|
||||
db.write_bytes(b"SQLite format 3 database")
|
||||
|
||||
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||
assert ok is True
|
||||
|
||||
# Safe JUnit reporting MUST be preserved
|
||||
assert junit.exists()
|
||||
assert "failures=\"1\"" in junit.read_text()
|
||||
|
||||
# All sensitive diagnostics MUST be purged
|
||||
assert not trace.exists()
|
||||
assert not har.exists()
|
||||
assert not png.exists()
|
||||
assert not tmp.exists()
|
||||
assert not db.exists()
|
||||
|
||||
|
||||
def test_sanitizes_trace_archive_when_disposable_ci_verified(tmp_path, monkeypatch):
|
||||
"""When running inside a verified disposable CI fixture, trace archives are sanitized fail-closed."""
|
||||
env = _setup_disposable_ci(tmp_path)
|
||||
for k, v in env.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "browser-junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||
|
||||
trace = results_dir / "trace.zip"
|
||||
with zipfile.ZipFile(trace, "w") as z:
|
||||
action = {"type": "action", "name": "fill", "params": {"value": "super_secret_password"}}
|
||||
z.writestr("0-trace.trace", json.dumps(action).encode() + b"\n")
|
||||
|
||||
png = results_dir / "failure.png"
|
||||
png.write_bytes(b"\x89PNG\r\n\x1a\nvalid screenshot payload")
|
||||
|
||||
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||
assert ok is True
|
||||
|
||||
assert junit.exists()
|
||||
assert trace.exists()
|
||||
assert png.exists()
|
||||
|
||||
with zipfile.ZipFile(trace, "r") as z:
|
||||
content = json.loads(z.read("0-trace.trace").decode().strip())
|
||||
assert content["params"]["value"] == "[REDACTED]"
|
||||
|
||||
|
||||
def test_malformed_trace_archive_purged_preserving_junit_xml(tmp_path, monkeypatch):
|
||||
"""Malformed trace archive is purged fail-closed without affecting JUnit XML."""
|
||||
env = _setup_disposable_ci(tmp_path)
|
||||
for k, v in env.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "browser-junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="1"/></testsuites>')
|
||||
|
||||
# Create trace archive with malformed JSONL line
|
||||
malformed_trace = results_dir / "trace.zip"
|
||||
with zipfile.ZipFile(malformed_trace, "w") as z:
|
||||
z.writestr("0-trace.trace", b'{"type": "action"}\nmalformed not json line\n')
|
||||
|
||||
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||
assert ok is True
|
||||
|
||||
# JUnit XML is preserved
|
||||
assert junit.exists()
|
||||
# Malformed archive is purged!
|
||||
assert not malformed_trace.exists()
|
||||
|
||||
|
||||
def test_unhandled_exception_fails_closed_purges_all_non_xml(tmp_path, monkeypatch):
|
||||
"""Unexpected exception during verification triggers emergency fail-closed purge of non-XML files."""
|
||||
env = _setup_disposable_ci(tmp_path)
|
||||
for k, v in env.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "browser-junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||
|
||||
trace = results_dir / "trace.zip"
|
||||
trace.write_bytes(b"PK0304 trace")
|
||||
|
||||
def broken_sanitize(zp):
|
||||
raise RuntimeError("Simulated unexpected crash in sanitizer")
|
||||
|
||||
monkeypatch.setattr(verify_module, "sanitize_trace_archive", broken_sanitize)
|
||||
|
||||
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||
assert ok is False
|
||||
# Safe JUnit reporting is preserved
|
||||
assert junit.exists()
|
||||
# Sensitive non-XML file is purged fail-closed!
|
||||
assert not trace.exists()
|
||||
|
||||
|
||||
def test_cli_invocation_returns_cleanly(tmp_path):
|
||||
"""Running the script as a CLI entrypoint succeeds with exit code 0."""
|
||||
results_dir = tmp_path / "test-results"
|
||||
results_dir.mkdir()
|
||||
junit = results_dir / "junit.xml"
|
||||
junit.write_text('<testsuites><testsuite name="native-node" tests="43" failures="0"/></testsuites>')
|
||||
|
||||
res = subprocess.run(
|
||||
[sys.executable, str(VERIFY_SCRIPT), str(results_dir)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert res.returncode == 0
|
||||
assert junit.exists()
|
||||
Reference in New Issue
Block a user