mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-31 10:12:21 +01:00
fix(ci): sanitize diagnostics fail-closed and purge malformed trace records
This commit is contained in:
+206
@@ -0,0 +1,206 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed independent verification and sanitization gate for test diagnostics.
|
||||||
|
|
||||||
|
Runs after test suite completion or abnormal runner termination (e.g. failures, timeouts, crashes).
|
||||||
|
Ensures that no unsanitized traces, raw credentials, temporary archives, or unauthorized
|
||||||
|
diagnostics can ever be published as CI artifacts, while preserving safe JUnit XML reporting.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Add repository root to Python path to import sanitizer
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
if str(REPO_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(REPO_ROOT))
|
||||||
|
|
||||||
|
try:
|
||||||
|
from scripts.sanitize_trace_artifacts import sanitize_trace_archive
|
||||||
|
except ImportError:
|
||||||
|
# If import fails, define fail-closed fallback
|
||||||
|
def sanitize_trace_archive(zp: str) -> bool:
|
||||||
|
if os.path.exists(zp):
|
||||||
|
try:
|
||||||
|
os.remove(zp)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return False
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||||
|
logger = logging.getLogger("verify_sanitized_artifacts")
|
||||||
|
|
||||||
|
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||||
|
|
||||||
|
|
||||||
|
def is_valid_disposable_ci_fixture() -> bool:
|
||||||
|
"""Verify that execution ran inside an authorized disposable synthetic fixture."""
|
||||||
|
is_ci = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS"))
|
||||||
|
if not is_ci:
|
||||||
|
# Local runs require explicit trace opt-in
|
||||||
|
trace_opt_in = os.environ.get("PLAYWRIGHT_ENABLE_TRACE", "").lower() in ("1", "true")
|
||||||
|
return trace_opt_in
|
||||||
|
|
||||||
|
if os.environ.get("ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS") != "1":
|
||||||
|
return False
|
||||||
|
if os.environ.get("PHOTO_EDITOR_E2E_AUTH") == "0":
|
||||||
|
return False
|
||||||
|
|
||||||
|
data_dir = os.environ.get("PHOTO_EDITOR_E2E_DATA_DIR")
|
||||||
|
if not data_dir:
|
||||||
|
return False
|
||||||
|
resolved = os.path.abspath(data_dir)
|
||||||
|
base_name = os.path.basename(resolved)
|
||||||
|
if not base_name.startswith("odysseus-photo-editor-e2e-"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
allowed_tmp = [os.path.abspath(p) for p in (tempfile.gettempdir(), "/tmp", "/var/tmp")]
|
||||||
|
if not any(resolved.startswith(b + os.sep) or resolved == b for b in allowed_tmp):
|
||||||
|
return False
|
||||||
|
|
||||||
|
db_path = os.environ.get("PHOTO_EDITOR_E2E_DB_PATH")
|
||||||
|
if db_path != os.path.join(resolved, "app.db"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
marker_path = os.path.join(resolved, ".disposable-ci-fixture.json")
|
||||||
|
if not os.path.exists(marker_path):
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(marker_path, "r", encoding="utf-8") as f:
|
||||||
|
fixture = json.load(f)
|
||||||
|
return (
|
||||||
|
fixture.get("synthetic") is True
|
||||||
|
and fixture.get("authenticated") is True
|
||||||
|
and fixture.get("database") == os.path.join(resolved, "app.db")
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def verify_and_sanitize_artifacts(target_dir: str) -> bool:
|
||||||
|
"""Independently verify and sanitize diagnostics fail-closed, preserving safe JUnit XML.
|
||||||
|
|
||||||
|
Returns True if verification succeeded, False if fail-closed purge was required.
|
||||||
|
"""
|
||||||
|
if not os.path.exists(target_dir):
|
||||||
|
logger.info("Target directory %s does not exist; nothing to sanitize.", target_dir)
|
||||||
|
return True
|
||||||
|
|
||||||
|
disposable_ci = is_valid_disposable_ci_fixture()
|
||||||
|
preserved_xmls = 0
|
||||||
|
sanitized_archives = 0
|
||||||
|
purged_files = 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
for root, _, files in os.walk(target_dir):
|
||||||
|
for filename in files:
|
||||||
|
filepath = os.path.join(root, filename)
|
||||||
|
name_lower = filename.lower()
|
||||||
|
|
||||||
|
# 1. Safe JUnit reporting: preserve valid XML reports
|
||||||
|
if name_lower.endswith(".xml"):
|
||||||
|
try:
|
||||||
|
ET.parse(filepath)
|
||||||
|
preserved_xmls += 1
|
||||||
|
continue
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Corrupted XML report %s: %s; purging fail-closed", filepath, exc)
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
purged_files += 1
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 2. Check if diagnostics are authorized (disposable synthetic fixture)
|
||||||
|
if not disposable_ci:
|
||||||
|
# In CI without disposable fixture, or local without opt-in: purge all diagnostics
|
||||||
|
logger.info("Purging diagnostic artifact %s (disposable CI fixture not verified)", filepath)
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
purged_files += 1
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 3. Disposable CI is authorized: enforce strict sanitization
|
||||||
|
if name_lower.endswith(".zip"):
|
||||||
|
ok = sanitize_trace_archive(filepath)
|
||||||
|
if ok and os.path.exists(filepath):
|
||||||
|
sanitized_archives += 1
|
||||||
|
else:
|
||||||
|
logger.warning("Trace archive %s failed sanitization; purged fail-closed", filepath)
|
||||||
|
if os.path.exists(filepath):
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
purged_files += 1
|
||||||
|
|
||||||
|
elif name_lower.endswith(".png"):
|
||||||
|
# Validate PNG magic bytes to ensure not a secret payload disguised as an image
|
||||||
|
try:
|
||||||
|
with open(filepath, "rb") as pf:
|
||||||
|
header = pf.read(8)
|
||||||
|
if header == PNG_MAGIC:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
logger.warning("Invalid or suspicious screenshot %s; purged fail-closed", filepath)
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
purged_files += 1
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Purge any .tmp, .har, .db, .sqlite, .log or unrecognized files
|
||||||
|
logger.info("Purging unauthorized diagnostic file %s", filepath)
|
||||||
|
try:
|
||||||
|
os.remove(filepath)
|
||||||
|
purged_files += 1
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Artifact verification complete for %s: preserved %d XML reports, sanitized %d archives, purged %d files.",
|
||||||
|
target_dir,
|
||||||
|
preserved_xmls,
|
||||||
|
sanitized_archives,
|
||||||
|
purged_files,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Unexpected error during artifact verification: %s; purging all non-XML files fail-closed", exc)
|
||||||
|
try:
|
||||||
|
for p in Path(target_dir).rglob("*"):
|
||||||
|
if p.is_file() and not p.name.lower().endswith(".xml"):
|
||||||
|
try:
|
||||||
|
p.unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("target_dir", nargs="?", default="test-results", help="Directory containing test results")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(args.target_dir)
|
||||||
|
# Always exit 0 so upload-artifact can safely publish verified reports
|
||||||
|
sys.exit(0 if ok else 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -39,6 +39,10 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
node .github/scripts/run_native_tests.mjs
|
node .github/scripts/run_native_tests.mjs
|
||||||
python .github/scripts/check_test_execution.py test-results/native-node/junit.xml --minimum 43
|
python .github/scripts/check_test_execution.py test-results/native-node/junit.xml --minimum 43
|
||||||
|
- name: Verify and sanitize native test results fail-closed
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
python .github/scripts/verify_sanitized_artifacts.py test-results/native-node/
|
||||||
- name: Save native test identities and results
|
- name: Save native test identities and results
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||||
@@ -95,6 +99,10 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
npm run test:photo-editor -- --reporter=line,junit
|
npm run test:photo-editor -- --reporter=line,junit
|
||||||
python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 137
|
python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 137
|
||||||
|
- name: Verify and sanitize failure diagnostics fail-closed
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
python .github/scripts/verify_sanitized_artifacts.py test-results/
|
||||||
- name: Save per-test results and sanitized failure diagnostics
|
- name: Save per-test results and sanitized failure diagnostics
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||||
|
|||||||
@@ -25,6 +25,10 @@ Default CI trace suppression remains intact. Only this job's explicit opt-in plu
|
|||||||
a newly generated authenticated synthetic fixture marker permits failure traces.
|
a newly generated authenticated synthetic fixture marker permits failure traces.
|
||||||
The existing structured sanitizer runs before artifacts are uploaded; invalid
|
The existing structured sanitizer runs before artifacts are uploaded; invalid
|
||||||
ownership, failed sanitization and missing tooling purge sensitive artifacts.
|
ownership, failed sanitization and missing tooling purge sensitive artifacts.
|
||||||
|
Unsupported or malformed sensitive records cause the affected archive to be removed
|
||||||
|
fail-closed. An independent fail-closed verification step (.github/scripts/verify_sanitized_artifacts.py)
|
||||||
|
runs after test suite execution or abnormal runner termination to prevent publishing
|
||||||
|
unsanitized diagnostics while preserving safe JUnit reporting.
|
||||||
Never upload the fixture directory, database or browser storage state.
|
Never upload the fixture directory, database or browser storage state.
|
||||||
|
|
||||||
The stroke budget now excludes measured synthetic event-delivery pacing and also
|
The stroke budget now excludes measured synthetic event-delivery pacing and also
|
||||||
|
|||||||
@@ -77,13 +77,14 @@ def sanitize_body_text(text: str) -> str:
|
|||||||
|
|
||||||
# 1. Try JSON
|
# 1. Try JSON
|
||||||
trimmed = text.strip()
|
trimmed = text.strip()
|
||||||
if (trimmed.startswith("{") and trimmed.endswith("}")) or (trimmed.startswith("[") and trimmed.endswith("]")):
|
if trimmed.startswith(("{", "[")):
|
||||||
try:
|
try:
|
||||||
parsed = json.loads(trimmed)
|
parsed = json.loads(trimmed)
|
||||||
sanitized = sanitize_structured_obj(parsed)
|
sanitized = sanitize_structured_obj(parsed)
|
||||||
return json.dumps(sanitized, ensure_ascii=False)
|
return json.dumps(sanitized, ensure_ascii=False)
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
pass
|
if SENSITIVE_KEY_PATTERN.search(trimmed) or any(p in trimmed.lower() for p in SENSITIVE_PARAM_NAMES):
|
||||||
|
raise ValueError(f"Malformed sensitive JSON request body: {exc}") from exc
|
||||||
|
|
||||||
# 2. Try URL-encoded form data
|
# 2. Try URL-encoded form data
|
||||||
if "=" in trimmed and not trimmed.startswith("<"):
|
if "=" in trimmed and not trimmed.startswith("<"):
|
||||||
@@ -98,8 +99,13 @@ def sanitize_body_text(text: str) -> str:
|
|||||||
else:
|
else:
|
||||||
out_pairs.append((k, v))
|
out_pairs.append((k, v))
|
||||||
return urllib.parse.urlencode(out_pairs)
|
return urllib.parse.urlencode(out_pairs)
|
||||||
except Exception:
|
except Exception as exc:
|
||||||
pass
|
if SENSITIVE_KEY_PATTERN.search(trimmed):
|
||||||
|
raise ValueError(f"Malformed sensitive form request body: {exc}") from exc
|
||||||
|
|
||||||
|
# If unparsed body text contains sensitive keys or tokens, fail closed
|
||||||
|
if SENSITIVE_KEY_PATTERN.search(trimmed) or any(p in trimmed.lower() for p in SENSITIVE_PARAM_NAMES):
|
||||||
|
raise ValueError("Malformed or unsupported sensitive request body in trace entry")
|
||||||
|
|
||||||
return text
|
return text
|
||||||
|
|
||||||
@@ -207,7 +213,11 @@ def sanitize_structured_obj(obj: Any) -> Any:
|
|||||||
|
|
||||||
|
|
||||||
def sanitize_jsonl_content(data: bytes) -> bytes:
|
def sanitize_jsonl_content(data: bytes) -> bytes:
|
||||||
"""Sanitize newline-delimited JSON Lines (.trace, .network, .stacks)."""
|
"""Sanitize newline-delimited JSON Lines (.trace, .network, .stacks).
|
||||||
|
|
||||||
|
If any non-empty record cannot be decoded as valid UTF-8 or parsed as valid JSON,
|
||||||
|
or is malformed, raise ValueError fail-closed so the affected archive is permanently removed.
|
||||||
|
"""
|
||||||
lines = data.split(b"\n")
|
lines = data.split(b"\n")
|
||||||
out_lines = []
|
out_lines = []
|
||||||
for line in lines:
|
for line in lines:
|
||||||
@@ -216,10 +226,8 @@ def sanitize_jsonl_content(data: bytes) -> bytes:
|
|||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
parsed = json.loads(line.decode("utf-8"))
|
parsed = json.loads(line.decode("utf-8"))
|
||||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
except (json.JSONDecodeError, UnicodeDecodeError) as err:
|
||||||
# Not valid JSON, keep as-is
|
raise ValueError(f"Malformed or unsupported JSONL trace record: {err}")
|
||||||
out_lines.append(line)
|
|
||||||
continue
|
|
||||||
sanitized = sanitize_structured_obj(parsed)
|
sanitized = sanitize_structured_obj(parsed)
|
||||||
out_lines.append(json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
out_lines.append(json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
||||||
return b"\n".join(out_lines)
|
return b"\n".join(out_lines)
|
||||||
@@ -275,11 +283,10 @@ def sanitize_trace_archive(zp: str) -> bool:
|
|||||||
elif filename_lower.endswith((".json", ".har")):
|
elif filename_lower.endswith((".json", ".har")):
|
||||||
try:
|
try:
|
||||||
parsed = json.loads(content.decode("utf-8"))
|
parsed = json.loads(content.decode("utf-8"))
|
||||||
sanitized = sanitize_structured_obj(parsed)
|
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||||
zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
raise ValueError(f"Malformed JSON in archive member {item.filename}: {exc}") from exc
|
||||||
except Exception:
|
sanitized = sanitize_structured_obj(parsed)
|
||||||
# If invalid JSON, treat as text line-by-line
|
zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8"))
|
||||||
zout.writestr(item, sanitize_jsonl_content(content))
|
|
||||||
else:
|
else:
|
||||||
# Unknown binary or text member: exclude to fail closed
|
# Unknown binary or text member: exclude to fail closed
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -242,3 +242,72 @@ def test_multiple_trace_archives_in_directory(tmp_path):
|
|||||||
with zipfile.ZipFile(z2, "r") as z:
|
with zipfile.ZipFile(z2, "r") as z:
|
||||||
content = json.loads(z.read("0.trace").decode().strip())
|
content = json.loads(z.read("0.trace").decode().strip())
|
||||||
assert content["params"]["value"] == "[REDACTED]"
|
assert content["params"]["value"] == "[REDACTED]"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_malformed_jsonl_record(tmp_path):
|
||||||
|
"""Malformed JSONL lines in .trace archives fail closed and delete the archive."""
|
||||||
|
zp = str(tmp_path / "malformed_jsonl.zip")
|
||||||
|
with zipfile.ZipFile(zp, "w") as z:
|
||||||
|
z.writestr(
|
||||||
|
"0-trace.trace",
|
||||||
|
b'{"type": "action", "params": {"value": "secret123"}}\n'
|
||||||
|
b'this is invalid json {"password": "leak_raw_pass"}\n'
|
||||||
|
)
|
||||||
|
ok = sanitize_trace_archive(zp)
|
||||||
|
assert ok is False
|
||||||
|
assert not os.path.exists(zp)
|
||||||
|
assert not os.path.exists(zp + ".tmp")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_malformed_jsonl_truncated(tmp_path):
|
||||||
|
"""Truncated JSONL records in .network archives fail closed and delete the archive."""
|
||||||
|
zp = str(tmp_path / "truncated_jsonl.zip")
|
||||||
|
with zipfile.ZipFile(zp, "w") as z:
|
||||||
|
z.writestr(
|
||||||
|
"0-trace.network",
|
||||||
|
b'{"type": "request", "request": {"url": "https://example.com/api/auth", "headers": [{"name": "Authorization", "value": "Bearer secret_tok"'
|
||||||
|
)
|
||||||
|
ok = sanitize_trace_archive(zp)
|
||||||
|
assert ok is False
|
||||||
|
assert not os.path.exists(zp)
|
||||||
|
assert not os.path.exists(zp + ".tmp")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_malformed_sensitive_body(tmp_path):
|
||||||
|
"""Malformed sensitive request body in trace archives fails closed and deletes archive."""
|
||||||
|
zp = str(tmp_path / "malformed_sensitive_body.zip")
|
||||||
|
with zipfile.ZipFile(zp, "w") as z:
|
||||||
|
record = {
|
||||||
|
"type": "request",
|
||||||
|
"request": {
|
||||||
|
"url": "https://example.com/api/login",
|
||||||
|
"postData": '{"password": "super_secret_pw", "corrupt_trailing": ',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
z.writestr("0-trace.network", json.dumps(record).encode() + b"\n")
|
||||||
|
ok = sanitize_trace_archive(zp)
|
||||||
|
assert ok is False
|
||||||
|
assert not os.path.exists(zp)
|
||||||
|
assert not os.path.exists(zp + ".tmp")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_malformed_har_member(tmp_path):
|
||||||
|
"""Malformed JSON in .har archives fails closed and deletes the archive."""
|
||||||
|
zp = str(tmp_path / "malformed_har.zip")
|
||||||
|
with zipfile.ZipFile(zp, "w") as z:
|
||||||
|
z.writestr("network.har", b'{"log": {"entries": [ malformed JSON content')
|
||||||
|
ok = sanitize_trace_archive(zp)
|
||||||
|
assert ok is False
|
||||||
|
assert not os.path.exists(zp)
|
||||||
|
assert not os.path.exists(zp + ".tmp")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_invalid_utf8_jsonl_record(tmp_path):
|
||||||
|
"""Non-UTF8 byte sequences in JSONL trace records fail closed and delete the archive."""
|
||||||
|
zp = str(tmp_path / "invalid_utf8.zip")
|
||||||
|
with zipfile.ZipFile(zp, "w") as z:
|
||||||
|
z.writestr("0-trace.trace", b"\xff\xfe\x00\x00 invalid utf8 raw bytes\n")
|
||||||
|
ok = sanitize_trace_archive(zp)
|
||||||
|
assert ok is False
|
||||||
|
assert not os.path.exists(zp)
|
||||||
|
assert not os.path.exists(zp + ".tmp")
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""Tests for the fail-closed artifact verification step (.github/scripts/verify_sanitized_artifacts.py).
|
||||||
|
|
||||||
|
Validates that:
|
||||||
|
1. Safe JUnit XML reporting is preserved across all scenarios.
|
||||||
|
2. Abnormal runner termination without verified disposable CI fixture purges all diagnostics.
|
||||||
|
3. Valid disposable CI environments sanitize trace archives fail-closed.
|
||||||
|
4. Malformed trace archives are deleted fail-closed without corrupting or deleting JUnit XML reports.
|
||||||
|
5. In-progress temporary files, HAR files, and database dumps are purged unconditionally.
|
||||||
|
6. Top-level exceptions fail closed to purge all non-XML diagnostics.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
VERIFY_SCRIPT = REPO_ROOT / ".github" / "scripts" / "verify_sanitized_artifacts.py"
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
|
||||||
|
spec = importlib.util.spec_from_file_location("verify_sanitized_artifacts", str(VERIFY_SCRIPT))
|
||||||
|
verify_module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules["verify_sanitized_artifacts"] = verify_module
|
||||||
|
spec.loader.exec_module(verify_module)
|
||||||
|
|
||||||
|
is_valid_disposable_ci_fixture = verify_module.is_valid_disposable_ci_fixture
|
||||||
|
verify_and_sanitize_artifacts = verify_module.verify_and_sanitize_artifacts
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_disposable_ci(tmp_path: Path) -> dict[str, str]:
|
||||||
|
data_dir = tmp_path / "odysseus-photo-editor-e2e-fixture"
|
||||||
|
data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
db_path = data_dir / "app.db"
|
||||||
|
db_path.write_bytes(b"SQLite format 3\x00")
|
||||||
|
marker_file = data_dir / ".disposable-ci-fixture.json"
|
||||||
|
marker_file.write_text(
|
||||||
|
json.dumps({
|
||||||
|
"synthetic": True,
|
||||||
|
"authenticated": True,
|
||||||
|
"database": str(db_path),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"CI": "true",
|
||||||
|
"GITHUB_ACTIONS": "true",
|
||||||
|
"ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS": "1",
|
||||||
|
"PHOTO_EDITOR_E2E_DATA_DIR": str(data_dir),
|
||||||
|
"PHOTO_EDITOR_E2E_DB_PATH": str(db_path),
|
||||||
|
"PHOTO_EDITOR_E2E_AUTH": "1",
|
||||||
|
"ODYSSEUS_TEST_PYTHON": sys.executable,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_preserves_valid_junit_xml_cleanly(tmp_path):
|
||||||
|
"""Valid JUnit XML files are preserved intact."""
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "browser-junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||||
|
assert ok is True
|
||||||
|
assert junit.exists()
|
||||||
|
assert "tests=\"137\"" in junit.read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_closed_abnormal_termination_purges_all_diagnostics(tmp_path, monkeypatch):
|
||||||
|
"""Abnormal runner termination without disposable fixture purges all diagnostics, keeping JUnit."""
|
||||||
|
# Simulate CI without disposable fixture
|
||||||
|
monkeypatch.setenv("CI", "true")
|
||||||
|
monkeypatch.setenv("GITHUB_ACTIONS", "true")
|
||||||
|
monkeypatch.delenv("ODYSSEUS_E2E_DISPOSABLE_ARTIFACTS", raising=False)
|
||||||
|
monkeypatch.delenv("PHOTO_EDITOR_E2E_DATA_DIR", raising=False)
|
||||||
|
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "browser-junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="photo-editor" tests="10" failures="1"/></testsuites>')
|
||||||
|
|
||||||
|
trace = results_dir / "trace.zip"
|
||||||
|
trace.write_bytes(b"PK0304 sensitive trace archive")
|
||||||
|
har = results_dir / "session.har"
|
||||||
|
har.write_text('{"log": {"entries": ["secret"]}}')
|
||||||
|
png = results_dir / "failure.png"
|
||||||
|
png.write_bytes(b"\x89PNG\r\n\x1a\nfake screenshot")
|
||||||
|
tmp = results_dir / "temp.tmp"
|
||||||
|
tmp.write_text("in-progress trace")
|
||||||
|
db = results_dir / "app.db"
|
||||||
|
db.write_bytes(b"SQLite format 3 database")
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||||
|
assert ok is True
|
||||||
|
|
||||||
|
# Safe JUnit reporting MUST be preserved
|
||||||
|
assert junit.exists()
|
||||||
|
assert "failures=\"1\"" in junit.read_text()
|
||||||
|
|
||||||
|
# All sensitive diagnostics MUST be purged
|
||||||
|
assert not trace.exists()
|
||||||
|
assert not har.exists()
|
||||||
|
assert not png.exists()
|
||||||
|
assert not tmp.exists()
|
||||||
|
assert not db.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sanitizes_trace_archive_when_disposable_ci_verified(tmp_path, monkeypatch):
|
||||||
|
"""When running inside a verified disposable CI fixture, trace archives are sanitized fail-closed."""
|
||||||
|
env = _setup_disposable_ci(tmp_path)
|
||||||
|
for k, v in env.items():
|
||||||
|
monkeypatch.setenv(k, v)
|
||||||
|
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "browser-junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||||
|
|
||||||
|
trace = results_dir / "trace.zip"
|
||||||
|
with zipfile.ZipFile(trace, "w") as z:
|
||||||
|
action = {"type": "action", "name": "fill", "params": {"value": "super_secret_password"}}
|
||||||
|
z.writestr("0-trace.trace", json.dumps(action).encode() + b"\n")
|
||||||
|
|
||||||
|
png = results_dir / "failure.png"
|
||||||
|
png.write_bytes(b"\x89PNG\r\n\x1a\nvalid screenshot payload")
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||||
|
assert ok is True
|
||||||
|
|
||||||
|
assert junit.exists()
|
||||||
|
assert trace.exists()
|
||||||
|
assert png.exists()
|
||||||
|
|
||||||
|
with zipfile.ZipFile(trace, "r") as z:
|
||||||
|
content = json.loads(z.read("0-trace.trace").decode().strip())
|
||||||
|
assert content["params"]["value"] == "[REDACTED]"
|
||||||
|
|
||||||
|
|
||||||
|
def test_malformed_trace_archive_purged_preserving_junit_xml(tmp_path, monkeypatch):
|
||||||
|
"""Malformed trace archive is purged fail-closed without affecting JUnit XML."""
|
||||||
|
env = _setup_disposable_ci(tmp_path)
|
||||||
|
for k, v in env.items():
|
||||||
|
monkeypatch.setenv(k, v)
|
||||||
|
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "browser-junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="1"/></testsuites>')
|
||||||
|
|
||||||
|
# Create trace archive with malformed JSONL line
|
||||||
|
malformed_trace = results_dir / "trace.zip"
|
||||||
|
with zipfile.ZipFile(malformed_trace, "w") as z:
|
||||||
|
z.writestr("0-trace.trace", b'{"type": "action"}\nmalformed not json line\n')
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||||
|
assert ok is True
|
||||||
|
|
||||||
|
# JUnit XML is preserved
|
||||||
|
assert junit.exists()
|
||||||
|
# Malformed archive is purged!
|
||||||
|
assert not malformed_trace.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_unhandled_exception_fails_closed_purges_all_non_xml(tmp_path, monkeypatch):
|
||||||
|
"""Unexpected exception during verification triggers emergency fail-closed purge of non-XML files."""
|
||||||
|
env = _setup_disposable_ci(tmp_path)
|
||||||
|
for k, v in env.items():
|
||||||
|
monkeypatch.setenv(k, v)
|
||||||
|
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "browser-junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="photo-editor" tests="137" failures="0"/></testsuites>')
|
||||||
|
|
||||||
|
trace = results_dir / "trace.zip"
|
||||||
|
trace.write_bytes(b"PK0304 trace")
|
||||||
|
|
||||||
|
def broken_sanitize(zp):
|
||||||
|
raise RuntimeError("Simulated unexpected crash in sanitizer")
|
||||||
|
|
||||||
|
monkeypatch.setattr(verify_module, "sanitize_trace_archive", broken_sanitize)
|
||||||
|
|
||||||
|
ok = verify_and_sanitize_artifacts(str(results_dir))
|
||||||
|
assert ok is False
|
||||||
|
# Safe JUnit reporting is preserved
|
||||||
|
assert junit.exists()
|
||||||
|
# Sensitive non-XML file is purged fail-closed!
|
||||||
|
assert not trace.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cli_invocation_returns_cleanly(tmp_path):
|
||||||
|
"""Running the script as a CLI entrypoint succeeds with exit code 0."""
|
||||||
|
results_dir = tmp_path / "test-results"
|
||||||
|
results_dir.mkdir()
|
||||||
|
junit = results_dir / "junit.xml"
|
||||||
|
junit.write_text('<testsuites><testsuite name="native-node" tests="43" failures="0"/></testsuites>')
|
||||||
|
|
||||||
|
res = subprocess.run(
|
||||||
|
[sys.executable, str(VERIFY_SCRIPT), str(results_dir)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert res.returncode == 0
|
||||||
|
assert junit.exists()
|
||||||
Reference in New Issue
Block a user