From 68aa830485e927daf42e6fdeb6f6bfee34e16528 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira Date: Sat, 31 Oct 2026 01:48:13 +0000 Subject: [PATCH] fix(ci): enforce restrictive published-JUnit representation and prune properties --- .github/scripts/verify_sanitized_artifacts.py | 137 +++++++++++------- tests/test_verify_sanitized_artifacts.py | 114 +++++++++++++++ 2 files changed, 198 insertions(+), 53 deletions(-) diff --git a/.github/scripts/verify_sanitized_artifacts.py b/.github/scripts/verify_sanitized_artifacts.py index abf35eb1c..be177b5e5 100755 --- a/.github/scripts/verify_sanitized_artifacts.py +++ b/.github/scripts/verify_sanitized_artifacts.py @@ -42,18 +42,20 @@ logger = logging.getLogger("verify_sanitized_artifacts") PNG_MAGIC = b"\x89PNG\r\n\x1a\n" ALLOWED_JUNIT_ROOT_TAGS = frozenset({"testsuites", "testsuite"}) -ALLOWED_JUNIT_ELEMENT_TAGS = frozenset({ - "testsuites", - "testsuite", - "testcase", - "failure", - "error", - "skipped", - "system-out", - "system-err", - "properties", - "property", -}) +ALLOWED_PUBLISHED_JUNIT_TAGS = frozenset({"testsuites", "testsuite", "testcase", "failure", "error", "skipped"}) +DISALLOWED_JUNIT_TAGS = frozenset({"properties", "property", "system-out", "system-err"}) + +ALLOWED_ATTRIBUTES_BY_TAG = { + "testsuites": frozenset({"name", "tests", "failures", "errors", "skipped", "time", "id", "disabled"}), + "testsuite": frozenset({ + "name", "tests", "failures", "errors", "skipped", "time", "timestamp", "hostname", + "id", "package", "disabled", "file" + }), + "testcase": frozenset({"name", "classname", "time", "file", "line", "assertions", "status"}), + "failure": frozenset({"message", "type"}), + "error": frozenset({"message", "type"}), + "skipped": frozenset({"message", "type"}), +} SENSITIVE_KEY_PATTERN = re.compile( r"(?i)(password|passwd|secret|token|api_key|apikey|access_token|refresh_token|" @@ -155,8 +157,67 @@ def sanitize_text_content(text: str) -> tuple[str, bool]: return out, modified +def prune_and_sanitize_junit_element(elem: ET.Element) -> bool: + """Recursively enforce restrictive published-JUnit representation. + + Removes properties, property entries, system-out, system-err, and any + unnecessary free-form content. Preserves test identities, counts, outcomes, + and safe failure classifications. + Returns True if any modifications were made. + """ + modified = False + + # 1. Prune child elements not permitted in restrictive published representation + for child in list(elem): + tag_lower = child.tag.lower() + if tag_lower in DISALLOWED_JUNIT_TAGS or tag_lower not in ALLOWED_PUBLISHED_JUNIT_TAGS: + elem.remove(child) + modified = True + else: + if prune_and_sanitize_junit_element(child): + modified = True + + tag_lower = elem.tag.lower() + + # 2. Prune attributes not permitted in restrictive published representation + allowed_attrs = ALLOWED_ATTRIBUTES_BY_TAG.get(tag_lower, frozenset()) + for attr_name in list(elem.attrib.keys()): + attr_lower = attr_name.lower() + if attr_lower not in allowed_attrs: + del elem.attrib[attr_name] + modified = True + else: + val = elem.attrib[attr_name] + if isinstance(val, str): + new_val, changed = sanitize_text_content(val) + if changed: + elem.attrib[attr_name] = new_val + modified = True + + # 3. Handle text content: structural elements do not carry free-form text + if tag_lower in ("testsuites", "testsuite", "testcase"): + if elem.text and elem.text.strip(): + elem.text = None + modified = True + elif tag_lower in ("failure", "error"): + if elem.text: + new_text, changed = sanitize_text_content(elem.text) + if changed: + elem.text = new_text + modified = True + + # Clean up tails + if elem.tail and elem.tail.strip(): + new_tail, changed = sanitize_text_content(elem.tail) + if changed: + elem.tail = new_tail + modified = True + + return modified + + def sanitize_and_verify_junit_xml(filepath: str | Path) -> bool: - """Verify that an XML file is an authentic JUnit report and redact credentials. + """Verify that an XML file is an authentic JUnit report and enforce published representation. Returns True if the XML file is a valid recognized JUnit report and was safely sanitized. Returns False if the file is invalid, unrecognized, corrupted, or failed sanitization. @@ -182,47 +243,17 @@ def sanitize_and_verify_junit_xml(filepath: str | Path) -> bool: logger.warning("XML report %s has invalid root element <%s>; not a JUnit report", filepath, root.tag) return False - # 4. Check element tags and redact sensitive content - file_modified = False - for elem in tree.iter(): - if elem.tag.lower() not in ALLOWED_JUNIT_ELEMENT_TAGS: - logger.warning("XML report %s contains unrecognized element tag <%s>", filepath, elem.tag) - return False + # 4. Enforce restrictive published representation on root and all descendants + prune_and_sanitize_junit_element(root) - # Sanitize attributes - for k, v in list(elem.attrib.items()): - if SENSITIVE_KEY_PATTERN.search(k.lower()): - elem.attrib[k] = "[REDACTED]" - file_modified = True - elif isinstance(v, str): - new_v, changed = sanitize_text_content(v) - if changed: - elem.attrib[k] = new_v - file_modified = True - - # Sanitize text - if elem.text: - new_text, changed = sanitize_text_content(elem.text) - if changed: - elem.text = new_text - file_modified = True - - # Sanitize tail - if elem.tail: - new_tail, changed = sanitize_text_content(elem.tail) - if changed: - elem.tail = new_tail - file_modified = True - - # 5. If content was sanitized, write back safely - if file_modified: - try: - tree.write(str(p), encoding="utf-8", xml_declaration=True) - # Verify rewritten XML parses cleanly - ET.parse(p) - except Exception as exc: - logger.error("Failed to write sanitized XML report %s: %s", filepath, exc) - return False + # 5. Normalize and rewrite sanitized report + try: + tree.write(str(p), encoding="utf-8", xml_declaration=True) + # Verify rewritten XML parses cleanly + ET.parse(p) + except Exception as exc: + logger.error("Failed to write sanitized XML report %s: %s", filepath, exc) + return False return True diff --git a/tests/test_verify_sanitized_artifacts.py b/tests/test_verify_sanitized_artifacts.py index 4dc4d96d1..488873437 100644 --- a/tests/test_verify_sanitized_artifacts.py +++ b/tests/test_verify_sanitized_artifacts.py @@ -136,6 +136,120 @@ def test_valid_xml_containing_synthetic_secret_not_published_unchanged(tmp_path) assert tree.getroot().tag == "testsuites" +def test_exact_property_api_key_cannot_appear_unchanged_in_uploaded_report(tmp_path): + """Regression: cannot appear unchanged in uploaded report.""" + results_dir = tmp_path / "test-results" + results_dir.mkdir() + junit = results_dir / "browser-junit.xml" + xml_with_property = """ + + + + + + + +""" + junit.write_text(xml_with_property) + + ok = verify_and_sanitize_artifacts(str(results_dir)) + assert ok is True + assert junit.exists() + + sanitized_xml = junit.read_text() + # The exact sensitive property and value MUST NOT appear anywhere in the published report + assert "sk-example-123" not in sanitized_xml + assert "api_key" not in sanitized_xml + assert " + + + +""") + + # 2. Standard Playwright report representation (with suites, timing, counts, and failure classifications) + browser_junit = results_dir / "browser-junit.xml" + browser_junit.write_text(""" + + + + + + + + + Error: expect(strokeDuration - controlDuration).toBeLessThan(5000) + at brush-engine.spec.js:84:10 + + Captured console stdout dump with session_token=abc12345 + Captured console stderr dump + + +""") + + ok_node = verify_and_sanitize_artifacts(str(node_dir)) + assert ok_node is True + ok_browser = verify_and_sanitize_artifacts(str(results_dir)) + assert ok_browser is True + + # Validate Node report + node_tree = ET.parse(node_junit) + node_cases = list(node_tree.iter("testcase")) + assert len(node_cases) == 2 + assert node_cases[0].get("name") == "research cards distinguish live work, handoff, completion and no evidence" + assert node_cases[1].get("name") == "Chat Photo Editor Action Integrity and Turn Scoping" + assert node_cases[0].get("file") == "tests/backgroundToolJobs.test.mjs" + + # Validate Playwright report + pw_tree = ET.parse(browser_junit) + pw_root = pw_tree.getroot() + assert pw_root.tag == "testsuites" + assert pw_root.get("tests") == "2" + assert pw_root.get("failures") == "1" + + pw_cases = list(pw_tree.iter("testcase")) + assert len(pw_cases) == 2 + assert pw_cases[0].get("name") == "brush presets and shared stroke controls remain reusable" + assert pw_cases[1].get("name") == "long sampled strokes stay responsive and undo atomically" + + # Verify failure classification is preserved + failures = list(pw_cases[1].iter("failure")) + assert len(failures) == 1 + assert "toBeLessThan(5000)" in failures[0].get("message") + assert failures[0].get("type") == "FAILURE" + + # Verify properties and system-out/err were pruned + pw_text = browser_junit.read_text() + assert "sk-example-123" not in pw_text + assert "api_key" not in pw_text + assert "