diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2ccf3ce1b..5a8226049 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -264,3 +264,20 @@ jobs:
ODYSSEUS_TEST_PYTHON=python ODYSSEUS_LAYOUT_BROWSER=chromium npx --no-install playwright test \
--config tests/e2e/ios-layout.config.cjs \
--reporter=line
+
+ benchmark-gfm-security:
+ name: Benchmark GFM rendering security
+ runs-on: ubuntu-24.04
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.11"
+ # Test-only renderer, pinned independently of application requirements.
+ - name: Install GitHub Flavored Markdown renderer
+ run: |
+ python -m pip install --disable-pip-version-check --only-binary=:all: "cmarkgfm==2025.10.22"
+ - name: Verify rendered benchmark report security
+ run: python tests/test_audit_c5_benchmark_evidence.py --gfm
diff --git a/scripts/audit_c5_benchmark_evidence.py b/scripts/audit_c5_benchmark_evidence.py
new file mode 100644
index 000000000..47d43e219
--- /dev/null
+++ b/scripts/audit_c5_benchmark_evidence.py
@@ -0,0 +1,1637 @@
+"""Offline benchmark evidence auditor for Odysseus C5 / E00 baseline reconstruction.
+
+Classifies historical evaluation artifacts using a strict evidence scheme.
+Never labels an artifact VERIFIED merely because its hash was calculated.
+VERIFIED requires an independently provided expected hash that matches.
+
+Separates artifact file integrity from campaign prerequisite completeness.
+A prerequisite is only complete when independently established expected sets
+are verified. Archive summaries and aggregate statistics do not prove complete
+task oracles or detailed latency records.
+
+Usage:
+ python3 scripts/audit_c5_benchmark_evidence.py MANIFEST ROOT [--output FILE] [--summary FILE] [--deterministic]
+
+Exit codes:
+ 0 - all artifacts VERIFIED or OBSERVED_ONLY
+ 1 - any MISSING, MISMATCH, or UNVERIFIABLE artifacts
+ 2 - INVALID manifest, preflight destination conflict, publication error, or fatal error
+"""
+from __future__ import annotations
+
+import argparse
+import errno
+import hashlib
+import json
+import os
+import posixpath
+import re
+import stat
+import sys
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Classification constants
+# ---------------------------------------------------------------------------
+
+VERIFIED = "VERIFIED"
+OBSERVED_ONLY = "OBSERVED_ONLY"
+MISSING = "MISSING"
+MISMATCH = "MISMATCH"
+INVALID = "INVALID"
+UNVERIFIABLE = "UNVERIFIABLE"
+PARTIAL = "PARTIAL"
+INCOMPLETE = "INCOMPLETE"
+UNKNOWN = "UNKNOWN"
+
+ALL_STATUSES = (VERIFIED, OBSERVED_ONLY, MISSING, MISMATCH, INVALID, UNVERIFIABLE)
+
+# Map artifact types to reproducibility prerequisite keys.
+_TYPE_TO_PREREQ: dict[str, str] = {
+ "model_artifact": "model_artifact_hash",
+ "source_revision": "source_checkpoint_revision",
+ "quantization_config": "quantization_info",
+ "backend_binary": "backend_executable_identity",
+ "chat_template": "chat_template_identity",
+ "run_config": "context_and_decoding_config",
+ "task_fixture": "task_fixture_identities",
+ "oracle_definition": "expected_outcomes_and_oracles",
+ "oracle_summary": "expected_outcomes_and_oracles",
+ "trajectory": "raw_trajectories",
+ "result_record": "result_and_latency_records",
+ "result_summary": "result_and_latency_records",
+ "aggregate_report": "result_and_latency_records",
+}
+
+_PREREQ_KEYS = (
+ "backend_executable_identity",
+ "chat_template_identity",
+ "context_and_decoding_config",
+ "expected_outcomes_and_oracles",
+ "model_artifact_hash",
+ "quantization_info",
+ "raw_trajectories",
+ "result_and_latency_records",
+ "source_checkpoint_revision",
+ "task_fixture_identities",
+)
+
+_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+def _valid_sha256(value: Any) -> bool:
+ if isinstance(value, bool) or not isinstance(value, str):
+ return False
+ return bool(_SHA256_RE.match(value))
+
+
+# ---------------------------------------------------------------------------
+# Path safety
+# ---------------------------------------------------------------------------
+
+def _is_safe_relative_path(raw: Any) -> bool:
+ """Return True only if raw is a non-empty, non-absolute relative path with no traversal."""
+ if isinstance(raw, bool) or not isinstance(raw, str):
+ return False
+ if "\0" in raw:
+ return False
+ try:
+ raw.encode("utf-8")
+ os.fsencode(raw)
+ except (UnicodeError, ValueError):
+ return False
+ clean = raw.strip()
+ if not clean or os.path.isabs(clean):
+ return False
+ try:
+ p = PurePosixPath(clean)
+ except Exception:
+ return False
+ # Reject dot-only paths and zero-component normalizations ('.', './', '././')
+ if not p.parts or not p.name or p.name in (".", ".."):
+ return False
+ if ".." in p.parts or "." in p.parts:
+ return False
+ if clean.endswith(("/.", "/..")) or clean in (".", ".."):
+ return False
+ # Reject backslash paths that could escape on Windows hosts.
+ if "\\" in clean:
+ return False
+ return True
+
+
+def _normalize_relative_path(raw: Any) -> str | None:
+ """Return normalized POSIX relative path string if safe, else None.
+
+ Normalizes redundant dot components and leading './' using posixpath.normpath
+ without following symlinks or escaping the root boundary.
+ """
+ if not _is_safe_relative_path(raw):
+ return None
+ try:
+ norm = posixpath.normpath(str(raw).strip())
+ except (UnicodeError, ValueError):
+ return None
+ if not _is_safe_relative_path(norm):
+ return None
+ return norm
+
+
+# ---------------------------------------------------------------------------
+# Secure File Descriptor Traversal and Hashing (O_NOFOLLOW / Descriptor-Relative)
+# ---------------------------------------------------------------------------
+
+def _open_dir_securely(path: Path) -> int:
+ """Securely open directory path descriptor-by-descriptor from root anchor with O_NOFOLLOW.
+
+ Anchors traversal at the filesystem root descriptor.
+ Rejects any path component (including the root itself) that is a symbolic link.
+ Caller owns the returned file descriptor and must close it.
+ """
+ try:
+ abs_p = path.absolute()
+ anchor = abs_p.anchor
+ parts = abs_p.relative_to(anchor).parts
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed directory path: {exc}") from exc
+
+ try:
+ fd = os.open(anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed root anchor: {exc}") from exc
+ open_fds: list[int] = [fd]
+ try:
+ curr = fd
+ for part in parts:
+ try:
+ st = os.lstat(part, dir_fd=curr)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed path component {part!r}: {exc}") from exc
+ if stat.S_ISLNK(st.st_mode):
+ raise OSError(f"symlink path component rejected by O_NOFOLLOW: {part!r}")
+ if not stat.S_ISDIR(st.st_mode):
+ raise OSError(f"path component is not a directory: {part!r}")
+ try:
+ next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=curr)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed path component {part!r}: {exc}") from exc
+ open_fds.append(next_fd)
+ curr = next_fd
+
+ open_fds.remove(curr)
+ return curr
+ finally:
+ for f in open_fds:
+ try:
+ os.close(f)
+ except OSError:
+ pass
+
+
+def _open_or_create_dir_securely(path: Path) -> int:
+ """Traverse and create path components descriptor-by-descriptor with O_NOFOLLOW.
+
+ Anchors traversal at the filesystem root descriptor.
+ Rejects any intermediate or final directory component that is a symbolic link.
+ Never reopens directory by full pathname.
+ Caller owns the returned file descriptor and must close it.
+ """
+ try:
+ abs_p = path.absolute()
+ anchor = abs_p.anchor
+ parts = abs_p.relative_to(anchor).parts
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed directory path: {exc}") from exc
+
+ try:
+ fd = os.open(anchor, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed root anchor: {exc}") from exc
+ open_fds: list[int] = [fd]
+ try:
+ curr = fd
+ for part in parts:
+ try:
+ try:
+ st = os.lstat(part, dir_fd=curr)
+ except FileNotFoundError:
+ try:
+ os.mkdir(part, 0o755, dir_fd=curr)
+ except FileExistsError:
+ pass
+ st = os.lstat(part, dir_fd=curr)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed path component {part!r}: {exc}") from exc
+
+ if stat.S_ISLNK(st.st_mode):
+ raise OSError(f"symlink directory component rejected by O_NOFOLLOW: {part!r}")
+ if not stat.S_ISDIR(st.st_mode):
+ raise OSError(f"path component is not a directory: {part!r}")
+
+ try:
+ next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=curr)
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"malformed path component {part!r}: {exc}") from exc
+ open_fds.append(next_fd)
+ curr = next_fd
+
+ open_fds.remove(curr)
+ return curr
+ finally:
+ for f in open_fds:
+ try:
+ os.close(f)
+ except OSError:
+ pass
+
+
+class ArtifactOpenResult(tuple):
+ """4-tuple (status_code, observed_sha256, size_bytes, note) carrying file_id."""
+ file_id: str | None = None
+
+
+def _open_and_hash_artifact(root: Path, raw_path: str) -> tuple[str, str | None, int | None, str | None]:
+ """Securely open artifact within root using descriptor-relative traversal and O_NOFOLLOW.
+
+ Enforces root boundary at point of access and protects against symlink TOCTOU replacement.
+ All symlinks within root and root itself are rejected as UNVERIFIABLE.
+
+ Returns:
+ (status_code, observed_sha256, size_bytes, note)
+ status_code is one of: 'PRESENT', 'MISSING', 'UNVERIFIABLE', 'INVALID'
+ """
+ if not _is_safe_relative_path(raw_path):
+ return INVALID, None, None, f"path has no valid filename component: {raw_path!r}"
+
+ try:
+ root_fd = _open_dir_securely(root)
+ except FileNotFoundError:
+ return MISSING, None, None, f"root directory not found: {root}"
+ except (OSError, ValueError, UnicodeError) as exc:
+ return UNVERIFIABLE, None, None, f"root directory is invalid or a symlink: {exc}"
+
+ open_fds: list[int] = [root_fd]
+ try:
+ try:
+ parts = list(PurePosixPath(raw_path.strip()).parts)
+ except (ValueError, UnicodeError):
+ return INVALID, None, None, f"path has no valid filename component: {raw_path!r}"
+ if not parts or not parts[-1] or parts[-1] in (".", ".."):
+ return INVALID, None, None, f"path has no valid filename component: {raw_path!r}"
+ curr_fd = root_fd
+
+ # Traverse directory components securely
+ for part in parts[:-1]:
+ try:
+ st = os.lstat(part, dir_fd=curr_fd)
+ if stat.S_ISLNK(st.st_mode):
+ return UNVERIFIABLE, None, None, f"symlink directory component rejected by O_NOFOLLOW: {part!r}"
+ if not stat.S_ISDIR(st.st_mode):
+ return UNVERIFIABLE, None, None, f"path component is not a directory: {part!r}"
+ next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=curr_fd)
+ open_fds.append(next_fd)
+ curr_fd = next_fd
+ except FileNotFoundError:
+ return MISSING, None, None, f"directory component not found: {part!r}"
+ except (ValueError, UnicodeError):
+ return INVALID, None, None, f"malformed directory component rejected: {part!r}"
+ except OSError as exc:
+ if exc.errno in (errno.ELOOP, errno.ENOTDIR):
+ return UNVERIFIABLE, None, None, f"symlink directory component rejected: {part!r}"
+ return UNVERIFIABLE, None, None, f"filesystem error accessing component {part!r}: {exc}"
+
+ # Access final artifact file securely
+ filename = parts[-1]
+ try:
+ st = os.lstat(filename, dir_fd=curr_fd)
+ if stat.S_ISLNK(st.st_mode):
+ return UNVERIFIABLE, None, None, f"symlink artifact rejected by O_NOFOLLOW: {filename!r}"
+ if not stat.S_ISREG(st.st_mode):
+ return UNVERIFIABLE, None, None, f"artifact is not a regular file: {filename!r}"
+ file_fd = os.open(filename, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=curr_fd)
+ open_fds.append(file_fd)
+ except FileNotFoundError:
+ return MISSING, None, None, f"artifact not found: {filename!r}"
+ except (ValueError, UnicodeError):
+ return INVALID, None, None, f"malformed artifact filename rejected: {filename!r}"
+ except OSError as exc:
+ if exc.errno in (errno.ELOOP, errno.ENOTDIR):
+ return UNVERIFIABLE, None, None, f"symlink artifact rejected: {filename!r}"
+ return UNVERIFIABLE, None, None, f"filesystem error opening artifact {filename!r}: {exc}"
+
+ # Stat opened file descriptor directly
+ try:
+ fst = os.fstat(file_fd)
+ if not stat.S_ISREG(fst.st_mode):
+ return UNVERIFIABLE, None, None, f"opened descriptor is not a regular file: {filename!r}"
+ size_bytes = fst.st_size
+ file_id = f"{fst.st_dev}:{fst.st_ino}"
+ h = hashlib.sha256()
+ read_bytes = 0
+ while True:
+ chunk = os.read(file_fd, 65536)
+ if not chunk:
+ break
+ read_bytes += len(chunk)
+ h.update(chunk)
+ res = ArtifactOpenResult(("PRESENT", h.hexdigest(), read_bytes, None))
+ res.file_id = file_id
+ return res
+ except OSError as exc:
+ return UNVERIFIABLE, None, None, f"filesystem error reading artifact: {exc}"
+ finally:
+ for fd in reversed(open_fds):
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Manifest and Expected-Sets Configuration Validation
+# ---------------------------------------------------------------------------
+
+def _validate_expected_sets_structure(esets: Any, label: str = "expected_sets") -> list[str]:
+ """Validate expected_sets mapping structure and types. Return list of error strings."""
+ errors: list[str] = []
+ if esets is None:
+ errors.append(f"{label} cannot be null")
+ return errors
+ if not isinstance(esets, dict):
+ errors.append(f"{label} must be a JSON object, got {type(esets).__name__}")
+ return errors
+ if not esets and label == "trusted_expected_sets":
+ errors.append(f"{label} cannot be an empty object")
+ return errors
+
+ for pkey, pdef in esets.items():
+ if not isinstance(pkey, str) or pkey not in _PREREQ_KEYS:
+ errors.append(f"unrecognized or unsupported prerequisite category in {label}: {pkey!r}")
+ continue
+ if not isinstance(pdef, dict):
+ errors.append(f"entry for category {pkey!r} in {label} must be a JSON object, got {type(pdef).__name__}")
+ continue
+
+ unsupported_fields = set(pdef.keys()) - {
+ "expected_paths",
+ "expected_count",
+ "provenance",
+ "independent_oracle",
+ "complete",
+ "expected_set_complete",
+ }
+ if unsupported_fields:
+ errors.append(f"unsupported field(s) in category {pkey!r}: {sorted(unsupported_fields)}")
+
+ has_paths = "expected_paths" in pdef
+ has_count = "expected_count" in pdef
+ if not has_paths and not has_count:
+ errors.append(f"entry for category {pkey!r} in {label} must specify 'expected_paths' or 'expected_count'")
+
+ if has_paths:
+ ep = pdef["expected_paths"]
+ if not isinstance(ep, list):
+ errors.append(f"expected_paths for category {pkey!r} in {label} must be a list of strings")
+ elif len(ep) == 0:
+ errors.append(f"expected_paths for category {pkey!r} in {label} cannot be empty")
+ else:
+ seen = set()
+ for item in ep:
+ if isinstance(item, bool) or not isinstance(item, str):
+ errors.append(f"invalid item in expected_paths for {pkey!r}: {item!r} (must be string)")
+ elif not _is_safe_relative_path(item):
+ errors.append(f"unsafe or invalid path in expected_paths for {pkey!r}: {item!r}")
+ else:
+ norm = _normalize_relative_path(item)
+ if norm in seen:
+ errors.append(f"duplicate path in expected_paths for {pkey!r}: {item!r}")
+ else:
+ seen.add(norm)
+
+ if has_count:
+ ec = pdef["expected_count"]
+ if isinstance(ec, bool) or not isinstance(ec, int) or ec <= 0:
+ errors.append(f"expected_count for category {pkey!r} in {label} must be a positive integer")
+
+ if has_paths and has_count and isinstance(pdef.get("expected_paths"), list) and isinstance(pdef.get("expected_count"), int) and not isinstance(pdef.get("expected_count"), bool):
+ if len(pdef["expected_paths"]) != pdef["expected_count"]:
+ errors.append(f"conflicting definitions in category {pkey!r}: len(expected_paths)={len(pdef['expected_paths'])} != expected_count={pdef['expected_count']}")
+
+ if "provenance" in pdef:
+ prov = pdef["provenance"]
+ if prov is not None and (isinstance(prov, bool) or not isinstance(prov, str) or not prov.strip()):
+ errors.append(f"invalid provenance for category {pkey!r} in {label}: {prov!r} (must be non-empty string)")
+
+ if "independent_oracle" in pdef:
+ io = pdef["independent_oracle"]
+ if not isinstance(io, bool):
+ errors.append(f"independent_oracle for category {pkey!r} in {label} must be a boolean")
+
+ if "complete" in pdef and not isinstance(pdef["complete"], bool):
+ errors.append(f"complete flag for category {pkey!r} in {label} must be a boolean")
+
+ if "expected_set_complete" in pdef and not isinstance(pdef["expected_set_complete"], bool):
+ errors.append(f"expected_set_complete flag for category {pkey!r} in {label} must be a boolean")
+
+ return errors
+
+
+def _validate_manifest(data: Any) -> list[str]:
+ """Return list of error strings; empty means valid."""
+ errors: list[str] = []
+ if not isinstance(data, dict):
+ errors.append("manifest must be a JSON object")
+ return errors
+ sv = data.get("schema_version")
+ if type(sv) is not int or sv != 1:
+ errors.append(f"unsupported schema_version: {sv!r} (expected 1)")
+ cid = data.get("campaign_id")
+ if isinstance(cid, bool) or not isinstance(cid, str) or not cid.strip():
+ errors.append("campaign_id must be a non-empty string")
+ arts = data.get("artifacts")
+ if not isinstance(arts, list):
+ errors.append("artifacts must be a JSON array")
+ if "expected_sets" in data:
+ errors.extend(_validate_expected_sets_structure(data["expected_sets"], label="expected_sets"))
+ return errors
+
+
+# ---------------------------------------------------------------------------
+# Single artifact audit
+# ---------------------------------------------------------------------------
+
+def _audit_artifact(entry: Any, root: Path) -> dict[str, Any]:
+ """Audit one artifact entry against the filesystem root."""
+ if not isinstance(entry, dict):
+ return {
+ "path": str(entry),
+ "type": None,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": None,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": None,
+ "notes": f"artifact entry must be a JSON object, got {type(entry).__name__}",
+ }
+
+ if "path" not in entry:
+ return {
+ "path": "",
+ "type": entry.get("type") if isinstance(entry.get("type"), str) else None,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": None,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": entry.get("provenance") if isinstance(entry.get("provenance"), str) else None,
+ "notes": "missing required 'path' field in artifact entry",
+ }
+
+ raw_path = entry["path"]
+ artifact_type = entry.get("type") if isinstance(entry.get("type"), str) else None
+ provenance = entry.get("provenance") if isinstance(entry.get("provenance"), str) else None
+ notes_in = entry.get("notes") if isinstance(entry.get("notes"), str) else None
+ mandatory = entry.get("mandatory", True) if not (entry.get("mandatory") is False or entry.get("optional") is True) else False
+ is_summary = bool(entry.get("is_summary") or entry.get("scope") in ("summary", "aggregate"))
+
+ # Validate path type before any operations
+ if isinstance(raw_path, bool) or not isinstance(raw_path, str):
+ return {
+ "path": str(raw_path),
+ "type": artifact_type,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": None,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": f"artifact path must be a string, got {type(raw_path).__name__}",
+ }
+
+ # Validate path safety
+ if not _is_safe_relative_path(raw_path):
+ return {
+ "path": raw_path,
+ "type": artifact_type,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": None,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": f"unsafe path rejected (absolute, traversal, or empty): {raw_path!r}",
+ }
+
+ # Validate expected hash format
+ expected_hash: str | None = None
+ if "expected_sha256" in entry and entry["expected_sha256"] is not None:
+ raw_hash = entry["expected_sha256"]
+ if isinstance(raw_hash, bool) or not isinstance(raw_hash, str):
+ return {
+ "path": raw_path,
+ "type": artifact_type,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": str(raw_hash),
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": f"expected_sha256 must be a string, got {type(raw_hash).__name__}",
+ }
+ if not _valid_sha256(raw_hash):
+ return {
+ "path": raw_path,
+ "type": artifact_type,
+ "status": INVALID,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": raw_hash,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": f"expected_sha256 is not a valid 64-hex-char SHA-256: {raw_hash!r}",
+ }
+ expected_hash = raw_hash
+
+ # Securely open and hash the artifact
+ open_res = _open_and_hash_artifact(root, raw_path)
+ open_status, observed_hash, size_bytes, err_note = open_res
+ file_id = getattr(open_res, "file_id", None)
+ norm_path = _normalize_relative_path(raw_path)
+
+ if open_status == "MISSING":
+ return {
+ "path": raw_path,
+ "normalized_path": norm_path,
+ "file_id": None,
+ "type": artifact_type,
+ "status": MISSING,
+ "present": False,
+ "observed_sha256": None,
+ "expected_sha256": expected_hash,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": err_note or notes_in,
+ }
+
+ if open_status == "UNVERIFIABLE":
+ return {
+ "path": raw_path,
+ "normalized_path": norm_path,
+ "file_id": None,
+ "type": artifact_type,
+ "status": UNVERIFIABLE,
+ "present": None,
+ "observed_sha256": None,
+ "expected_sha256": expected_hash,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": err_note or notes_in,
+ }
+
+ if open_status == "INVALID":
+ return {
+ "path": raw_path,
+ "normalized_path": norm_path,
+ "file_id": None,
+ "type": artifact_type,
+ "status": INVALID,
+ "present": True,
+ "observed_sha256": None,
+ "expected_sha256": expected_hash,
+ "hash_match": False,
+ "size_bytes": None,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": err_note or notes_in,
+ }
+
+ # Artifact is present regular file
+ if expected_hash is None:
+ status = OBSERVED_ONLY
+ hash_match = False
+ note = "no independent expected hash provided; observed hash recorded but not verified"
+ elif observed_hash == expected_hash:
+ status = VERIFIED
+ hash_match = True
+ note = notes_in
+ else:
+ status = MISMATCH
+ hash_match = False
+ note = "observed SHA-256 does not match independently provided expected hash"
+
+ res = {
+ "path": raw_path,
+ "normalized_path": norm_path,
+ "file_id": file_id,
+ "type": artifact_type,
+ "status": status,
+ "present": True,
+ "observed_sha256": observed_hash,
+ "expected_sha256": expected_hash,
+ "hash_match": hash_match,
+ "size_bytes": size_bytes,
+ "provenance": provenance,
+ "mandatory": mandatory,
+ "is_summary": is_summary,
+ "notes": note,
+ }
+ # Preserve explicit expected_set properties when present
+ for k in ("expected_set_id", "expected_set_provenance", "expected_set_complete", "independent_oracle"):
+ if k in entry:
+ res[k] = entry[k]
+ return res
+
+
+# ---------------------------------------------------------------------------
+# Reproducibility prerequisites conservative completeness model
+# ---------------------------------------------------------------------------
+
+def _build_prereqs(
+ artifact_results: list[dict[str, Any]],
+ expected_sets_def: Any = None,
+ trusted_sources: list[str] | set[str] | None = None,
+ trusted_expected_sets: dict[str, Any] | None = None,
+) -> tuple[dict[str, str], dict[str, dict[str, Any]]]:
+ """Derive reproducibility prerequisite completeness and artifact integrity.
+
+ Separates artifact checksum integrity from campaign completeness.
+ A prerequisite is complete only when sufficient evidence establishes that its
+ full required set is present, valid, and supported by independent provenance.
+
+ - Manifest-provided assertions (e.g. independent_oracle: true, fabricated URL)
+ are never treated as independent proof without a separately trusted source.
+ - Summary documents do not prove task-level oracle validity.
+ - Aggregate statistics do not prove complete latency records.
+ - Unverified self-assertions of completeness are rejected as UNVERIFIED_CLAIM.
+ - Unverified declared expected sets report UNVERIFIED, producing PARTIAL, not VERIFIED.
+ """
+ # Normalize trusted_sources
+ trusted_src_set: set[str] = set()
+ if trusted_sources:
+ if isinstance(trusted_sources, (list, tuple, set)):
+ trusted_src_set = {str(s).strip() for s in trusted_sources if s and isinstance(s, str)}
+ elif isinstance(trusted_sources, str):
+ trusted_src_set = {trusted_sources.strip()}
+
+ external_sets_mode = False
+ active_expected_sets: dict[str, Any] = {}
+ if trusted_expected_sets is not None:
+ if isinstance(trusted_expected_sets, dict):
+ active_expected_sets = trusted_expected_sets
+ external_sets_mode = True
+ else:
+ active_expected_sets = {}
+ elif isinstance(expected_sets_def, dict):
+ active_expected_sets = expected_sets_def
+ elif expected_sets_def is None:
+ active_expected_sets = {}
+ else:
+ # expected_sets_def was provided but is not a dict (e.g. [] or "string")
+ prereqs = {k: INVALID for k in sorted(_PREREQ_KEYS)}
+ details = {
+ k: {
+ "status": INVALID,
+ "artifact_integrity": INVALID,
+ "expected_set_status": "INVALID",
+ "expected_set_source": "none",
+ "prerequisite_complete": False,
+ "blocking": [],
+ "missing_or_unverified": [],
+ "evidence_provenance": "none",
+ "notes": f"expected_sets definition must be a JSON object, got {type(expected_sets_def).__name__}",
+ }
+ for k in sorted(_PREREQ_KEYS)
+ }
+ return prereqs, details
+
+ by_prereq: dict[str, list[dict[str, Any]]] = {k: [] for k in _PREREQ_KEYS}
+ for art in artifact_results:
+ atype = art.get("type")
+ pkey = _TYPE_TO_PREREQ.get(atype or "", "")
+ if pkey:
+ by_prereq[pkey].append(art)
+
+ prereqs: dict[str, str] = {}
+ details: dict[str, dict[str, Any]] = {}
+
+ for pkey in sorted(_PREREQ_KEYS):
+ arts = by_prereq[pkey]
+ es_info = active_expected_sets.get(pkey)
+
+ if es_info is not None and not isinstance(es_info, dict):
+ prereqs[pkey] = INVALID
+ details[pkey] = {
+ "status": INVALID,
+ "artifact_integrity": INVALID,
+ "expected_set_status": "INVALID",
+ "expected_set_source": "externally_verified" if external_sets_mode else "declared",
+ "prerequisite_complete": False,
+ "blocking": [],
+ "missing_or_unverified": [],
+ "evidence_provenance": "none",
+ "notes": f"expected_sets entry for {pkey!r} must be a JSON object, got {type(es_info).__name__}",
+ }
+ continue
+
+ if not arts and not es_info:
+ prereqs[pkey] = UNKNOWN
+ details[pkey] = {
+ "status": UNKNOWN,
+ "artifact_integrity": UNKNOWN,
+ "expected_set_status": UNKNOWN,
+ "expected_set_source": "none",
+ "prerequisite_complete": False,
+ "blocking": [],
+ "missing_or_unverified": [],
+ "evidence_provenance": "none",
+ "notes": "no artifacts or expected set declared for this prerequisite",
+ }
+ continue
+
+ mandatory = [a for a in arts if a.get("mandatory", True) is not False]
+ optional = [a for a in arts if a.get("mandatory") is False]
+
+ # 1. Determine artifact integrity for the cohort
+ statuses = [a.get("status", UNKNOWN) for a in arts]
+ if not arts:
+ artifact_integrity = UNKNOWN
+ elif any(s == MISMATCH for s in statuses):
+ artifact_integrity = MISMATCH
+ elif any(s == INVALID for s in statuses):
+ artifact_integrity = INVALID
+ elif any(s == UNVERIFIABLE for s in statuses):
+ artifact_integrity = UNVERIFIABLE
+ elif any(s == MISSING for s in (a.get("status") for a in mandatory)):
+ artifact_integrity = INCOMPLETE
+ elif all(s == VERIFIED for s in statuses):
+ artifact_integrity = VERIFIED
+ elif all(s == OBSERVED_ONLY for s in statuses):
+ artifact_integrity = OBSERVED_ONLY
+ elif any(s == VERIFIED for s in statuses) and any(s == OBSERVED_ONLY for s in statuses):
+ artifact_integrity = PARTIAL
+ else:
+ artifact_integrity = INCOMPLETE
+
+ # 2. Validate expected set definition structure
+ es_invalid = False
+ es_invalid_reason = ""
+ exp_paths_list: list[str] | None = None
+ exp_count_val: int | None = None
+
+ if es_info:
+ if "expected_paths" in es_info:
+ ep = es_info["expected_paths"]
+ if not isinstance(ep, list):
+ es_invalid = True
+ es_invalid_reason = f"expected_paths for {pkey} must be a list"
+ elif len(ep) == 0:
+ es_invalid = True
+ es_invalid_reason = f"expected_paths for {pkey} cannot be empty"
+ else:
+ seen = set()
+ for item in ep:
+ if isinstance(item, bool) or not isinstance(item, str):
+ es_invalid = True
+ es_invalid_reason = f"invalid item in expected_paths for {pkey}: {item!r} (must be string)"
+ break
+ if not _is_safe_relative_path(item):
+ es_invalid = True
+ es_invalid_reason = f"unsafe or invalid path in expected_paths for {pkey}: {item!r}"
+ break
+ norm = _normalize_relative_path(item)
+ if norm in seen:
+ es_invalid = True
+ es_invalid_reason = f"duplicate path in expected_paths for {pkey}: {item!r}"
+ break
+ seen.add(norm)
+ if not es_invalid:
+ exp_paths_list = ep
+
+ if not es_invalid and "expected_count" in es_info:
+ ec = es_info["expected_count"]
+ if isinstance(ec, bool) or not isinstance(ec, int) or ec <= 0:
+ es_invalid = True
+ es_invalid_reason = f"expected_count for {pkey} must be a positive integer"
+ else:
+ exp_count_val = ec
+
+ if es_invalid:
+ prereqs[pkey] = INVALID
+ details[pkey] = {
+ "status": INVALID,
+ "artifact_integrity": artifact_integrity if artifact_integrity != VERIFIED else INVALID,
+ "expected_set_status": "INVALID",
+ "expected_set_source": "externally_verified" if external_sets_mode else "declared",
+ "prerequisite_complete": False,
+ "blocking": [],
+ "missing_or_unverified": [],
+ "evidence_provenance": "none",
+ "notes": es_invalid_reason,
+ }
+ continue
+
+ # 3. Detect duplicate logical and physical records within category
+ seen_norm_paths: dict[str, str] = {}
+ seen_file_ids: dict[str, str] = {}
+ duplicate_records: list[dict[str, str]] = []
+ distinct_verified_norm_paths: set[str] = set()
+ distinct_verified_file_ids: set[str] = set()
+
+ for a in arts:
+ raw_p = str(a.get("path") or "")
+ norm_p = a.get("normalized_path") or _normalize_relative_path(raw_p)
+ fid = a.get("file_id")
+ is_dup = False
+ dup_reason = ""
+
+ if norm_p is not None:
+ if norm_p in seen_norm_paths:
+ is_dup = True
+ orig_p = seen_norm_paths[norm_p]
+ if orig_p == raw_p:
+ dup_reason = f"exact duplicate artifact path {raw_p!r}"
+ else:
+ dup_reason = f"normalized path alias {raw_p!r} duplicates {orig_p!r}"
+ else:
+ seen_norm_paths[norm_p] = raw_p
+
+ if not is_dup and fid is not None:
+ if fid in seen_file_ids:
+ is_dup = True
+ orig_p = seen_file_ids[fid]
+ dup_reason = f"physical file alias {raw_p!r} hardlinks {orig_p!r}"
+ else:
+ seen_file_ids[fid] = raw_p
+
+ if is_dup:
+ duplicate_records.append({
+ "path": raw_p,
+ "reason": dup_reason,
+ })
+ else:
+ if a.get("status") == VERIFIED:
+ if norm_p:
+ distinct_verified_norm_paths.add(norm_p)
+ if fid:
+ distinct_verified_file_ids.add(fid)
+
+ distinct_verified_count = (
+ min(len(distinct_verified_norm_paths), len(distinct_verified_file_ids))
+ if distinct_verified_file_ids
+ else len(distinct_verified_norm_paths)
+ )
+
+ expected_set_status = UNKNOWN
+ expected_set_source = "none"
+ prerequisite_complete = False
+ evidence_provenance = "none"
+ notes = None
+
+ if es_info:
+ prov = es_info.get("provenance")
+ self_claim = bool(es_info.get("complete") or es_info.get("expected_set_complete"))
+
+ # Determine whether the verification source is separately trusted
+ is_trusted_source = False
+ if external_sets_mode:
+ is_trusted_source = True
+ expected_set_source = "externally_verified"
+ evidence_provenance = str(prov or "external_trusted_expected_sets")
+ elif prov and isinstance(prov, str) and prov.strip() in trusted_src_set:
+ is_trusted_source = True
+ expected_set_source = "externally_verified"
+ evidence_provenance = prov.strip()
+ else:
+ expected_set_source = "declared"
+ evidence_provenance = str(prov or "manifest_declared")
+
+ if self_claim and not is_trusted_source:
+ expected_set_status = "UNVERIFIED_CLAIM"
+ notes = "completeness self-asserted in manifest without separately trusted verification source"
+ elif not is_trusted_source:
+ expected_set_status = "UNVERIFIED"
+ notes = "expected set declared in manifest lacks separately trusted verification source"
+ else:
+ # Separately trusted verification source exists
+ if exp_paths_list is not None:
+ norm_exp_paths = set()
+ for p in exp_paths_list:
+ np = _normalize_relative_path(p)
+ if np:
+ norm_exp_paths.add(np)
+
+ if (
+ norm_exp_paths
+ and norm_exp_paths.issubset(distinct_verified_norm_paths)
+ and all(a.get("status") == VERIFIED for a in arts)
+ and not duplicate_records
+ ):
+ expected_set_status = "COMPLETE"
+ prerequisite_complete = True
+ else:
+ expected_set_status = "INCOMPLETE"
+ prerequisite_complete = False
+ elif exp_count_val is not None:
+ if (
+ distinct_verified_count >= exp_count_val
+ and len(arts) == distinct_verified_count
+ and all(a.get("status") == VERIFIED for a in arts)
+ and not duplicate_records
+ ):
+ expected_set_status = "COMPLETE"
+ prerequisite_complete = True
+ else:
+ expected_set_status = "INCOMPLETE"
+ prerequisite_complete = False
+ else:
+ expected_set_status = "UNKNOWN"
+ else:
+ expected_set_status = "UNKNOWN"
+ expected_set_source = "none"
+
+ if duplicate_records:
+ dup_msg = f"duplicate artifact records detected: {'; '.join(d['reason'] for d in duplicate_records)}"
+ notes = f"{notes}; {dup_msg}" if notes else dup_msg
+
+ # 4. Special scientific domain boundaries
+ # Requirement 8: Generic summary documents do not prove task-level oracle validity
+ if pkey == "expected_outcomes_and_oracles" and arts:
+ all_summaries = all(
+ a.get("is_summary") is True
+ or a.get("type") in ("oracle_summary", "campaign_manifest")
+ or "freeze-manifest" in a.get("path", "").lower()
+ for a in arts
+ )
+ if all_summaries:
+ expected_set_status = "SUMMARY_ONLY"
+ prerequisite_complete = False
+ notes = "summary document does not prove task-level oracle validity"
+
+ # Requirement 9: Aggregate statistics do not prove complete latency records
+ if pkey == "result_and_latency_records" and arts:
+ all_aggregates = all(
+ a.get("is_summary") is True
+ or a.get("type") in ("result_summary", "aggregate_report", "campaign_manifest")
+ or "report" in a.get("path", "").lower()
+ or "inventory" in a.get("path", "").lower()
+ or "status" in a.get("path", "").lower()
+ for a in arts
+ )
+ if all_aggregates:
+ expected_set_status = "AGGREGATE_ONLY"
+ prerequisite_complete = False
+ notes = "aggregate statistics do not prove complete latency records"
+
+ # 5. Compute missing or unverified artifacts
+ missing_or_unverified: list[str] = [
+ a.get("path", "")
+ for a in mandatory
+ if a.get("status") != VERIFIED
+ ]
+ if exp_paths_list:
+ for p in exp_paths_list:
+ np = _normalize_relative_path(p)
+ if np and np not in distinct_verified_norm_paths and p not in missing_or_unverified:
+ missing_or_unverified.append(p)
+
+ # 6. Determine top-level aggregate status string
+ if prerequisite_complete and artifact_integrity == VERIFIED:
+ status = VERIFIED
+ elif artifact_integrity == MISMATCH:
+ status = MISMATCH
+ elif artifact_integrity == INVALID:
+ status = INVALID
+ elif artifact_integrity == UNVERIFIABLE:
+ status = UNVERIFIABLE
+ elif artifact_integrity == INCOMPLETE or expected_set_status == "INCOMPLETE":
+ status = INCOMPLETE
+ elif expected_set_status in ("SUMMARY_ONLY", "AGGREGATE_ONLY"):
+ status = OBSERVED_ONLY
+ elif artifact_integrity == OBSERVED_ONLY:
+ status = OBSERVED_ONLY
+ elif artifact_integrity == VERIFIED and not prerequisite_complete:
+ # Verified files present, but expected-set completeness is not verified
+ status = PARTIAL
+ elif artifact_integrity == PARTIAL:
+ status = PARTIAL
+ else:
+ status = UNKNOWN
+
+ prereqs[pkey] = status
+ details[pkey] = {
+ "status": status,
+ "artifact_integrity": artifact_integrity,
+ "expected_set_status": expected_set_status,
+ "expected_set_source": expected_set_source,
+ "prerequisite_complete": prerequisite_complete,
+ "distinct_verified_count": distinct_verified_count,
+ "duplicate_artifacts": [d["reason"] for d in duplicate_records],
+ "blocking": sorted(str(p) for p in missing_or_unverified),
+ "missing_or_unverified": sorted(str(p) for p in missing_or_unverified),
+ "evidence_provenance": evidence_provenance,
+ "notes": notes,
+ }
+
+ return prereqs, details
+
+
+# ---------------------------------------------------------------------------
+# Output Publication & Safety (TOCTOU and Descriptor Anchoring)
+# ---------------------------------------------------------------------------
+
+def _collect_protected_targets(
+ manifest_path: str,
+ root_path: str,
+ artifact_results: list[dict[str, Any]],
+ extra_input_paths: list[str | Path | None] | None = None,
+) -> tuple[dict[Path, str], dict[tuple[int, int], str]]:
+ """Collect resolved paths and file identities of protected inputs and manifest."""
+ protected_paths: dict[Path, str] = {}
+ protected_inodes: dict[tuple[int, int], str] = {}
+
+ try:
+ root = Path(root_path).resolve()
+ except (ValueError, UnicodeError, OSError):
+ root = Path(root_path)
+
+ try:
+ mpath = Path(manifest_path).resolve()
+ protected_paths[mpath] = "input manifest"
+ if mpath.exists():
+ try:
+ st = mpath.stat()
+ protected_inodes[(st.st_dev, st.st_ino)] = "input manifest"
+ except OSError:
+ pass
+ except (ValueError, UnicodeError, OSError):
+ pass
+
+ for art in artifact_results:
+ raw = art.get("path")
+ if isinstance(raw, str) and _is_safe_relative_path(raw):
+ try:
+ art_full = (root / raw).resolve()
+ label = f"audited artifact '{raw}'"
+ protected_paths[art_full] = label
+ if art_full.exists():
+ try:
+ st = art_full.stat()
+ protected_inodes[(st.st_dev, st.st_ino)] = label
+ except OSError:
+ pass
+ except (ValueError, UnicodeError, OSError):
+ pass
+
+ if extra_input_paths:
+ for p in extra_input_paths:
+ if p is not None and str(p).strip():
+ try:
+ p_obj = Path(p).resolve()
+ label = f"trusted input file '{p}'"
+ protected_paths[p_obj] = label
+ if p_obj.exists():
+ try:
+ st = p_obj.stat()
+ protected_inodes[(st.st_dev, st.st_ino)] = label
+ except OSError:
+ pass
+ except (ValueError, UnicodeError, OSError):
+ pass
+
+ return protected_paths, protected_inodes
+
+
+def _validate_output_destinations(
+ manifest_path: str,
+ root_path: str,
+ output_path: str | None,
+ summary_path: str | None,
+ artifact_results: list[dict[str, Any]],
+ extra_input_paths: list[str | Path | None] | None = None,
+) -> list[str]:
+ """Validate that output and summary destinations do not overwrite input evidence or manifest."""
+ errors: list[str] = []
+ if not output_path and not summary_path:
+ return errors
+
+ # Conflict between output and summary
+ if output_path and summary_path:
+ try:
+ out_p = Path(output_path).resolve()
+ sum_p = Path(summary_path).resolve()
+ if out_p == sum_p:
+ errors.append(f"output and summary cannot target the same destination: {output_path}")
+ elif out_p.exists() and sum_p.exists():
+ try:
+ if os.path.samefile(out_p, sum_p):
+ errors.append("output and summary target the same file (alias detected)")
+ except OSError:
+ pass
+ except (ValueError, UnicodeError, OSError):
+ pass
+
+ protected_paths, protected_inodes = _collect_protected_targets(
+ manifest_path,
+ root_path,
+ artifact_results,
+ extra_input_paths=extra_input_paths,
+ )
+
+ dests_to_check = []
+ if output_path:
+ dests_to_check.append(("output", Path(output_path)))
+ if summary_path:
+ dests_to_check.append(("summary", Path(summary_path)))
+
+ for label, dest in dests_to_check:
+ try:
+ res_dest = dest.resolve()
+ if res_dest in protected_paths:
+ errors.append(f"{label} destination targets protected {protected_paths[res_dest]}: {dest}")
+ if dest.exists() or dest.is_symlink():
+ try:
+ if dest.is_symlink():
+ link_target = dest.resolve()
+ if link_target in protected_paths:
+ errors.append(f"{label} destination symlink targets protected {protected_paths[link_target]}: {dest}")
+ dst_st = dest.stat()
+ ident = (dst_st.st_dev, dst_st.st_ino)
+ if ident in protected_inodes:
+ errors.append(f"{label} destination hardlink/alias targets protected {protected_inodes[ident]}: {dest}")
+ except OSError:
+ pass
+ except (ValueError, UnicodeError, OSError) as exc:
+ errors.append(f"{label} destination path is invalid: {dest}")
+
+ return errors
+
+
+def _publish_file_securely(
+ dest_path_str: str,
+ content_str: str,
+ protected_paths: dict[Path, str],
+ protected_inodes: dict[tuple[int, int], str],
+) -> None:
+ """Publish content atomically and securely, protecting against symlink substitution.
+
+ 1. Checks parent path components without resolving symlinks; rejects symlinked directories.
+ 2. Opens parent directory descriptor with O_NOFOLLOW.
+ 3. Verifies destination is not a symlink or hardlink pointing to protected targets.
+ 4. Creates temporary file exclusively with O_EXCL and O_NOFOLLOW in the parent directory.
+ 5. Writes and flushes complete content.
+ 6. Atomically replaces destination using descriptor-anchored renameat (os.replace).
+ 7. Cleans up temporary files and descriptors on success and failure.
+ """
+ raw_p = Path(dest_path_str)
+ abs_p = raw_p.absolute()
+ parent = abs_p.parent
+
+ # Traverse/create parent directory components descriptor-by-descriptor with O_NOFOLLOW
+ # Never reopen parent through full pathname!
+ parent_fd = _open_or_create_dir_securely(parent)
+ base_name = abs_p.name
+ temp_name = f".tmp_{uuid.uuid4().hex}_{base_name}"
+ temp_fd: int | None = None
+
+ try:
+ # Check destination in parent directory before creating temp file
+ try:
+ st = os.lstat(base_name, dir_fd=parent_fd)
+ ident = (st.st_dev, st.st_ino)
+ if ident in protected_inodes:
+ raise OSError(f"Destination is protected alias: {dest_path_str}")
+ if stat.S_ISLNK(st.st_mode):
+ target = (parent / os.readlink(base_name, dir_fd=parent_fd)).resolve()
+ if target in protected_paths or (st.st_dev, st.st_ino) in protected_inodes:
+ raise OSError(f"Destination symlink targets protected path: {dest_path_str}")
+ except FileNotFoundError:
+ pass
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"Invalid destination filename: {exc}") from exc
+
+ # Create temporary file exclusively in parent directory
+ temp_fd = os.open(
+ temp_name,
+ os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW | os.O_CLOEXEC,
+ 0o644,
+ dir_fd=parent_fd,
+ )
+ data = content_str.encode("utf-8", errors="backslashreplace")
+ total = 0
+ while total < len(data):
+ n = os.write(temp_fd, data[total:])
+ total += n
+ os.fsync(temp_fd)
+ os.close(temp_fd)
+ temp_fd = None
+
+ # Verify destination before atomic replace
+ try:
+ st = os.lstat(base_name, dir_fd=parent_fd)
+ ident = (st.st_dev, st.st_ino)
+ if ident in protected_inodes:
+ raise OSError(f"Destination changed to protected file: {dest_path_str}")
+ if stat.S_ISLNK(st.st_mode):
+ target = (parent / os.readlink(base_name, dir_fd=parent_fd)).resolve()
+ if target in protected_paths:
+ raise OSError(f"Destination changed to symlink targeting protected path: {dest_path_str}")
+ except FileNotFoundError:
+ pass
+ except (ValueError, UnicodeError) as exc:
+ raise OSError(f"Invalid destination filename: {exc}") from exc
+
+ # Atomic rename anchored to directory descriptor
+ os.replace(temp_name, base_name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
+ finally:
+ if temp_fd is not None:
+ try:
+ os.close(temp_fd)
+ except OSError:
+ pass
+ try:
+ os.unlink(temp_name, dir_fd=parent_fd)
+ except OSError:
+ pass
+ os.close(parent_fd)
+
+
+# ---------------------------------------------------------------------------
+# Core audit function
+# ---------------------------------------------------------------------------
+
+def audit(
+ manifest_path: str,
+ root_path: str,
+ deterministic: bool = False,
+ trusted_sources: list[str] | set[str] | None = None,
+ trusted_expected_sets: dict[str, Any] | str | Path | None = None,
+) -> dict[str, Any]:
+ """Run the full evidence audit and return a result dict.
+
+ This function never raises; all errors are captured in the returned dict.
+ It is safe to call repeatedly without side effects.
+ """
+ if deterministic:
+ audited_at = "1970-01-01T00:00:00Z"
+ else:
+ audited_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ base: dict[str, Any] = {
+ "schema_version": 1,
+ "audited_at": audited_at,
+ "timestamp_kind": "deterministic_placeholder" if deterministic else "live_execution",
+ "campaign_id": None,
+ "root": str(root_path),
+ "manifest_path": str(manifest_path),
+ "totals": {s: 0 for s in ALL_STATUSES} | {"total": 0},
+ "artifacts": [],
+ "reproducibility_prerequisites": {k: UNKNOWN for k in sorted(_PREREQ_KEYS)},
+ "reproducibility_details": {},
+ "errors": [],
+ "manifest_valid": False,
+ "configuration_valid": True,
+ }
+
+ try:
+ root = Path(root_path).absolute()
+ mpath = Path(manifest_path)
+ except (ValueError, UnicodeError, OSError) as exc:
+ base["errors"].append(f"invalid path: {exc}")
+ return base
+
+ base["root"] = str(root)
+ base["manifest_path"] = str(mpath)
+
+ # Load manifest
+ try:
+ raw = mpath.read_bytes()
+ except (OSError, PermissionError, ValueError, UnicodeError) as exc:
+ base["errors"].append(f"cannot read manifest: {exc}")
+ return base
+
+ try:
+ data = json.loads(raw)
+ except (json.JSONDecodeError, ValueError) as exc:
+ base["errors"].append(f"manifest JSON parse error: {exc}")
+ return base
+
+ # Validate manifest structure
+ validation_errors = _validate_manifest(data)
+ if validation_errors:
+ base["errors"].extend(validation_errors)
+ base["campaign_id"] = data.get("campaign_id") if isinstance(data, dict) and not isinstance(data.get("campaign_id"), bool) else None
+ return base
+
+ base["manifest_valid"] = True
+ base["campaign_id"] = data["campaign_id"]
+
+ # Audit each artifact
+ artifact_entries = data.get("artifacts", [])
+ results: list[dict[str, Any]] = []
+ for entry in artifact_entries:
+ result = _audit_artifact(entry, root)
+ results.append(result)
+
+ # Deterministic stable sort by path string
+ results.sort(key=lambda r: str(r.get("path") if r.get("path") is not None else ""))
+
+ # Compute totals
+ totals: dict[str, int] = {s: 0 for s in ALL_STATUSES}
+ totals["total"] = len(results)
+ for r in results:
+ s = r.get("status", INVALID)
+ if s in totals:
+ totals[s] += 1
+
+ loaded_trusted_sets: dict[str, Any] | None = None
+ configuration_valid = True
+ if trusted_expected_sets is not None:
+ if isinstance(trusted_expected_sets, (str, Path)):
+ try:
+ with open(str(trusted_expected_sets), "rb") as f:
+ loaded_trusted_sets = json.load(f)
+ except Exception as exc:
+ base["errors"].append(f"cannot load trusted_expected_sets: {exc}")
+ configuration_valid = False
+ elif isinstance(trusted_expected_sets, dict):
+ loaded_trusted_sets = trusted_expected_sets
+ else:
+ base["errors"].append(f"trusted_expected_sets must be a path or dict, got {type(trusted_expected_sets).__name__}")
+ configuration_valid = False
+
+ if configuration_valid:
+ struct_errors = _validate_expected_sets_structure(loaded_trusted_sets, label="trusted_expected_sets")
+ if struct_errors:
+ base["errors"].extend(struct_errors)
+ configuration_valid = False
+
+ base["configuration_valid"] = configuration_valid
+ if not configuration_valid:
+ prereqs = {k: INVALID for k in sorted(_PREREQ_KEYS)}
+ details = {k: {"status": INVALID, "notes": "invalid trusted_expected_sets configuration"} for k in sorted(_PREREQ_KEYS)}
+ else:
+ prereqs, details = _build_prereqs(
+ results,
+ data.get("expected_sets", {}),
+ trusted_sources=trusted_sources,
+ trusted_expected_sets=loaded_trusted_sets,
+ )
+
+ base["artifacts"] = results
+ base["totals"] = totals
+ base["reproducibility_prerequisites"] = prereqs
+ base["reproducibility_details"] = details
+ return base
+
+
+# ---------------------------------------------------------------------------
+# Markdown summary
+# ---------------------------------------------------------------------------
+
+def _markdown_display_value(value: Any) -> str:
+ """Make controls and invalid Unicode visible before code-span quoting."""
+ raw = str(value).encode("utf-8", errors="backslashreplace").decode("utf-8")
+ safe = []
+ for char in raw:
+ if char == "\n":
+ safe.append(r"\n")
+ elif char == "\r":
+ safe.append(r"\r")
+ elif char == "\t":
+ safe.append(r"\t")
+ elif ord(char) < 32 or ord(char) == 127:
+ safe.append(f"\\x{ord(char):02x}")
+ elif (ord(char) in (0x061C, 0x200E, 0x200F)
+ or 0x202A <= ord(char) <= 0x202E
+ or 0x2066 <= ord(char) <= 0x2069):
+ safe.append(f"\\u{ord(char):04x}")
+ elif char in ("\u0085", "\u2028", "\u2029"):
+ safe.append(f"\\u{ord(char):04x}")
+ else:
+ safe.append(char)
+ return "".join(safe)
+
+
+def _markdown_code(value: Any, *, table_cell: bool = False) -> str:
+ """Quote literal text using a native GFM code span, not inline HTML."""
+ text = _markdown_display_value(value)
+ if not text:
+ return ""
+ if table_cell:
+ # GFM splits cells before parsing code spans and removes this one escape.
+ text = text.replace("|", r"\|")
+ # cmark-gfm limits delimiter lengths. Keep spans short and emit input
+ # backticks as literal entities outside them, never as Markdown delimiters.
+ spans = []
+ for part in text.split("`"):
+ # One padding space is stripped by GFM, except for all-space spans.
+ padding = " " if part.strip(" ") else ""
+ spans.append(f"`{padding}{part}{padding}`" if part else "")
+ return "`".join(spans)
+
+
+def _markdown_text(value: Any, *, table_cell: bool = False) -> str:
+ return _markdown_code(value, table_cell=table_cell)
+
+
+def _markdown_summary(result: dict[str, Any]) -> str:
+ lines: list[str] = []
+ lines.append("# Benchmark Evidence Audit Summary")
+ lines.append("")
+ lines.append(f"- **Campaign**: {_markdown_code(result.get('campaign_id', 'unknown'))}")
+ lines.append(f"- **Audited at**: {result.get('audited_at', '')}")
+ if result.get("timestamp_kind") == "deterministic_placeholder":
+ lines.append("- **Timestamp note**: Fixed reproducibility placeholder (1970-01-01T00:00:00Z)")
+ lines.append(f"- **Root**: {_markdown_code(result.get('root', ''))}")
+ lines.append(f"- **Manifest valid**: {result.get('manifest_valid', False)}")
+ lines.append("")
+
+ t = result.get("totals", {})
+ lines.append("## Totals")
+ lines.append("")
+ lines.append("| Status | Count |")
+ lines.append("|---|---|")
+ lines.append(f"| Total | {t.get('total', 0)} |")
+ for s in ALL_STATUSES:
+ lines.append(f"| {s} | {t.get(s, 0)} |")
+ lines.append("")
+
+ errors = result.get("errors", [])
+ if errors:
+ lines.append("## Errors")
+ lines.append("")
+ for e in errors:
+ lines.append(f"- {_markdown_text(e)}")
+ lines.append("")
+
+ arts = result.get("artifacts", [])
+ if arts:
+ lines.append("## Artifacts")
+ lines.append("")
+ lines.append("| Path | Type | Status | Present | Size (bytes) |")
+ lines.append("|---|---|---|---|---|")
+ for a in arts:
+ path = a.get("path", "")
+ display_path = _markdown_code(path, table_cell=True)
+ atype = a.get("type") or ""
+ status = a.get("status", "")
+ present = "yes" if a.get("present") else ("no" if a.get("present") is False else "?")
+ size = str(a.get("size_bytes")) if a.get("size_bytes") is not None else "-"
+ lines.append(f"| {display_path} | {_markdown_text(atype, table_cell=True)} | **{status}** | {present} | {size} |")
+ lines.append("")
+
+ prereqs = result.get("reproducibility_prerequisites", {})
+ details = result.get("reproducibility_details", {})
+ if prereqs:
+ lines.append("## Reproducibility Prerequisites")
+ lines.append("")
+ lines.append("| Prerequisite | Prerequisite Status | Artifact Integrity | Expected Set | Complete | Notes |")
+ lines.append("|---|---|---|---|---|---|")
+ for k, v in sorted(prereqs.items()):
+ det = details.get(k, {})
+ integrity = det.get("artifact_integrity", "UNKNOWN")
+ es_status = det.get("expected_set_status", "UNKNOWN")
+ comp = "yes" if det.get("prerequisite_complete") else "no"
+ notes = det.get("notes") or ""
+ lines.append(f"| {k} | **{v}** | {integrity} | {es_status} | {comp} | {_markdown_text(notes, table_cell=True)} |")
+ lines.append("")
+
+ return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument("manifest", help="Path to the artifact manifest JSON file")
+ parser.add_argument("root", help="Designated filesystem root for artifact lookup")
+ parser.add_argument(
+ "--output", "-o", metavar="FILE",
+ help="Write JSON output to FILE instead of stdout",
+ )
+ parser.add_argument(
+ "--summary", "-s", metavar="FILE",
+ help="Also write a Markdown summary to FILE",
+ )
+ parser.add_argument(
+ "--deterministic", "-d", action="store_true",
+ help="Use a fixed timestamp (1970-01-01T00:00:00Z) as reproducibility placeholder",
+ )
+ parser.add_argument(
+ "--trusted-source", "-t", action="append", default=[],
+ help="Designate an external trusted verification source identifier or URL",
+ )
+ parser.add_argument(
+ "--trusted-expected-sets", metavar="FILE",
+ help="Path to an independently established trusted expected-sets JSON file",
+ )
+ args = parser.parse_args(argv)
+
+ result = audit(
+ args.manifest,
+ args.root,
+ deterministic=args.deterministic,
+ trusted_sources=args.trusted_source,
+ trusted_expected_sets=args.trusted_expected_sets,
+ )
+
+ extra_inputs = [args.trusted_expected_sets] if args.trusted_expected_sets else None
+
+ # Preflight output destination validation
+ preflight_errors = _validate_output_destinations(
+ args.manifest,
+ args.root,
+ args.output,
+ args.summary,
+ result.get("artifacts", []),
+ extra_input_paths=extra_inputs,
+ )
+ if preflight_errors:
+ # Input paths may contain credentials. Do not echo validation details to stderr.
+ print("preflight error: unsafe or conflicting output destination", file=sys.stderr)
+ return 2
+
+ if not result.get("manifest_valid") or result.get("configuration_valid") is False or result.get("errors"):
+ # The structured audit result retains diagnostics for trusted callers.
+ # Never log its untrusted error strings, paths, or configuration values.
+ if not result.get("manifest_valid"):
+ print("audit error: manifest is unreadable or invalid", file=sys.stderr)
+ elif result.get("configuration_valid") is False:
+ print("audit error: invalid trusted expected-sets configuration", file=sys.stderr)
+ else:
+ print("audit error: invalid audit input", file=sys.stderr)
+ return 2
+
+ output_json = json.dumps(result, indent=2, sort_keys=False) + "\n"
+ summary_md = _markdown_summary(result) + "\n" if args.summary else ""
+
+ # Atomic publication using descriptor-anchored safe writer
+ protected_paths, protected_inodes = _collect_protected_targets(
+ args.manifest,
+ args.root,
+ result.get("artifacts", []),
+ extra_input_paths=extra_inputs,
+ )
+
+ if args.output:
+ try:
+ _publish_file_securely(args.output, output_json, protected_paths, protected_inodes)
+ except OSError:
+ print("publication error: unable to write output", file=sys.stderr)
+ return 2
+ else:
+ print(output_json, end="")
+
+ if args.summary:
+ try:
+ _publish_file_securely(args.summary, summary_md, protected_paths, protected_inodes)
+ except OSError:
+ print("publication error: unable to write summary", file=sys.stderr)
+ return 2
+
+ # Determine exit code
+ totals = result.get("totals", {})
+ if totals.get(INVALID, 0) > 0:
+ return 2
+ prereqs = result.get("reproducibility_prerequisites", {})
+ if any(status == INVALID for status in prereqs.values()):
+ return 2
+ if any(totals.get(s, 0) > 0 for s in (MISSING, MISMATCH, UNVERIFIABLE)):
+ return 1
+ if any(status in (INCOMPLETE, MISMATCH, UNVERIFIABLE) for status in prereqs.values()):
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_audit_c5_benchmark_evidence.py b/tests/test_audit_c5_benchmark_evidence.py
new file mode 100644
index 000000000..da92f24a5
--- /dev/null
+++ b/tests/test_audit_c5_benchmark_evidence.py
@@ -0,0 +1,3020 @@
+"""Deterministic tests for audit_c5_benchmark_evidence.py.
+
+All tests use temporary synthetic fixtures - no network, no runtime, no model.
+Tests are self-contained and safe to run repeatedly without changing source data.
+
+GFM integration checks are separate from the dependency-free unit suite:
+ python -m venv /tmp/c5-gfm
+ /tmp/c5-gfm/bin/pip install cmarkgfm==2025.10.22
+ /tmp/c5-gfm/bin/python tests/test_audit_c5_benchmark_evidence.py --gfm
+The explicit --gfm command fails if cmarkgfm is unavailable; it never skips.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+# Ensure project root is on path for import without conftest.
+_ROOT = Path(__file__).resolve().parents[1]
+if str(_ROOT) not in sys.path:
+ sys.path.insert(0, str(_ROOT))
+
+import scripts.audit_c5_benchmark_evidence as auditor
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _sha256(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def _write(path: Path, content: bytes) -> str:
+ """Write bytes to path and return its sha256."""
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(content)
+ return _sha256(content)
+
+
+def _manifest(artifacts: list, campaign_id: str = "TEST-CAMPAIGN-01", expected_sets: dict | None = None) -> dict:
+ d = {
+ "schema_version": 1,
+ "campaign_id": campaign_id,
+ "description": "Synthetic test manifest",
+ "artifacts": artifacts,
+ }
+ if expected_sets:
+ d["expected_sets"] = expected_sets
+ return d
+
+
+def _run_audit(
+ tmp: Path,
+ manifest_dict: dict,
+ deterministic: bool = False,
+ trusted_sources: list[str] | None = None,
+ trusted_expected_sets: dict | None = None,
+) -> dict:
+ """Write manifest to tmp, run audit, return parsed result."""
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(manifest_dict))
+ return auditor.audit(
+ str(mpath),
+ str(tmp),
+ deterministic=deterministic,
+ trusted_sources=trusted_sources,
+ trusted_expected_sets=trusted_expected_sets,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Baseline Integrity and Safety Tests
+# ---------------------------------------------------------------------------
+
+class TestValidMatchingHash(unittest.TestCase):
+ """VERIFIED: artifact present, observed hash matches expected hash."""
+
+ def test_verified_when_observed_matches_expected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"hello world"
+ h = _write(tmp / "file.txt", content)
+ result = _run_audit(tmp, _manifest([{
+ "path": "file.txt",
+ "type": "result_record",
+ "expected_sha256": h,
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "VERIFIED")
+ self.assertTrue(art["present"])
+ self.assertEqual(art["observed_sha256"], h)
+ self.assertEqual(art["expected_sha256"], h)
+ self.assertTrue(art["hash_match"])
+ self.assertEqual(result["totals"]["VERIFIED"], 1)
+
+
+class TestMismatchedHash(unittest.TestCase):
+ """MISMATCH: artifact present, but observed hash differs from expected."""
+
+ def test_mismatch_when_hash_differs(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"actual content"
+ _write(tmp / "data.json", content)
+ wrong_hash = _sha256(b"different content")
+ result = _run_audit(tmp, _manifest([{
+ "path": "data.json",
+ "type": "campaign_manifest",
+ "expected_sha256": wrong_hash,
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "MISMATCH")
+ self.assertTrue(art["present"])
+ self.assertNotEqual(art["observed_sha256"], art["expected_sha256"])
+ self.assertFalse(art["hash_match"])
+ self.assertEqual(result["totals"]["MISMATCH"], 1)
+
+
+class TestMissingArtifact(unittest.TestCase):
+ """MISSING: expected artifact is absent from the filesystem."""
+
+ def test_missing_when_file_absent(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([{
+ "path": "nonexistent/file.json",
+ "type": "trajectory",
+ "expected_sha256": "a" * 64,
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "MISSING")
+ self.assertFalse(art["present"])
+ self.assertIsNone(art["observed_sha256"])
+ self.assertIsNone(art["size_bytes"])
+ self.assertEqual(result["totals"]["MISSING"], 1)
+
+
+class TestObservedOnlyNoExpectedHash(unittest.TestCase):
+ """OBSERVED_ONLY: artifact exists but manifest has no expected hash."""
+
+ def test_observed_only_when_no_expected_hash(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "report.md", b"# Report\n")
+ result = _run_audit(tmp, _manifest([{
+ "path": "report.md",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "OBSERVED_ONLY")
+ self.assertTrue(art["present"])
+ self.assertIsNotNone(art["observed_sha256"])
+ self.assertIsNone(art["expected_sha256"])
+ self.assertFalse(art["hash_match"])
+ self.assertEqual(result["totals"]["OBSERVED_ONLY"], 1)
+
+ def test_observed_only_when_expected_hash_is_null(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "report.md", b"# Report\n")
+ result = _run_audit(tmp, _manifest([{
+ "path": "report.md",
+ "type": "result_record",
+ "expected_sha256": None,
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "OBSERVED_ONLY")
+
+
+class TestInvalidManifest(unittest.TestCase):
+ """INVALID: manifest fails validation."""
+
+ def test_invalid_missing_schema_version(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps({"campaign_id": "X", "artifacts": []}))
+ result = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(result["manifest_valid"])
+ self.assertGreater(len(result["errors"]), 0)
+
+ def test_invalid_wrong_schema_version(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, {"schema_version": 99, "campaign_id": "X", "artifacts": []})
+ self.assertFalse(result["manifest_valid"])
+
+ def test_invalid_artifacts_not_a_list(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, {"schema_version": 1, "campaign_id": "X", "artifacts": "oops"})
+ self.assertFalse(result["manifest_valid"])
+
+ def test_invalid_malformed_json(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text("{not valid json}")
+ result = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(result["manifest_valid"])
+ self.assertGreater(len(result["errors"]), 0)
+
+ def test_invalid_missing_manifest_file(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "does_not_exist.json"
+ result = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(result["manifest_valid"])
+ self.assertGreater(len(result["errors"]), 0)
+
+
+class TestPathTraversal(unittest.TestCase):
+ """INVALID: path traversal attempts must be rejected."""
+
+ def test_reject_dotdot_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([{
+ "path": "../escape.txt",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "INVALID")
+
+ def test_reject_absolute_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([{
+ "path": "/etc/passwd",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "INVALID")
+
+ def test_reject_nested_dotdot(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([{
+ "path": "subdir/../../etc/passwd",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertEqual(art["status"], "INVALID")
+
+
+class TestSymlinkEscape(unittest.TestCase):
+ """UNVERIFIABLE: symlinks that escape the designated root."""
+
+ def test_symlink_escape_is_unverifiable(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ link = tmp / "escape_link.txt"
+ link.symlink_to("/etc/hostname")
+ result = _run_audit(tmp, _manifest([{
+ "path": "escape_link.txt",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertIn(art["status"], ("UNVERIFIABLE", "INVALID"))
+
+ def test_internal_symlink_is_handled(self):
+ """A symlink that stays within root is rejected as UNVERIFIABLE under O_NOFOLLOW boundary."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ target = tmp / "real.txt"
+ target.write_bytes(b"data")
+ link = tmp / "link.txt"
+ link.symlink_to(target)
+ result = _run_audit(tmp, _manifest([{
+ "path": "link.txt",
+ "type": "result_record",
+ }]))
+ art = result["artifacts"][0]
+ self.assertNotEqual(art["status"], "INVALID")
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+
+
+class TestStableRepeatedOutput(unittest.TestCase):
+ """Stable: running the audit twice must produce identical output."""
+
+ def test_identical_output_on_repeated_runs(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"stable content"
+ h = _write(tmp / "a.json", content)
+ _write(tmp / "b.json", b"other")
+ manifest = _manifest([
+ {"path": "a.json", "type": "result_record", "expected_sha256": h},
+ {"path": "b.json", "type": "campaign_manifest"},
+ {"path": "missing.json", "type": "trajectory", "expected_sha256": "b" * 64},
+ ])
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(manifest))
+
+ r1 = auditor.audit(str(mpath), str(tmp), deterministic=True)
+ r2 = auditor.audit(str(mpath), str(tmp), deterministic=True)
+
+ self.assertEqual(
+ json.dumps(r1, sort_keys=True),
+ json.dumps(r2, sort_keys=True),
+ "Repeated runs produced different output",
+ )
+
+ def test_artifacts_sorted_by_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "zzz.txt", b"z")
+ _write(tmp / "aaa.txt", b"a")
+ result = _run_audit(tmp, _manifest([
+ {"path": "zzz.txt", "type": "result_record"},
+ {"path": "aaa.txt", "type": "result_record"},
+ ]))
+ paths = [a["path"] for a in result["artifacts"]]
+ self.assertEqual(paths, sorted(paths))
+
+
+class TestEmptyManifest(unittest.TestCase):
+ """Empty artifact list is valid but produces zero results."""
+
+ def test_empty_artifacts_list(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([]))
+ self.assertTrue(result["manifest_valid"])
+ self.assertEqual(result["artifacts"], [])
+ self.assertEqual(result["totals"]["total"], 0)
+
+
+class TestMixedOutcomes(unittest.TestCase):
+ """Multiple artifacts with mixed verification outcomes."""
+
+ def test_mixed_verified_missing_mismatch_observed(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ good_content = b"good"
+ good_h = _write(tmp / "good.json", good_content)
+ bad_content = b"actual"
+ _write(tmp / "bad.json", bad_content)
+ _write(tmp / "noexpect.json", b"whatever")
+ wrong_h = _sha256(b"expected but wrong")
+
+ result = _run_audit(tmp, _manifest([
+ {"path": "good.json", "type": "result_record", "expected_sha256": good_h},
+ {"path": "bad.json", "type": "result_record", "expected_sha256": wrong_h},
+ {"path": "missing.json", "type": "trajectory", "expected_sha256": "c" * 64},
+ {"path": "noexpect.json", "type": "campaign_manifest"},
+ ]))
+
+ statuses = {a["path"]: a["status"] for a in result["artifacts"]}
+ self.assertEqual(statuses["good.json"], "VERIFIED")
+ self.assertEqual(statuses["bad.json"], "MISMATCH")
+ self.assertEqual(statuses["missing.json"], "MISSING")
+ self.assertEqual(statuses["noexpect.json"], "OBSERVED_ONLY")
+
+ t = result["totals"]
+ self.assertEqual(t["VERIFIED"], 1)
+ self.assertEqual(t["MISMATCH"], 1)
+ self.assertEqual(t["MISSING"], 1)
+ self.assertEqual(t["OBSERVED_ONLY"], 1)
+ self.assertEqual(t["total"], 4)
+
+
+class TestNeverVerifiedFromOwnHash(unittest.TestCase):
+ """The utility must not auto-verify by computing and comparing its own hash."""
+
+ def test_no_expected_hash_means_not_verified(self):
+ """An artifact with no expected_sha256 must never be VERIFIED."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "auto.txt", b"some data")
+ result = _run_audit(tmp, _manifest([{
+ "path": "auto.txt",
+ "type": "model_artifact",
+ }]))
+ art = result["artifacts"][0]
+ self.assertNotEqual(art["status"], "VERIFIED",
+ "Artifact must not be VERIFIED without an independently provided expected hash")
+ self.assertEqual(art["status"], "OBSERVED_ONLY")
+
+
+class TestReproducibilityPrerequisites(unittest.TestCase):
+ """Reproducibility prerequisites section is present and populated."""
+
+ def test_reproducibility_keys_present(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([]))
+ prereqs = result.get("reproducibility_prerequisites", {})
+ expected_keys = {
+ "model_artifact_hash",
+ "source_checkpoint_revision",
+ "quantization_info",
+ "backend_executable_identity",
+ "chat_template_identity",
+ "context_and_decoding_config",
+ "task_fixture_identities",
+ "expected_outcomes_and_oracles",
+ "raw_trajectories",
+ "result_and_latency_records",
+ }
+ self.assertEqual(set(prereqs.keys()), expected_keys)
+
+ def test_model_artifact_hash_unknown_without_artifact(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([]))
+ self.assertEqual(
+ result["reproducibility_prerequisites"]["model_artifact_hash"],
+ "UNKNOWN",
+ )
+
+ def test_model_artifact_hash_observed_when_type_present(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "model.gguf.txt", b"placeholder")
+ result = _run_audit(tmp, _manifest([{
+ "path": "model.gguf.txt",
+ "type": "model_artifact",
+ }]))
+ val = result["reproducibility_prerequisites"]["model_artifact_hash"]
+ self.assertNotEqual(val, "UNKNOWN",
+ "model_artifact_hash should not be UNKNOWN when a model_artifact is present")
+
+
+class TestOutputStructure(unittest.TestCase):
+ """Output JSON has required top-level fields."""
+
+ def test_required_top_level_fields(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([]))
+ for key in ("schema_version", "audited_at", "campaign_id", "root",
+ "manifest_path", "totals", "artifacts",
+ "reproducibility_prerequisites", "reproducibility_details",
+ "errors", "manifest_valid"):
+ self.assertIn(key, result, f"Missing top-level key: {key}")
+
+ def test_schema_version_is_one(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ result = _run_audit(tmp, _manifest([]))
+ self.assertEqual(result["schema_version"], 1)
+
+
+# ---------------------------------------------------------------------------
+# Finding 1: Output Destination Overwrite & TOCTOU Protection
+# ---------------------------------------------------------------------------
+
+class TestFinding1OutputProtection(unittest.TestCase):
+ """Regression tests for Finding 1 (P1): Output publication remains vulnerable to TOCTOU."""
+
+ def test_output_targeting_manifest_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ content = json.dumps(_manifest([])).encode()
+ mpath.write_bytes(content)
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(mpath)])
+ self.assertEqual(code, 2)
+ self.assertEqual(mpath.read_bytes(), content, "Manifest was modified by output destination")
+
+ def test_summary_targeting_audited_artifact_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ art_file = tmp / "evidence.dat"
+ original_bytes = b"precious_historical_evidence"
+ art_file.write_bytes(original_bytes)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{"path": "evidence.dat", "type": "result_record"}])))
+
+ code = auditor.main([str(mpath), str(tmp), "--summary", str(art_file)])
+ self.assertEqual(code, 2)
+ self.assertEqual(art_file.read_bytes(), original_bytes, "Audited artifact was overwritten")
+
+ def test_output_and_summary_same_destination_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ dest = tmp / "collision.txt"
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(dest), "--summary", str(dest)])
+ self.assertEqual(code, 2)
+ self.assertFalse(dest.exists(), "File should not be created on destination collision")
+
+ def test_existing_symlink_alias_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ original = json.dumps(_manifest([])).encode()
+ mpath.write_bytes(original)
+ link_out = tmp / "link_manifest.json"
+ link_out.symlink_to(mpath)
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(link_out)])
+ self.assertEqual(code, 2)
+ self.assertEqual(mpath.read_bytes(), original)
+
+ def test_existing_hardlink_alias_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ art_file = tmp / "original.txt"
+ original_bytes = b"immutable_artifact"
+ art_file.write_bytes(original_bytes)
+ hardlink_file = tmp / "hardlink.txt"
+ os.link(art_file, hardlink_file)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{"path": "original.txt", "type": "result_record"}])))
+
+ code = auditor.main([str(mpath), str(tmp), "--summary", str(hardlink_file)])
+ self.assertEqual(code, 2)
+ self.assertEqual(art_file.read_bytes(), original_bytes)
+
+ def test_injected_destination_replacement_after_preflight_blocks_overwrite(self):
+ """Adversarial TOCTOU test: destination is substituted with symlink after preflight."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ art_file = tmp / "protected.dat"
+ original_bytes = b"IMMUTABLE_HISTORICAL_DATA"
+ art_file.write_bytes(original_bytes)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{"path": "protected.dat", "type": "result_record"}])))
+
+ target_out = tmp / "safe_destination.json"
+
+ # Inject symlink substitution after preflight succeeds
+ orig_validate = auditor._validate_output_destinations
+
+ def injected_validation(*args, **kwargs):
+ errs = orig_validate(*args, **kwargs)
+ # Inject malicious symlink pointing to protected artifact after preflight check
+ if target_out.exists():
+ target_out.unlink()
+ target_out.symlink_to(art_file)
+ return errs
+
+ auditor._validate_output_destinations = injected_validation
+ try:
+ code = auditor.main([str(mpath), str(tmp), "--output", str(target_out)])
+ self.assertEqual(code, 2, "CLI should reject publication when destination points to protected artifact")
+ self.assertEqual(art_file.read_bytes(), original_bytes, "Protected artifact was overwritten!")
+ finally:
+ auditor._validate_output_destinations = orig_validate
+
+ def test_nested_output_directory_symlink_replacement_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+
+ outside = tmp / "outside"
+ outside.mkdir()
+ sub = tmp / "nested_dir"
+ sub.symlink_to(outside)
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(sub / "report.json")])
+ self.assertEqual(code, 2)
+ self.assertFalse((outside / "report.json").exists())
+
+ def test_failed_publication_preserves_existing_protected_input_bytes(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ art = tmp / "artifact.dat"
+ sentinel = b"SENTINEL_BYTES_PRESERVED"
+ art.write_bytes(sentinel)
+ mpath = tmp / "manifest.json"
+ m_bytes = json.dumps(_manifest([{"path": "artifact.dat", "type": "result_record"}])).encode()
+ mpath.write_bytes(m_bytes)
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(art)])
+ self.assertEqual(code, 2)
+ self.assertEqual(art.read_bytes(), sentinel)
+ self.assertEqual(mpath.read_bytes(), m_bytes)
+
+ def test_temporary_files_are_cleaned_up_after_failure(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ art = tmp / "artifact.dat"
+ art.write_bytes(b"data")
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{"path": "artifact.dat", "type": "result_record"}])))
+
+ out_dir = tmp / "out_dir"
+ out_dir.mkdir()
+ out_file = out_dir / "target.json"
+ out_file.symlink_to(art)
+
+ auditor.main([str(mpath), str(tmp), "--output", str(out_file)])
+ # Verify no .tmp_* files remain in out_dir
+ leftover_tmps = [p for p in out_dir.iterdir() if p.name.startswith(".tmp_")]
+ self.assertEqual(leftover_tmps, [], "Temporary files leaked in output directory")
+
+ def test_valid_distinct_json_output_destination_works(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ out = tmp / "out.json"
+
+ code = auditor.main([str(mpath), str(tmp), "--output", str(out)])
+ self.assertEqual(code, 0)
+ self.assertTrue(out.is_file())
+ parsed = json.loads(out.read_text())
+ self.assertTrue(parsed["manifest_valid"])
+
+ def test_valid_distinct_markdown_summary_destination_works(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ summ = tmp / "summary.md"
+
+ code = auditor.main([str(mpath), str(tmp), "--summary", str(summ)])
+ self.assertEqual(code, 0)
+ self.assertTrue(summ.is_file())
+ self.assertIn("Benchmark Evidence Audit Summary", summ.read_text())
+
+ def test_repeated_successful_publication_produces_complete_valid_files(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ out = tmp / "out.json"
+
+ code1 = auditor.main([str(mpath), str(tmp), "--output", str(out), "--deterministic"])
+ bytes1 = out.read_bytes()
+ code2 = auditor.main([str(mpath), str(tmp), "--output", str(out), "--deterministic"])
+ bytes2 = out.read_bytes()
+
+ self.assertEqual(code1, 0)
+ self.assertEqual(code2, 0)
+ self.assertEqual(bytes1, bytes2)
+
+
+# ---------------------------------------------------------------------------
+# Finding 2: Conservative Prerequisite Completeness Model
+# ---------------------------------------------------------------------------
+
+class TestFinding2PrerequisiteCompleteness(unittest.TestCase):
+ """Mandatory regressions for Finding 2 (P2): Campaign prerequisite completeness."""
+
+ def test_1_one_verified_trajectory_without_completeness_evidence(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h = _write(tmp / "t1.json", b"single_trajectory")
+ res = _run_audit(tmp, _manifest([{"path": "t1.json", "type": "trajectory", "expected_sha256": h}]))
+
+ # Prerequisite completeness is NOT verified when expected-set completeness is unknown
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["artifact_integrity"], "VERIFIED")
+ self.assertEqual(det["expected_set_status"], "UNKNOWN")
+ self.assertFalse(det["prerequisite_complete"])
+
+ def test_2_two_verified_trajectories_with_unknown_expected_set_completeness(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ h2 = _write(tmp / "t2.json", b"traj2")
+ res = _run_audit(tmp, _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2},
+ ]))
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["artifact_integrity"], "VERIFIED")
+ self.assertEqual(det["expected_set_status"], "UNKNOWN")
+ self.assertFalse(det["prerequisite_complete"])
+
+ def test_3_complete_independently_established_expected_set_all_verify(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ h2 = _write(tmp / "t2.json", b"traj2")
+ trusted_src = "https://specs.example.org/qwen-e00-trajectories.json"
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ "provenance": trusted_src,
+ "independent_oracle": True,
+ }
+ }
+ res = _run_audit(
+ tmp,
+ _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2},
+ ], expected_sets=expected_sets),
+ trusted_sources=[trusted_src],
+ )
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "VERIFIED")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["artifact_integrity"], "VERIFIED")
+ self.assertEqual(det["expected_set_status"], "COMPLETE")
+ self.assertTrue(det["prerequisite_complete"])
+ self.assertEqual(det["missing_or_unverified"], [])
+
+ def test_4_established_expected_set_with_one_missing_artifact(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ trusted_src = "https://specs.example.org/qwen-e00-trajectories.json"
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2_missing.json"],
+ "provenance": trusted_src,
+ "independent_oracle": True,
+ }
+ }
+ res = _run_audit(
+ tmp,
+ _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2_missing.json", "type": "trajectory", "expected_sha256": "0" * 64},
+ ], expected_sets=expected_sets),
+ trusted_sources=[trusted_src],
+ )
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "INCOMPLETE")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("t2_missing.json", det["missing_or_unverified"])
+
+ def test_5_established_expected_set_with_one_mismatch_artifact(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ _write(tmp / "t2.json", b"actual_content")
+ trusted_src = "https://specs.example.org/qwen-e00-trajectories.json"
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ "provenance": trusted_src,
+ "independent_oracle": True,
+ }
+ }
+ res = _run_audit(
+ tmp,
+ _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": "f" * 64},
+ ], expected_sets=expected_sets),
+ trusted_sources=[trusted_src],
+ )
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "MISMATCH")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("t2.json", det["missing_or_unverified"])
+
+ def test_6_verified_plus_observed_only(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ _write(tmp / "t2.json", b"traj2")
+ res = _run_audit(tmp, _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory"},
+ ]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+
+ def test_7_optional_verified_artifacts_with_missing_mandatory_artifact(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h_opt = _write(tmp / "optional.json", b"opt")
+ res = _run_audit(tmp, _manifest([
+ {"path": "mandatory.json", "type": "trajectory", "expected_sha256": "1" * 64, "mandatory": True},
+ {"path": "optional.json", "type": "trajectory", "expected_sha256": h_opt, "mandatory": False},
+ ]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "INCOMPLETE")
+
+ def test_8_empty_declared_artifact_set(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "UNKNOWN")
+
+ def test_9_incomplete_archive_with_self_declared_completeness_flag(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json"],
+ "complete": True,
+ "provenance": "internal-collector-self-claim",
+ "independent_oracle": False, # Not independently verified!
+ }
+ }
+ res = _run_audit(tmp, _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ ], expected_sets=expected_sets))
+
+ # Unverified self-claim must not be accepted as verified completeness
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["expected_set_status"], "UNVERIFIED_CLAIM")
+ self.assertFalse(det["prerequisite_complete"])
+
+ def test_10_checksum_matched_summary_document_does_not_prove_task_level_oracles(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b'{"task_specifications": "summary_only"}'
+ h = _write(tmp / "freeze-manifest.json", content)
+ res = _run_audit(tmp, _manifest([
+ {"path": "freeze-manifest.json", "type": "oracle_definition", "expected_sha256": h, "is_summary": True},
+ ]))
+
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "VERIFIED", "File checksum integrity verified")
+
+ # Prerequisite completeness is NOT verified: summary does not prove task-level oracles
+ self.assertEqual(res["reproducibility_prerequisites"]["expected_outcomes_and_oracles"], "OBSERVED_ONLY")
+ det = res["reproducibility_details"]["expected_outcomes_and_oracles"]
+ self.assertEqual(det["expected_set_status"], "SUMMARY_ONLY")
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("summary document does not prove", det["notes"])
+
+ def test_11_checksum_matched_aggregate_report_does_not_prove_complete_latency_records(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"# Aggregate Benchmark Report\nOverall latency: 120ms\n"
+ h = _write(tmp / "campaign-report.md", content)
+ res = _run_audit(tmp, _manifest([
+ {"path": "campaign-report.md", "type": "result_record", "expected_sha256": h, "is_summary": True},
+ ]))
+
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "VERIFIED", "Report file checksum integrity verified")
+
+ # Prerequisite completeness is NOT verified: aggregate statistics do not prove per-turn latency
+ self.assertEqual(res["reproducibility_prerequisites"]["result_and_latency_records"], "OBSERVED_ONLY")
+ det = res["reproducibility_details"]["result_and_latency_records"]
+ self.assertEqual(det["expected_set_status"], "AGGREGATE_ONLY")
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("aggregate statistics do not prove", det["notes"])
+
+ def test_12_stable_output_regardless_of_manifest_ordering(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ _write(tmp / "t2.json", b"traj2")
+ e1 = {"path": "t1.json", "type": "trajectory", "expected_sha256": h1}
+ e2 = {"path": "t2.json", "type": "trajectory", "expected_sha256": "0" * 64}
+
+ res1 = _run_audit(tmp, _manifest([e1, e2], campaign_id="STABLE"))
+ res2 = _run_audit(tmp, _manifest([e2, e1], campaign_id="STABLE"))
+
+ self.assertEqual(
+ res1["reproducibility_prerequisites"],
+ res2["reproducibility_prerequisites"],
+ )
+ self.assertEqual(
+ res1["reproducibility_details"],
+ res2["reproducibility_details"],
+ )
+
+
+# ---------------------------------------------------------------------------
+# Finding 3: Symlink TOCTOU and O_NOFOLLOW Enforcement
+# ---------------------------------------------------------------------------
+
+class TestFinding3SymlinkTOCTOU(unittest.TestCase):
+ """Regression tests for Finding 3 (P2): Symlink time-of-check/time-of-use vulnerability."""
+
+ def test_ordinary_regular_file_inside_root(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"regular_inside_root"
+ h = _write(tmp / "subdir" / "data.txt", content)
+ res = _run_audit(tmp, _manifest([{
+ "path": "subdir/data.txt",
+ "type": "result_record",
+ "expected_sha256": h,
+ }]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "VERIFIED")
+ self.assertEqual(art["observed_sha256"], h)
+
+ def test_direct_symlink_escaping_root(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ link = tmp / "escape_passwd"
+ link.symlink_to("/etc/passwd")
+ res = _run_audit(tmp, _manifest([{"path": "escape_passwd", "type": "result_record"}]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+ self.assertIsNone(art["observed_sha256"])
+
+ def test_nested_symlink_escaping_root(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ sub = tmp / "a" / "b"
+ sub.mkdir(parents=True)
+ link = sub / "nested_escape"
+ link.symlink_to("/etc/passwd")
+ res = _run_audit(tmp, _manifest([{"path": "a/b/nested_escape", "type": "result_record"}]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+ self.assertIsNone(art["observed_sha256"])
+
+ def test_symlink_directory_component_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ outside = Path(tempfile.mkdtemp())
+ try:
+ (outside / "secret.txt").write_bytes(b"secret")
+ (tmp / "linkdir").symlink_to(outside)
+ res = _run_audit(tmp, _manifest([{"path": "linkdir/secret.txt", "type": "result_record"}]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+ self.assertIsNone(art["observed_sha256"])
+ finally:
+ import shutil
+ shutil.rmtree(outside)
+
+ def test_safe_handling_of_missing_or_replaced_files(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": "missing/dir/file.txt", "type": "result_record"}]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "MISSING")
+
+ def test_no_outside_root_file_access(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ secret_outside = Path(tempfile.mkdtemp())
+ try:
+ secret_file = secret_outside / "secret.dat"
+ secret_file.write_bytes(b"TOP_SECRET_DATA")
+ secret_hash = _sha256(b"TOP_SECRET_DATA")
+
+ link = tmp / "secret_link"
+ link.symlink_to(secret_file)
+
+ res = _run_audit(tmp, _manifest([{
+ "path": "secret_link",
+ "type": "result_record",
+ "expected_sha256": secret_hash,
+ }]))
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+ self.assertIsNone(art["observed_sha256"], "Outside-root content was read or hashed")
+ finally:
+ import shutil
+ shutil.rmtree(secret_outside)
+
+ def test_no_descriptor_leaks_after_failure(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ (tmp / "bad_link").symlink_to("/etc/passwd")
+
+ fd_dir = Path(f"/proc/{os.getpid()}/fd")
+ if not fd_dir.exists():
+ self.skipTest("/proc/self/fd not accessible on this platform")
+
+ # Warm-up run
+ _run_audit(tmp, _manifest([{"path": "bad_link", "type": "result_record"}]))
+
+ fds_before = len(list(fd_dir.iterdir()))
+ for _ in range(10):
+ _run_audit(tmp, _manifest([
+ {"path": "bad_link", "type": "result_record"},
+ {"path": "missing.txt", "type": "result_record"},
+ {"path": "../outside.txt", "type": "result_record"},
+ ]))
+ fds_after = len(list(fd_dir.iterdir()))
+ self.assertEqual(fds_before, fds_after, "File descriptor leak detected")
+
+
+# ---------------------------------------------------------------------------
+# Finding 4: Malformed Manifest Handling
+# ---------------------------------------------------------------------------
+
+class TestFinding4MalformedManifest(unittest.TestCase):
+ """Regression tests for Finding 4 (P2): Malformed manifest handling."""
+
+ def test_invalid_list_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": ["invalid", "list"], "type": "result_record"}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_integer_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": 12345, "type": "result_record"}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_null_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": None, "type": "result_record"}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_boolean_expected_hash(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "file.txt", b"data")
+ res_false = _run_audit(tmp, _manifest([{"path": "file.txt", "type": "result_record", "expected_sha256": False}]))
+ self.assertEqual(res_false["totals"]["INVALID"], 1)
+ self.assertEqual(res_false["artifacts"][0]["status"], "INVALID")
+
+ res_true = _run_audit(tmp, _manifest([{"path": "file.txt", "type": "result_record", "expected_sha256": True}]))
+ self.assertEqual(res_true["totals"]["INVALID"], 1)
+ self.assertEqual(res_true["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_numeric_expected_hash(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "file.txt", b"data")
+ res = _run_audit(tmp, _manifest([{"path": "file.txt", "type": "result_record", "expected_sha256": 1234567890}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_malformed_hexadecimal_hash(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "file.txt", b"data")
+ res = _run_audit(tmp, _manifest([{"path": "file.txt", "type": "result_record", "expected_sha256": "g" * 64}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_wrong_length_hash(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ _write(tmp / "file.txt", b"data")
+ res = _run_audit(tmp, _manifest([{"path": "file.txt", "type": "result_record", "expected_sha256": "a" * 32}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_invalid_artifact_record_structure(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest(["not_a_dict", 42, None]))
+ self.assertEqual(res["totals"]["INVALID"], 3)
+ for art in res["artifacts"]:
+ self.assertEqual(art["status"], "INVALID")
+
+ def test_missing_required_path_field(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"type": "result_record"}]))
+ self.assertEqual(res["totals"]["INVALID"], 1)
+ self.assertEqual(res["artifacts"][0]["status"], "INVALID")
+
+ def test_mixed_valid_and_invalid_entries(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h = _write(tmp / "good.txt", b"content")
+ res = _run_audit(tmp, _manifest([
+ {"path": "good.txt", "type": "result_record", "expected_sha256": h},
+ {"path": ["bad", "list"], "type": "result_record"},
+ {"path": "bad_hash.txt", "type": "result_record", "expected_sha256": False},
+ ]))
+ self.assertEqual(res["totals"]["VERIFIED"], 1)
+ self.assertEqual(res["totals"]["INVALID"], 2)
+
+
+# ---------------------------------------------------------------------------
+# Deterministic Output Mode
+# ---------------------------------------------------------------------------
+
+class TestDeterministicOutputMode(unittest.TestCase):
+ """Test deterministic CLI mode generates identical output bytes."""
+
+ def test_deterministic_cli_flag_produces_identical_bytes(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ out1 = tmp / "out1.json"
+ out2 = tmp / "out2.json"
+
+ code1 = auditor.main([str(mpath), str(tmp), "--output", str(out1), "--deterministic"])
+ code2 = auditor.main([str(mpath), str(tmp), "--output", str(out2), "--deterministic"])
+
+ self.assertEqual(code1, 0)
+ self.assertEqual(code2, 0)
+ self.assertEqual(out1.read_bytes(), out2.read_bytes(), "Deterministic runs differed in output bytes")
+
+
+# ---------------------------------------------------------------------------
+# Additional Regressions for Final Security & Evidence Findings
+# ---------------------------------------------------------------------------
+
+class TestFinding1ParentDirectorySubstitution(unittest.TestCase):
+ """P1: Output parent directory substitution protection."""
+
+ def test_intermediate_directory_symlink_substitution_blocked_and_target_preserved(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ # Create a protected target directory containing sensitive files
+ protected_dir = tmp / "protected"
+ protected_dir.mkdir()
+ protected_file = protected_dir / "secret_evidence.dat"
+ original_bytes = b"ORIGINAL PROTECTED EVIDENCE - DO NOT OVERWRITE"
+ protected_file.write_bytes(original_bytes)
+
+ # Create an intermediate directory structure
+ base_out = tmp / "output_root"
+ base_out.mkdir()
+ # Replace intermediate component with a symlink pointing to the protected directory
+ inter_symlink = base_out / "intermediate"
+ os.symlink(str(protected_dir), str(inter_symlink))
+
+ # Attempt publication targeting a path through the intermediate symlink
+ dest = inter_symlink / "secret_evidence.dat"
+ protected_paths, protected_inodes = auditor._collect_protected_targets(
+ str(tmp / "manifest.json"),
+ str(tmp),
+ [{"path": "secret_evidence.dat", "type": "result_record"}],
+ )
+ # Explicitly register protected file
+ protected_paths[protected_file.resolve()] = "protected file"
+ st = protected_file.stat()
+ protected_inodes[(st.st_dev, st.st_ino)] = "protected file"
+
+ with self.assertRaises(OSError) as ctx:
+ auditor._publish_file_securely(
+ str(dest),
+ "MALICIOUS OVERWRITE CONTENT",
+ protected_paths,
+ protected_inodes,
+ )
+
+ self.assertIn("symlink", str(ctx.exception).lower())
+ # Verify the protected file bytes are completely unchanged
+ self.assertEqual(protected_file.read_bytes(), original_bytes)
+
+ def test_cli_intermediate_directory_symlink_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+
+ outside = tmp / "outside"
+ outside.mkdir()
+ canary = outside / "canary.txt"
+ canary.write_bytes(b"CANARY UNTOUCHED")
+
+ out_tree = tmp / "out_tree"
+ out_tree.mkdir()
+ sym_dir = out_tree / "sym_sub"
+ os.symlink(str(outside), str(sym_dir))
+
+ target_out = sym_dir / "out.json"
+ code = auditor.main([str(mpath), str(tmp), "--output", str(target_out)])
+ self.assertEqual(code, 2)
+ self.assertEqual(canary.read_bytes(), b"CANARY UNTOUCHED")
+
+
+class TestFinding2ArtifactRootSubstitution(unittest.TestCase):
+ """P2: Artifact root substitution and outside-root read protection."""
+
+ def test_root_directory_symlink_rejected_and_outside_file_not_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ # Create outside secret directory
+ outside_dir = tmp / "outside_secret"
+ outside_dir.mkdir()
+ secret_file = outside_dir / "confidential.dat"
+ secret_bytes = b"CONFIDENTIAL SYSTEM DATA"
+ secret_file.write_bytes(secret_bytes)
+ expected_hash = _sha256(secret_bytes)
+
+ # Create symlink root pointing to outside directory
+ symlink_root = tmp / "symlink_root"
+ os.symlink(str(outside_dir), str(symlink_root))
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{
+ "path": "confidential.dat",
+ "type": "result_record",
+ "expected_sha256": expected_hash,
+ }])))
+
+ # Audit against symlink root
+ res = auditor.audit(str(mpath), str(symlink_root))
+
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], "UNVERIFIABLE")
+ self.assertFalse(art["present"])
+ self.assertIsNone(art["observed_sha256"])
+ self.assertFalse(art["hash_match"])
+ self.assertIn("root directory is invalid or a symlink", art["notes"])
+ self.assertEqual(res["totals"]["VERIFIED"], 0)
+ self.assertEqual(res["totals"]["UNVERIFIABLE"], 1)
+
+ def test_root_replaced_with_symlink_during_audit(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ real_root = tmp / "root_dir"
+ real_root.mkdir()
+ outside = tmp / "outside"
+ outside.mkdir()
+ h = _write(outside / "data.txt", b"outside data")
+
+ # Replace real root with symlink
+ real_root.rmdir()
+ os.symlink(str(outside), str(real_root))
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{
+ "path": "data.txt",
+ "type": "result_record",
+ "expected_sha256": h,
+ }])))
+
+ res = auditor.audit(str(mpath), str(real_root))
+ self.assertEqual(res["artifacts"][0]["status"], "UNVERIFIABLE")
+ self.assertEqual(res["totals"]["VERIFIED"], 0)
+
+ def test_no_descriptor_leaks_after_root_symlink_rejections(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ sym_root = tmp / "sym_root"
+ os.symlink(str(tmp), str(sym_root))
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{"path": "file.txt", "type": "result_record"}])))
+
+ fd_dir = Path(f"/proc/{os.getpid()}/fd")
+ if not fd_dir.exists():
+ self.skipTest("/proc/self/fd not accessible on this platform")
+
+ # Warm-up run
+ auditor.audit(str(mpath), str(sym_root))
+ fds_before = len(list(fd_dir.iterdir()))
+
+ for _ in range(10):
+ auditor.audit(str(mpath), str(sym_root))
+
+ fds_after = len(list(fd_dir.iterdir()))
+ self.assertEqual(fds_before, fds_after, "Descriptor leak detected after root symlink rejections")
+
+
+class TestFinding3ProvenanceTrust(unittest.TestCase):
+ """P2: Self-attested independent provenance and trusted verification source."""
+
+ def test_fabricated_provenance_url_cannot_produce_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ h2 = _write(tmp / "t2.json", b"traj2")
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ "provenance": "https://fabricated-external-oracle.org/claims.json",
+ "independent_oracle": True,
+ }
+ }
+ # Run without trusted source
+ res = _run_audit(tmp, _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2},
+ ], expected_sets=expected_sets))
+
+ # Must NOT produce VERIFIED
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["artifact_integrity"], "VERIFIED")
+ self.assertEqual(det["expected_set_status"], "UNVERIFIED")
+ self.assertEqual(det["expected_set_source"], "declared")
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("lacks separately trusted verification source", det["notes"])
+
+ def test_self_declared_completeness_flag_cannot_produce_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json"],
+ "complete": True,
+ "provenance": "internal-self-assertion",
+ }
+ }
+ res = _run_audit(tmp, _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ ], expected_sets=expected_sets))
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["expected_set_status"], "UNVERIFIED_CLAIM")
+ self.assertFalse(det["prerequisite_complete"])
+
+ def test_partial_manifest_with_trusted_source_cannot_produce_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ trusted_src = "https://official-specs.org/e00.json"
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2_missing.json"],
+ "provenance": trusted_src,
+ "independent_oracle": True,
+ }
+ }
+ res = _run_audit(
+ tmp,
+ _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ ], expected_sets=expected_sets),
+ trusted_sources=[trusted_src],
+ )
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "INCOMPLETE")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertIn("t2_missing.json", det["missing_or_unverified"])
+
+ def test_untrusted_provenance_mismatch_cannot_produce_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ expected_sets = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json"],
+ "provenance": "https://unapproved-oracle.org",
+ "independent_oracle": True,
+ }
+ }
+ res = _run_audit(
+ tmp,
+ _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ ], expected_sets=expected_sets),
+ trusted_sources=["https://approved-oracle.org"],
+ )
+
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "PARTIAL")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["expected_set_status"], "UNVERIFIED")
+ self.assertFalse(det["prerequisite_complete"])
+
+ def test_external_trusted_expected_sets_file_works(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h1 = _write(tmp / "t1.json", b"traj1")
+ h2 = _write(tmp / "t2.json", b"traj2")
+ trusted_sets_file = tmp / "trusted_sets.json"
+ trusted_sets_file.write_text(json.dumps({
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ "provenance": "official-e00-spec",
+ }
+ }))
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2},
+ ])))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(trusted_sets_file))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "VERIFIED")
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertEqual(det["expected_set_status"], "COMPLETE")
+ self.assertEqual(det["expected_set_source"], "externally_verified")
+ self.assertTrue(det["prerequisite_complete"])
+
+
+class TestFinding4MalformedExpectedSets(unittest.TestCase):
+ """P2: Malformed expected-set definitions and structural validation."""
+
+ def test_expected_sets_as_list_handled_without_attribute_error(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mdata = _manifest([])
+ mdata["expected_sets"] = [] # list instead of dict
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(res["manifest_valid"])
+ self.assertTrue(any("expected_sets must be a JSON object" in e for e in res["errors"]))
+
+ # Direct call to _build_prereqs with list must also not raise AttributeError
+ prereqs, details = auditor._build_prereqs([], [])
+ self.assertEqual(prereqs["raw_trajectories"], "INVALID")
+ self.assertEqual(details["raw_trajectories"]["status"], "INVALID")
+
+ def test_expected_paths_containing_dict_object_handled_without_type_error(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mdata = _manifest([])
+ mdata["expected_sets"] = {
+ "raw_trajectories": {
+ "expected_paths": [{"nested": "unhashable_dict"}],
+ }
+ }
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(res["manifest_valid"])
+ self.assertTrue(any("must be string" in e for e in res["errors"]))
+
+ # Direct call to _build_prereqs must also handle safely
+ prereqs, details = auditor._build_prereqs([], mdata["expected_sets"])
+ self.assertEqual(prereqs["raw_trajectories"], "INVALID")
+ self.assertEqual(details["raw_trajectories"]["expected_set_status"], "INVALID")
+
+ def test_empty_expected_paths_list_does_not_produce_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ h = _write(tmp / "t1.json", b"content")
+ trusted_src = "https://specs.example.org/oracle.json"
+ mdata = _manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h},
+ ])
+ mdata["expected_sets"] = {
+ "raw_trajectories": {
+ "expected_paths": [], # Empty list!
+ "provenance": trusted_src,
+ "independent_oracle": True,
+ }
+ }
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_sources=[trusted_src])
+ self.assertNotEqual(res["reproducibility_prerequisites"]["raw_trajectories"], "VERIFIED")
+ self.assertFalse(res["manifest_valid"])
+ self.assertTrue(any("cannot be empty" in e for e in res["errors"]))
+
+ # Direct call to _build_prereqs must also handle safely without producing VERIFIED
+ prereqs, details = auditor._build_prereqs(
+ [{"path": "t1.json", "type": "trajectory", "status": "VERIFIED"}],
+ mdata["expected_sets"],
+ trusted_sources=[trusted_src],
+ )
+ self.assertNotEqual(prereqs["raw_trajectories"], "VERIFIED")
+ self.assertEqual(prereqs["raw_trajectories"], "INVALID")
+ self.assertFalse(details["raw_trajectories"]["prerequisite_complete"])
+
+ def test_expected_paths_with_non_string_items_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mdata = _manifest([])
+ mdata["expected_sets"] = {
+ "raw_trajectories": {
+ "expected_paths": [123, True, None],
+ }
+ }
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(res["manifest_valid"])
+
+ def test_expected_paths_with_duplicate_paths_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mdata = _manifest([])
+ mdata["expected_sets"] = {
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t1.json"],
+ }
+ }
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(res["manifest_valid"])
+ self.assertTrue(any("duplicate path" in e for e in res["errors"]))
+
+ def test_expected_count_invalid_type_or_negative_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mdata = _manifest([])
+ mdata["expected_sets"] = {
+ "raw_trajectories": {
+ "expected_count": -5,
+ }
+ }
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(mdata))
+
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertFalse(res["manifest_valid"])
+ self.assertTrue(any("positive integer" in e for e in res["errors"]))
+
+
+# ---------------------------------------------------------------------------
+# Finding 1: P1, Trusted Input Overwrite Protection
+# ---------------------------------------------------------------------------
+
+class TestFinding1TrustedInputProtection(unittest.TestCase):
+ """Output publication must never overwrite any trusted input file."""
+
+ def test_output_targeting_trusted_expected_sets_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted_sets.json"
+ tbytes = json.dumps({"raw_trajectories": {"expected_count": 1}}).encode()
+ tpath.write_bytes(tbytes)
+
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--output", str(tpath),
+ ])
+ self.assertEqual(code, 2)
+ self.assertEqual(tpath.read_bytes(), tbytes, "Trusted expected sets file was overwritten")
+
+ def test_summary_targeting_trusted_expected_sets_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted_sets.json"
+ tbytes = json.dumps({"raw_trajectories": {"expected_count": 1}}).encode()
+ tpath.write_bytes(tbytes)
+
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--summary", str(tpath),
+ ])
+ self.assertEqual(code, 2)
+ self.assertEqual(tpath.read_bytes(), tbytes, "Trusted expected sets file was overwritten by summary")
+
+ def test_output_symlink_alias_targeting_trusted_expected_sets_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted_sets.json"
+ tbytes = json.dumps({"raw_trajectories": {"expected_count": 1}}).encode()
+ tpath.write_bytes(tbytes)
+ link_out = tmp / "link_trusted.json"
+ link_out.symlink_to(tpath)
+
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--output", str(link_out),
+ ])
+ self.assertEqual(code, 2)
+ self.assertEqual(tpath.read_bytes(), tbytes, "Trusted expected sets overwritten via symlink alias")
+
+ def test_output_hardlink_alias_targeting_trusted_expected_sets_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted_sets.json"
+ tbytes = json.dumps({"raw_trajectories": {"expected_count": 1}}).encode()
+ tpath.write_bytes(tbytes)
+ hlink_out = tmp / "hlink_trusted.json"
+ os.link(tpath, hlink_out)
+
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--output", str(hlink_out),
+ ])
+ self.assertEqual(code, 2)
+ self.assertEqual(tpath.read_bytes(), tbytes, "Trusted expected sets overwritten via hardlink alias")
+
+ def test_distinct_legitimate_destinations_with_trusted_sets_succeeds(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b'{"task": "c5"}'
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "t1.json").write_bytes(content)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{
+ "path": "t1.json",
+ "type": "trajectory",
+ "expected_sha256": chash,
+ "size_bytes": len(content),
+ }])))
+ tpath = tmp / "trusted_sets.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": ["t1.json"]}}))
+
+ out = tmp / "out.json"
+ summ = tmp / "summary.md"
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--output", str(out),
+ "--summary", str(summ),
+ "--deterministic",
+ ])
+ self.assertEqual(code, 0)
+ self.assertTrue(out.is_file())
+ self.assertTrue(summ.is_file())
+
+ def test_adversarial_post_preflight_substitution_targeting_trusted_input_blocked(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ tpath = tmp / "trusted_sets.json"
+ original_bytes = b'{"immutable": "evidence"}'
+ tpath.write_bytes(original_bytes)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+
+ protected_paths, protected_inodes = auditor._collect_protected_targets(
+ str(mpath),
+ str(tmp),
+ [],
+ extra_input_paths=[tpath],
+ )
+ self.assertIn(tpath.resolve(), protected_paths)
+ self.assertIn((tpath.stat().st_dev, tpath.stat().st_ino), protected_inodes)
+
+ out_dest = tmp / "attack_out.json"
+ out_dest.symlink_to(tpath)
+
+ with self.assertRaises(OSError):
+ auditor._publish_file_securely(
+ str(out_dest),
+ "attacker payload",
+ protected_paths,
+ protected_inodes,
+ )
+ self.assertEqual(tpath.read_bytes(), original_bytes)
+
+
+# ---------------------------------------------------------------------------
+# Finding 2: P2, Dot-Only Path Crash Rejection
+# ---------------------------------------------------------------------------
+
+class TestFinding2DotPathRejection(unittest.TestCase):
+ """Dot-only path components must be rejected cleanly as INVALID without IndexError."""
+
+ def test_dot_path_in_manifest_classified_as_invalid(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": ".", "type": "trajectory"}]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], auditor.INVALID)
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+
+ def test_dot_slash_path_in_manifest_classified_as_invalid(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": "./", "type": "trajectory"}]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], auditor.INVALID)
+
+ def test_dot_slash_dot_path_classified_as_invalid(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": "././", "type": "trajectory"}]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+ def test_multi_dot_slash_path_classified_as_invalid(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": ".///.", "type": "trajectory"}]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+ def test_empty_path_classified_as_invalid(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ res = _run_audit(tmp, _manifest([{"path": "", "type": "trajectory"}]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+ def test_valid_nested_path_accepted(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ sub = tmp / "nested"
+ sub.mkdir()
+ content = b"trajectory_content"
+ (sub / "traj.json").write_bytes(content)
+ chash = hashlib.sha256(content).hexdigest()
+ res = _run_audit(tmp, _manifest([{
+ "path": "nested/traj.json",
+ "type": "trajectory",
+ "expected_sha256": chash,
+ "size_bytes": len(content),
+ }]))
+ self.assertEqual(len(res["artifacts"]), 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.VERIFIED)
+
+ def test_mixed_manifest_with_dot_path_and_valid_path(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"valid_data"
+ (tmp / "valid.json").write_bytes(content)
+ chash = hashlib.sha256(content).hexdigest()
+
+ res = _run_audit(tmp, _manifest([
+ {"path": ".", "type": "trajectory"},
+ {"path": "valid.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ]))
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ self.assertEqual(res["totals"][auditor.VERIFIED], 1)
+ by_path = {a["path"]: a["status"] for a in res["artifacts"]}
+ self.assertEqual(by_path["."], auditor.INVALID)
+ self.assertEqual(by_path["valid.json"], auditor.VERIFIED)
+
+ def test_open_and_hash_artifact_direct_dot_calls_never_raise_index_error(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ for bad_path in [".", "./", "././", ".///.", "", " "]:
+ status, chash, size, note = auditor._open_and_hash_artifact(tmp, bad_path)
+ self.assertEqual(status, auditor.INVALID)
+ self.assertIsNone(chash)
+ self.assertIsNone(size)
+ self.assertIsNotNone(note)
+
+
+# ---------------------------------------------------------------------------
+# Finding 3: P2, Unchecked Trusted-Expected-Sets Structure
+# ---------------------------------------------------------------------------
+
+class TestFinding3MalformedTrustedExpectedSets(unittest.TestCase):
+ """External trusted expected-sets configuration must be strictly validated."""
+
+ def test_trusted_sets_top_level_list_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text("[]")
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("must be a JSON object" in e for e in res["errors"]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INVALID)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_top_level_null_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text("null")
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("cannot be null" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_top_level_empty_dict_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text("{}")
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("cannot be an empty object" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_invalid_json_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text("THIS IS NOT JSON {{{{")
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("cannot load trusted_expected_sets" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_unknown_category_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"fabricated_unknown_category": {"expected_count": 1}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("unrecognized or unsupported prerequisite category" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_empty_expected_paths_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": []}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("cannot be empty" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_non_string_paths_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": [42, False, None]}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("must be string" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_duplicate_paths_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": ["a.json", "a.json"]}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("duplicate path" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_unsafe_path_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": ["../escape.json"]}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("unsafe or invalid path" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_invalid_expected_count_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": -3}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("positive integer" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_conflicting_paths_and_count_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({
+ "raw_trajectories": {
+ "expected_paths": ["a.json", "b.json"],
+ "expected_count": 5,
+ }
+ }))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("conflicting definitions" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_invalid_provenance_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 1, "provenance": " "}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("invalid provenance" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_trusted_sets_unsupported_fields_rejected(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 1, "illegal_field": True}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("unsupported field(s)" in e for e in res["errors"]))
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2)
+
+ def test_valid_trusted_sets_with_complete_evidence_verified(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"evidence_payload"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "task1.json").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{
+ "path": "task1.json",
+ "type": "trajectory",
+ "expected_sha256": chash,
+ "size_bytes": len(content),
+ }])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_paths": ["task1.json"]}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertTrue(res["configuration_valid"])
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.VERIFIED)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 0)
+
+ def test_valid_trusted_sets_with_incomplete_evidence_yields_code_1_not_2(self):
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"partial_data"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "t1.json").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([{
+ "path": "t1.json",
+ "type": "trajectory",
+ "expected_sha256": chash,
+ "size_bytes": len(content),
+ }])))
+
+ # Trusted expected sets requires 5 trajectories, but only 1 is present
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 5}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertTrue(res["configuration_valid"])
+ self.assertEqual(len(res["errors"]), 0)
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ # Valid configuration with incomplete evidence must return exit code 1, NOT exit code 2
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1)
+
+
+# ---------------------------------------------------------------------------
+# P2: Duplicate Evidence and Normalized Path Protection
+# ---------------------------------------------------------------------------
+
+class TestP2DuplicateEvidenceProtection(unittest.TestCase):
+ """Campaign completeness must never be established by duplicate evidence."""
+
+ def test_1_physical_file_declared_twice_with_exact_same_path_cannot_satisfy_count(self):
+ """Case A: One physical file declared twice cannot satisfy an expected count of two."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b'{"trajectory_id": "traj_01"}'
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "t1.json").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "INCOMPLETE")
+ self.assertEqual(det["distinct_verified_count"], 1)
+ self.assertTrue(len(det["duplicate_artifacts"]) > 0)
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ # CLI execution assertion
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1, "Duplicate evidence should return exit code 1 (incomplete evidence)")
+
+ def test_2_file_declared_as_plain_and_dot_slash_path_cannot_be_counted_separately(self):
+ """Case B: 'artifact.bin' and './artifact.bin' cannot be counted separately."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"artifact_binary_payload"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "artifact.bin").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "artifact.bin", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": "./artifact.bin", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "INCOMPLETE")
+ self.assertEqual(det["distinct_verified_count"], 1)
+ self.assertTrue(any("normalized path alias" in d for d in det["duplicate_artifacts"]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1)
+
+ def test_3_duplicate_nested_relative_paths_with_redundant_dot_slash(self):
+ """Duplicate nested relative paths containing redundant dot components are detected."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ sub = tmp / "nested"
+ sub.mkdir()
+ content = b"nested_trajectory_data"
+ chash = hashlib.sha256(content).hexdigest()
+ (sub / "traj.json").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "nested/traj.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": "./nested/./traj.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["distinct_verified_count"], 1)
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ def test_4_expected_count_of_two_with_only_one_distinct_verified_artifact(self):
+ """Expected count of two with only one distinct artifact reports incomplete."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"only_one_file"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "single.json").write_bytes(content)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "single.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "INCOMPLETE")
+ self.assertEqual(det["distinct_verified_count"], 1)
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1)
+
+ def test_5_expected_set_containing_normalized_duplicate_paths_rejected(self):
+ """Expected-set definition containing normalized duplicates is rejected as configuration error."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({
+ "raw_trajectories": {
+ "expected_paths": ["sub/traj.json", "./sub/./traj.json"],
+ }
+ }))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ self.assertFalse(res["configuration_valid"])
+ self.assertTrue(any("duplicate path in expected_paths" in e for e in res["errors"]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INVALID)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 2, "Normalized duplicate in expected_paths must exit code 2")
+
+ def test_6_two_genuinely_distinct_checksum_verified_artifacts_satisfy_count(self):
+ """Case C: Two genuinely distinct verified artifacts satisfy an expected count of two."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ c1 = b'{"task": "distinct_01"}'
+ c2 = b'{"task": "distinct_02"}'
+ h1 = hashlib.sha256(c1).hexdigest()
+ h2 = hashlib.sha256(c2).hexdigest()
+ (tmp / "t1.json").write_bytes(c1)
+ (tmp / "t2.json").write_bytes(c2)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1, "size_bytes": len(c1)},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2, "size_bytes": len(c2)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ out = tmp / "out.json"
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--trusted-expected-sets", str(tpath),
+ "--output", str(out),
+ ])
+ self.assertEqual(code, 0)
+ res = json.loads(out.read_text())
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertTrue(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "COMPLETE")
+ self.assertEqual(det["distinct_verified_count"], 2)
+ self.assertEqual(det["duplicate_artifacts"], [])
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.VERIFIED)
+
+ def test_7_explicit_expected_paths_where_one_required_artifact_is_absent(self):
+ """Explicit expected paths where one required artifact is absent remains incomplete."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ c1 = b"present_payload"
+ h1 = hashlib.sha256(c1).hexdigest()
+ (tmp / "t1.json").write_bytes(c1)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "t1.json", "type": "trajectory", "expected_sha256": h1, "size_bytes": len(c1)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ }
+ }))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "INCOMPLETE")
+ self.assertIn("t2.json", det["missing_or_unverified"])
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1)
+
+ def test_8_explicit_expected_paths_where_every_required_artifact_verifies(self):
+ """Explicit expected paths where every required distinct artifact verifies produces complete."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ c1 = b"payload_01"
+ c2 = b"payload_02"
+ h1 = hashlib.sha256(c1).hexdigest()
+ h2 = hashlib.sha256(c2).hexdigest()
+ (tmp / "t1.json").write_bytes(c1)
+ (tmp / "t2.json").write_bytes(c2)
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "./t1.json", "type": "trajectory", "expected_sha256": h1, "size_bytes": len(c1)},
+ {"path": "t2.json", "type": "trajectory", "expected_sha256": h2, "size_bytes": len(c2)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({
+ "raw_trajectories": {
+ "expected_paths": ["t1.json", "t2.json"],
+ }
+ }))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertTrue(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "COMPLETE")
+ self.assertEqual(det["missing_or_unverified"], [])
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.VERIFIED)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 0)
+
+ def test_9_manifest_entry_ordering_does_not_change_results(self):
+ """Manifest entry ordering produces identical deterministic audit outcomes."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ c1 = b"first_file"
+ c2 = b"second_file"
+ h1 = hashlib.sha256(c1).hexdigest()
+ h2 = hashlib.sha256(c2).hexdigest()
+ (tmp / "a1.json").write_bytes(c1)
+ (tmp / "b2.json").write_bytes(c2)
+
+ e1 = {"path": "a1.json", "type": "trajectory", "expected_sha256": h1, "size_bytes": len(c1)}
+ e2 = {"path": "b2.json", "type": "trajectory", "expected_sha256": h2, "size_bytes": len(c2)}
+
+ mpath = tmp / "manifest.json"
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ out1 = tmp / "out1.json"
+ out2 = tmp / "out2.json"
+
+ mpath.write_text(json.dumps(_manifest([e1, e2])))
+ code1 = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath), "--output", str(out1), "--deterministic"])
+
+ mpath.write_text(json.dumps(_manifest([e2, e1])))
+ code2 = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath), "--output", str(out2), "--deterministic"])
+
+ self.assertEqual(code1, 0)
+ self.assertEqual(code2, 0)
+ self.assertEqual(out1.read_bytes(), out2.read_bytes())
+
+ def test_10_duplicate_evidence_never_produces_prerequisite_complete_or_verified(self):
+ """Duplicate evidence never produces prerequisite_complete: true or VERIFIED status."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"sole_evidence_payload"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "sole.json").write_bytes(content)
+
+ # Even if expected_count is 1, a manifest with duplicate entries must not report prerequisite_complete
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "sole.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": "sole.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 1}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"], "Duplicate evidence must not produce prerequisite_complete: true")
+ self.assertNotEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.VERIFIED)
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ def test_11_hardlink_alias_detected_and_cannot_satisfy_expected_count(self):
+ """Filesystem hardlink alias to the same inode cannot satisfy an expected count of two."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ content = b"hardlink_target_bytes"
+ chash = hashlib.sha256(content).hexdigest()
+ (tmp / "orig.json").write_bytes(content)
+ os.link(tmp / "orig.json", tmp / "hardlink_alias.json")
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": "orig.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": "hardlink_alias.json", "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ ])))
+
+ tpath = tmp / "trusted.json"
+ tpath.write_text(json.dumps({"raw_trajectories": {"expected_count": 2}}))
+
+ res = auditor.audit(str(mpath), str(tmp), trusted_expected_sets=str(tpath))
+ det = res["reproducibility_details"]["raw_trajectories"]
+ self.assertFalse(det["prerequisite_complete"])
+ self.assertEqual(det["expected_set_status"], "INCOMPLETE")
+ self.assertEqual(det["distinct_verified_count"], 1)
+ self.assertTrue(any("hardlinks" in d for d in det["duplicate_artifacts"]))
+ self.assertEqual(res["reproducibility_prerequisites"]["raw_trajectories"], auditor.INCOMPLETE)
+
+ code = auditor.main([str(mpath), str(tmp), "--trusted-expected-sets", str(tpath)])
+ self.assertEqual(code, 1)
+
+
+# ---------------------------------------------------------------------------
+# P2: Malformed Filename Rejection (Embedded NUL and Unpaired Unicode Surrogates)
+# ---------------------------------------------------------------------------
+
+class TestMalformedFilenameRejection(unittest.TestCase):
+ """Malformed filesystem filenames must be rejected cleanly without traceback."""
+
+ def test_1_filename_containing_nul_classified_invalid(self):
+ """Case 1: Filename containing embedded NUL is classified as INVALID without crash."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ bad_path = "artifact\x00.txt"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_path, "type": "result_record"}
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ self.assertEqual(len(res["artifacts"]), 1)
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], auditor.INVALID)
+ self.assertFalse(art["present"])
+ self.assertEqual(art["path"], bad_path)
+
+ def test_2_filename_containing_unpaired_high_surrogate_classified_invalid(self):
+ """Case 2: Filename containing unpaired high surrogate U+D800 is classified as INVALID."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ bad_path = f"artifact{chr(0xd800)}.txt"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_path, "type": "result_record"}
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], auditor.INVALID)
+ self.assertFalse(art["present"])
+
+ def test_3_filename_containing_unpaired_low_surrogate_classified_invalid(self):
+ """Case 3: Filename containing unpaired low surrogate (U+DC00 / U+DFFF) is classified as INVALID."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ for surrogate_cp in (0xdc00, 0xdc80, 0xdfff):
+ mpath = tmp / f"manifest_{surrogate_cp}.json"
+ bad_name = f"artifact{chr(surrogate_cp)}.txt"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_name, "type": "result_record"}
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+ def test_4_nul_in_nested_relative_path_component(self):
+ """Case 4: NUL in nested directory or filename relative path component is classified as INVALID."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ bad_path = "nested/dir\x00component/artifact.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_path, "type": "result_record"}
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+ def test_5_valid_unicode_filename_with_non_ascii_characters(self):
+ """Case 5: Valid Unicode filename with non-ASCII characters is accepted and verified."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ valid_name = "café_測試_файл_🚀.json"
+ content = b'{"status": "ok", "experiment": "valid_unicode"}'
+ (tmp / valid_name).write_bytes(content)
+ chash = hashlib.sha256(content).hexdigest()
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": valid_name, "type": "result_record", "expected_sha256": chash, "size_bytes": len(content)}
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.VERIFIED], 1)
+ self.assertEqual(res["totals"][auditor.INVALID], 0)
+ art = res["artifacts"][0]
+ self.assertEqual(art["status"], auditor.VERIFIED)
+ self.assertTrue(art["present"])
+ self.assertTrue(art["hash_match"])
+ self.assertEqual(art["observed_sha256"], chash)
+
+ def test_6_mixed_manifest_containing_one_valid_and_one_malformed_artifact(self):
+ """Case 6: Mixed manifest continues processing other entries when one is invalid."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ valid_name = "valid_evidence.txt"
+ content = b"clean_verified_evidence_data"
+ (tmp / valid_name).write_bytes(content)
+ chash = hashlib.sha256(content).hexdigest()
+
+ bad_nul = "malformed\x00_entry.dat"
+ bad_surrogate = f"malformed{chr(0xd800)}_entry.dat"
+
+ mpath = tmp / "manifest.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": valid_name, "type": "trajectory", "expected_sha256": chash, "size_bytes": len(content)},
+ {"path": bad_nul, "type": "result_record"},
+ {"path": bad_surrogate, "type": "result_record"},
+ ])))
+ res = auditor.audit(str(mpath), str(tmp))
+ self.assertEqual(res["totals"][auditor.VERIFIED], 1)
+ self.assertEqual(res["totals"][auditor.INVALID], 2)
+ by_path = {a["path"]: a["status"] for a in res["artifacts"]}
+ self.assertEqual(by_path[valid_name], auditor.VERIFIED)
+ self.assertEqual(by_path[bad_nul], auditor.INVALID)
+ self.assertEqual(by_path[bad_surrogate], auditor.INVALID)
+
+ def test_7_invalid_paths_never_reach_filesystem_calls(self):
+ """Case 7: Invalid paths with NUL or surrogates never reach filesystem stat/open calls."""
+ from unittest.mock import patch
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ mpath = tmp / "manifest.json"
+ bad_paths = ["file\x00.txt", f"file{chr(0xd800)}.txt", "nested\x00/file.txt"]
+ mpath.write_text(json.dumps(_manifest([
+ {"path": p, "type": "result_record"} for p in bad_paths
+ ])))
+
+ with patch("os.lstat", wraps=os.lstat) as mock_lstat, patch("os.open", wraps=os.open) as mock_open:
+ res = auditor.audit(str(mpath), str(tmp))
+ for bad_p in bad_paths:
+ for c in mock_lstat.call_args_list:
+ self.assertNotIn(bad_p, str(c))
+ for c in mock_open.call_args_list:
+ self.assertNotIn(bad_p, str(c))
+ self.assertEqual(res["totals"][auditor.INVALID], len(bad_paths))
+
+ # Direct reader invocation also avoids filesystem calls
+ with patch("os.lstat", wraps=os.lstat) as mock_lstat, patch("os.open", wraps=os.open) as mock_open:
+ for bad_p in bad_paths:
+ status, chash, size, note = auditor._open_and_hash_artifact(tmp, bad_p)
+ self.assertEqual(status, auditor.INVALID)
+ for c in mock_lstat.call_args_list:
+ self.assertNotIn(bad_p, str(c))
+ for c in mock_open.call_args_list:
+ self.assertNotIn(bad_p, str(c))
+
+ def test_8_direct_invocation_of_path_validator_and_artifact_reader(self):
+ """Case 8: Path validator and reader return safe results when invoked directly."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ bad_nul = "file\x00.txt"
+ bad_high = f"file{chr(0xd800)}.txt"
+ bad_low1 = f"file{chr(0xdc00)}.txt"
+ bad_low_esc = f"file{chr(0xdc80)}.txt"
+ bad_low2 = f"file{chr(0xdfff)}.txt"
+ bad_nested_nul = "dir/sub\x00/file.txt"
+ bad_nested_surr = f"dir/sub{chr(0xd800)}/file.txt"
+
+ # Validator returns False
+ self.assertFalse(auditor._is_safe_relative_path(bad_nul))
+ self.assertFalse(auditor._is_safe_relative_path(bad_high))
+ self.assertFalse(auditor._is_safe_relative_path(bad_low1))
+ self.assertFalse(auditor._is_safe_relative_path(bad_low_esc))
+ self.assertFalse(auditor._is_safe_relative_path(bad_low2))
+ self.assertFalse(auditor._is_safe_relative_path(bad_nested_nul))
+ self.assertFalse(auditor._is_safe_relative_path(bad_nested_surr))
+
+ # Normalizer returns None
+ self.assertIsNone(auditor._normalize_relative_path(bad_nul))
+ self.assertIsNone(auditor._normalize_relative_path(bad_high))
+ self.assertIsNone(auditor._normalize_relative_path(bad_low1))
+ self.assertIsNone(auditor._normalize_relative_path(bad_low_esc))
+
+ # Valid Unicode paths accepted
+ self.assertTrue(auditor._is_safe_relative_path("café_測試.json"))
+ self.assertEqual(auditor._normalize_relative_path("./café_測試.json"), "café_測試.json")
+
+ # Direct reader returns INVALID
+ for bad in [bad_nul, bad_high, bad_low1, bad_nested_nul]:
+ status, chash, size, note = auditor._open_and_hash_artifact(tmp, bad)
+ self.assertEqual(status, auditor.INVALID)
+ self.assertIsNone(chash)
+ self.assertIsNone(size)
+ self.assertIsNotNone(note)
+
+ def test_9_actual_cli_execution_with_malformed_paths(self):
+ """Case 9: CLI execution returns exit code 2 without traceback for malformed paths."""
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ for bad_name in ["file\x00.txt", f"file{chr(0xd800)}.txt", "nested\x00/file.txt"]:
+ mpath = tmp / "manifest.json"
+ out_path = tmp / "out.json"
+ sum_path = tmp / "summary.md"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_name, "type": "result_record"}
+ ])))
+
+ code = auditor.main([
+ str(mpath), str(tmp),
+ "--output", str(out_path),
+ "--summary", str(sum_path),
+ ])
+ self.assertEqual(code, 2, f"Expected exit code 2 for malformed path: {bad_name!r}")
+ self.assertTrue(out_path.exists())
+ self.assertTrue(sum_path.exists())
+ out_path.unlink()
+ sum_path.unlink()
+
+ def test_10_no_traceback_correct_classification_and_expected_exit_code(self):
+ """Case 10: Subprocess CLI execution produces no traceback, controlled INVALID, and exit code 2."""
+ import subprocess
+ with tempfile.TemporaryDirectory() as td:
+ tmp = Path(td)
+ cases = [
+ ("embedded_nul", "bad\x00filename.json"),
+ ("unpaired_high_surrogate", f"bad{chr(0xd800)}filename.json"),
+ ("unpaired_low_surrogate", f"bad{chr(0xdc00)}filename.json"),
+ ]
+ for case_label, bad_name in cases:
+ mpath = tmp / f"manifest_{case_label}.json"
+ mpath.write_text(json.dumps(_manifest([
+ {"path": bad_name, "type": "result_record"}
+ ])))
+ proc = subprocess.run(
+ [sys.executable, str(_ROOT / "scripts" / "audit_c5_benchmark_evidence.py"), str(mpath), str(tmp)],
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(proc.returncode, 2, f"Case {case_label} must exit with code 2")
+ self.assertNotIn("Traceback", proc.stderr, f"Case {case_label} stderr must not contain traceback")
+ res = json.loads(proc.stdout)
+ self.assertEqual(res["totals"][auditor.INVALID], 1)
+ self.assertEqual(res["artifacts"][0]["status"], auditor.INVALID)
+
+
+# ---------------------------------------------------------------------------
+# Security: command-line diagnostics must not disclose untrusted input values
+# ---------------------------------------------------------------------------
+
+class TestCliDiagnosticConfidentiality(unittest.TestCase):
+ """Keep detailed diagnostics in the structured API, not in terminal logs."""
+
+ def test_invalid_manifest_category_does_not_leak_into_stderr(self):
+ import subprocess
+ marker = "SENSITIVE_MARKER_CATEGORY_0183"
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([], expected_sets={marker: {"expected_count": 1}})))
+ result = auditor.audit(str(manifest), str(root))
+ self.assertTrue(any(marker in err for err in result["errors"]))
+ proc = subprocess.run(
+ [sys.executable, str(_ROOT / "scripts" / "audit_c5_benchmark_evidence.py"), str(manifest), str(root)],
+ capture_output=True, text=True,
+ )
+ self.assertEqual(proc.returncode, 2)
+ self.assertIn("audit error: manifest is unreadable or invalid", proc.stderr)
+ self.assertNotIn(marker, proc.stderr)
+ self.assertNotIn("Traceback", proc.stderr)
+
+ def test_unreadable_trusted_config_path_does_not_leak_into_stderr(self):
+ import subprocess
+ marker = "SENSITIVE_MARKER_TRUSTED_PATH_0294"
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([])))
+ trusted_path = root / (marker + ".json")
+ proc = subprocess.run(
+ [sys.executable, str(_ROOT / "scripts" / "audit_c5_benchmark_evidence.py"), str(manifest), str(root),
+ "--trusted-expected-sets", str(trusted_path)],
+ capture_output=True, text=True,
+ )
+ self.assertEqual(proc.returncode, 2)
+ self.assertIn("audit error: invalid trusted expected-sets configuration", proc.stderr)
+ self.assertNotIn(marker, proc.stderr)
+ self.assertNotIn("Traceback", proc.stderr)
+
+ def test_preflight_conflict_does_not_leak_path_or_modify_manifest(self):
+ import subprocess
+ marker = "SENSITIVE_MARKER_DESTINATION_0312"
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ manifest = root / (marker + ".json")
+ manifest.write_text(json.dumps(_manifest([])))
+ original = manifest.read_bytes()
+ proc = subprocess.run(
+ [sys.executable, str(_ROOT / "scripts" / "audit_c5_benchmark_evidence.py"), str(manifest), str(root),
+ "--output", str(manifest)],
+ capture_output=True, text=True,
+ )
+ self.assertEqual(proc.returncode, 2)
+ self.assertIn("preflight error: unsafe or conflicting output destination", proc.stderr)
+ self.assertNotIn(marker, proc.stderr)
+ self.assertNotIn("Traceback", proc.stderr)
+ self.assertEqual(manifest.read_bytes(), original)
+
+ def test_publication_error_does_not_leak_exception_text(self):
+ from contextlib import redirect_stderr
+ from io import StringIO
+ from unittest.mock import patch
+ marker = "SENSITIVE_MARKER_PUBLISH_0489"
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([])))
+ for option, expected in (
+ ("--output", "publication error: unable to write output"),
+ ("--summary", "publication error: unable to write summary"),
+ ):
+ stderr = StringIO()
+ with patch.object(auditor, "_publish_file_securely", side_effect=OSError(marker)):
+ with redirect_stderr(stderr):
+ code = auditor.main([str(manifest), str(root), option, str(root / "output.txt")])
+ self.assertEqual(code, 2)
+ self.assertIn(expected, stderr.getvalue())
+ self.assertNotIn(marker, stderr.getvalue())
+ self.assertNotIn("Traceback", stderr.getvalue())
+
+
+# ---------------------------------------------------------------------------
+# Markdown report integrity: untrusted values must remain within their cells.
+# ---------------------------------------------------------------------------
+
+class TestMarkdownReportIntegrity(unittest.TestCase):
+ def test_newline_and_pipe_in_missing_path_cannot_forge_verified_row(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ bad_name = "not-found\n| apparent-success | result_record | **VERIFIED** | yes | 20 |"
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([{
+ "path": bad_name, "type": "result_record"
+ }])))
+ result = auditor.audit(str(manifest), str(root), deterministic=True)
+ self.assertEqual(result["artifacts"][0]["status"], auditor.MISSING)
+ summary = auditor._markdown_summary(result)
+ artifact_section = summary.split("## Artifacts\n", 1)[1].split("## Reproducibility Prerequisites", 1)[0]
+ rows = [row for row in artifact_section.splitlines() if row.startswith("|")]
+ self.assertEqual(len(rows), 3)
+ self.assertNotIn("\n| apparent-success", summary)
+ self.assertIn(r"\| apparent-success", summary)
+ self.assertIn("**MISSING**", summary)
+
+ def test_code_delimiters_and_html_are_escaped(self):
+ value = "bad`
|name"
+ safe = auditor._markdown_code(value)
+ self.assertEqual(safe, '` bad ``` |name `')
+ self.assertEqual(auditor._markdown_code(value, table_cell=True),
+ '` bad ``` \\|name `')
+
+ def test_campaign_and_notes_cannot_insert_headings_or_columns(self):
+ result = {
+ "campaign_id": "benchmark\n## Forged status",
+ "root": "/tmp/a|b",
+ "manifest_valid": True,
+ "totals": {status: 0 for status in auditor.ALL_STATUSES} | {"total": 0},
+ "reproducibility_prerequisites": {"raw_trajectories": auditor.INCOMPLETE},
+ "reproducibility_details": {"raw_trajectories": {"notes": "missing\n| VERIFIED | yes |"}},
+ }
+ summary = auditor._markdown_summary(result)
+ self.assertNotIn("\n## Forged status", summary)
+ prerequisite_section = summary.split("## Reproducibility Prerequisites", 1)[1]
+ self.assertNotIn("\n| VERIFIED", prerequisite_section)
+ self.assertIn("benchmark\\n## Forged status", summary)
+ self.assertIn(r"missing\n\| VERIFIED", summary)
+
+ def test_cli_published_summary_does_not_forge_missing_artifact(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ missing = "missing\n| forged | result_record | **VERIFIED** | yes | 99 |"
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([{"path": missing, "type": "result_record"}])))
+ output = root / "audit.json"
+ summary = root / "audit.md"
+ code = auditor.main([str(manifest), str(root), "--output", str(output), "--summary", str(summary), "--deterministic"])
+ self.assertEqual(code, 1)
+ self.assertEqual(json.loads(output.read_text())["artifacts"][0]["status"], auditor.MISSING)
+ text = summary.read_text()
+ section = text.split("## Artifacts\n", 1)[1].split("## Reproducibility Prerequisites", 1)[0]
+ self.assertEqual(sum(line.startswith("|") for line in section.splitlines()), 3)
+ self.assertNotIn("\n| forged", text)
+ self.assertIn("**MISSING**", text)
+
+ def test_unicode_line_separators_do_not_create_new_rows(self):
+ for separator in (chr(0x85), chr(0x2028), chr(0x2029)):
+ safe = auditor._markdown_display_value("before" + separator + "after")
+ self.assertNotIn(separator, safe)
+ self.assertIn("after", safe)
+
+ def test_unicode_and_surrogates_remain_publishable(self):
+ text = auditor._markdown_code("café_測試_🚀" + chr(0xd800))
+ self.assertIn("café", text)
+ self.assertIn("測試", text)
+ self.assertIn("\\ud800", text)
+ text.encode("utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Markdown inline safety: untrusted text uses native code spans.
+# ---------------------------------------------------------------------------
+
+class TestMarkdownInlineSafety(unittest.TestCase):
+ def test_code_field_escapes_link_image_emphasis_and_backticks(self):
+ malicious = (
+ "[**VERIFIED**](https://example.invalid/track) "
+ " "
+ "`fake_code`"
+ )
+ safe = auditor._markdown_code(malicious)
+ self.assertEqual(safe, "` " + malicious.split("`")[0] + " ``` fake_code ``")
+
+ def test_report_renders_potential_links_as_literal_syntax(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ malicious_path = "[VERIFIED](https://example.invalid/track)"
+ malicious_campaign = ""
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest(
+ [{"path": malicious_path, "type": "result_record"}],
+ campaign_id=malicious_campaign,
+ )))
+ result = auditor.audit(str(manifest), str(root), deterministic=True)
+ self.assertEqual(result["artifacts"][0]["status"], auditor.MISSING)
+ summary = auditor._markdown_summary(result)
+ self.assertIn("` " + malicious_path + " `", summary)
+ self.assertIn("` " + malicious_campaign + " `", summary)
+ self.assertIn("**MISSING**", summary)
+
+ def test_raw_url_email_and_strikethrough_are_not_active_markdown(self):
+ value = "https://example.invalid/track www.example.invalid mail@example.invalid ~~VERIFIED~~"
+ safe = auditor._markdown_code(value)
+ self.assertEqual(safe, "` " + value + " `")
+
+ def test_bidi_formatting_controls_cannot_reorder_display(self):
+ for codepoint in (0x061C, 0x200E, 0x200F, 0x202A, 0x202B, 0x202C,
+ 0x202D, 0x202E, 0x2066, 0x2067, 0x2068, 0x2069):
+ value = "start" + chr(codepoint) + "finish"
+ escaped = auditor._markdown_display_value(value)
+ self.assertNotIn(chr(codepoint), escaped)
+ self.assertIn(f"\\u{codepoint:04x}", escaped)
+ self.assertIn("finish", escaped)
+
+
+
+_MARKDOWN_CASES = (
+ ("[VERIFIED](https://example.invalid)", "[VERIFIED](https://example.invalid)"),
+ ("", ""),
+ ("https://example.invalid/a", "https://example.invalid/a"),
+ ("http://example.invalid/a", "http://example.invalid/a"),
+ ("www.example.invalid", "www.example.invalid"),
+ ("eval_user@example.invalid", "eval_user@example.invalid"),
+ ("", ""),
+ ("mailto:user@example.invalid", "mailto:user@example.invalid"),
+ ("@ . | `", "@ . | `"),
+ ("@ . | `", "@ . | `"),
+ ("user@example.invalid", "user@example.invalid"),
+ ("<img> & @ |", "<img> & @ |"),
+ ("**VERIFIED** *em* _em_ __strong__", "**VERIFIED** *em* _em_ __strong__"),
+ ("~~VERIFIED~~", "~~VERIFIED~~"),
+ ("` `` ``` **VERIFIED**", "` `` ``` **VERIFIED**"),
+ ("`", "`"),
+ ("````", "````"),
+ (r"a|b\|c\\|d|", r"a|b\|c\\|d|"),
+ ("a\n| forged |\r\n## Forged\rheading", r"a\n| forged |\r\n## Forged\rheading"),
+ ("a\u0085b\u2028c\u2029d", r"a\u0085b\u2028c\u2029d"),
+ ("a\u202eb\u2066c\u2069d", r"a\u202eb\u2066c\u2069d"),
+ ("a\ud800\udcffz", r"a\ud800\udcffz"),
+ ('fake',
+ 'fake'),
+ ("a\x00b\t\x1b\x7f", r"a\x00b\t\x1b\x7f"),
+ (" leading and trailing ", " leading and trailing "),
+ (" ", " "),
+ ("", ""),
+ ("\\[link](https://example.invalid)\\", "\\[link](https://example.invalid)\\"),
+ ("café_測試_" + "long." * 250, "café_測試_" + "long." * 250),
+)
+
+
+def _markdown_fixture(payload: str) -> dict:
+ return {
+ "campaign_id": payload, "root": payload,
+ "audited_at": "1970-01-01T00:00:00Z", "manifest_valid": True,
+ "timestamp_kind": "deterministic_placeholder",
+ "totals": {s: 0 for s in auditor.ALL_STATUSES} | {"total": 1, auditor.MISSING: 1},
+ "errors": [payload],
+ "artifacts": [{"path": payload, "type": payload, "status": auditor.MISSING,
+ "present": False, "size_bytes": None}],
+ "reproducibility_prerequisites": {"raw_trajectories": auditor.INCOMPLETE},
+ "reproducibility_details": {"raw_trajectories": {
+ "artifact_integrity": auditor.MISSING, "expected_set_status": "INCOMPLETE",
+ "prerequisite_complete": False, "notes": payload,
+ }},
+ }
+
+
+class TestMarkdownNativeSpans(unittest.TestCase):
+ def test_display_values_preserve_punctuation_and_expose_controls(self):
+ for value, display in _MARKDOWN_CASES:
+ with self.subTest(value=value):
+ self.assertEqual(auditor._markdown_display_value(value), display)
+
+ def test_code_and_text_use_the_same_quoting_in_each_context(self):
+ for value, _ in _MARKDOWN_CASES:
+ for table_cell in (False, True):
+ with self.subTest(value=value, table_cell=table_cell):
+ self.assertEqual(auditor._markdown_text(value, table_cell=table_cell),
+ auditor._markdown_code(value, table_cell=table_cell))
+
+ def test_backticks_are_literal_entities_between_short_spans(self):
+ for length in (1, 2, 3, 8, 64, 256, 10000):
+ value = "`" * length + " text " + "`" * length
+ self.assertEqual(auditor._markdown_code(value),
+ "`" * length + "` text `" + "`" * length)
+
+ def test_table_pipe_escape_preserves_existing_backslashes(self):
+ for length in range(16):
+ value = "a" + "\\" * length + "|b"
+ self.assertEqual(auditor._markdown_code(value), "` " + value + " `")
+ self.assertEqual(auditor._markdown_code(value, table_cell=True),
+ "` a" + "\\" * (length + 1) + "|b `")
+
+ def test_empty_and_all_space_values_do_not_gain_padding(self):
+ self.assertEqual(auditor._markdown_code(""), "")
+ self.assertEqual(auditor._markdown_code(" "), "` `")
+ self.assertEqual(auditor._markdown_code(" x "), "` x `")
+
+ def test_reports_quote_each_untrusted_field_without_mutating_json(self):
+ for value, _ in _MARKDOWN_CASES:
+ result = _markdown_fixture(value)
+ before = json.dumps(result, sort_keys=True)
+ summary = auditor._markdown_summary(result)
+ self.assertIn("- **Campaign**: " + auditor._markdown_code(value), summary)
+ self.assertIn("- **Root**: " + auditor._markdown_code(value), summary)
+ self.assertIn("- " + auditor._markdown_text(value), summary)
+ self.assertIn("| " + auditor._markdown_code(value, table_cell=True) + " | "
+ + auditor._markdown_text(value, table_cell=True) + " | **MISSING** | no | - |", summary)
+ self.assertEqual(json.dumps(result, sort_keys=True), before)
+
+ def test_cli_json_and_markdown_are_deterministic(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ manifest = root / "manifest.json"
+ manifest.write_text(json.dumps(_manifest([
+ {"path": "missing|user@example.invalid", "type": "~~VERIFIED~~"},
+ ], campaign_id="user@example.invalid")))
+ outputs = []
+ for index in range(2):
+ output, summary = root / f"{index}.json", root / f"{index}.md"
+ self.assertEqual(auditor.main([str(manifest), str(root), "--output", str(output),
+ "--summary", str(summary), "--deterministic"]), 1)
+ outputs.append((output.read_bytes(), summary.read_bytes()))
+ self.assertEqual(outputs[0], outputs[1])
+
+
+def _run_gfm_checks() -> None:
+ # Kept explicit because cmarkgfm is a verification tool, not an app dependency.
+ # Import failure must fail this command, rather than silently skip security tests.
+ from cmarkgfm import github_flavored_markdown_to_html
+ from cmarkgfm.cmark import Options
+ from xml.etree import ElementTree as ET
+
+ def render(markdown):
+ # Allow raw HTML so a broken escape cannot be hidden by sanitization.
+ html = github_flavored_markdown_to_html(markdown, options=Options.CMARK_OPT_UNSAFE)
+ return ET.fromstring("" + html + "")
+
+ class TestMarkdownGFM(unittest.TestCase):
+ def test_renderer_positive_controls(self):
+ dom = render("https://example.invalid user@example.invalid ~~deleted~~ **bold** *em*\n\n"
+ ' HTML\n\n'
+ "| A | B |\n|---|---|\n| x | y |")
+ for tag in ("a", "del", "strong", "em", "img", "span", "table"):
+ self.assertIsNotNone(dom.find(".//" + tag), tag)
+ self.assertTrue(any(a.get("href", "").startswith("mailto:") for a in dom.iter("a")))
+ # Reproduce the original decoded-entity autolink, not just bare URLs.
+ original = render("eval_user@example.invalid")
+ self.assertEqual(original.find(".//code/a").get("href"), "mailto:eval_user@example.invalid")
+
+ def test_helpers_render_as_literal_text(self):
+ for value, display in _MARKDOWN_CASES:
+ for helper in (auditor._markdown_code, auditor._markdown_text):
+ for table_cell in (False, True):
+ with self.subTest(value=value, helper=helper.__name__, table_cell=table_cell):
+ quoted = helper(value, table_cell=table_cell)
+ markdown = "| Value |\n|---|\n| " + quoted + " |" if table_cell else quoted
+ dom = render(markdown)
+ parent = dom.find(".//td") if table_cell else dom.find("p")
+ self.assertEqual("".join(parent.itertext()) if parent is not None else "", display)
+ self.assertTrue(all(len(code) == 0 for code in dom.iter("code")))
+ expected_tags = (["report", "table", "thead", "tr", "th", "tbody", "tr", "td"]
+ if table_cell else (["report", "p"] if display else ["report"]))
+ self.assertEqual([node.tag for node in dom.iter() if node.tag != "code"], expected_tags)
+
+ def test_backtick_lengths_and_backslashes_before_table_pipes(self):
+ for length in list(range(65)) + [80, 81, 128, 256, 1000, 10000]:
+ value = "`" * length + "a" + "\\" * length + "| **fake** |" + "`" * length
+ for table_cell in (False, True):
+ quoted = auditor._markdown_code(value, table_cell=table_cell)
+ markdown = "| A | B |\n|---|---|\n| " + quoted + " | end |" if table_cell else quoted
+ dom = render(markdown)
+ parent = dom.find(".//td") if table_cell else dom.find("p")
+ self.assertEqual("".join(parent.itertext()), value)
+ self.assertTrue(all(len(code) == 0 for code in dom.iter("code")))
+ for tag in ("strong", "em", "a", "del"):
+ self.assertIsNone(dom.find(".//" + tag), tag)
+ if table_cell:
+ self.assertEqual(len(dom.findall(".//tbody/tr")), 1)
+ self.assertEqual(len(dom.findall(".//td")), 2)
+ self.assertEqual(dom.findall(".//td")[1].text, "end")
+
+ def test_every_short_backtick_length_and_very_long_runs(self):
+ for value in (
+ " ".join("`" * n for n in range(1, 130)) + " https://example.invalid **fake**",
+ "`" * 100000 + " user@example.invalid ~~fake~~ " + "`" * 100000,
+ ):
+ for table_cell in (False, True):
+ quoted = auditor._markdown_code(value, table_cell=table_cell)
+ markdown = "| A |\n|---|\n| " + quoted + " |" if table_cell else quoted
+ dom = render(markdown)
+ parent = dom.find(".//td") if table_cell else dom.find("p")
+ self.assertEqual("".join(parent.itertext()), value)
+ self.assertTrue(all(n.tag in {"report", "p", "code", "table", "thead",
+ "tbody", "tr", "th", "td"} for n in dom.iter()))
+
+ def assert_report(self, result):
+ summary = auditor._markdown_summary(result)
+ dom = render(summary)
+ # Compare the entire element structure with the same report using safe text.
+ baseline = render(auditor._markdown_summary(_markdown_fixture("safe")))
+ self.assertEqual([n.tag for n in dom.iter() if n.tag != "code"],
+ [n.tag for n in baseline.iter() if n.tag != "code"])
+ self.assertTrue(all(len(code) == 0 for code in dom.iter("code")))
+ tables = dom.findall(".//table")
+ self.assertEqual([len(t.findall(".//tbody/tr")) for t in tables], [7, 1, 1])
+ self.assertEqual([[len(row) for row in t.findall(".//tr")] for t in tables],
+ [[2] * 8, [5] * 2, [6] * 2])
+ self.assertEqual([n.text for n in dom.iter("strong")],
+ ["Campaign", "Audited at", "Timestamp note", "Root", "Manifest valid",
+ auditor.MISSING, auditor.INCOMPLETE])
+ return dom
+
+ def test_complete_reports_keep_classification_and_structure(self):
+ for value, display in _MARKDOWN_CASES:
+ if not value:
+ continue # Empty cells are checked by the helper test.
+ with self.subTest(value=value):
+ dom = self.assert_report(_markdown_fixture(value))
+ items = dom.findall(".//li")
+ self.assertEqual("".join(items[0].itertext()), "Campaign: " + display)
+ self.assertEqual("".join(items[3].itertext()), "Root: " + display)
+ self.assertEqual("".join(items[5].itertext()), display)
+ tables = dom.findall(".//table")
+ cells = tables[1].findall(".//td")
+ self.assertEqual(["".join(c.itertext()) for c in cells],
+ [display, display, auditor.MISSING, "no", "-"])
+ self.assertEqual("".join(tables[2].findall(".//td")[-1].itertext()), display)
+
+ def test_actual_cli_reports_match_authoritative_json(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td) / "`root`|admin@example.invalid"
+ root.mkdir()
+ manifest = root / "manifest.json"
+ path = "missing`|user@example.invalid\n## Forged"
+ manifest.write_text(json.dumps(_manifest([
+ {"path": path, "type": "https://example.invalid ~~VERIFIED~~"},
+ ], campaign_id="admin@example.invalid")))
+ output, summary = root / "audit.json", root / "audit.md"
+ self.assertEqual(auditor.main([str(manifest), str(root), "--output", str(output),
+ "--summary", str(summary), "--deterministic"]), 1)
+ result = json.loads(output.read_text())
+ self.assertEqual(result["artifacts"][0]["status"], auditor.MISSING)
+ dom = render(summary.read_text())
+ for tag in ("a", "img", "del", "em", "span", "script"):
+ self.assertIsNone(dom.find(".//" + tag), tag)
+ rows = dom.findall(".//table")[1].findall(".//tbody/tr")
+ self.assertEqual(len(rows), 1)
+ self.assertEqual(["".join(cell.itertext()) for cell in rows[0]],
+ [r"missing`|user@example.invalid\n## Forged",
+ result["artifacts"][0]["type"], auditor.MISSING, "no", "-"])
+ self.assertEqual(len(dom.findall(".//h1")), 1)
+ self.assertEqual(len(dom.findall(".//h2")), 3)
+ items = ["".join(n.itertext()) for n in dom.iter("li")]
+ self.assertIn("Root: " + str(root), items)
+ self.assertIn("Campaign: " + result["campaign_id"], items)
+
+ def test_real_duplicate_diagnostics_and_validation_errors(self):
+ with tempfile.TemporaryDirectory() as td:
+ root = Path(td)
+ path = "user@example.invalid|~~VERIFIED~~"
+ entry = {"path": path, "type": "trajectory"}
+ result = _run_audit(root, _manifest([entry, entry]), deterministic=True)
+ notes = result["reproducibility_details"]["raw_trajectories"]["notes"]
+ self.assertIn("duplicate", notes)
+ self.assertIn(path, notes)
+ dom = render(auditor._markdown_summary(result))
+ self.assertIn(notes, [n.text for n in dom.iter("code")])
+ bad = _manifest([], expected_sets={"https://example.invalid ~~VERIFIED~~": {}})
+ result = _run_audit(root, bad, deterministic=True)
+ self.assertTrue(result["errors"])
+ dom = render(auditor._markdown_summary(result))
+ self.assertIn(result["errors"][0], [n.text for n in dom.iter("code")])
+ for tag in ("a", "img", "del", "em"):
+ self.assertIsNone(dom.find(".//" + tag), tag)
+
+ suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestMarkdownGFM)
+ result = unittest.TextTestRunner(verbosity=2).run(suite)
+ raise SystemExit(not result.wasSuccessful())
+
+
+if __name__ == "__main__":
+ if sys.argv[1:] == ["--gfm"]:
+ _run_gfm_checks()
+ else:
+ unittest.main()