diff --git a/tests/run-visual b/tests/run-visual index 0be6a50f0..98b99c950 100755 --- a/tests/run-visual +++ b/tests/run-visual @@ -210,6 +210,11 @@ if [[ "$MODE" == "screenshot" ]]; then PNG="$CACHE_DIR/$TARGET.png" if [[ -f "$PNG" ]]; then echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)" + # Advisory here rather than fatal — an ad-hoc capture of a rung that + # genuinely renders nothing is a legitimate thing to want to look at + # (that is how the empty deep rungs were found). But say so out loud, + # because file size alone reads as success. + "$ROOT/tooling/visual-blank-check" "$PNG" --quiet || true else echo "Error: capture failed — $PNG not found" >&2 exit 1 @@ -298,6 +303,27 @@ for scenario in "${SCENARIOS[@]}"; do continue fi + # Verify the renderer actually drew something. + # + # "Non-empty file" was the only content check until 2026-08-06, and a blank + # screen is a perfectly valid ~19 KB PNG. Worse, once a blank capture was + # recorded as a golden, every later blank capture matched it at 0.0% and + # the scenario PASSED — atlas_GJ338Bd_Block and atlas_GJ445c-m1_Chunk were + # green against blank goldens. A test that cannot fail is worse than no + # test, because it is counted as coverage. + # + # Checked in BOTH modes, and the update mode matters most: refusing to + # RECORD a blank golden is what stops the trap being re-armed. + set +e + BLANK_OUT=$("$ROOT/tooling/visual-blank-check" "$CAPTURED" 2>&1) + BLANK_RC=$? + set -e + if [[ $BLANK_RC -ne 0 ]]; then + echo " FAIL: $BLANK_OUT" >&2 + FAILED=$((FAILED + 1)) + continue + fi + if [[ "$MODE" == "update" ]]; then cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png" echo " Updated golden: $GOLDEN_DIR/$scenario.png" diff --git a/tooling/visual-blank-check b/tooling/visual-blank-check new file mode 100755 index 000000000..8e202db76 --- /dev/null +++ b/tooling/visual-blank-check @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Reject a screen capture in which the renderer drew nothing. + +WHY THIS EXISTS (2026-08-06). `tests/run-visual` verified only that the +captured PNG was non-empty *as a file*. A capture of a blank screen is a +perfectly valid ~19 KB PNG, so it passed — and then, because the golden had +been recorded from an equally blank capture, `visual-diff` reported 0.0% +difference and the scenario PASSED. Two Atlas scenarios +(`atlas_GJ338Bd_Block`, `atlas_GJ445c-m1_Chunk`) sat green for months against +blank goldens. This is the same failure `e024cfb3f` recorded once already +("the Atlas Global goldens have been measuring nothing"); it kept recurring +because nothing checked the property, only the file. + +A green comparison between two blank images is the worst kind of test result: +it is indistinguishable from success and it is load-bearing for exactly the +work it fails to cover. + +THE METRIC. Fraction of the frame occupied by the single most common colour. +Measured on this project's real captures: + + Global (real world map) 38.7% modal + Region (flat colour wash) 7.2% modal <- dither; least uniform of all + District 45.4% modal + Block / Chunk / Quarter 92.9-94.6% modal <- nothing drawn + +Nothing falls between 45% and 93%, so the default 0.85 threshold sits in open +space rather than being tuned to a boundary case. + +Note what this deliberately does NOT do: it does not judge whether a render is +GOOD. The Region wash is a real product gap (T-1213), and it scores 7.2% — +comfortably "content". The question here is only "did the renderer put a world +on the screen", never "is the world any good". Keep it that way; a check that +creeps toward aesthetics will start failing legitimate frames and get muted. + +Usage: + tooling/visual-blank-check [--max-modal 0.85] [--quiet] + +Exit: 0 = has content, 1 = blank, 2 = error (unreadable/missing). +""" + +import sys +from pathlib import Path + +DEFAULT_MAX_MODAL = 0.85 + + +def modal_fraction(path: Path) -> tuple[float, int]: + """Return (modal colour's share of the frame, distinct colour count).""" + try: + from PIL import Image + except ImportError: + print("visual-blank-check: Pillow not available", file=sys.stderr) + raise SystemExit(2) + + with Image.open(path) as im: + rgb = im.convert("RGB") + total = rgb.size[0] * rgb.size[1] + if total == 0: + return 1.0, 0 + # maxcolors high enough that a real frame is never bucketed away; + # getcolors returns None if it overflows, which itself means the frame + # is richly coloured and therefore certainly not blank. + colors = rgb.getcolors(maxcolors=1 << 24) + if colors is None: + return 0.0, 1 << 24 + top = max(colors)[0] + return top / total, len(colors) + + +def main(argv: list[str]) -> int: + args = [a for a in argv if not a.startswith("--")] + if not args: + print(__doc__, file=sys.stderr) + return 2 + + max_modal = DEFAULT_MAX_MODAL + for i, a in enumerate(argv): + if a == "--max-modal" and i + 1 < len(argv): + max_modal = float(argv[i + 1]) + quiet = "--quiet" in argv + + path = Path(args[0]) + if not path.is_file(): + print(f"visual-blank-check: no such file: {path}", file=sys.stderr) + return 2 + + modal, distinct = modal_fraction(path) + + if modal > max_modal: + print( + f"BLANK: {path.name} is {modal:.1%} a single colour " + f"({distinct} distinct) — the renderer drew nothing but chrome. " + f"A golden recorded from this would pass against any other blank " + f"capture forever.", + file=sys.stderr, + ) + return 1 + + if not quiet: + print(f"content: {path.name} modal={modal:.1%} distinct={distinct}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))