#!/usr/bin/env bash # tests/run-visual: Visual regression test suite. # Captures scenarios from tests/visual.json, compares against golden PNGs. # # Modes: # (no args) Run all scenario golden comparisons (xvfb-wrapped) # --screenshot NAME Ad-hoc single capture to .cache/screenshots/ (no xvfb) # --movie NAME Flow capture to .cache/screenshots/ (no xvfb) # --update Regenerate all goldens and stage for commit # --filter PATTERN Accepted and ignored (compat with run-all) # # Exit: 0=pass (or skip), 1=fail, 2=error # Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N} set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" CONFIG="$ROOT/tests/visual.json" GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)" CACHE_DIR="$ROOT/.cache/screenshots" DIFF_DIR="$ROOT/.cache/visual-diff" MODE="golden" # golden | screenshot | movie | update TARGET="" INTERVAL="" # Server state for live scenarios SERVER_BIN="" SERVER_PID="" SERVER_PORT="" # Cleanup server on exit trap '[[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || true' EXIT # -- Parse args ---------------------------------------------------------------- while [[ $# -gt 0 ]]; do case "$1" in --screenshot) MODE="screenshot"; TARGET="${2:-atlas_GJ820Bc_Global}"; shift 2 ;; --movie) MODE="movie"; TARGET="${2:-flow_dialogue}"; shift 2 ;; --update) MODE="update"; shift ;; --filter) shift 2 ;; # Accept and ignore (run-all compat) --filter=*) shift ;; --interval) INTERVAL="${2:-3}"; shift 2 ;; # Override the config resolution for ONE ad-hoc capture (T-1239). The # Atlas derives canvas extent, and therefore metres-per-gridunit, from # the viewport — so "does this defect depend on resolution?" is a real # diagnostic question, and answering it by hand-editing the committed # tests/visual.json invites leaving it edited. Goldens are shot at the # config resolution; this flag deliberately does not touch them. --resolution) RESOLUTION_OVERRIDE="${2:-}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done # -- Preflight ----------------------------------------------------------------- if [[ -z "$GODOT" ]]; then echo "Warning: Godot not found — skipping visual tests" >&2 printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' exit 0 fi if ! python3 -c "pass" 2>/dev/null; then echo "Warning: Python 3 not found — skipping visual tests" >&2 printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' exit 0 fi if [[ ! -f "$CONFIG" ]]; then echo "Error: $CONFIG not found" >&2 exit 2 fi # Read config values GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")" TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")" RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")" if [[ -n "${RESOLUTION_OVERRIDE:-}" ]]; then RESOLUTION="$RESOLUTION_OVERRIDE" echo "Note: resolution overridden to $RESOLUTION (goldens are shot at the config resolution)" >&2 fi # Read scenario names from config SCENARIOS=($(python3 -c " import json c = json.load(open('$CONFIG')) for name in c.get('scenarios', {}): print(name) ")) # -- Helpers ------------------------------------------------------------------- # Offscreen wrapper for EVERY capture path. # # Captures must never steal the desktop: this box is also the developer's own # session. Until 2026-08-06 only the golden path even tried, via xvfb-run — # which is not installed here, so it hit the "using visible window" fallback, # and --screenshot/--movie never wrapped at all. Every capture grabbed focus. # # gamescope is preferred, for two independent reasons: # 1. It renders on the REAL GPU. The goldens are pinned to this box's # Mesa/AMD output (T-1121 — they do not port across rendering stacks), and # xvfb-run would fall back to llvmpipe software rendering, shifting every # pixel. Offscreen must not mean a different renderer. # 2. It is the only option that lets us SET the output size. cage is also # installed and also GPU-backed, but it is a kiosk compositor: it forces # its client to the headless output's default 1280x720 and silently # overrides --resolution (measured — a 960x540 request produced a 1280x720 # PNG). A wrapper that quietly changes resolution is worse than none, # because RESOLUTION is load-bearing here: D-255's extent inversion makes # the shorter viewport axis span exactly one cell of the rung, so window # size decides how much world a rung shows. cage is deliberately not used. CAPTURE_PREFIX=() if command -v gamescope >/dev/null 2>&1; then CAPTURE_PREFIX=(gamescope --backend headless -W "${RESOLUTION%x*}" -H "${RESOLUTION#*x}" --) elif command -v xvfb-run >/dev/null 2>&1; then echo "Note: gamescope not found — falling back to xvfb-run (software GL; goldens may drift)" >&2 CAPTURE_PREFIX=(xvfb-run -a --server-args="-screen 0 ${RESOLUTION}x24") else echo "Warning: no offscreen compositor (gamescope/xvfb-run) — capturing in a VISIBLE window" >&2 fi godot_capture() { local mode_flag="$1" # --scenario or --flow local name="$2" local output="$3" local extra_args=("${@:4}") # Isolate `user://` per run (T-1239). Godot resolves user:// under # XDG_DATA_HOME, which is how the Atlas disk cache (D-255, T-1183) persisted # across captures — including across the server changes that made its # contents wrong. A capture then rendered a canvas generated by a build that # no longer existed: Ferrath Global replayed a pre-T-1237 canvas from weeks # earlier and showed 375 hop-fragments where the live server produces 73 # whole rivers, and every golden shot in that window silently inherited it. # A visual test must exercise the code in the tree, so the cache it warms # must not outlive the run. The dir is recreated fresh each capture; the # user's real cache at ~/.local/share/godot is never touched. local user_data="$ROOT/.cache/visual-user-data" rm -rf "$user_data" mkdir -p "$user_data" XDG_DATA_HOME="$user_data" \ "${CAPTURE_PREFIX[@]}" "$GODOT" --rendering-driver opengl3 --fixed-fps 60 \ --resolution "$RESOLUTION" \ --path "$ROOT/client" -s res://tests/visual_capture.gd -- \ $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1 } # Retained as the golden-path name; offscreen handling now lives in # godot_capture, so both paths get it. xvfb_capture() { godot_capture "$@" } # -- Server lifecycle (live scenarios) ----------------------------------------- # Check if a scenario has "live": true in config is_live_scenario() { python3 -c " import json, sys c = json.load(open('${CONFIG}')) s = c.get('scenarios', {}).get('${1}', {}) sys.exit(0 if s.get('live') else 1) " } # Build server binary (once, cached) ensure_server_built() { if [[ -n "$SERVER_BIN" ]]; then return 0; fi echo " Building server for live visual tests..." (cd "$ROOT/server" && cargo build --bin settled-reach-server 2>&1) || { echo "Error: server build failed" >&2 return 1 } SERVER_BIN="$ROOT/server/target/debug/settled-reach-server" } # Start server with --test-mode --port 0, parse LISTENING:{port} start_server() { # Keep the server's own output (T-1239). This was `mktemp` + `2>/dev/null` # + `rm` as soon as LISTENING was parsed, which meant two things: every # tracing::warn!/error! the simulation emitted was discarded, and everything # after startup went to an unlinked file. A server quietly degrading mid- # capture looked identical to a healthy one — the capture only ever showed # what the CLIENT thought it received. Now stderr is merged in and the log # lives at a stable path that survives the run for inspection. local stdout_log="$ROOT/.cache/visual-server.log" mkdir -p "$ROOT/.cache" "$SERVER_BIN" --test-mode --port 0 >"$stdout_log" 2>&1 & SERVER_PID=$! local attempts=0 while [[ $attempts -lt 150 ]]; do if ! kill -0 "$SERVER_PID" 2>/dev/null; then echo " Error: server exited unexpectedly — see $stdout_log" >&2 SERVER_PID="" return 1 fi if grep -q "^LISTENING:" "$stdout_log" 2>/dev/null; then SERVER_PORT=$(sed -n 's/^LISTENING://p' "$stdout_log") echo " Server started: pid=$SERVER_PID port=$SERVER_PORT log=$stdout_log" return 0 fi sleep 0.1 attempts=$((attempts + 1)) done echo " Error: no LISTENING signal after 15s — see $stdout_log" >&2 kill "$SERVER_PID" 2>/dev/null || true SERVER_PID="" return 1 } # Stop server (called after each live capture; server may have exited on disconnect) stop_server() { if [[ -n "$SERVER_PID" ]]; then kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true SERVER_PID="" SERVER_PORT="" fi } # -- Screenshot mode ----------------------------------------------------------- if [[ "$MODE" == "screenshot" ]]; then mkdir -p "$CACHE_DIR" echo "Capturing scenario: $TARGET" if is_live_scenario "$TARGET"; then ensure_server_built || exit 2 start_server || exit 2 export SR_LIVE=1 SR_PORT="$SERVER_PORT" fi godot_capture --scenario "$TARGET" "$CACHE_DIR" stop_server unset SR_LIVE SR_PORT 2>/dev/null || true PNG="$CACHE_DIR/$TARGET.png" if [[ -f "$PNG" ]]; then echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)" # Advisory here rather than fatal — an ad-hoc capture of a rung that # genuinely renders nothing is a legitimate thing to want to look at # (that is how the empty deep rungs were found). But say so out loud, # because file size alone reads as success. reach --no-input visual blank-check "$PNG" --quiet || true else echo "Error: capture failed — $PNG not found" >&2 exit 1 fi exit 0 fi # -- Movie mode ---------------------------------------------------------------- if [[ "$MODE" == "movie" ]]; then mkdir -p "$CACHE_DIR" echo "Capturing flow: $TARGET" EXTRA=() [[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL") godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}" FLOW_DIR="$CACHE_DIR/$TARGET" if [[ -d "$FLOW_DIR" ]]; then FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l) echo "Flow: $FRAME_COUNT frames in $FLOW_DIR" # Generate contact sheet if visual-thumbnail is available if command -v reach >/dev/null 2>&1; then reach --no-input visual thumbnail "$FLOW_DIR" --config "$CONFIG" SHEET="$FLOW_DIR/${TARGET}_sheet.png" [[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET" fi else echo "Error: flow capture failed — $FLOW_DIR not found" >&2 exit 1 fi exit 0 fi # -- Golden mode (default) / Update mode -------------------------------------- START_MS=$(date +%s%3N) TOTAL=0 PASSED=0 FAILED=0 mkdir -p "$CACHE_DIR" "$DIFF_DIR" if [[ "$MODE" == "update" ]]; then mkdir -p "$GOLDEN_DIR" fi for scenario in "${SCENARIOS[@]}"; do TOTAL=$((TOTAL + 1)) echo "--- $scenario ---" # Start server for live scenarios IS_LIVE=false if is_live_scenario "$scenario"; then IS_LIVE=true ensure_server_built || { FAILED=$((FAILED + 1)); continue; } start_server || { FAILED=$((FAILED + 1)); continue; } export SR_LIVE=1 SR_PORT="$SERVER_PORT" fi # Capture set +e CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1) CAPTURE_RC=$? set -e # Stop server after capture (server exits on client disconnect anyway) if [[ "$IS_LIVE" == "true" ]]; then unset SR_LIVE SR_PORT 2>/dev/null || true stop_server fi CAPTURED="$CACHE_DIR/$scenario.png" if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then echo " FAIL: capture failed (exit $CAPTURE_RC)" >&2 echo "$CAPTURE_OUT" >&2 FAILED=$((FAILED + 1)) continue fi # Verify non-empty if [[ ! -s "$CAPTURED" ]]; then echo " FAIL: captured PNG is empty" >&2 FAILED=$((FAILED + 1)) continue fi # Verify the renderer actually drew something. # # "Non-empty file" was the only content check until 2026-08-06, and a blank # screen is a perfectly valid ~19 KB PNG. Worse, once a blank capture was # recorded as a golden, every later blank capture matched it at 0.0% and # the scenario PASSED — atlas_GJ338Bd_Block and atlas_GJ445c-m1_Chunk were # green against blank goldens. A test that cannot fail is worse than no # test, because it is counted as coverage. # # Checked in BOTH modes, and the update mode matters most: refusing to # RECORD a blank golden is what stops the trap being re-armed. set +e BLANK_OUT=$(reach --no-input visual blank-check "$CAPTURED" 2>&1) BLANK_RC=$? set -e if [[ $BLANK_RC -ne 0 ]]; then echo " FAIL: $BLANK_OUT" >&2 FAILED=$((FAILED + 1)) continue fi if [[ "$MODE" == "update" ]]; then cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png" echo " Updated golden: $GOLDEN_DIR/$scenario.png" PASSED=$((PASSED + 1)) else GOLDEN="$GOLDEN_DIR/$scenario.png" if [[ ! -f "$GOLDEN" ]]; then echo " FAIL: golden not found — run 'make visual-update' first" >&2 FAILED=$((FAILED + 1)) continue fi # Compare set +e DIFF_OUT=$(reach --no-input visual diff "$GOLDEN" "$CAPTURED" \ --tolerance "$TOLERANCE" \ --diff-output "$DIFF_DIR/$scenario-diff.png" \ --config "$CONFIG" 2>&1) DIFF_RC=$? set -e if [[ $DIFF_RC -eq 0 ]]; then echo " $DIFF_OUT" PASSED=$((PASSED + 1)) elif [[ $DIFF_RC -eq 1 ]]; then echo " $DIFF_OUT" echo " Diff image: $DIFF_DIR/$scenario-diff.png" FAILED=$((FAILED + 1)) else echo " ERROR: visual-diff failed (exit $DIFF_RC)" >&2 echo " $DIFF_OUT" >&2 FAILED=$((FAILED + 1)) fi fi done END_MS=$(date +%s%3N) DURATION_MS=$((END_MS - START_MS)) if [[ "$MODE" == "update" ]]; then # Stage golden files cd "$ROOT" git add "$GOLDEN_DIR/" 2>/dev/null || true echo "" echo "=== Visual goldens updated ($PASSED of $TOTAL) ===" echo "Review with: git diff --cached -- $GOLDEN_DIR/" fi printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \ "$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS" [[ $FAILED -gt 0 ]] && exit 1 exit 0