jpmschweitzerandClaude Opus 5
a384ec0c7c
reach godot parse-sweep / cold-parse, reach visual diff / blank-check / thumbnail. Five scripts retired, and the callers rewired — tests/run-visual invoked three of them by path at four sites, which is a wider blast radius than the make targets were. The godot pair were grep pipelines encoding five hard-won lessons as comments nobody could test. They are Python filters now, with the reasons attached, and the engine invocation is a guarded exec. Verified on the real client: 229 scripts, clean. Their three not-ok states stay distinct, because only one is a verdict about the code. An engine that crashed or is missing is not a parse failure — reporting it as one blames the tree for a broken toolchain. A sweep that emitted no completion marker checked nothing, and zero errors from a check that never ran reads as clean, which is the false-green the sweep exists to close. The deliberate asymmetry between the two checks is preserved and documented: cold-parse filters "Cannot infer the type", the sweep does not, because that suppression is why cold-parse stayed silent about a helper that genuinely does not parse. All three visual scripts carried the same root bug as validate-checklist: Path(__file__).parent.parent, correct at tooling/ and two levels too deep at tooling/domains/visual. Fixed during the move rather than after, having learned that it fails silently — paths resolve to nothing, the work appears to have nothing to do, and the tool reports success. Three domains now where that would have shipped a false pass. Two bugs my own transformation introduced, both found by running rather than reading. Multi-line print(..., file=sys.stderr) became console.event(..., file=sys.stderr), and console puts unknown kwargs into the payload — a file object would have reached json.dumps at the exact moment something was already being reported as an error. And the replacement script wrote escaped quotes into three files. Mechanical transformations need mechanical verification. sys.exit removed from four sites: a service must not end the process. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
410 lines
15 KiB
Bash
Executable File
410 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# tests/run-visual: Visual regression test suite.
|
|
# Captures scenarios from tests/visual.json, compares against golden PNGs.
|
|
#
|
|
# Modes:
|
|
# (no args) Run all scenario golden comparisons (xvfb-wrapped)
|
|
# --screenshot NAME Ad-hoc single capture to .cache/screenshots/ (no xvfb)
|
|
# --movie NAME Flow capture to .cache/screenshots/ (no xvfb)
|
|
# --update Regenerate all goldens and stage for commit
|
|
# --filter PATTERN Accepted and ignored (compat with run-all)
|
|
#
|
|
# Exit: 0=pass (or skip), 1=fail, 2=error
|
|
# Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N}
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
CONFIG="$ROOT/tests/visual.json"
|
|
|
|
GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)"
|
|
CACHE_DIR="$ROOT/.cache/screenshots"
|
|
DIFF_DIR="$ROOT/.cache/visual-diff"
|
|
|
|
MODE="golden" # golden | screenshot | movie | update
|
|
TARGET=""
|
|
INTERVAL=""
|
|
|
|
# Server state for live scenarios
|
|
SERVER_BIN=""
|
|
SERVER_PID=""
|
|
SERVER_PORT=""
|
|
|
|
# Cleanup server on exit
|
|
trap '[[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || true' EXIT
|
|
|
|
# -- Parse args ----------------------------------------------------------------
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--screenshot) MODE="screenshot"; TARGET="${2:-atlas_GJ820Bc_Global}"; shift 2 ;;
|
|
--movie) MODE="movie"; TARGET="${2:-flow_dialogue}"; shift 2 ;;
|
|
--update) MODE="update"; shift ;;
|
|
--filter) shift 2 ;; # Accept and ignore (run-all compat)
|
|
--filter=*) shift ;;
|
|
--interval) INTERVAL="${2:-3}"; shift 2 ;;
|
|
# Override the config resolution for ONE ad-hoc capture (T-1239). The
|
|
# Atlas derives canvas extent, and therefore metres-per-gridunit, from
|
|
# the viewport — so "does this defect depend on resolution?" is a real
|
|
# diagnostic question, and answering it by hand-editing the committed
|
|
# tests/visual.json invites leaving it edited. Goldens are shot at the
|
|
# config resolution; this flag deliberately does not touch them.
|
|
--resolution) RESOLUTION_OVERRIDE="${2:-}"; shift 2 ;;
|
|
*) echo "Unknown argument: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
# -- Preflight -----------------------------------------------------------------
|
|
|
|
if [[ -z "$GODOT" ]]; then
|
|
echo "Warning: Godot not found — skipping visual tests" >&2
|
|
printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
|
|
exit 0
|
|
fi
|
|
|
|
if ! python3 -c "pass" 2>/dev/null; then
|
|
echo "Warning: Python 3 not found — skipping visual tests" >&2
|
|
printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ! -f "$CONFIG" ]]; then
|
|
echo "Error: $CONFIG not found" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Read config values
|
|
GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")"
|
|
TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")"
|
|
RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")"
|
|
if [[ -n "${RESOLUTION_OVERRIDE:-}" ]]; then
|
|
RESOLUTION="$RESOLUTION_OVERRIDE"
|
|
echo "Note: resolution overridden to $RESOLUTION (goldens are shot at the config resolution)" >&2
|
|
fi
|
|
|
|
# Read scenario names from config
|
|
SCENARIOS=($(python3 -c "
|
|
import json
|
|
c = json.load(open('$CONFIG'))
|
|
for name in c.get('scenarios', {}):
|
|
print(name)
|
|
"))
|
|
|
|
# -- Helpers -------------------------------------------------------------------
|
|
|
|
# Offscreen wrapper for EVERY capture path.
|
|
#
|
|
# Captures must never steal the desktop: this box is also the developer's own
|
|
# session. Until 2026-08-06 only the golden path even tried, via xvfb-run —
|
|
# which is not installed here, so it hit the "using visible window" fallback,
|
|
# and --screenshot/--movie never wrapped at all. Every capture grabbed focus.
|
|
#
|
|
# gamescope is preferred, for two independent reasons:
|
|
# 1. It renders on the REAL GPU. The goldens are pinned to this box's
|
|
# Mesa/AMD output (T-1121 — they do not port across rendering stacks), and
|
|
# xvfb-run would fall back to llvmpipe software rendering, shifting every
|
|
# pixel. Offscreen must not mean a different renderer.
|
|
# 2. It is the only option that lets us SET the output size. cage is also
|
|
# installed and also GPU-backed, but it is a kiosk compositor: it forces
|
|
# its client to the headless output's default 1280x720 and silently
|
|
# overrides --resolution (measured — a 960x540 request produced a 1280x720
|
|
# PNG). A wrapper that quietly changes resolution is worse than none,
|
|
# because RESOLUTION is load-bearing here: D-255's extent inversion makes
|
|
# the shorter viewport axis span exactly one cell of the rung, so window
|
|
# size decides how much world a rung shows. cage is deliberately not used.
|
|
CAPTURE_PREFIX=()
|
|
if command -v gamescope >/dev/null 2>&1; then
|
|
CAPTURE_PREFIX=(gamescope --backend headless -W "${RESOLUTION%x*}" -H "${RESOLUTION#*x}" --)
|
|
elif command -v xvfb-run >/dev/null 2>&1; then
|
|
echo "Note: gamescope not found — falling back to xvfb-run (software GL; goldens may drift)" >&2
|
|
CAPTURE_PREFIX=(xvfb-run -a --server-args="-screen 0 ${RESOLUTION}x24")
|
|
else
|
|
echo "Warning: no offscreen compositor (gamescope/xvfb-run) — capturing in a VISIBLE window" >&2
|
|
fi
|
|
|
|
godot_capture() {
|
|
local mode_flag="$1" # --scenario or --flow
|
|
local name="$2"
|
|
local output="$3"
|
|
local extra_args=("${@:4}")
|
|
|
|
# Isolate `user://` per run (T-1239). Godot resolves user:// under
|
|
# XDG_DATA_HOME, which is how the Atlas disk cache (D-255, T-1183) persisted
|
|
# across captures — including across the server changes that made its
|
|
# contents wrong. A capture then rendered a canvas generated by a build that
|
|
# no longer existed: Ferrath Global replayed a pre-T-1237 canvas from weeks
|
|
# earlier and showed 375 hop-fragments where the live server produces 73
|
|
# whole rivers, and every golden shot in that window silently inherited it.
|
|
# A visual test must exercise the code in the tree, so the cache it warms
|
|
# must not outlive the run. The dir is recreated fresh each capture; the
|
|
# user's real cache at ~/.local/share/godot is never touched.
|
|
local user_data="$ROOT/.cache/visual-user-data"
|
|
rm -rf "$user_data"
|
|
mkdir -p "$user_data"
|
|
|
|
XDG_DATA_HOME="$user_data" \
|
|
"${CAPTURE_PREFIX[@]}" "$GODOT" --rendering-driver opengl3 --fixed-fps 60 \
|
|
--resolution "$RESOLUTION" \
|
|
--path "$ROOT/client" -s res://tests/visual_capture.gd -- \
|
|
$mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1
|
|
}
|
|
|
|
# Retained as the golden-path name; offscreen handling now lives in
|
|
# godot_capture, so both paths get it.
|
|
xvfb_capture() {
|
|
godot_capture "$@"
|
|
}
|
|
|
|
# -- Server lifecycle (live scenarios) -----------------------------------------
|
|
|
|
# Check if a scenario has "live": true in config
|
|
is_live_scenario() {
|
|
python3 -c "
|
|
import json, sys
|
|
c = json.load(open('${CONFIG}'))
|
|
s = c.get('scenarios', {}).get('${1}', {})
|
|
sys.exit(0 if s.get('live') else 1)
|
|
"
|
|
}
|
|
|
|
# Build server binary (once, cached)
|
|
ensure_server_built() {
|
|
if [[ -n "$SERVER_BIN" ]]; then return 0; fi
|
|
echo " Building server for live visual tests..."
|
|
(cd "$ROOT/server" && cargo build --bin settled-reach-server 2>&1) || {
|
|
echo "Error: server build failed" >&2
|
|
return 1
|
|
}
|
|
SERVER_BIN="$ROOT/server/target/debug/settled-reach-server"
|
|
}
|
|
|
|
# Start server with --test-mode --port 0, parse LISTENING:{port}
|
|
start_server() {
|
|
# Keep the server's own output (T-1239). This was `mktemp` + `2>/dev/null`
|
|
# + `rm` as soon as LISTENING was parsed, which meant two things: every
|
|
# tracing::warn!/error! the simulation emitted was discarded, and everything
|
|
# after startup went to an unlinked file. A server quietly degrading mid-
|
|
# capture looked identical to a healthy one — the capture only ever showed
|
|
# what the CLIENT thought it received. Now stderr is merged in and the log
|
|
# lives at a stable path that survives the run for inspection.
|
|
local stdout_log="$ROOT/.cache/visual-server.log"
|
|
mkdir -p "$ROOT/.cache"
|
|
"$SERVER_BIN" --test-mode --port 0 >"$stdout_log" 2>&1 &
|
|
SERVER_PID=$!
|
|
|
|
local attempts=0
|
|
while [[ $attempts -lt 150 ]]; do
|
|
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
echo " Error: server exited unexpectedly — see $stdout_log" >&2
|
|
SERVER_PID=""
|
|
return 1
|
|
fi
|
|
if grep -q "^LISTENING:" "$stdout_log" 2>/dev/null; then
|
|
SERVER_PORT=$(sed -n 's/^LISTENING://p' "$stdout_log")
|
|
echo " Server started: pid=$SERVER_PID port=$SERVER_PORT log=$stdout_log"
|
|
return 0
|
|
fi
|
|
sleep 0.1
|
|
attempts=$((attempts + 1))
|
|
done
|
|
|
|
echo " Error: no LISTENING signal after 15s — see $stdout_log" >&2
|
|
kill "$SERVER_PID" 2>/dev/null || true
|
|
SERVER_PID=""
|
|
return 1
|
|
}
|
|
|
|
# Stop server (called after each live capture; server may have exited on disconnect)
|
|
stop_server() {
|
|
if [[ -n "$SERVER_PID" ]]; then
|
|
kill "$SERVER_PID" 2>/dev/null || true
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
SERVER_PID=""
|
|
SERVER_PORT=""
|
|
fi
|
|
}
|
|
|
|
# -- Screenshot mode -----------------------------------------------------------
|
|
|
|
if [[ "$MODE" == "screenshot" ]]; then
|
|
mkdir -p "$CACHE_DIR"
|
|
echo "Capturing scenario: $TARGET"
|
|
if is_live_scenario "$TARGET"; then
|
|
ensure_server_built || exit 2
|
|
start_server || exit 2
|
|
export SR_LIVE=1 SR_PORT="$SERVER_PORT"
|
|
fi
|
|
godot_capture --scenario "$TARGET" "$CACHE_DIR"
|
|
stop_server
|
|
unset SR_LIVE SR_PORT 2>/dev/null || true
|
|
PNG="$CACHE_DIR/$TARGET.png"
|
|
if [[ -f "$PNG" ]]; then
|
|
echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)"
|
|
# Advisory here rather than fatal — an ad-hoc capture of a rung that
|
|
# genuinely renders nothing is a legitimate thing to want to look at
|
|
# (that is how the empty deep rungs were found). But say so out loud,
|
|
# because file size alone reads as success.
|
|
reach --no-input visual blank-check "$PNG" --quiet || true
|
|
else
|
|
echo "Error: capture failed — $PNG not found" >&2
|
|
exit 1
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# -- Movie mode ----------------------------------------------------------------
|
|
|
|
if [[ "$MODE" == "movie" ]]; then
|
|
mkdir -p "$CACHE_DIR"
|
|
echo "Capturing flow: $TARGET"
|
|
EXTRA=()
|
|
[[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL")
|
|
godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}"
|
|
|
|
FLOW_DIR="$CACHE_DIR/$TARGET"
|
|
if [[ -d "$FLOW_DIR" ]]; then
|
|
FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l)
|
|
echo "Flow: $FRAME_COUNT frames in $FLOW_DIR"
|
|
|
|
# Generate contact sheet if visual-thumbnail is available
|
|
if command -v reach >/dev/null 2>&1; then
|
|
reach --no-input visual thumbnail "$FLOW_DIR" --config "$CONFIG"
|
|
SHEET="$FLOW_DIR/${TARGET}_sheet.png"
|
|
[[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET"
|
|
fi
|
|
else
|
|
echo "Error: flow capture failed — $FLOW_DIR not found" >&2
|
|
exit 1
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# -- Golden mode (default) / Update mode --------------------------------------
|
|
|
|
START_MS=$(date +%s%3N)
|
|
TOTAL=0
|
|
PASSED=0
|
|
FAILED=0
|
|
|
|
mkdir -p "$CACHE_DIR" "$DIFF_DIR"
|
|
|
|
if [[ "$MODE" == "update" ]]; then
|
|
mkdir -p "$GOLDEN_DIR"
|
|
fi
|
|
|
|
for scenario in "${SCENARIOS[@]}"; do
|
|
TOTAL=$((TOTAL + 1))
|
|
echo "--- $scenario ---"
|
|
|
|
# Start server for live scenarios
|
|
IS_LIVE=false
|
|
if is_live_scenario "$scenario"; then
|
|
IS_LIVE=true
|
|
ensure_server_built || { FAILED=$((FAILED + 1)); continue; }
|
|
start_server || { FAILED=$((FAILED + 1)); continue; }
|
|
export SR_LIVE=1 SR_PORT="$SERVER_PORT"
|
|
fi
|
|
|
|
# Capture
|
|
set +e
|
|
CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1)
|
|
CAPTURE_RC=$?
|
|
set -e
|
|
|
|
# Stop server after capture (server exits on client disconnect anyway)
|
|
if [[ "$IS_LIVE" == "true" ]]; then
|
|
unset SR_LIVE SR_PORT 2>/dev/null || true
|
|
stop_server
|
|
fi
|
|
|
|
CAPTURED="$CACHE_DIR/$scenario.png"
|
|
|
|
if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then
|
|
echo " FAIL: capture failed (exit $CAPTURE_RC)" >&2
|
|
echo "$CAPTURE_OUT" >&2
|
|
FAILED=$((FAILED + 1))
|
|
continue
|
|
fi
|
|
|
|
# Verify non-empty
|
|
if [[ ! -s "$CAPTURED" ]]; then
|
|
echo " FAIL: captured PNG is empty" >&2
|
|
FAILED=$((FAILED + 1))
|
|
continue
|
|
fi
|
|
|
|
# Verify the renderer actually drew something.
|
|
#
|
|
# "Non-empty file" was the only content check until 2026-08-06, and a blank
|
|
# screen is a perfectly valid ~19 KB PNG. Worse, once a blank capture was
|
|
# recorded as a golden, every later blank capture matched it at 0.0% and
|
|
# the scenario PASSED — atlas_GJ338Bd_Block and atlas_GJ445c-m1_Chunk were
|
|
# green against blank goldens. A test that cannot fail is worse than no
|
|
# test, because it is counted as coverage.
|
|
#
|
|
# Checked in BOTH modes, and the update mode matters most: refusing to
|
|
# RECORD a blank golden is what stops the trap being re-armed.
|
|
set +e
|
|
BLANK_OUT=$(reach --no-input visual blank-check "$CAPTURED" 2>&1)
|
|
BLANK_RC=$?
|
|
set -e
|
|
if [[ $BLANK_RC -ne 0 ]]; then
|
|
echo " FAIL: $BLANK_OUT" >&2
|
|
FAILED=$((FAILED + 1))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$MODE" == "update" ]]; then
|
|
cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png"
|
|
echo " Updated golden: $GOLDEN_DIR/$scenario.png"
|
|
PASSED=$((PASSED + 1))
|
|
else
|
|
GOLDEN="$GOLDEN_DIR/$scenario.png"
|
|
if [[ ! -f "$GOLDEN" ]]; then
|
|
echo " FAIL: golden not found — run 'make visual-update' first" >&2
|
|
FAILED=$((FAILED + 1))
|
|
continue
|
|
fi
|
|
|
|
# Compare
|
|
set +e
|
|
DIFF_OUT=$(reach --no-input visual diff "$GOLDEN" "$CAPTURED" \
|
|
--tolerance "$TOLERANCE" \
|
|
--diff-output "$DIFF_DIR/$scenario-diff.png" \
|
|
--config "$CONFIG" 2>&1)
|
|
DIFF_RC=$?
|
|
set -e
|
|
|
|
if [[ $DIFF_RC -eq 0 ]]; then
|
|
echo " $DIFF_OUT"
|
|
PASSED=$((PASSED + 1))
|
|
elif [[ $DIFF_RC -eq 1 ]]; then
|
|
echo " $DIFF_OUT"
|
|
echo " Diff image: $DIFF_DIR/$scenario-diff.png"
|
|
FAILED=$((FAILED + 1))
|
|
else
|
|
echo " ERROR: visual-diff failed (exit $DIFF_RC)" >&2
|
|
echo " $DIFF_OUT" >&2
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
fi
|
|
done
|
|
|
|
END_MS=$(date +%s%3N)
|
|
DURATION_MS=$((END_MS - START_MS))
|
|
|
|
if [[ "$MODE" == "update" ]]; then
|
|
# Stage golden files
|
|
cd "$ROOT"
|
|
git add "$GOLDEN_DIR/" 2>/dev/null || true
|
|
echo ""
|
|
echo "=== Visual goldens updated ($PASSED of $TOTAL) ==="
|
|
echo "Review with: git diff --cached -- $GOLDEN_DIR/"
|
|
fi
|
|
|
|
printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \
|
|
"$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS"
|
|
|
|
[[ $FAILED -gt 0 ]] && exit 1
|
|
exit 0
|