#!/usr/bin/env bash # tests/run-visual: Visual regression test suite. # Captures scenarios from tests/visual.json, compares against golden PNGs. # # Modes: # (no args) Run all scenario golden comparisons (xvfb-wrapped) # --screenshot NAME Ad-hoc single capture to .cache/screenshots/ (no xvfb) # --movie NAME Flow capture to .cache/screenshots/ (no xvfb) # --update Regenerate all goldens and stage for commit # --filter PATTERN Accepted and ignored (compat with run-all) # # Exit: 0=pass (or skip), 1=fail, 2=error # Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N} set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" CONFIG="$ROOT/tests/visual.json" GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)" CACHE_DIR="$ROOT/.cache/screenshots" DIFF_DIR="$ROOT/.cache/visual-diff" MODE="golden" # golden | screenshot | movie | update TARGET="" INTERVAL="" # Server state for live scenarios SERVER_BIN="" SERVER_PID="" SERVER_PORT="" # Cleanup server on exit trap '[[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || true' EXIT # -- Parse args ---------------------------------------------------------------- while [[ $# -gt 0 ]]; do case "$1" in --screenshot) MODE="screenshot"; TARGET="${2:-fog_3state}"; shift 2 ;; --movie) MODE="movie"; TARGET="${2:-flow_dialogue}"; shift 2 ;; --update) MODE="update"; shift ;; --filter) shift 2 ;; # Accept and ignore (run-all compat) --filter=*) shift ;; --interval) INTERVAL="${2:-3}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done # -- Preflight ----------------------------------------------------------------- if [[ -z "$GODOT" ]]; then echo "Warning: Godot not found — skipping visual tests" >&2 printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' exit 0 fi if ! python3 -c "pass" 2>/dev/null; then echo "Warning: Python 3 not found — skipping visual tests" >&2 printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' exit 0 fi if [[ ! -f "$CONFIG" ]]; then echo "Error: $CONFIG not found" >&2 exit 2 fi # Read config values GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")" TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")" RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")" # Read scenario names from config SCENARIOS=($(python3 -c " import json c = json.load(open('$CONFIG')) for name in c.get('scenarios', {}): print(name) ")) # -- Helpers ------------------------------------------------------------------- godot_capture() { local mode_flag="$1" # --scenario or --flow local name="$2" local output="$3" local extra_args=("${@:4}") "$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \ --path "$ROOT/client" -s res://tests/visual_capture.gd -- \ $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1 } xvfb_capture() { local mode_flag="$1" local name="$2" local output="$3" local extra_args=("${@:4}") # Try xvfb-run for deterministic captures if command -v xvfb-run >/dev/null 2>&1; then xvfb-run -a --server-args="-screen 0 ${RESOLUTION/x/x}x24" \ "$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \ --path "$ROOT/client" -s res://tests/visual_capture.gd -- \ $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1 else echo "Warning: xvfb-run not found — using visible window" >&2 godot_capture "$mode_flag" "$name" "$output" "${extra_args[@]}" fi } # -- Server lifecycle (live scenarios) ----------------------------------------- # Check if a scenario has "live": true in config is_live_scenario() { python3 -c " import json, sys c = json.load(open('${CONFIG}')) s = c.get('scenarios', {}).get('${1}', {}) sys.exit(0 if s.get('live') else 1) " } # Build server binary (once, cached) ensure_server_built() { if [[ -n "$SERVER_BIN" ]]; then return 0; fi echo " Building server for live visual tests..." (cd "$ROOT/server" && cargo build --bin settled-reach-server 2>&1) || { echo "Error: server build failed" >&2 return 1 } SERVER_BIN="$ROOT/server/target/debug/settled-reach-server" } # Start server with --test-mode --port 0, parse LISTENING:{port} start_server() { local stdout_log stdout_log=$(mktemp) "$SERVER_BIN" --test-mode --port 0 >"$stdout_log" 2>/dev/null & SERVER_PID=$! local attempts=0 while [[ $attempts -lt 150 ]]; do if ! kill -0 "$SERVER_PID" 2>/dev/null; then echo " Error: server exited unexpectedly" >&2 rm -f "$stdout_log" SERVER_PID="" return 1 fi if grep -q "^LISTENING:" "$stdout_log" 2>/dev/null; then SERVER_PORT=$(sed -n 's/^LISTENING://p' "$stdout_log") rm -f "$stdout_log" echo " Server started: pid=$SERVER_PID port=$SERVER_PORT" return 0 fi sleep 0.1 attempts=$((attempts + 1)) done echo " Error: no LISTENING signal after 15s" >&2 kill "$SERVER_PID" 2>/dev/null || true rm -f "$stdout_log" SERVER_PID="" return 1 } # Stop server (called after each live capture; server may have exited on disconnect) stop_server() { if [[ -n "$SERVER_PID" ]]; then kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true SERVER_PID="" SERVER_PORT="" fi } # -- Screenshot mode ----------------------------------------------------------- if [[ "$MODE" == "screenshot" ]]; then mkdir -p "$CACHE_DIR" echo "Capturing scenario: $TARGET" if is_live_scenario "$TARGET"; then ensure_server_built || exit 2 start_server || exit 2 export SR_LIVE=1 SR_PORT="$SERVER_PORT" fi godot_capture --scenario "$TARGET" "$CACHE_DIR" stop_server unset SR_LIVE SR_PORT 2>/dev/null || true PNG="$CACHE_DIR/$TARGET.png" if [[ -f "$PNG" ]]; then echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)" else echo "Error: capture failed — $PNG not found" >&2 exit 1 fi exit 0 fi # -- Movie mode ---------------------------------------------------------------- if [[ "$MODE" == "movie" ]]; then mkdir -p "$CACHE_DIR" echo "Capturing flow: $TARGET" EXTRA=() [[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL") godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}" FLOW_DIR="$CACHE_DIR/$TARGET" if [[ -d "$FLOW_DIR" ]]; then FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l) echo "Flow: $FRAME_COUNT frames in $FLOW_DIR" # Generate contact sheet if visual-thumbnail is available if [[ -x "$ROOT/tooling/visual-thumbnail" ]]; then "$ROOT/tooling/visual-thumbnail" "$FLOW_DIR" --config "$CONFIG" SHEET="$FLOW_DIR/${TARGET}_sheet.png" [[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET" fi else echo "Error: flow capture failed — $FLOW_DIR not found" >&2 exit 1 fi exit 0 fi # -- Golden mode (default) / Update mode -------------------------------------- START_MS=$(date +%s%3N) TOTAL=0 PASSED=0 FAILED=0 mkdir -p "$CACHE_DIR" "$DIFF_DIR" if [[ "$MODE" == "update" ]]; then mkdir -p "$GOLDEN_DIR" fi for scenario in "${SCENARIOS[@]}"; do TOTAL=$((TOTAL + 1)) echo "--- $scenario ---" # Start server for live scenarios IS_LIVE=false if is_live_scenario "$scenario"; then IS_LIVE=true ensure_server_built || { FAILED=$((FAILED + 1)); continue; } start_server || { FAILED=$((FAILED + 1)); continue; } export SR_LIVE=1 SR_PORT="$SERVER_PORT" fi # Capture set +e CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1) CAPTURE_RC=$? set -e # Stop server after capture (server exits on client disconnect anyway) if [[ "$IS_LIVE" == "true" ]]; then unset SR_LIVE SR_PORT 2>/dev/null || true stop_server fi CAPTURED="$CACHE_DIR/$scenario.png" if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then echo " FAIL: capture failed (exit $CAPTURE_RC)" >&2 echo "$CAPTURE_OUT" >&2 FAILED=$((FAILED + 1)) continue fi # Verify non-empty if [[ ! -s "$CAPTURED" ]]; then echo " FAIL: captured PNG is empty" >&2 FAILED=$((FAILED + 1)) continue fi if [[ "$MODE" == "update" ]]; then cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png" echo " Updated golden: $GOLDEN_DIR/$scenario.png" PASSED=$((PASSED + 1)) else GOLDEN="$GOLDEN_DIR/$scenario.png" if [[ ! -f "$GOLDEN" ]]; then echo " FAIL: golden not found — run 'make visual-update' first" >&2 FAILED=$((FAILED + 1)) continue fi # Compare set +e DIFF_OUT=$("$ROOT/tooling/visual-diff" "$GOLDEN" "$CAPTURED" \ --tolerance "$TOLERANCE" \ --diff-output "$DIFF_DIR/$scenario-diff.png" \ --config "$CONFIG" 2>&1) DIFF_RC=$? set -e if [[ $DIFF_RC -eq 0 ]]; then echo " $DIFF_OUT" PASSED=$((PASSED + 1)) elif [[ $DIFF_RC -eq 1 ]]; then echo " $DIFF_OUT" echo " Diff image: $DIFF_DIR/$scenario-diff.png" FAILED=$((FAILED + 1)) else echo " ERROR: visual-diff failed (exit $DIFF_RC)" >&2 echo " $DIFF_OUT" >&2 FAILED=$((FAILED + 1)) fi fi done END_MS=$(date +%s%3N) DURATION_MS=$((END_MS - START_MS)) if [[ "$MODE" == "update" ]]; then # Stage golden files cd "$ROOT" git add "$GOLDEN_DIR/" 2>/dev/null || true echo "" echo "=== Visual goldens updated ($PASSED of $TOTAL) ===" echo "Review with: git diff --cached -- $GOLDEN_DIR/" fi printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \ "$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS" [[ $FAILED -gt 0 ]] && exit 1 exit 0