feat(client): add visual test harness with golden regression
Gives Claude eyes: `make screenshot` captures a rendered frame, `make test-visual` compares against golden PNGs, `make visual-update` regenerates goldens. Built to debug the Sprint 22 fog regression and prevent future visual regressions across fog, HUD, dialogue, and UI. Config-driven via tests/visual.json (11 scenarios, 2 flows). Capture engine boots main.tscn with real GPU rendering (not --headless), waits for NoiseTexture2D async gen, uses deterministic shader time. Components: - visual_capture.gd: SceneTree capture engine (scenario + movie modes) - visual_scenarios.gd: per-scenario setup hooks - tooling/visual-diff: pixel comparator (PIL primary, struct fallback) - tooling/visual-thumbnail: contact sheet + crop tool - tests/run-visual: suite script (xvfb wrapping, golden workflow) - fog_state.gd: override_time for deterministic captures - fog_shader.gd: fog_noise_ready signal for settle sequencing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -25,6 +25,7 @@ SUITES=(
|
||||
run-ipc-fixtures
|
||||
run-ipc-protocol
|
||||
run-ipc-integration
|
||||
run-visual
|
||||
)
|
||||
|
||||
START_MS=$(date +%s%3N)
|
||||
|
||||
Executable
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env bash
|
||||
# tests/run-visual: Visual regression test suite.
|
||||
# Captures scenarios from tests/visual.json, compares against golden PNGs.
|
||||
#
|
||||
# Modes:
|
||||
# (no args) Run all scenario golden comparisons (xvfb-wrapped)
|
||||
# --screenshot NAME Ad-hoc single capture to .cache/screenshots/ (no xvfb)
|
||||
# --movie NAME Flow capture to .cache/screenshots/ (no xvfb)
|
||||
# --update Regenerate all goldens and stage for commit
|
||||
# --filter PATTERN Accepted and ignored (compat with run-all)
|
||||
#
|
||||
# Exit: 0=pass (or skip), 1=fail, 2=error
|
||||
# Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N}
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CONFIG="$ROOT/tests/visual.json"
|
||||
|
||||
GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)"
|
||||
CACHE_DIR="$ROOT/.cache/screenshots"
|
||||
DIFF_DIR="$ROOT/.cache/visual-diff"
|
||||
|
||||
MODE="golden" # golden | screenshot | movie | update
|
||||
TARGET=""
|
||||
INTERVAL=""
|
||||
|
||||
# -- Parse args ----------------------------------------------------------------
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--screenshot) MODE="screenshot"; TARGET="${2:-fog_3state}"; shift 2 ;;
|
||||
--movie) MODE="movie"; TARGET="${2:-flow_dialogue}"; shift 2 ;;
|
||||
--update) MODE="update"; shift ;;
|
||||
--filter) shift 2 ;; # Accept and ignore (run-all compat)
|
||||
--filter=*) shift ;;
|
||||
--interval) INTERVAL="${2:-3}"; shift 2 ;;
|
||||
*) echo "Unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# -- Preflight -----------------------------------------------------------------
|
||||
|
||||
if [[ -z "$GODOT" ]]; then
|
||||
echo "Warning: Godot not found — skipping visual tests" >&2
|
||||
printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! python3 -c "pass" 2>/dev/null; then
|
||||
echo "Warning: Python 3 not found — skipping visual tests" >&2
|
||||
printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -f "$CONFIG" ]]; then
|
||||
echo "Error: $CONFIG not found" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Read config values
|
||||
GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")"
|
||||
TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")"
|
||||
RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")"
|
||||
|
||||
# Read scenario names from config
|
||||
SCENARIOS=($(python3 -c "
|
||||
import json
|
||||
c = json.load(open('$CONFIG'))
|
||||
for name in c.get('scenarios', {}):
|
||||
print(name)
|
||||
"))
|
||||
|
||||
# -- Helpers -------------------------------------------------------------------
|
||||
|
||||
godot_capture() {
|
||||
local mode_flag="$1" # --scenario or --flow
|
||||
local name="$2"
|
||||
local output="$3"
|
||||
local extra_args=("${@:4}")
|
||||
|
||||
"$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \
|
||||
--path "$ROOT/client" -s res://tests/visual_capture.gd -- \
|
||||
$mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1
|
||||
}
|
||||
|
||||
xvfb_capture() {
|
||||
local mode_flag="$1"
|
||||
local name="$2"
|
||||
local output="$3"
|
||||
local extra_args=("${@:4}")
|
||||
|
||||
# Try xvfb-run for deterministic captures
|
||||
if command -v xvfb-run >/dev/null 2>&1; then
|
||||
xvfb-run -a --server-args="-screen 0 ${RESOLUTION/x/x}x24" \
|
||||
"$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \
|
||||
--path "$ROOT/client" -s res://tests/visual_capture.gd -- \
|
||||
$mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1
|
||||
else
|
||||
echo "Warning: xvfb-run not found — using visible window" >&2
|
||||
godot_capture "$mode_flag" "$name" "$output" "${extra_args[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
# -- Screenshot mode -----------------------------------------------------------
|
||||
|
||||
if [[ "$MODE" == "screenshot" ]]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
echo "Capturing scenario: $TARGET"
|
||||
godot_capture --scenario "$TARGET" "$CACHE_DIR"
|
||||
PNG="$CACHE_DIR/$TARGET.png"
|
||||
if [[ -f "$PNG" ]]; then
|
||||
echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)"
|
||||
else
|
||||
echo "Error: capture failed — $PNG not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -- Movie mode ----------------------------------------------------------------
|
||||
|
||||
if [[ "$MODE" == "movie" ]]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
echo "Capturing flow: $TARGET"
|
||||
EXTRA=()
|
||||
[[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL")
|
||||
godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}"
|
||||
|
||||
FLOW_DIR="$CACHE_DIR/$TARGET"
|
||||
if [[ -d "$FLOW_DIR" ]]; then
|
||||
FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l)
|
||||
echo "Flow: $FRAME_COUNT frames in $FLOW_DIR"
|
||||
|
||||
# Generate contact sheet if visual-thumbnail is available
|
||||
if [[ -x "$ROOT/tooling/visual-thumbnail" ]]; then
|
||||
"$ROOT/tooling/visual-thumbnail" "$FLOW_DIR" --config "$CONFIG"
|
||||
SHEET="$FLOW_DIR/${TARGET}_sheet.png"
|
||||
[[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET"
|
||||
fi
|
||||
else
|
||||
echo "Error: flow capture failed — $FLOW_DIR not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -- Golden mode (default) / Update mode --------------------------------------
|
||||
|
||||
START_MS=$(date +%s%3N)
|
||||
TOTAL=0
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
|
||||
mkdir -p "$CACHE_DIR" "$DIFF_DIR"
|
||||
|
||||
if [[ "$MODE" == "update" ]]; then
|
||||
mkdir -p "$GOLDEN_DIR"
|
||||
fi
|
||||
|
||||
for scenario in "${SCENARIOS[@]}"; do
|
||||
TOTAL=$((TOTAL + 1))
|
||||
echo "--- $scenario ---"
|
||||
|
||||
# Capture
|
||||
set +e
|
||||
CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1)
|
||||
CAPTURE_RC=$?
|
||||
set -e
|
||||
|
||||
CAPTURED="$CACHE_DIR/$scenario.png"
|
||||
|
||||
if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then
|
||||
echo " FAIL: capture failed (exit $CAPTURE_RC)" >&2
|
||||
echo "$CAPTURE_OUT" >&2
|
||||
FAILED=$((FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify non-empty
|
||||
if [[ ! -s "$CAPTURED" ]]; then
|
||||
echo " FAIL: captured PNG is empty" >&2
|
||||
FAILED=$((FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$MODE" == "update" ]]; then
|
||||
cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png"
|
||||
echo " Updated golden: $GOLDEN_DIR/$scenario.png"
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
GOLDEN="$GOLDEN_DIR/$scenario.png"
|
||||
if [[ ! -f "$GOLDEN" ]]; then
|
||||
echo " FAIL: golden not found — run 'make visual-update' first" >&2
|
||||
FAILED=$((FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Compare
|
||||
set +e
|
||||
DIFF_OUT=$("$ROOT/tooling/visual-diff" "$GOLDEN" "$CAPTURED" \
|
||||
--tolerance "$TOLERANCE" \
|
||||
--diff-output "$DIFF_DIR/$scenario-diff.png" \
|
||||
--config "$CONFIG" 2>&1)
|
||||
DIFF_RC=$?
|
||||
set -e
|
||||
|
||||
if [[ $DIFF_RC -eq 0 ]]; then
|
||||
echo " $DIFF_OUT"
|
||||
PASSED=$((PASSED + 1))
|
||||
elif [[ $DIFF_RC -eq 1 ]]; then
|
||||
echo " $DIFF_OUT"
|
||||
echo " Diff image: $DIFF_DIR/$scenario-diff.png"
|
||||
FAILED=$((FAILED + 1))
|
||||
else
|
||||
echo " ERROR: visual-diff failed (exit $DIFF_RC)" >&2
|
||||
echo " $DIFF_OUT" >&2
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
END_MS=$(date +%s%3N)
|
||||
DURATION_MS=$((END_MS - START_MS))
|
||||
|
||||
if [[ "$MODE" == "update" ]]; then
|
||||
# Stage golden files
|
||||
cd "$ROOT"
|
||||
git add "$GOLDEN_DIR/" 2>/dev/null || true
|
||||
echo ""
|
||||
echo "=== Visual goldens updated ($PASSED of $TOTAL) ==="
|
||||
echo "Review with: git diff --cached -- $GOLDEN_DIR/"
|
||||
fi
|
||||
|
||||
printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \
|
||||
"$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS"
|
||||
|
||||
[[ $FAILED -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"resolution": [960, 540],
|
||||
"settle_frames": 30,
|
||||
"tolerance": 5,
|
||||
"golden_dir": "client/tests/golden/visual",
|
||||
"scenarios": {
|
||||
"fog_3state": {
|
||||
"ticks": 5,
|
||||
"description": "All 3 fog states — clear/explored/unexplored"
|
||||
},
|
||||
"fog_diagonal": {
|
||||
"ticks": 10,
|
||||
"description": "Diagonal LOS boundary (staircase regression target)"
|
||||
},
|
||||
"fog_boundary": {
|
||||
"ticks": 8,
|
||||
"description": "Explored/unexplored transition centered in frame"
|
||||
},
|
||||
"fog_zone_tint": {
|
||||
"ticks": 5,
|
||||
"description": "Bar zone warm amber tint in explored fog (D-046)"
|
||||
},
|
||||
"fog_debug": {
|
||||
"ticks": 5,
|
||||
"description": "Raw exploration overlay (green/blue/red)"
|
||||
},
|
||||
"npc_in_fog": {
|
||||
"ticks": 5,
|
||||
"description": "NPC sprite visible above fog overlay (D-033)"
|
||||
},
|
||||
"hud_default": {
|
||||
"ticks": 1,
|
||||
"description": "Full HUD: time, Careful stance (green), minimap, monologue"
|
||||
},
|
||||
"dialogue_open": {
|
||||
"ticks": 3,
|
||||
"description": "Dialogue box + interaction prompt + key bar positioning"
|
||||
},
|
||||
"dialogue_with_monologue": {
|
||||
"ticks": 4,
|
||||
"description": "Confrontation beat: dialogue + monologue simultaneously"
|
||||
},
|
||||
"minimap_stance": {
|
||||
"ticks": 2,
|
||||
"description": "Minimap + Sprint stance indicator overlap (top-right)"
|
||||
},
|
||||
"cursor_menu": {
|
||||
"ticks": 3,
|
||||
"description": "Cursor rendering over dialogue option"
|
||||
}
|
||||
},
|
||||
"flows": {
|
||||
"flow_dialogue": {
|
||||
"interval": 3,
|
||||
"steps": [
|
||||
{ "action": "MoveSouth", "label": "approaching NPC" },
|
||||
{ "action": "Interact", "label": "dialogue opens" },
|
||||
{ "action": "wait", "label": "reading options" },
|
||||
{ "action": "select_option_1", "label": "option selected" },
|
||||
{ "action": "wait", "label": "NPC response" },
|
||||
{ "action": "MoveNorth", "label": "walking away" }
|
||||
]
|
||||
},
|
||||
"flow_explore": {
|
||||
"interval": 3,
|
||||
"steps": [
|
||||
{ "action": "MoveNorth", "label": "moving north" },
|
||||
{ "action": "MoveNorth", "label": "fog revealing" },
|
||||
{ "action": "MoveNorth", "label": "near wall" },
|
||||
{ "action": "MoveSouth", "label": "fog hiding" }
|
||||
]
|
||||
}
|
||||
},
|
||||
"crops": {
|
||||
"top-right": [760, 0, 200, 200],
|
||||
"top-left": [0, 0, 250, 150],
|
||||
"bottom": [0, 340, 960, 200],
|
||||
"full": [0, 0, 960, 540]
|
||||
},
|
||||
"thumbnail": {
|
||||
"width": 240,
|
||||
"height": 135,
|
||||
"columns": 4
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user