From 6bcdc484124695dd737d12626b9947fa14a85c01 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Sun, 1 Mar 2026 16:33:55 +0100 Subject: [PATCH] feat(client): add visual test harness with golden regression Gives Claude eyes: `make screenshot` captures a rendered frame, `make test-visual` compares against golden PNGs, `make visual-update` regenerates goldens. Built to debug the Sprint 22 fog regression and prevent future visual regressions across fog, HUD, dialogue, and UI. Config-driven via tests/visual.json (11 scenarios, 2 flows). Capture engine boots main.tscn with real GPU rendering (not --headless), waits for NoiseTexture2D async gen, uses deterministic shader time. Components: - visual_capture.gd: SceneTree capture engine (scenario + movie modes) - visual_scenarios.gd: per-scenario setup hooks - tooling/visual-diff: pixel comparator (PIL primary, struct fallback) - tooling/visual-thumbnail: contact sheet + crop tool - tests/run-visual: suite script (xvfb wrapping, golden workflow) - fog_state.gd: override_time for deterministic captures - fog_shader.gd: fog_noise_ready signal for settle sequencing Co-Authored-By: Claude Opus 4.6 --- Makefile | 22 +- client/scripts/autoloads/fog_state.gd | 5 + client/scripts/rendering/fog_shader.gd | 9 +- client/tests/visual_capture.gd | 270 ++++++++++++++++++++ client/tests/visual_scenarios.gd | 143 +++++++++++ tests/run-all | 1 + tests/run-visual | 239 ++++++++++++++++++ tests/visual.json | 85 +++++++ tooling/visual-diff | 335 +++++++++++++++++++++++++ tooling/visual-thumbnail | 217 ++++++++++++++++ 10 files changed, 1323 insertions(+), 3 deletions(-) create mode 100644 client/tests/visual_capture.gd create mode 100644 client/tests/visual_scenarios.gd create mode 100755 tests/run-visual create mode 100644 tests/visual.json create mode 100755 tooling/visual-diff create mode 100755 tooling/visual-thumbnail diff --git a/Makefile b/Makefile index 772b8ffb0..e8080ffe9 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,8 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null) fixtures-client golden-diff golden-update \ checklist-validate checklist-generate \ perf-baseline debug-schedule \ - test-ipc-fixtures test-ipc-protocol test-ipc-integration test-ipc-benchmark + test-ipc-fixtures test-ipc-protocol test-ipc-integration test-ipc-benchmark \ + screenshot visual-movie test-visual visual-update # --- Configuration --- @@ -53,6 +54,11 @@ help: @echo " make checklist-generate Validate checklists + print condition summary" @echo " make perf-baseline Run performance benchmarks and save baseline" @echo "" + @echo " make screenshot Ad-hoc visual capture (SCENARIO=name, default: fog_3state)" + @echo " make visual-movie Flow capture with contact sheet (FLOW=name)" + @echo " make test-visual Run visual golden regression tests" + @echo " make visual-update Regenerate visual goldens and stage for commit" + @echo "" @echo " make pre-pr Run all pre-PR checks (lint, build, test, validate, fixtures)" @echo " make pre-pr-server Server-scoped pre-PR (lint, build, test, fixtures)" @echo " make pre-pr-client Client-scoped pre-PR (lint, build, test)" @@ -318,6 +324,20 @@ debug-schedule: @echo "Dumping bevy_ecs schedule graph..." @cd server && cargo run --bin settled-reach-server -- --dump-schedule +# --- Visual test harness --- + +screenshot: + @tests/run-visual --screenshot $(SCENARIO) + +visual-movie: + @tests/run-visual --movie $(FLOW) + +test-visual: + @tests/run-visual + +visual-update: + @tests/run-visual --update + content-ron: cd tooling/content-converter && cargo build --release tooling/content-converter/target/release/content-converter --input content --output content-ron --verbose diff --git a/client/scripts/autoloads/fog_state.gd b/client/scripts/autoloads/fog_state.gd index 7ae8ec65a..32252f5d5 100644 --- a/client/scripts/autoloads/fog_state.gd +++ b/client/scripts/autoloads/fog_state.gd @@ -52,6 +52,11 @@ var _prev_visible: Dictionary = {} # Tiles visible last frame (for incremental ## Toggle via FogState.debug_exploration = true in the console. var debug_exploration: bool = false +## Deterministic shader time for visual test captures. +## When >= 0, fog_shader.gd uses this instead of Time.get_ticks_msec(). +## Set before settle frames so noise phase is reproducible across runs. +var override_time: float = -1.0 + func _ready() -> void: _resize(Rect2i(0, 0, 64, 64)) diff --git a/client/scripts/rendering/fog_shader.gd b/client/scripts/rendering/fog_shader.gd index 97586aeb9..24b423fe0 100644 --- a/client/scripts/rendering/fog_shader.gd +++ b/client/scripts/rendering/fog_shader.gd @@ -4,6 +4,9 @@ extends Node2D ## Reads textures from FogState autoload, positions rect to cover viewport. ## Architecture: docs/architecture/fog-shader-spec.md +signal fog_noise_ready +var _noise_ready: bool = false + var _fog_rect: ColorRect var _shader_mat: ShaderMaterial @@ -36,10 +39,11 @@ func _ready() -> void: noise_tex.width = 256 noise_tex.height = 256 noise_tex.seamless = true + noise_tex.changed.connect(func(): _noise_ready = true; fog_noise_ready.emit()) _shader_mat.set_shader_parameter("noise_tex", noise_tex) _shader_mat.set_shader_parameter("tile_size", TILE_SIZE) - print("FogShader: Initialized (D-059 5-layer)") + print("FogShader: Initialized (D-059 3-state)") func update_fog() -> void: @@ -68,5 +72,6 @@ func update_fog() -> void: _shader_mat.set_shader_parameter("rect_sz", _fog_rect.size) _shader_mat.set_shader_parameter("map_offset", Vector2(FogState.map_bounds.position)) _shader_mat.set_shader_parameter("map_size", Vector2(FogState.map_bounds.size)) - _shader_mat.set_shader_parameter("time", Time.get_ticks_msec() / 1000.0) + var t: float = FogState.override_time if FogState.override_time >= 0.0 else Time.get_ticks_msec() / 1000.0 + _shader_mat.set_shader_parameter("time", t) _shader_mat.set_shader_parameter("debug_exploration", FogState.debug_exploration) diff --git a/client/tests/visual_capture.gd b/client/tests/visual_capture.gd new file mode 100644 index 000000000..8e9f4550e --- /dev/null +++ b/client/tests/visual_capture.gd @@ -0,0 +1,270 @@ +## Visual test capture engine — boots main.tscn, runs a named scenario or flow, +## captures viewport PNGs for golden comparison or ad-hoc inspection. +## +## Usage: +## godot --rendering-driver opengl3 --fixed-fps 60 --resolution 960x540 \ +## --path client -s res://tests/visual_capture.gd -- \ +## --scenario fog_3state --output /abs/path +## +## godot ... -- --flow flow_dialogue --output /abs/path --interval 3 +## godot ... -- --list +## +## Follows gen_client_fixtures.gd pattern: extends SceneTree, _init -> _run, quit(). +## Requires a real rendering driver (not --headless) for shader execution. +extends SceneTree + +# Load VisualScenarios explicitly — class_name registry not populated in -s mode. +# Must create an instance: static methods on loaded GDScript resources aren't +# callable via variable (only via class_name, which isn't available in -s mode). +var _scenarios: RefCounted = null + +var _output_dir: String = "" +var _scenario: String = "" +var _flow: String = "" +var _interval: float = 3.0 +var _list_mode: bool = false +var _config: Dictionary = {} + + +func _init(): + _run.call_deferred() + + +func _run(): + # Parse CLI args (after -- separator) + _parse_args() + + # Load scenarios script and create instance (class_name not available in -s mode) + var scenarios_script: GDScript = load("res://tests/visual_scenarios.gd") + _scenarios = scenarios_script.new() + + # Load config + _config = _load_config() + if _config.is_empty(): + push_error("visual_capture: failed to load tests/visual.json") + quit(1) + return + + # List mode: print scenario/flow names and exit + if _list_mode: + _print_list() + quit() + return + + # Validate mode + if _scenario.is_empty() and _flow.is_empty(): + push_error("visual_capture: specify --scenario NAME or --flow NAME (or --list)") + quit(1) + return + + if _output_dir.is_empty(): + push_error("visual_capture: --output DIR is required") + quit(1) + return + + # Ensure output directory exists + DirAccess.make_dir_recursive_absolute(_output_dir) + + # Load main scene (autoloads already initialized from project.godot) + var main_scene = load("res://scenes/main.tscn") + if main_scene == null: + push_error("visual_capture: failed to load res://scenes/main.tscn") + quit(1) + return + + var main_node = main_scene.instantiate() + root.add_child(main_node) + + # Wait for NoiseTexture2D async generation + var fog_overlay = main_node.get_node_or_null("World/FogOverlay") + if fog_overlay and not fog_overlay._noise_ready: + print("visual_capture: waiting for fog noise texture...") + await fog_overlay.fog_noise_ready + print("visual_capture: fog noise ready") + + # Set deterministic shader time BEFORE settle frames + # Autoload globals not available as identifiers in -s mode — use tree path. + root.get_node("/root/FogState").override_time = 1.0 + + # Settle frames (camera smoothing, fog uniform init, UI layout) + var settle_count: int = _config.get("settle_frames", 30) + print("visual_capture: settling %d frames..." % settle_count) + for i in range(settle_count): + await process_frame + + if not _scenario.is_empty(): + await _run_scenario(main_node) + elif not _flow.is_empty(): + await _run_flow(main_node) + + quit() + + +func _run_scenario(_main_node: Node) -> void: + var scenarios: Dictionary = _config.get("scenarios", {}) + if not scenarios.has(_scenario): + push_error("visual_capture: unknown scenario '%s'" % _scenario) + quit(1) + return + + var scenario_cfg: Dictionary = scenarios[_scenario] + var ticks: int = scenario_cfg.get("ticks", 5) + + print("visual_capture: scenario '%s' — %d ticks" % [_scenario, ticks]) + + # Apply scenario setup hook (pass root for autoload access) + if not _scenarios.apply_setup(_scenario, root): + push_warning("visual_capture: no setup hook for '%s'" % _scenario) + + # Advance simulation ticks (each process_frame polls one snapshot) + for i in range(ticks): + await process_frame + + # Post-tick setup (e.g. zone tint patching) + _scenarios.post_setup(_scenario, root) + + # Extra frames for state propagation + viewport texture lag + await process_frame + await process_frame + + # Capture + await _capture("%s/%s.png" % [_output_dir, _scenario]) + + +func _run_flow(_main_node: Node) -> void: + var flows: Dictionary = _config.get("flows", {}) + if not flows.has(_flow): + push_error("visual_capture: unknown flow '%s'" % _flow) + quit(1) + return + + var flow_cfg: Dictionary = flows[_flow] + var steps: Array = flow_cfg.get("steps", []) + var interval: float = flow_cfg.get("interval", _interval) + + print("visual_capture: flow '%s' — %d steps, %.1fs interval" % [_flow, steps.size(), interval]) + + # Create output subdirectory for flow frames + var flow_dir := "%s/%s" % [_output_dir, _flow] + DirAccess.make_dir_recursive_absolute(flow_dir) + + var manifest_lines: PackedStringArray = [] + var frame_idx: int = 0 + + for step in steps: + if not step is Dictionary: + continue + var action: String = step.get("action", "wait") + var label: String = step.get("label", action) + + # Apply action + _scenarios.apply_flow_action(action, root) + + # Wait for interval (timer yields back to main loop — frames keep rendering) + await create_timer(interval).timeout + + # Extra frames for viewport texture to catch up + await process_frame + await process_frame + + # Capture frame + var frame_name := "%s_%03d.png" % [_flow, frame_idx] + await _capture("%s/%s" % [flow_dir, frame_name]) + + # Build manifest line + var total_seconds: int = int((frame_idx + 1) * interval) + var timecode := "%d:%02d" % [total_seconds / 60, total_seconds % 60] + manifest_lines.append("%03d %s %s" % [frame_idx, timecode, label]) + + frame_idx += 1 + + # Write manifest sidecar + var manifest_path := "%s/%s_manifest.txt" % [flow_dir, _flow] + var f := FileAccess.open(manifest_path, FileAccess.WRITE) + if f: + f.store_string("\n".join(manifest_lines) + "\n") + f.close() + print("visual_capture: manifest -> %s" % manifest_path) + + +func _capture(path: String) -> void: + # Wait for GPU to finish rendering current frame + await RenderingServer.frame_post_draw + + var image := root.get_viewport().get_texture().get_image() + if image == null: + push_error("visual_capture: viewport image is null — GPU rendering may not be available") + quit(1) + return + + var err := image.save_png(path) + if err != OK: + push_error("visual_capture: failed to save PNG to %s (error %d)" % [path, err]) + quit(1) + return + + print("visual_capture: captured -> %s (%dx%d)" % [path, image.get_width(), image.get_height()]) + + +func _load_config() -> Dictionary: + # Config is at tests/visual.json relative to repo root. + # Repo root = parent of Godot project root (client/). + var project_root := ProjectSettings.globalize_path("res://") + var repo_root := project_root.rstrip("/").get_base_dir() + var config_path := repo_root.path_join("tests/visual.json") + + var f := FileAccess.open(config_path, FileAccess.READ) + if f == null: + push_error("visual_capture: cannot open %s" % config_path) + return {} + + var json_text := f.get_as_text() + f.close() + + var parsed = JSON.parse_string(json_text) + if parsed == null or not parsed is Dictionary: + push_error("visual_capture: invalid JSON in %s" % config_path) + return {} + + return parsed + + +func _parse_args() -> void: + var args := OS.get_cmdline_user_args() + var i := 0 + while i < args.size(): + match args[i]: + "--scenario": + i += 1 + if i < args.size(): + _scenario = args[i] + "--flow": + i += 1 + if i < args.size(): + _flow = args[i] + "--output": + i += 1 + if i < args.size(): + _output_dir = args[i] + "--interval": + i += 1 + if i < args.size(): + _interval = float(args[i]) + "--list": + _list_mode = true + _: + push_warning("visual_capture: unknown arg '%s'" % args[i]) + i += 1 + + +func _print_list() -> void: + var scenarios: Dictionary = _config.get("scenarios", {}) + var flows: Dictionary = _config.get("flows", {}) + print("scenarios:") + for name in scenarios: + var desc: String = scenarios[name].get("description", "") + print(" %s — %s" % [name, desc]) + print("flows:") + for name in flows: + var steps: Array = flows[name].get("steps", []) + print(" %s — %d steps" % [name, steps.size()]) diff --git a/client/tests/visual_scenarios.gd b/client/tests/visual_scenarios.gd new file mode 100644 index 000000000..0732fe0f4 --- /dev/null +++ b/client/tests/visual_scenarios.gd @@ -0,0 +1,143 @@ +extends RefCounted +## Setup hooks for visual test scenarios. +## Config (tests/visual.json) controls WHAT to run; this file controls HOW to +## set up each scenario in-engine. Called by visual_capture.gd after main.tscn +## is loaded and settled. +## +## Autoload globals (FogState, SimBridge, GameState) are NOT available as +## compile-time identifiers in -s mode. All methods receive tree_root and +## look up autoloads via get_node("/root/..."). + +## Apply scenario-specific setup before tick advancement. +## Returns true if the scenario is recognized, false otherwise. +func apply_setup(scenario_name: String, tree_root: Node) -> bool: + var sim_bridge := tree_root.get_node("/root/SimBridge") + var fog_state := tree_root.get_node("/root/FogState") + var game_state := tree_root.get_node("/root/GameState") + + match scenario_name: + "fog_3state": + # Default TestHarness state — player at (10,10), 8x8 room. + # All 3 fog states naturally visible: clear (cone), explored (out of cone), unexplored. + pass + + "fog_diagonal": + # Diagonal LOS boundary — staircase regression target. + # Default room works: the diamond-shaped visibility radius creates diagonal edges. + # 10 ticks builds enough explored tiles to show the boundary. + pass + + "fog_boundary": + # Explored/unexplored transition centered. + # Move player south so the northern explored boundary is center-frame. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(10, 12) + + "fog_zone_tint": + # Bar zone warm amber tint (D-046). + # Inject zone_id into visible_tiles so fog_state.gd writes bar tint. + # TestHarness tiles don't include zone_id by default — patch them. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(10, 10) + # Zone tint is written from visible_tiles with zone_id field. + # We patch GameState.visible_tiles after the first snapshot in post_setup(). + pass + + "fog_debug": + # Raw exploration overlay (green/blue/red). + fog_state.debug_exploration = true + + "npc_in_fog": + # NPC sprite visible above fog (D-033). + # Position player so NPC at (12,9) is at the fog edge. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(10, 10) + + "hud_default": + # Full HUD: time, Careful stance, minimap, monologue. + # Tick 1 triggers monologue from TestHarness. + # Inject Careful stance for visible green indicator. + game_state.player_stance = "Careful" + + "dialogue_open": + # Dialogue box + interaction prompt + key bar. + # Move adjacent to NPC and interact. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(11, 9) + sim_bridge.harness.process_input("Interact") + + "dialogue_with_monologue": + # Confrontation beat: dialogue + monologue overlap. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(11, 9) + sim_bridge.harness.process_input("Interact") + # Monologue will be injected in post_setup after dialogue is active. + + "minimap_stance": + # Minimap + Sprint stance indicator overlap (top-right). + game_state.player_stance = "Sprint" + + "cursor_menu": + # Cursor rendering over dialogue option area. + if sim_bridge.harness: + sim_bridge.harness.player_pos = Vector2i(11, 9) + sim_bridge.harness.process_input("Interact") + + _: + push_warning("VisualScenarios: unknown scenario '%s'" % scenario_name) + return false + + return true + + +## Post-tick setup — called after N ticks have been advanced. +## Use for state that depends on tick processing (e.g. zone tint patching). +func post_setup(scenario_name: String, tree_root: Node) -> void: + var fog_state := tree_root.get_node("/root/FogState") + var game_state := tree_root.get_node("/root/GameState") + + match scenario_name: + "fog_zone_tint": + # Patch visible_tiles with zone_id for fog_state.gd zone tint pipeline. + var patched: Array = [] + for tile in game_state.visible_tiles: + if tile is Dictionary: + var t: Dictionary = tile.duplicate() + t["zone_id"] = "bar" + patched.append(t) + game_state.visible_tiles = patched + # Force fog_state to re-process with zone tint data + fog_state.update_from_state() + + "dialogue_with_monologue": + # Inject monologue to create the overlap condition. + game_state.current_monologue = { + "id": "test_confrontation", + "text": "Something about that answer doesn't add up.", + "duration_seconds": 5.0, + "priority": "high", + "is_urgent": true, + } + + "hud_default": + # Ensure stance is still Careful after ticks (TestHarness may override). + game_state.player_stance = "Careful" + + +## Apply flow action — called for each step in a movie flow. +func apply_flow_action(action: String, tree_root: Node) -> void: + var sim_bridge := tree_root.get_node("/root/SimBridge") + + match action: + "wait": + pass # No-op — just captures current state + + "select_option_1": + # Simulate selecting the first dialogue option + if sim_bridge.harness: + sim_bridge.harness.process_input("DialogueOption1") + + _: + # Movement actions and Interact pass through to TestHarness + if sim_bridge.harness: + sim_bridge.harness.process_input(action) diff --git a/tests/run-all b/tests/run-all index b296cfb22..6d7706667 100755 --- a/tests/run-all +++ b/tests/run-all @@ -25,6 +25,7 @@ SUITES=( run-ipc-fixtures run-ipc-protocol run-ipc-integration + run-visual ) START_MS=$(date +%s%3N) diff --git a/tests/run-visual b/tests/run-visual new file mode 100755 index 000000000..24ee6fb32 --- /dev/null +++ b/tests/run-visual @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# tests/run-visual: Visual regression test suite. +# Captures scenarios from tests/visual.json, compares against golden PNGs. +# +# Modes: +# (no args) Run all scenario golden comparisons (xvfb-wrapped) +# --screenshot NAME Ad-hoc single capture to .cache/screenshots/ (no xvfb) +# --movie NAME Flow capture to .cache/screenshots/ (no xvfb) +# --update Regenerate all goldens and stage for commit +# --filter PATTERN Accepted and ignored (compat with run-all) +# +# Exit: 0=pass (or skip), 1=fail, 2=error +# Stdout (golden mode): {"suite":"visual","total":N,"passed":N,"failed":N,"duration_ms":N} +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG="$ROOT/tests/visual.json" + +GODOT="$(command -v godot4 2>/dev/null || command -v godot 2>/dev/null || true)" +CACHE_DIR="$ROOT/.cache/screenshots" +DIFF_DIR="$ROOT/.cache/visual-diff" + +MODE="golden" # golden | screenshot | movie | update +TARGET="" +INTERVAL="" + +# -- Parse args ---------------------------------------------------------------- + +while [[ $# -gt 0 ]]; do + case "$1" in + --screenshot) MODE="screenshot"; TARGET="${2:-fog_3state}"; shift 2 ;; + --movie) MODE="movie"; TARGET="${2:-flow_dialogue}"; shift 2 ;; + --update) MODE="update"; shift ;; + --filter) shift 2 ;; # Accept and ignore (run-all compat) + --filter=*) shift ;; + --interval) INTERVAL="${2:-3}"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +# -- Preflight ----------------------------------------------------------------- + +if [[ -z "$GODOT" ]]; then + echo "Warning: Godot not found — skipping visual tests" >&2 + printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' + exit 0 +fi + +if ! python3 -c "pass" 2>/dev/null; then + echo "Warning: Python 3 not found — skipping visual tests" >&2 + printf '{"suite":"visual","total":0,"passed":0,"failed":0,"skipped":1,"duration_ms":0}\n' + exit 0 +fi + +if [[ ! -f "$CONFIG" ]]; then + echo "Error: $CONFIG not found" >&2 + exit 2 +fi + +# Read config values +GOLDEN_DIR="$ROOT/$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('golden_dir','client/tests/golden/visual'))")" +TOLERANCE="$(python3 -c "import json; c=json.load(open('$CONFIG')); print(c.get('tolerance', 5))")" +RESOLUTION="$(python3 -c "import json; c=json.load(open('$CONFIG')); r=c.get('resolution',[960,540]); print(f'{r[0]}x{r[1]}')")" + +# Read scenario names from config +SCENARIOS=($(python3 -c " +import json +c = json.load(open('$CONFIG')) +for name in c.get('scenarios', {}): + print(name) +")) + +# -- Helpers ------------------------------------------------------------------- + +godot_capture() { + local mode_flag="$1" # --scenario or --flow + local name="$2" + local output="$3" + local extra_args=("${@:4}") + + "$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \ + --path "$ROOT/client" -s res://tests/visual_capture.gd -- \ + $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1 +} + +xvfb_capture() { + local mode_flag="$1" + local name="$2" + local output="$3" + local extra_args=("${@:4}") + + # Try xvfb-run for deterministic captures + if command -v xvfb-run >/dev/null 2>&1; then + xvfb-run -a --server-args="-screen 0 ${RESOLUTION/x/x}x24" \ + "$GODOT" --rendering-driver opengl3 --fixed-fps 60 --resolution "$RESOLUTION" \ + --path "$ROOT/client" -s res://tests/visual_capture.gd -- \ + $mode_flag "$name" --output "$output" "${extra_args[@]}" 2>&1 + else + echo "Warning: xvfb-run not found — using visible window" >&2 + godot_capture "$mode_flag" "$name" "$output" "${extra_args[@]}" + fi +} + +# -- Screenshot mode ----------------------------------------------------------- + +if [[ "$MODE" == "screenshot" ]]; then + mkdir -p "$CACHE_DIR" + echo "Capturing scenario: $TARGET" + godot_capture --scenario "$TARGET" "$CACHE_DIR" + PNG="$CACHE_DIR/$TARGET.png" + if [[ -f "$PNG" ]]; then + echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)" + else + echo "Error: capture failed — $PNG not found" >&2 + exit 1 + fi + exit 0 +fi + +# -- Movie mode ---------------------------------------------------------------- + +if [[ "$MODE" == "movie" ]]; then + mkdir -p "$CACHE_DIR" + echo "Capturing flow: $TARGET" + EXTRA=() + [[ -n "$INTERVAL" ]] && EXTRA+=(--interval "$INTERVAL") + godot_capture --flow "$TARGET" "$CACHE_DIR" "${EXTRA[@]}" + + FLOW_DIR="$CACHE_DIR/$TARGET" + if [[ -d "$FLOW_DIR" ]]; then + FRAME_COUNT=$(find "$FLOW_DIR" -name "*.png" | wc -l) + echo "Flow: $FRAME_COUNT frames in $FLOW_DIR" + + # Generate contact sheet if visual-thumbnail is available + if [[ -x "$ROOT/tooling/visual-thumbnail" ]]; then + "$ROOT/tooling/visual-thumbnail" "$FLOW_DIR" --config "$CONFIG" + SHEET="$FLOW_DIR/${TARGET}_sheet.png" + [[ -f "$SHEET" ]] && echo "Contact sheet: $SHEET" + fi + else + echo "Error: flow capture failed — $FLOW_DIR not found" >&2 + exit 1 + fi + exit 0 +fi + +# -- Golden mode (default) / Update mode -------------------------------------- + +START_MS=$(date +%s%3N) +TOTAL=0 +PASSED=0 +FAILED=0 + +mkdir -p "$CACHE_DIR" "$DIFF_DIR" + +if [[ "$MODE" == "update" ]]; then + mkdir -p "$GOLDEN_DIR" +fi + +for scenario in "${SCENARIOS[@]}"; do + TOTAL=$((TOTAL + 1)) + echo "--- $scenario ---" + + # Capture + set +e + CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1) + CAPTURE_RC=$? + set -e + + CAPTURED="$CACHE_DIR/$scenario.png" + + if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then + echo " FAIL: capture failed (exit $CAPTURE_RC)" >&2 + echo "$CAPTURE_OUT" >&2 + FAILED=$((FAILED + 1)) + continue + fi + + # Verify non-empty + if [[ ! -s "$CAPTURED" ]]; then + echo " FAIL: captured PNG is empty" >&2 + FAILED=$((FAILED + 1)) + continue + fi + + if [[ "$MODE" == "update" ]]; then + cp "$CAPTURED" "$GOLDEN_DIR/$scenario.png" + echo " Updated golden: $GOLDEN_DIR/$scenario.png" + PASSED=$((PASSED + 1)) + else + GOLDEN="$GOLDEN_DIR/$scenario.png" + if [[ ! -f "$GOLDEN" ]]; then + echo " FAIL: golden not found — run 'make visual-update' first" >&2 + FAILED=$((FAILED + 1)) + continue + fi + + # Compare + set +e + DIFF_OUT=$("$ROOT/tooling/visual-diff" "$GOLDEN" "$CAPTURED" \ + --tolerance "$TOLERANCE" \ + --diff-output "$DIFF_DIR/$scenario-diff.png" \ + --config "$CONFIG" 2>&1) + DIFF_RC=$? + set -e + + if [[ $DIFF_RC -eq 0 ]]; then + echo " $DIFF_OUT" + PASSED=$((PASSED + 1)) + elif [[ $DIFF_RC -eq 1 ]]; then + echo " $DIFF_OUT" + echo " Diff image: $DIFF_DIR/$scenario-diff.png" + FAILED=$((FAILED + 1)) + else + echo " ERROR: visual-diff failed (exit $DIFF_RC)" >&2 + echo " $DIFF_OUT" >&2 + FAILED=$((FAILED + 1)) + fi + fi +done + +END_MS=$(date +%s%3N) +DURATION_MS=$((END_MS - START_MS)) + +if [[ "$MODE" == "update" ]]; then + # Stage golden files + cd "$ROOT" + git add "$GOLDEN_DIR/" 2>/dev/null || true + echo "" + echo "=== Visual goldens updated ($PASSED of $TOTAL) ===" + echo "Review with: git diff --cached -- $GOLDEN_DIR/" +fi + +printf '{"suite":"visual","total":%d,"passed":%d,"failed":%d,"duration_ms":%d}\n' \ + "$TOTAL" "$PASSED" "$FAILED" "$DURATION_MS" + +[[ $FAILED -gt 0 ]] && exit 1 +exit 0 diff --git a/tests/visual.json b/tests/visual.json new file mode 100644 index 000000000..be5f52007 --- /dev/null +++ b/tests/visual.json @@ -0,0 +1,85 @@ +{ + "resolution": [960, 540], + "settle_frames": 30, + "tolerance": 5, + "golden_dir": "client/tests/golden/visual", + "scenarios": { + "fog_3state": { + "ticks": 5, + "description": "All 3 fog states — clear/explored/unexplored" + }, + "fog_diagonal": { + "ticks": 10, + "description": "Diagonal LOS boundary (staircase regression target)" + }, + "fog_boundary": { + "ticks": 8, + "description": "Explored/unexplored transition centered in frame" + }, + "fog_zone_tint": { + "ticks": 5, + "description": "Bar zone warm amber tint in explored fog (D-046)" + }, + "fog_debug": { + "ticks": 5, + "description": "Raw exploration overlay (green/blue/red)" + }, + "npc_in_fog": { + "ticks": 5, + "description": "NPC sprite visible above fog overlay (D-033)" + }, + "hud_default": { + "ticks": 1, + "description": "Full HUD: time, Careful stance (green), minimap, monologue" + }, + "dialogue_open": { + "ticks": 3, + "description": "Dialogue box + interaction prompt + key bar positioning" + }, + "dialogue_with_monologue": { + "ticks": 4, + "description": "Confrontation beat: dialogue + monologue simultaneously" + }, + "minimap_stance": { + "ticks": 2, + "description": "Minimap + Sprint stance indicator overlap (top-right)" + }, + "cursor_menu": { + "ticks": 3, + "description": "Cursor rendering over dialogue option" + } + }, + "flows": { + "flow_dialogue": { + "interval": 3, + "steps": [ + { "action": "MoveSouth", "label": "approaching NPC" }, + { "action": "Interact", "label": "dialogue opens" }, + { "action": "wait", "label": "reading options" }, + { "action": "select_option_1", "label": "option selected" }, + { "action": "wait", "label": "NPC response" }, + { "action": "MoveNorth", "label": "walking away" } + ] + }, + "flow_explore": { + "interval": 3, + "steps": [ + { "action": "MoveNorth", "label": "moving north" }, + { "action": "MoveNorth", "label": "fog revealing" }, + { "action": "MoveNorth", "label": "near wall" }, + { "action": "MoveSouth", "label": "fog hiding" } + ] + } + }, + "crops": { + "top-right": [760, 0, 200, 200], + "top-left": [0, 0, 250, 150], + "bottom": [0, 340, 960, 200], + "full": [0, 0, 960, 540] + }, + "thumbnail": { + "width": 240, + "height": 135, + "columns": 4 + } +} diff --git a/tooling/visual-diff b/tooling/visual-diff new file mode 100755 index 000000000..c1a9c7fa0 --- /dev/null +++ b/tooling/visual-diff @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Pixel-level visual diff for golden image comparison. + +Compares two PNG images per-channel with configurable tolerance. +Reads default tolerance from tests/visual.json if available. + +Usage: + tooling/visual-diff EXPECTED ACTUAL [--tolerance N] [--diff-output PATH] [--config PATH] + +Exit codes: + 0 = images match (all pixels within tolerance) + 1 = images differ + 2 = size mismatch or fatal error +""" + +import argparse +import json +import struct +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CONFIG = ROOT / "tests" / "visual.json" +DEFAULT_TOLERANCE = 5 + +# --------------------------------------------------------------------------- +# PNG reading — prefer PIL, fallback to pure stdlib +# --------------------------------------------------------------------------- + +_USE_PIL = False +try: + from PIL import Image as _PILImage + + _USE_PIL = True +except ImportError: + pass + + +def _read_png_pil(path: str) -> tuple[int, int, bytes]: + """Read PNG via Pillow, return (width, height, RGBA bytes).""" + img = _PILImage.open(path).convert("RGBA") + return img.width, img.height, img.tobytes() + + +def _paeth(a: int, b: int, c: int) -> int: + p = a + b - c + pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + if pa <= pb and pa <= pc: + return a + if pb <= pc: + return b + return c + + +def _read_png_stdlib(path: str) -> tuple[int, int, bytes]: + """Read an RGBA (color type 6) PNG using only struct + zlib. + + Handles multiple IDAT chunks and all 5 PNG filter types. + """ + with open(path, "rb") as f: + sig = f.read(8) + if sig != b"\x89PNG\r\n\x1a\n": + print(f"ERROR: {path} is not a valid PNG", file=sys.stderr) + sys.exit(2) + + width = height = 0 + bit_depth = color_type = 0 + idat_chunks: list[bytes] = [] + + while True: + header = f.read(8) + if len(header) < 8: + break + length, chunk_type = struct.unpack(">I4s", header) + data = f.read(length) + _crc = f.read(4) + + if chunk_type == b"IHDR": + width, height, bit_depth, color_type = struct.unpack( + ">IIBB", data[:10] + ) + if color_type != 6: + print( + f"ERROR: {path} has color type {color_type}, expected 6 (RGBA)", + file=sys.stderr, + ) + sys.exit(2) + if bit_depth != 8: + print( + f"ERROR: {path} has bit depth {bit_depth}, expected 8", + file=sys.stderr, + ) + sys.exit(2) + elif chunk_type == b"IDAT": + idat_chunks.append(data) + elif chunk_type == b"IEND": + break + + raw = zlib.decompress(b"".join(idat_chunks)) + + bpp = 4 # RGBA = 4 bytes per pixel + stride = width * bpp + pixels = bytearray(height * stride) + + pos = 0 + for y in range(height): + filter_type = raw[pos] + pos += 1 + row_start = y * stride + + for x in range(stride): + cur = raw[pos] + pos += 1 + + a = pixels[row_start + x - bpp] if x >= bpp else 0 + b = pixels[row_start - stride + x] if y > 0 else 0 + c = ( + pixels[row_start - stride + x - bpp] + if y > 0 and x >= bpp + else 0 + ) + + if filter_type == 0: # None + val = cur + elif filter_type == 1: # Sub + val = (cur + a) & 0xFF + elif filter_type == 2: # Up + val = (cur + b) & 0xFF + elif filter_type == 3: # Average + val = (cur + ((a + b) >> 1)) & 0xFF + elif filter_type == 4: # Paeth + val = (cur + _paeth(a, b, c)) & 0xFF + else: + print( + f"ERROR: unknown PNG filter type {filter_type} at row {y}", + file=sys.stderr, + ) + sys.exit(2) + + pixels[row_start + x] = val + + return width, height, bytes(pixels) + + +def read_png(path: str) -> tuple[int, int, bytes]: + """Read PNG, return (width, height, RGBA bytes).""" + if _USE_PIL: + return _read_png_pil(path) + return _read_png_stdlib(path) + + +# --------------------------------------------------------------------------- +# Diff PNG writing — prefer PIL, fallback to pure stdlib +# --------------------------------------------------------------------------- + + +def _write_png_pil(path: str, width: int, height: int, rgba: bytes) -> None: + img = _PILImage.frombytes("RGBA", (width, height), rgba) + img.save(path) + + +def _write_png_stdlib( + path: str, width: int, height: int, rgba: bytes +) -> None: + """Write a minimal RGBA PNG using zlib + struct (filter type 0/None).""" + + def _chunk(chunk_type: bytes, data: bytes) -> bytes: + crc = zlib.crc32(chunk_type + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + chunk_type + data + struct.pack(">I", crc) + + # IHDR: width, height, bit_depth=8, color_type=6, compress=0, filter=0, interlace=0 + ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0) + + # Build raw scanlines with filter byte 0 (None) per row + stride = width * 4 + raw = bytearray() + for y in range(height): + raw.append(0) # filter type None + offset = y * stride + raw.extend(rgba[offset : offset + stride]) + + compressed = zlib.compress(bytes(raw)) + + with open(path, "wb") as f: + f.write(b"\x89PNG\r\n\x1a\n") + f.write(_chunk(b"IHDR", ihdr_data)) + f.write(_chunk(b"IDAT", compressed)) + f.write(_chunk(b"IEND", b"")) + + +def write_png(path: str, width: int, height: int, rgba: bytes) -> None: + if _USE_PIL: + _write_png_pil(path, width, height, rgba) + else: + _write_png_stdlib(path, width, height, rgba) + + +# --------------------------------------------------------------------------- +# Comparison +# --------------------------------------------------------------------------- + + +def compare( + expected: bytes, + actual: bytes, + width: int, + height: int, + tolerance: int, +) -> tuple[int, bytes | None]: + """Compare two RGBA buffers. Returns (diff_count, diff_rgba_or_None).""" + total = width * height + diff_count = 0 + diff_buf = bytearray(total * 4) + + for i in range(total): + off = i * 4 + er, eg, eb, ea = expected[off], expected[off + 1], expected[off + 2], expected[off + 3] + ar, ag, ab, aa = actual[off], actual[off + 1], actual[off + 2], actual[off + 3] + + if ( + abs(er - ar) > tolerance + or abs(eg - ag) > tolerance + or abs(eb - ab) > tolerance + or abs(ea - aa) > tolerance + ): + diff_count += 1 + diff_buf[off] = 0xFF + diff_buf[off + 1] = 0x00 + diff_buf[off + 2] = 0x00 + diff_buf[off + 3] = 0xFF + # else: remains (0, 0, 0, 0) — transparent + + return diff_count, bytes(diff_buf) + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + + +def load_tolerance(config_path: Path | None) -> int: + """Read tolerance from config JSON, return DEFAULT_TOLERANCE on failure.""" + if config_path is None: + config_path = DEFAULT_CONFIG + if not config_path.exists(): + return DEFAULT_TOLERANCE + try: + with open(config_path) as f: + data = json.load(f) + return int(data.get("tolerance", DEFAULT_TOLERANCE)) + except (json.JSONDecodeError, ValueError, OSError): + return DEFAULT_TOLERANCE + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Pixel-level visual diff for golden image comparison." + ) + parser.add_argument("expected", help="Path to golden PNG") + parser.add_argument("actual", help="Path to captured PNG") + parser.add_argument( + "--tolerance", + type=int, + default=None, + help="Per-channel pixel tolerance (default: from config or 5)", + ) + parser.add_argument( + "--diff-output", + default=None, + help="Path to write diff PNG highlighting changed pixels", + ) + parser.add_argument( + "--config", + default=None, + help="Path to tests/visual.json (default: auto-detect)", + ) + args = parser.parse_args() + + # Resolve tolerance: CLI > config > fallback + config_path = Path(args.config) if args.config else None + tolerance = args.tolerance if args.tolerance is not None else load_tolerance(config_path) + + # Read images + try: + ew, eh, epx = read_png(args.expected) + except FileNotFoundError: + print(f"ERROR: expected image not found: {args.expected}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"ERROR: failed to read expected image: {exc}", file=sys.stderr) + return 2 + + try: + aw, ah, apx = read_png(args.actual) + except FileNotFoundError: + print(f"ERROR: actual image not found: {args.actual}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"ERROR: failed to read actual image: {exc}", file=sys.stderr) + return 2 + + # Size check + if ew != aw or eh != ah: + print( + f"ERROR: size mismatch — expected {ew}x{eh}, actual {aw}x{ah}", + file=sys.stderr, + ) + return 2 + + # Compare + diff_count, diff_buf = compare(epx, apx, ew, eh, tolerance) + total = ew * eh + + if diff_count == 0: + print(f"PASS: images match ({ew}x{eh})") + return 0 + + pct = diff_count / total * 100 + print(f"FAIL: {diff_count} of {total} pixels differ ({pct:.1f}%)") + + if args.diff_output and diff_buf: + Path(args.diff_output).parent.mkdir(parents=True, exist_ok=True) + write_png(args.diff_output, ew, eh, diff_buf) + + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tooling/visual-thumbnail b/tooling/visual-thumbnail new file mode 100755 index 000000000..f660eff2b --- /dev/null +++ b/tooling/visual-thumbnail @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Contact sheet and crop tool for visual QA flow captures. + +Two modes: + + Grid mode (default): + tooling/visual-thumbnail DIR [--config PATH] + Reads {flow}_{NNN}.png frames + {flow}_manifest.txt sidecar from DIR, + generates a contact sheet grid with timecodes and labels. + Output: DIR/{flow}_sheet.png + + Crop mode: + tooling/visual-thumbnail --crop REGION IMAGE [--config PATH] + Extracts a named region from IMAGE at 1:1 scale. + Output: IMAGE_crop_{REGION}.png + +Config: reads thumbnail dimensions, columns, and crop regions from +tests/visual.json (auto-detected from script location, or --config). +""" + +import argparse +import json +import sys +from pathlib import Path + +try: + from PIL import Image, ImageDraw +except ImportError: + print("visual-thumbnail requires Pillow: pip install Pillow", file=sys.stderr) + sys.exit(1) + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CONFIG = ROOT / "tests" / "visual.json" + +# Fallbacks when config keys are missing +DEFAULT_THUMB_WIDTH = 320 +DEFAULT_THUMB_HEIGHT = 180 +DEFAULT_COLUMNS = 4 +LABEL_HEIGHT = 24 # pixels reserved below each thumbnail for text + + +def load_config(config_path: Path) -> dict: + """Load configuration from JSON file.""" + if not config_path.exists(): + print(f"Config not found: {config_path}", file=sys.stderr) + print("Continuing with built-in defaults.", file=sys.stderr) + return {} + with open(config_path) as f: + return json.load(f) + + +def parse_manifest(manifest_path: Path) -> list[dict]: + """Parse a flow manifest file. + + Each line: NNN TIMECODE LABEL + Example: 001 0:03 dialogue opens + """ + entries = [] + with open(manifest_path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split(None, 2) + if len(parts) < 2: + continue + entry = { + "frame": parts[0], + "timecode": parts[1], + "label": parts[2] if len(parts) > 2 else "", + } + entries.append(entry) + return entries + + +def detect_flow(directory: Path) -> str | None: + """Detect flow name from manifest sidecar in directory.""" + manifests = list(directory.glob("*_manifest.txt")) + if len(manifests) == 1: + # {flow}_manifest.txt -> flow + stem = manifests[0].stem + return stem.removesuffix("_manifest") + if len(manifests) > 1: + print(f"Multiple manifests found in {directory}:", file=sys.stderr) + for m in manifests: + print(f" {m.name}", file=sys.stderr) + return None + return None + + +def grid_mode(directory: Path, config: dict) -> int: + """Generate a contact sheet from flow captures.""" + directory = directory.resolve() + if not directory.is_dir(): + print(f"Not a directory: {directory}", file=sys.stderr) + return 1 + + flow = detect_flow(directory) + if flow is None: + print(f"No manifest found in {directory}. Expected {{flow}}_manifest.txt", file=sys.stderr) + return 1 + + manifest_path = directory / f"{flow}_manifest.txt" + entries = parse_manifest(manifest_path) + if not entries: + print(f"Empty manifest: {manifest_path}", file=sys.stderr) + return 1 + + # Read thumbnail config + thumb_cfg = config.get("thumbnail", {}) + tw = thumb_cfg.get("width", DEFAULT_THUMB_WIDTH) + th = thumb_cfg.get("height", DEFAULT_THUMB_HEIGHT) + cols = thumb_cfg.get("columns", DEFAULT_COLUMNS) + + rows = (len(entries) + cols - 1) // cols + cell_h = th + LABEL_HEIGHT + + sheet_w = tw * cols + sheet_h = cell_h * rows + sheet = Image.new("RGB", (sheet_w, sheet_h), color=(30, 30, 30)) + draw = ImageDraw.Draw(sheet) + + for idx, entry in enumerate(entries): + frame_file = directory / f"{flow}_{entry['frame']}.png" + if not frame_file.exists(): + print(f" Missing frame: {frame_file.name}", file=sys.stderr) + continue + + img = Image.open(frame_file) + img.thumbnail((tw, th), Image.LANCZOS) + + col = idx % cols + row = idx // cols + x = col * tw + y = row * cell_h + + # Center thumbnail within its cell if it's smaller than tw x th + offset_x = x + (tw - img.width) // 2 + offset_y = y + (th - img.height) // 2 + sheet.paste(img, (offset_x, offset_y)) + + # Draw timecode + label below thumbnail + text = entry["timecode"] + if entry["label"]: + text += f" {entry['label']}" + text_y = y + th + 2 + draw.text((x + 4, text_y), text, fill=(200, 200, 200)) + + output_path = directory / f"{flow}_sheet.png" + sheet.save(output_path) + print(f"Sheet: {output_path}") + return 0 + + +def crop_mode(region_name: str, image_path: Path, config: dict) -> int: + """Extract a named crop region from an image at 1:1 scale.""" + image_path = image_path.resolve() + if not image_path.exists(): + print(f"Image not found: {image_path}", file=sys.stderr) + return 1 + + crops = config.get("crops", {}) + if region_name not in crops: + available = ", ".join(sorted(crops.keys())) if crops else "(none)" + print(f"Unknown crop region: {region_name}", file=sys.stderr) + print(f"Available regions: {available}", file=sys.stderr) + return 1 + + coords = crops[region_name] + if not isinstance(coords, list) or len(coords) != 4: + print(f"Invalid crop coords for '{region_name}': expected [x, y, w, h]", file=sys.stderr) + return 1 + + x, y, w, h = coords + img = Image.open(image_path) + cropped = img.crop((x, y, x + w, y + h)) + + stem = image_path.stem + suffix = image_path.suffix + output_path = image_path.parent / f"{stem}_crop_{region_name}{suffix}" + cropped.save(output_path) + print(output_path) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Contact sheet and crop tool for visual QA flow captures.", + ) + parser.add_argument( + "--config", type=Path, default=DEFAULT_CONFIG, + help=f"Config JSON path (default: {DEFAULT_CONFIG.relative_to(ROOT)})", + ) + + # Crop mode + parser.add_argument( + "--crop", metavar="REGION", + help="Crop mode: extract named region from IMAGE", + ) + + # Positional: DIR (grid mode) or IMAGE (crop mode) + parser.add_argument( + "target", type=Path, + help="Directory of flow captures (grid mode) or image file (crop mode)", + ) + + args = parser.parse_args() + config = load_config(args.config) + + if args.crop: + return crop_mode(args.crop, args.target, config) + else: + return grid_mode(args.target, config) + + +if __name__ == "__main__": + sys.exit(main())