feat(engine): live server visual tests and gauntlet snapshot replay

Add live server lifecycle to tests/run-visual (start/stop server per
scenario, parse LISTENING:{port}). Add MessagePack snapshot replay to
visual_capture.gd via Protocol.decode_snapshot() — exercises the full
client pipeline from wire bytes to rendered fog. Three replay scenarios
(hub_spawn, fog_theater, hub_after_movement) plus one live scenario
(fog_live_hub). Add gen_gauntlet_fixtures.rs to produce .msgpack fixtures
from the Gauntlet test world. Add max_diff_pct threshold to visual-diff.
Makefile: add fixtures-gauntlet target, fix build-client double-import,
preserve .godot cache in clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-05 08:41:09 +01:00
co-authored by Claude Opus 4.6
parent 2189b00c6f
commit ac763fef97
16 changed files with 9034 additions and 9 deletions
+8 -2
View File
@@ -5,7 +5,7 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null)
db-backup db-install validate-content content-ron check-fact-ids setup-hooks \
pre-pr pre-pr-lint pre-pr-build pre-pr-test pre-pr-validate pre-pr-fixtures \
pre-pr-server pre-pr-client pre-pr-content \
fixtures-client golden-diff golden-update \
fixtures-client fixtures-gauntlet golden-diff golden-update \
checklist-validate checklist-generate \
perf-baseline debug-schedule \
test-ipc-fixtures test-ipc-protocol test-ipc-integration test-ipc-benchmark \
@@ -110,6 +110,9 @@ build-server:
build-client:
@test -n "$(GODOT)" || { echo "Godot not found. Run 'make setup' first."; exit 1; }
@# First import may error on theme/font loading before the import scan completes.
@# Run twice: first pass generates imports silently, second pass validates clean.
@$(GODOT) --headless --path client --import --quit 2>/dev/null || true
$(GODOT) --headless --path client --import --quit
# --- Run ---
@@ -144,6 +147,9 @@ test-server:
fixtures:
cd server && cargo test --test gen_fixtures -- --ignored
fixtures-gauntlet:
cd server && cargo test --test gen_gauntlet_fixtures -- --ignored
fixtures-client:
@test -n "$(GODOT)" || { echo "Godot not found. Run 'make setup' first."; exit 1; }
@echo "Generating GDScript fixtures for Rust decoder..."
@@ -347,5 +353,5 @@ content-ron:
clean:
cd server && cargo clean || true
rm -rf .cache/*
rm -rf client/.godot/* client/reports
rm -rf client/reports
@echo "Clean complete."
+78
View File
@@ -24,6 +24,7 @@ var _flow: String = ""
var _interval: float = 3.0
var _list_mode: bool = false
var _config: Dictionary = {}
var _is_live: bool = false
func _init():
@@ -73,6 +74,19 @@ func _run():
return
var main_node = main_scene.instantiate()
# Live mode: configure server port BEFORE main.gd._ready() calls connect_to_sim()
_is_live = OS.get_environment("SR_LIVE") == "1"
if _is_live:
var sim_bridge := root.get_node("/root/SimBridge")
var port_env := OS.get_environment("SR_PORT")
if port_env.is_empty():
push_error("visual_capture: SR_LIVE=1 but SR_PORT not set")
quit(1)
return
sim_bridge.server_port = int(port_env)
print("visual_capture: live mode — server port %d" % sim_bridge.server_port)
root.add_child(main_node)
# Wait for NoiseTexture2D async generation
@@ -92,6 +106,36 @@ func _run():
for i in range(settle_count):
await process_frame
# Live mode: wait for server connection and first snapshot
if _is_live:
var sim_bridge := root.get_node("/root/SimBridge")
var game_state := root.get_node("/root/GameState")
print("visual_capture: waiting for server connection...")
var max_frames := 300 # 5 seconds at 60fps
var waited := 0
while sim_bridge.state != sim_bridge.ConnectionState.CONNECTED:
if sim_bridge.state == sim_bridge.ConnectionState.ERROR:
push_error("visual_capture: server connection failed")
quit(1)
return
await process_frame
waited += 1
if waited >= max_frames:
push_error("visual_capture: connection timeout after %d frames" % waited)
quit(1)
return
print("visual_capture: connected after %d frames" % waited)
# Wait for first snapshot from server
waited = 0
while game_state.current_tick == 0:
await process_frame
waited += 1
if waited >= max_frames:
push_error("visual_capture: no snapshot after %d frames" % waited)
quit(1)
return
print("visual_capture: first snapshot tick=%d (%d frames)" % [game_state.current_tick, waited])
if not _scenario.is_empty():
await _run_scenario(main_node)
elif not _flow.is_empty():
@@ -123,6 +167,40 @@ func _run_scenario(_main_node: Node) -> void:
# Post-tick setup (e.g. zone tint patching)
_scenarios.post_setup(_scenario, root)
# Replay snapshot: inject a real server snapshot through the FULL client pipeline.
# Loads MessagePack bytes (exact wire format from server), decodes via Protocol.gd,
# then applies through GameState → FogState → shader — same path as live game.
var replay_path: String = scenario_cfg.get("replay_snapshot", "")
if not replay_path.is_empty():
var project_root := ProjectSettings.globalize_path("res://")
var repo_root := project_root.rstrip("/").get_base_dir()
var abs_path := repo_root.path_join(replay_path)
var rf := FileAccess.open(abs_path, FileAccess.READ)
if rf == null:
push_error("visual_capture: cannot open replay snapshot %s" % abs_path)
quit(1)
return
var replay_bytes := rf.get_buffer(rf.get_length())
rf.close()
# Decode through Protocol.decode_snapshot() — same as live IPC receive path.
# This exercises: msgpack decode → entity decode → tile_kind→type mapping → etc.
var replay_data: Variant = Protocol.decode_snapshot(replay_bytes)
if replay_data == null or not replay_data is Dictionary:
push_error("visual_capture: Protocol.decode_snapshot failed for %s" % abs_path)
quit(1)
return
print("visual_capture: replaying %s (%d bytes, tick=%s, %d tiles)" % [
replay_path, replay_bytes.size(),
str(replay_data.get("tick", "?")),
replay_data.get("visible_tiles", []).size()])
var game_state := root.get_node("/root/GameState")
var fog_state := root.get_node("/root/FogState")
game_state.apply_snapshot(replay_data)
fog_state.update_from_state()
# Extra frames for fog uniform propagation
for i in range(4):
await process_frame
# Extra frames for state propagation + viewport texture lag
await process_frame
await process_frame
+1
View File
@@ -0,0 +1 @@
uid://pd1qpgxiodig
+11
View File
@@ -83,6 +83,17 @@ func apply_setup(scenario_name: String, tree_root: Node) -> bool:
sim_bridge.harness.player_pos = Vector2i(11, 9)
sim_bridge.harness.process_input("Interact")
"fog_live_replay", "fog_theater_replay", "fog_boundary_replay":
# Replay real server snapshots via MessagePack → Protocol.decode_snapshot().
# Setup handled by visual_capture.gd (reads replay_snapshot from config).
pass
"fog_live_hub":
# Live server connection — Hub spawn position.
# No setup needed: server starts in --test-mode with Gauntlet,
# player spawns at Hub (50,58). Captures real fog pipeline output.
pass
_:
push_warning("VisualScenarios: unknown scenario '%s'" % scenario_name)
return false
+1
View File
@@ -0,0 +1 @@
uid://dna10a0ln5pd0
+185
View File
@@ -0,0 +1,185 @@
//! Generate snapshot fixtures from the Gauntlet test world (D-030 full pipeline).
//!
//! Runs the full Gauntlet simulation pipeline — same code path as --test-mode —
//! then serializes ObserverSnapshots to both MessagePack (wire format) and JSON
//! (human-readable debug) for client-side visual testing.
//!
//! The .msgpack files are the actual wire-format bytes the server sends over IPC.
//! Client visual tests load them via Protocol.decode_snapshot() → GameState.apply_snapshot(),
//! exercising the exact same pipeline as the live game.
//!
//! The .json files are for human inspection only.
//!
//! Run with: cargo test --test gen_gauntlet_fixtures -- --ignored
//! Or: make fixtures-gauntlet
//!
//! Output: tests/fixtures/gauntlet/*.{msgpack,json} (relative to repo root)
use settled_reach_server::bridge::types::*;
use settled_reach_server::bridge::{BridgePlugin, SnapshotBuffer};
use settled_reach_server::knowledge::KnowledgePlugin;
use settled_reach_server::npc::NpcPlugin;
use settled_reach_server::perception::vision_cone::Facing;
use settled_reach_server::simulation::movement::{PlayerCharacter, TilePosition};
use settled_reach_server::simulation::rng::SimRng;
use settled_reach_server::simulation::SimulationPlugin;
use settled_reach_server::test_world;
use bevy_app::prelude::*;
use bevy_ecs::prelude::*;
use serde_json::Value;
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
const SEED: u64 = 42;
const FIXTURE_DIR: &str = "../tests/fixtures/gauntlet";
/// Build a deterministic Gauntlet simulation app.
/// Identical to what the server runs in --test-mode.
fn build_gauntlet(seed: u64) -> App {
let mut app = App::new();
app.add_plugins(SimulationPlugin);
app.add_plugins(BridgePlugin);
app.add_plugins(KnowledgePlugin);
app.add_plugins(NpcPlugin);
app.insert_resource(SimRng::new(seed));
test_world::setup_gauntlet(&mut app);
app
}
/// Teleport the player to a specific position and facing.
/// Directly modifies ECS components — same effect as PlayerAction::Teleport
/// but without needing a target room action.
fn teleport_player(app: &mut App, pos: TilePosition, facing: Facing) {
let player = {
let mut query = app
.world_mut()
.query_filtered::<Entity, With<PlayerCharacter>>();
query.single(app.world()).expect("player entity must exist")
};
app.world_mut()
.entity_mut(player)
.insert((pos, facing));
}
/// Run N ticks, feeding inputs each tick, return the last snapshot.
fn run_ticks(app: &mut App, inputs: &[Vec<PlayerInput>]) -> ObserverSnapshot {
let mut last_snapshot: Option<ObserverSnapshot> = None;
for tick_inputs in inputs {
{
let mut queue = app
.world_mut()
.resource_mut::<settled_reach_server::simulation::input::InputQueue>();
for input in tick_inputs {
queue.push(input.clone());
}
}
app.update();
let buffer = app.world().resource::<SnapshotBuffer>();
if let Some(snapshot) = &buffer.snapshot {
last_snapshot = Some(snapshot.clone());
}
}
last_snapshot.expect("no snapshot produced")
}
/// Recursively sort all object keys for deterministic JSON output.
fn sort_json_keys(value: &Value) -> Value {
match value {
Value::Object(map) => {
let sorted: BTreeMap<String, Value> = map
.iter()
.map(|(k, v)| (k.clone(), sort_json_keys(v)))
.collect();
Value::Object(sorted.into_iter().collect())
}
Value::Array(arr) => Value::Array(arr.iter().map(sort_json_keys).collect()),
other => other.clone(),
}
}
fn write_fixture(name: &str, snapshot: &ObserverSnapshot) {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_DIR);
fs::create_dir_all(&dir).expect("create fixture dir");
// MessagePack — exact wire-format bytes the server sends over IPC.
// Client visual tests load these via Protocol.decode_snapshot().
let msgpack = rmp_serde::to_vec_named(snapshot).expect("serialize to MessagePack");
let msgpack_path = dir.join(format!("{}.msgpack", name));
fs::write(&msgpack_path, &msgpack).expect("write msgpack fixture");
// JSON — human-readable debug companion (not loaded by tests).
let value: Value = serde_json::to_value(snapshot).expect("serialize to JSON");
let sorted = sort_json_keys(&value);
let json = serde_json::to_string_pretty(&sorted).expect("format JSON") + "\n";
let json_path = dir.join(format!("{}.json", name));
fs::write(&json_path, &json).expect("write json fixture");
eprintln!(
"Wrote {} ({} bytes msgpack, {} bytes json, {} visible_tiles, {} entities)",
name,
msgpack.len(),
json.len(),
snapshot.visible_tiles.len(),
snapshot.entities.len(),
);
}
#[test]
#[ignore] // Run manually: cargo test --test gen_gauntlet_fixtures -- --ignored
fn generate_gauntlet_snapshot_fixtures() {
// --- Hub (default spawn position) ---
// Player at (50, 58) facing North — tests fog rendering at the starting location.
// This is the exact state a new player sees on connect.
{
let mut app = build_gauntlet(SEED);
// Run 3 idle ticks to stabilize (cognitive delay, vision cone init)
let inputs: Vec<Vec<PlayerInput>> = vec![vec![], vec![], vec![]];
let snapshot = run_ticks(&mut app, &inputs);
write_fixture("hub_spawn", &snapshot);
}
// --- Fog Theater (observer position) ---
// Player at (56, 18) facing South — large open room with NPCs at varying distances.
// Tests visibility cone, fog layers, and distance-based fog rendering.
{
let mut app = build_gauntlet(SEED);
let fog_theater = test_world::constants::FOG_THEATER;
teleport_player(&mut app, fog_theater.observer, fog_theater.observer_facing);
let inputs: Vec<Vec<PlayerInput>> = vec![vec![], vec![], vec![]];
let snapshot = run_ticks(&mut app, &inputs);
write_fixture("fog_theater", &snapshot);
}
// --- Hub after movement (explored tiles + visible tiles differ) ---
// Player moves south from hub, creating a mix of explored-but-not-visible
// and currently-visible tiles — the exact fog boundary condition.
{
let mut app = build_gauntlet(SEED);
let inputs: Vec<Vec<PlayerInput>> = vec![
vec![],
vec![PlayerInput {
tick: 1,
action: PlayerAction::MoveSouth,
}],
vec![PlayerInput {
tick: 2,
action: PlayerAction::MoveSouth,
}],
vec![PlayerInput {
tick: 3,
action: PlayerAction::MoveSouth,
}],
vec![], // idle — snapshot has explored + visible tiles that differ
];
let snapshot = run_ticks(&mut app, &inputs);
write_fixture("hub_after_movement", &snapshot);
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
Binary file not shown.
File diff suppressed because it is too large Load Diff
Binary file not shown.
File diff suppressed because it is too large Load Diff
Binary file not shown.
+95
View File
@@ -25,6 +25,14 @@ MODE="golden" # golden | screenshot | movie | update
TARGET=""
INTERVAL=""
# Server state for live scenarios
SERVER_BIN=""
SERVER_PID=""
SERVER_PORT=""
# Cleanup server on exit
trap '[[ -n "${SERVER_PID:-}" ]] && kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || true' EXIT
# -- Parse args ----------------------------------------------------------------
while [[ $# -gt 0 ]]; do
@@ -102,12 +110,84 @@ xvfb_capture() {
fi
}
# -- Server lifecycle (live scenarios) -----------------------------------------
# Check if a scenario has "live": true in config
is_live_scenario() {
python3 -c "
import json, sys
c = json.load(open('${CONFIG}'))
s = c.get('scenarios', {}).get('${1}', {})
sys.exit(0 if s.get('live') else 1)
"
}
# Build server binary (once, cached)
ensure_server_built() {
if [[ -n "$SERVER_BIN" ]]; then return 0; fi
echo " Building server for live visual tests..."
(cd "$ROOT/server" && cargo build --bin settled-reach-server 2>&1) || {
echo "Error: server build failed" >&2
return 1
}
SERVER_BIN="$ROOT/server/target/debug/settled-reach-server"
}
# Start server with --test-mode --port 0, parse LISTENING:{port}
start_server() {
local stdout_log
stdout_log=$(mktemp)
"$SERVER_BIN" --test-mode --port 0 >"$stdout_log" 2>/dev/null &
SERVER_PID=$!
local attempts=0
while [[ $attempts -lt 150 ]]; do
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
echo " Error: server exited unexpectedly" >&2
rm -f "$stdout_log"
SERVER_PID=""
return 1
fi
if grep -q "^LISTENING:" "$stdout_log" 2>/dev/null; then
SERVER_PORT=$(sed -n 's/^LISTENING://p' "$stdout_log")
rm -f "$stdout_log"
echo " Server started: pid=$SERVER_PID port=$SERVER_PORT"
return 0
fi
sleep 0.1
attempts=$((attempts + 1))
done
echo " Error: no LISTENING signal after 15s" >&2
kill "$SERVER_PID" 2>/dev/null || true
rm -f "$stdout_log"
SERVER_PID=""
return 1
}
# Stop server (called after each live capture; server may have exited on disconnect)
stop_server() {
if [[ -n "$SERVER_PID" ]]; then
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
SERVER_PID=""
SERVER_PORT=""
fi
}
# -- Screenshot mode -----------------------------------------------------------
if [[ "$MODE" == "screenshot" ]]; then
mkdir -p "$CACHE_DIR"
echo "Capturing scenario: $TARGET"
if is_live_scenario "$TARGET"; then
ensure_server_built || exit 2
start_server || exit 2
export SR_LIVE=1 SR_PORT="$SERVER_PORT"
fi
godot_capture --scenario "$TARGET" "$CACHE_DIR"
stop_server
unset SR_LIVE SR_PORT 2>/dev/null || true
PNG="$CACHE_DIR/$TARGET.png"
if [[ -f "$PNG" ]]; then
echo "Screenshot: $PNG ($(stat -c%s "$PNG" 2>/dev/null || stat -f%z "$PNG") bytes)"
@@ -162,12 +242,27 @@ for scenario in "${SCENARIOS[@]}"; do
TOTAL=$((TOTAL + 1))
echo "--- $scenario ---"
# Start server for live scenarios
IS_LIVE=false
if is_live_scenario "$scenario"; then
IS_LIVE=true
ensure_server_built || { FAILED=$((FAILED + 1)); continue; }
start_server || { FAILED=$((FAILED + 1)); continue; }
export SR_LIVE=1 SR_PORT="$SERVER_PORT"
fi
# Capture
set +e
CAPTURE_OUT=$(xvfb_capture --scenario "$scenario" "$CACHE_DIR" 2>&1)
CAPTURE_RC=$?
set -e
# Stop server after capture (server exits on client disconnect anyway)
if [[ "$IS_LIVE" == "true" ]]; then
unset SR_LIVE SR_PORT 2>/dev/null || true
stop_server
fi
CAPTURED="$CACHE_DIR/$scenario.png"
if [[ $CAPTURE_RC -ne 0 ]] || [[ ! -f "$CAPTURED" ]]; then
+21
View File
@@ -2,6 +2,7 @@
"resolution": [960, 540],
"settle_frames": 30,
"tolerance": 5,
"max_diff_pct": 0.5,
"golden_dir": "client/tests/golden/visual",
"scenarios": {
"fog_3state": {
@@ -47,6 +48,26 @@
"cursor_menu": {
"ticks": 3,
"description": "Cursor rendering over dialogue option"
},
"fog_live_replay": {
"ticks": 1,
"description": "Replay real server hub snapshot — full pipeline fog test",
"replay_snapshot": "tests/fixtures/gauntlet/hub_spawn.msgpack"
},
"fog_theater_replay": {
"ticks": 1,
"description": "Replay Fog Theater — large room with NPCs at varying distances",
"replay_snapshot": "tests/fixtures/gauntlet/fog_theater.msgpack"
},
"fog_boundary_replay": {
"ticks": 1,
"description": "Replay hub after movement — explored/visible tile boundary",
"replay_snapshot": "tests/fixtures/gauntlet/hub_after_movement.msgpack"
},
"fog_live_hub": {
"ticks": 10,
"live": true,
"description": "Live server: Hub spawn fog — real pipeline regression test"
}
},
"flows": {
+24 -7
View File
@@ -239,18 +239,22 @@ def compare(
# ---------------------------------------------------------------------------
def load_tolerance(config_path: Path | None) -> int:
"""Read tolerance from config JSON, return DEFAULT_TOLERANCE on failure."""
def load_config(config_path: Path | None) -> dict:
"""Read visual test config, return dict with tolerance and max_diff_pct."""
if config_path is None:
config_path = DEFAULT_CONFIG
defaults = {"tolerance": DEFAULT_TOLERANCE, "max_diff_pct": 0.0}
if not config_path.exists():
return DEFAULT_TOLERANCE
return defaults
try:
with open(config_path) as f:
data = json.load(f)
return int(data.get("tolerance", DEFAULT_TOLERANCE))
return {
"tolerance": int(data.get("tolerance", DEFAULT_TOLERANCE)),
"max_diff_pct": float(data.get("max_diff_pct", 0.0)),
}
except (json.JSONDecodeError, ValueError, OSError):
return DEFAULT_TOLERANCE
return defaults
# ---------------------------------------------------------------------------
@@ -270,6 +274,12 @@ def main() -> int:
default=None,
help="Per-channel pixel tolerance (default: from config or 5)",
)
parser.add_argument(
"--max-diff-pct",
type=float,
default=None,
help="Max allowed diff percentage (default: from config or 0.0)",
)
parser.add_argument(
"--diff-output",
default=None,
@@ -282,9 +292,11 @@ def main() -> int:
)
args = parser.parse_args()
# Resolve tolerance: CLI > config > fallback
# Resolve settings: CLI > config > fallback
config_path = Path(args.config) if args.config else None
tolerance = args.tolerance if args.tolerance is not None else load_tolerance(config_path)
cfg = load_config(config_path)
tolerance = args.tolerance if args.tolerance is not None else cfg["tolerance"]
max_diff_pct = args.max_diff_pct if args.max_diff_pct is not None else cfg["max_diff_pct"]
# Read images
try:
@@ -322,6 +334,11 @@ def main() -> int:
return 0
pct = diff_count / total * 100
if pct <= max_diff_pct:
print(f"PASS: {diff_count} of {total} pixels differ ({pct:.1f}%, within {max_diff_pct}% threshold)")
return 0
print(f"FAIL: {diff_count} of {total} pixels differ ({pct:.1f}%)")
if args.diff_output and diff_buf: