From 08052c6459782bb76fd9c733c45455ad32d6b176 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 18 Feb 2026 12:42:09 +0100 Subject: [PATCH] feat(ci): add performance baseline tooling (#499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `make perf-baseline` — boots the full server plugin stack with real content, measures 50 ticks (5 warmup), captures per-tick timing, entity counts, and process RSS. Includes shadowcast benchmarks. Saves structured JSON to tests/perf/baseline.json for regression detection. Supports --compare mode (>20% threshold). First baseline: mean 366µs, p95 526µs (0.5% of D-026 100ms budget). Co-Authored-By: Claude Opus 4.6 --- Makefile | 7 +- server/Cargo.lock | 2 +- server/tests/perf_bench.rs | 211 ++++++++++++++++++++++++++++ tests/perf/baseline.json | 116 ++++++++++++++++ tooling/perf-baseline | 273 +++++++++++++++++++++++++++++++++++++ 5 files changed, 607 insertions(+), 2 deletions(-) create mode 100644 server/tests/perf_bench.rs create mode 100644 tests/perf/baseline.json create mode 100755 tooling/perf-baseline diff --git a/Makefile b/Makefile index a836d0d38..bd6ef6196 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null) pre-pr pre-pr-lint pre-pr-build pre-pr-test pre-pr-validate pre-pr-fixtures \ pre-pr-server pre-pr-client pre-pr-content \ fixtures-client golden-diff golden-update \ - checklist-validate checklist-generate + checklist-validate checklist-generate \ + perf-baseline # --- Configuration --- @@ -44,6 +45,7 @@ help: @echo " make golden-update Regenerate golden file and stage for commit" @echo " make checklist-validate Validate checklist YAML against schema" @echo " make checklist-generate Validate checklists + print condition summary" + @echo " make perf-baseline Run performance benchmarks and save baseline" @echo "" @echo " make pre-pr Run all pre-PR checks (lint, build, test, validate, fixtures)" @echo " make pre-pr-server Server-scoped pre-PR (lint, build, test, fixtures)" @@ -280,6 +282,9 @@ checklist-validate: checklist-generate: @tooling/validate-checklist +perf-baseline: + @tooling/perf-baseline + content-ron: cd tooling/content-converter && cargo build --release tooling/content-converter/target/release/content-converter --input content --output content-ron --verbose diff --git a/server/Cargo.lock b/server/Cargo.lock index 88dc1b8fd..ae9e05069 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -978,7 +978,7 @@ dependencies = [ [[package]] name = "settled-reach-server" -version = "0.1.0" +version = "0.1.9" dependencies = [ "bevy_app", "bevy_ecs", diff --git a/server/tests/perf_bench.rs b/server/tests/perf_bench.rs new file mode 100644 index 000000000..788ea311a --- /dev/null +++ b/server/tests/perf_bench.rs @@ -0,0 +1,211 @@ +//! Performance benchmarks: tick timing and memory usage +//! +//! Run with: cargo test --release --test perf_bench -- --ignored --nocapture +//! Output: PERF_RESULT:{json} lines for tooling/perf-baseline to parse. +//! +//! Tick budget target: 100ms (D-026) + +use bevy_app::prelude::*; +use std::io::{BufReader, BufWriter}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::thread; +use std::time::{Duration, Instant}; + +use settled_reach_server::bridge::framing::{read_framed, write_framed}; +use settled_reach_server::bridge::tcp::TcpBridge; +use settled_reach_server::bridge::types::*; +use settled_reach_server::bridge::{BridgePlugin, BridgeResource}; +use settled_reach_server::content::{ContentConfig, ContentPlugin}; +use settled_reach_server::knowledge::registry::EntityRegistry; +use settled_reach_server::knowledge::KnowledgeGraph; +use settled_reach_server::perception::cognitive_delay::CognitiveDelay; +use settled_reach_server::perception::vision_cone::Facing; +use settled_reach_server::simulation::interaction::NearbyInteractionBuffer; +use settled_reach_server::simulation::listening::ListeningFocus; +use settled_reach_server::simulation::monologue::{ + MonologueBuffer, MonologueState, SprintAnomalyQueue, +}; +use settled_reach_server::simulation::movement::{PlayerCharacter, TilePosition, WalkabilityMap}; +use settled_reach_server::simulation::stance::{MovementProfile, PlayerMoveCooldown}; +use settled_reach_server::simulation::SimulationPlugin; + +const WARMUP_TICKS: usize = 5; +const MEASURE_TICKS: usize = 50; +const TOTAL_TICKS: usize = WARMUP_TICKS + MEASURE_TICKS; + +fn content_root() -> PathBuf { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + PathBuf::from(manifest_dir).join("../content") +} + +fn read_rss_kb() -> Option { + std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("VmRSS:")) + .and_then(|l| l.split_whitespace().nth(1)) + .and_then(|v| v.parse().ok()) + }) +} + +/// Full plugin stack tick benchmark with real content. +/// +/// Boots the server with production content, runs WARMUP_TICKS to stabilize, +/// then measures MEASURE_TICKS of app.update() wall-clock time. Reports entity +/// counts from observer snapshots and process RSS. +#[test] +#[ignore] +fn perf_tick_timing() { + let root = content_root(); + if !root.join("content.yaml").exists() { + eprintln!("Skipping: content directory not found at {:?}", root); + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener"); + let server_addr = listener.local_addr().expect("get local addr"); + + // Server thread: full plugin stack with real content, timed ticks + let server_root = root.clone(); + let server_handle = thread::spawn(move || -> Vec { + let bridge = TcpBridge::accept_on(listener).expect("accept connection"); + + let mut app = App::new(); + app.add_plugins(SimulationPlugin); + app.add_plugins(BridgePlugin); + app.add_plugins(settled_reach_server::knowledge::KnowledgePlugin); + app.add_plugins(settled_reach_server::npc::NpcPlugin); + app.insert_resource(ContentConfig { + content_root: server_root, + ..Default::default() + }); + app.add_plugins(ContentPlugin); + app.insert_resource(BridgeResource::new(bridge)); + app.insert_resource(WalkabilityMap::new(32, 32, 1)); + + let profile = MovementProfile::smuggler(); + let mut registry = EntityRegistry::new(0); + let player = app + .world_mut() + .spawn(( + PlayerCharacter, + TilePosition::new(16, 16, 0), + Facing::default(), + KnowledgeGraph::new(), + NearbyInteractionBuffer::default(), + MonologueState::default(), + MonologueBuffer::default(), + SprintAnomalyQueue::default(), + CognitiveDelay::default(), + ListeningFocus::new(TilePosition::new(16, 16, 0)), + profile, + profile.initial_stance(), + PlayerMoveCooldown::default(), + )) + .id(); + registry.register(player); + app.insert_resource(registry); + + let mut timings = Vec::with_capacity(TOTAL_TICKS); + for _ in 0..TOTAL_TICKS { + let start = Instant::now(); + app.update(); + timings.push(start.elapsed()); + } + timings + }); + + // Client: pump protocol — read snapshots, send empty inputs + let stream = TcpStream::connect(server_addr).expect("client connect"); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .expect("set read timeout"); + let mut reader = BufReader::new(stream.try_clone().expect("clone for reader")); + let mut writer = BufWriter::new(stream); + + let mut entity_counts: Vec = Vec::with_capacity(TOTAL_TICKS); + for tick in 0..TOTAL_TICKS { + // Server may close connection after its last tick — handle gracefully + let payload = match read_framed(&mut reader) { + Ok(Some(p)) => p, + Ok(None) | Err(_) => break, + }; + + let snapshot: ObserverSnapshot = rmp_serde::from_slice(&payload) + .unwrap_or_else(|e| panic!("deserialization error at tick {}: {}", tick, e)); + + entity_counts.push(snapshot.entities.len()); + + let empty: Vec = vec![]; + let input_payload = rmp_serde::to_vec(&empty).expect("serialize empty input"); + if write_framed(&mut writer, &input_payload).is_err() { + break; + } + } + + // Must have received enough data past warmup for meaningful results + assert!( + entity_counts.len() > WARMUP_TICKS, + "Only received {} snapshots, need at least {} (warmup) + 1", + entity_counts.len(), + WARMUP_TICKS + ); + + drop(reader); + drop(writer); + + let timings = server_handle + .join() + .expect("server thread panicked during tick benchmark"); + + // Analyze measured ticks (skip warmup) + let measured_us: Vec = timings + .iter() + .skip(WARMUP_TICKS) + .map(|d| d.as_micros() as u64) + .collect(); + + let min = *measured_us.iter().min().unwrap(); + let max = *measured_us.iter().max().unwrap(); + let sum: u64 = measured_us.iter().sum(); + let mean = sum / measured_us.len() as u64; + + let mut sorted = measured_us.clone(); + sorted.sort(); + let p95_idx = ((sorted.len() as f64 * 0.95).ceil() as usize).saturating_sub(1); + let p95 = sorted[p95_idx.min(sorted.len() - 1)]; + + let entity_counts_measured: Vec = + entity_counts.iter().skip(WARMUP_TICKS).copied().collect(); + let avg_entities = + entity_counts_measured.iter().sum::() / entity_counts_measured.len().max(1); + let max_entities = entity_counts_measured.iter().max().copied().unwrap_or(0); + + let rss_kb = read_rss_kb(); + + let result = serde_json::json!({ + "tick_timing": { + "warmup_ticks": WARMUP_TICKS, + "measured_ticks": measured_us.len(), + "min_us": min, + "max_us": max, + "mean_us": mean, + "p95_us": p95, + "all_us": measured_us, + }, + "entities": { + "avg_per_snapshot": avg_entities, + "max_per_snapshot": max_entities, + }, + "memory": { + "rss_kb": rss_kb, + }, + }); + + println!( + "PERF_RESULT:{}", + serde_json::to_string(&result).unwrap() + ); +} diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json new file mode 100644 index 000000000..04824102e --- /dev/null +++ b/tests/perf/baseline.json @@ -0,0 +1,116 @@ +{ + "timestamp": "2026-02-18T11:39:46.913254+00:00", + "git": { + "commit": "b4a3784", + "branch": "ci" + }, + "tick_timing": { + "max_us": 537, + "mean_us": 366, + "measured_ticks": 50, + "min_us": 314, + "p95_us": 526, + "warmup_ticks": 5 + }, + "entities": { + "avg_per_snapshot": 21, + "max_per_snapshot": 21 + }, + "memory": { + "rss_kb": 6972 + }, + "shadowcast": { + "configs": [ + { + "map_size": 32, + "density": "open field", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 62.2, + "symmetric_per_call_us": 62.2, + "recursive_total_ms": 107.25, + "recursive_per_call_us": 107.25 + }, + { + "map_size": 32, + "density": "moderate corridors", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 60.15, + "symmetric_per_call_us": 60.15, + "recursive_total_ms": 211.61, + "recursive_per_call_us": 211.61 + }, + { + "map_size": 32, + "density": "dense rooms", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 24.28, + "symmetric_per_call_us": 24.28, + "recursive_total_ms": 107.95, + "recursive_per_call_us": 107.95 + }, + { + "map_size": 64, + "density": "open field", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 75.38, + "symmetric_per_call_us": 75.38, + "recursive_total_ms": 108.22, + "recursive_per_call_us": 108.22 + }, + { + "map_size": 64, + "density": "moderate corridors", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 48.72, + "symmetric_per_call_us": 48.72, + "recursive_total_ms": 202.49, + "recursive_per_call_us": 202.49 + }, + { + "map_size": 64, + "density": "dense rooms", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 14.5, + "symmetric_per_call_us": 14.5, + "recursive_total_ms": 90.33, + "recursive_per_call_us": 90.33 + }, + { + "map_size": 150, + "density": "open field", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 59.7, + "symmetric_per_call_us": 59.7, + "recursive_total_ms": 103.09, + "recursive_per_call_us": 103.09 + }, + { + "map_size": 150, + "density": "moderate corridors", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 43.64, + "symmetric_per_call_us": 43.64, + "recursive_total_ms": 178.09, + "recursive_per_call_us": 178.09 + }, + { + "map_size": 150, + "density": "dense rooms", + "range": 20, + "iterations": 1000, + "symmetric_total_ms": 12.63, + "symmetric_per_call_us": 12.63, + "recursive_total_ms": 79.87, + "recursive_per_call_us": 79.87 + } + ] + } +} diff --git a/tooling/perf-baseline b/tooling/perf-baseline new file mode 100755 index 000000000..2b9079976 --- /dev/null +++ b/tooling/perf-baseline @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Performance baseline tooling. + +Runs the server benchmark suite, captures tick timing, memory usage, and entity +count scaling metrics, outputs results to tests/perf/. + +Usage: + tooling/perf-baseline Run benchmarks and save baseline + tooling/perf-baseline --compare Compare current run against saved baseline (no save) + +Exit code 0 = success, 1 = failure or regression detected (--compare mode). +""" + +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +PERF_DIR = ROOT / "tests" / "perf" +BASELINE_FILE = PERF_DIR / "baseline.json" + +# Tick budget from D-026 +TICK_BUDGET_US = 100_000 # 100ms + + +def run_command(cmd, **kwargs): + """Run a command in the server directory and return the result.""" + return subprocess.run( + cmd, capture_output=True, text=True, cwd=ROOT / "server", **kwargs + ) + + +def get_git_info(): + """Get current git commit and branch.""" + commit = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, text=True, cwd=ROOT, + ).stdout.strip() + branch = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, text=True, cwd=ROOT, + ).stdout.strip() + return {"commit": commit, "branch": branch} + + +def run_tick_benchmark(): + """Run perf_tick_timing test and parse PERF_RESULT JSON.""" + print(" Running tick timing benchmark (release mode)...") + result = run_command([ + "cargo", "test", "--release", "--test", "perf_bench", + "--", "--ignored", "--nocapture", "perf_tick_timing", + ]) + + if result.returncode != 0: + print(f" FAILED: tick benchmark exited {result.returncode}") + if result.stderr: + # Print last 20 lines of stderr for diagnostics + lines = result.stderr.strip().splitlines() + for line in lines[-20:]: + print(f" {line}") + return None + + # Parse PERF_RESULT: line from stdout + for line in result.stdout.splitlines(): + if line.startswith("PERF_RESULT:"): + json_str = line[len("PERF_RESULT:"):] + return json.loads(json_str) + + print(" WARNING: No PERF_RESULT found in test output") + return None + + +def run_shadowcast_benchmark(): + """Run shadowcast benchmark and parse structured output.""" + print(" Running shadowcast benchmark (release mode)...") + result = run_command([ + "cargo", "test", "--release", "--test", "shadowcast_bench", + "--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive", + ]) + + if result.returncode != 0: + print(f" FAILED: shadowcast benchmark exited {result.returncode}") + return None + + configs = [] + current = {} + for line in result.stdout.splitlines(): + line = line.strip() + + m = re.match( + r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)", + line, + ) + if m: + current = { + "map_size": int(m.group(1)), + "density": m.group(3), + "range": int(m.group(4)), + "iterations": int(m.group(5)), + } + continue + + m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) + if m: + current["symmetric_total_ms"] = float(m.group(1)) + current["symmetric_per_call_us"] = float(m.group(2)) + continue + + m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) + if m: + current["recursive_total_ms"] = float(m.group(1)) + current["recursive_per_call_us"] = float(m.group(2)) + if current.get("map_size"): + configs.append(current) + current = {} + continue + + return {"configs": configs} if configs else None + + +def compare_baselines(old, new): + """Compare two baselines and report regressions. Returns list of regression strings.""" + regressions = [] + improvements = [] + + old_tick = old.get("tick_timing", {}) + new_tick = new.get("tick_timing", {}) + + if old_tick and new_tick: + # Mean tick time regression (>20% = warning) + old_mean = old_tick.get("mean_us", 0) + new_mean = new_tick.get("mean_us", 0) + if old_mean > 0: + change = (new_mean - old_mean) / old_mean * 100 + if change > 20: + regressions.append( + f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)" + ) + elif change < -20: + improvements.append( + f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)" + ) + + # p95 tick time regression + old_p95 = old_tick.get("p95_us", 0) + new_p95 = new_tick.get("p95_us", 0) + if old_p95 > 0: + change = (new_p95 - old_p95) / old_p95 * 100 + if change > 20: + regressions.append( + f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)" + ) + elif change < -20: + improvements.append( + f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)" + ) + + # Absolute budget check + new_p95 = new.get("tick_timing", {}).get("p95_us", 0) + if new_p95 > TICK_BUDGET_US: + regressions.append( + f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)" + ) + + return regressions, improvements + + +def main(): + compare_mode = "--compare" in sys.argv + + print("=== Performance Baseline ===\n") + + # Build in release mode first + print("Building server (release)...") + build = run_command(["cargo", "build", "--release"]) + if build.returncode != 0: + print("BUILD FAILED") + lines = build.stderr.strip().splitlines() + for line in lines[-20:]: + print(f" {line}") + return 1 + + print("\nRunning benchmarks...\n") + + tick_results = run_tick_benchmark() + shadowcast_results = run_shadowcast_benchmark() + + if not tick_results: + print("\nFATAL: tick benchmark failed -- no baseline generated") + return 1 + + # Assemble baseline + git_info = get_git_info() + baseline = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "git": git_info, + "tick_timing": tick_results.get("tick_timing", {}), + "entities": tick_results.get("entities", {}), + "memory": tick_results.get("memory", {}), + } + if shadowcast_results: + baseline["shadowcast"] = shadowcast_results + + # Report + tt = baseline["tick_timing"] + print(f"\n--- Results ---") + print(f"Git: {git_info['commit']} ({git_info['branch']})") + print(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, " + f"{tt.get('warmup_ticks', '?')} warmup):") + print(f" min: {tt.get('min_us', '?')}us") + print(f" mean: {tt.get('mean_us', '?')}us") + print(f" p95: {tt.get('p95_us', '?')}us") + print(f" max: {tt.get('max_us', '?')}us") + + ent = baseline["entities"] + print(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, " + f"max {ent.get('max_per_snapshot', '?')}") + + mem = baseline["memory"] + rss = mem.get("rss_kb") + if rss: + print(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)") + + if shadowcast_results: + n = len(shadowcast_results.get("configs", [])) + print(f"Shadowcast: {n} configurations benchmarked") + + # Budget check + p95 = tt.get("p95_us", 0) + if p95 > TICK_BUDGET_US: + print(f"\nBUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)") + else: + budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0 + print(f"\nBudget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)") + + # Compare with previous baseline if it exists + if BASELINE_FILE.exists(): + with open(BASELINE_FILE) as f: + old_baseline = json.load(f) + old_commit = old_baseline.get("git", {}).get("commit", "?") + print(f"\n--- Comparison vs {old_commit} ---") + regressions, improvements = compare_baselines(old_baseline, baseline) + for r in regressions: + print(f" REGRESSION: {r}") + for i in improvements: + print(f" IMPROVEMENT: {i}") + if not regressions and not improvements: + print(" No significant changes.") + if compare_mode and regressions: + print(f"\n{len(regressions)} regression(s) detected.") + return 1 + + if compare_mode: + return 0 + + # Save baseline (strip per-tick array — too noisy for git diffs) + PERF_DIR.mkdir(parents=True, exist_ok=True) + committed = json.loads(json.dumps(baseline)) + committed["tick_timing"].pop("all_us", None) + + with open(BASELINE_FILE, "w") as f: + json.dump(committed, f, indent=2) + f.write("\n") + + print(f"\nBaseline written to {BASELINE_FILE.relative_to(ROOT)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())