#!/usr/bin/env python3 """Performance baseline tooling. Runs the server benchmark suite, captures tick timing, memory usage, and entity count scaling metrics, outputs results to tests/perf/. Usage: tooling/perf-baseline Run benchmarks and save baseline tooling/perf-baseline --compare Compare current run against saved baseline (no save) Exit code 0 = success, 1 = failure or regression detected (--compare mode). """ import json import re import subprocess import sys from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parent.parent PERF_DIR = ROOT / "tests" / "perf" BASELINE_FILE = PERF_DIR / "baseline.json" # Tick budget from D-026: 100ms per tick at 10 tps floor (D-031). # If tick rate changes, update this constant. TICK_BUDGET_US = 100_000 # 100ms def run_command(cmd, **kwargs): """Run a command in the server directory and return the result.""" return subprocess.run( cmd, capture_output=True, text=True, cwd=ROOT / "server", **kwargs ) def get_git_info(): """Get current git commit and branch.""" commit = subprocess.run( ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, cwd=ROOT, ).stdout.strip() branch = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True, cwd=ROOT, ).stdout.strip() return {"commit": commit, "branch": branch} def run_tick_benchmark(): """Run perf_tick_timing test and parse PERF_RESULT JSON.""" print(" Running tick timing benchmark (release mode)...") result = run_command([ "cargo", "test", "--release", "--test", "perf_bench", "--", "--ignored", "--nocapture", "perf_tick_timing", ]) if result.returncode != 0: print(f" FAILED: tick benchmark exited {result.returncode}") if result.stderr: # Print last 20 lines of stderr for diagnostics lines = result.stderr.strip().splitlines() for line in lines[-20:]: print(f" {line}") return None # Parse PERF_RESULT: line from stdout for line in result.stdout.splitlines(): if line.startswith("PERF_RESULT:"): json_str = line[len("PERF_RESULT:"):] return json.loads(json_str) print(" WARNING: No PERF_RESULT found in test output") return None def run_shadowcast_benchmark(): """Run shadowcast benchmark and parse structured output.""" print(" Running shadowcast benchmark (release mode)...") result = run_command([ "cargo", "test", "--release", "--test", "shadowcast_bench", "--", "--ignored", "--nocapture", "benchmark_symmetric_vs_recursive", ]) if result.returncode != 0: print(f" FAILED: shadowcast benchmark exited {result.returncode}") return None configs = [] current = {} for line in result.stdout.splitlines(): line = line.strip() m = re.match( r"Map: (\d+)x(\d+), Density: (.+), Range: (\d+), Iterations: (\d+)", line, ) if m: # New config block — flush previous if complete if current.get("map_size"): configs.append(current) current = { "map_size": int(m.group(1)), "density": m.group(3), "range": int(m.group(4)), "iterations": int(m.group(5)), } continue m = re.match(r"Symmetric:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) if m: current["symmetric_total_ms"] = float(m.group(1)) current["symmetric_per_call_us"] = float(m.group(2)) continue m = re.match(r"Recursive:\s+([0-9.]+)ms total, ([0-9.]+).s/call", line) if m: current["recursive_total_ms"] = float(m.group(1)) current["recursive_per_call_us"] = float(m.group(2)) continue # Flush last config if current.get("map_size"): configs.append(current) return {"configs": configs} if configs else None def compare_baselines(old, new): """Compare two baselines and report regressions. Returns list of regression strings.""" regressions = [] improvements = [] old_tick = old.get("tick_timing", {}) new_tick = new.get("tick_timing", {}) if old_tick and new_tick: # Mean tick time regression (>20% = warning) old_mean = old_tick.get("mean_us", 0) new_mean = new_tick.get("mean_us", 0) if old_mean > 0: change = (new_mean - old_mean) / old_mean * 100 if change > 20: regressions.append( f"mean tick time {old_mean}us -> {new_mean}us (+{change:.1f}%)" ) elif change < -20: improvements.append( f"mean tick time {old_mean}us -> {new_mean}us ({change:.1f}%)" ) # p95 tick time regression old_p95 = old_tick.get("p95_us", 0) new_p95 = new_tick.get("p95_us", 0) if old_p95 > 0: change = (new_p95 - old_p95) / old_p95 * 100 if change > 20: regressions.append( f"p95 tick time {old_p95}us -> {new_p95}us (+{change:.1f}%)" ) elif change < -20: improvements.append( f"p95 tick time {old_p95}us -> {new_p95}us ({change:.1f}%)" ) # Absolute budget check new_p95 = new.get("tick_timing", {}).get("p95_us", 0) if new_p95 > TICK_BUDGET_US: regressions.append( f"p95 {new_p95}us exceeds {TICK_BUDGET_US}us tick budget (D-026)" ) return regressions, improvements def main(): compare_mode = "--compare" in sys.argv print("=== Performance Baseline ===\n") # Build in release mode first print("Building server (release)...") build = run_command(["cargo", "build", "--release"]) if build.returncode != 0: print("BUILD FAILED") lines = build.stderr.strip().splitlines() for line in lines[-20:]: print(f" {line}") return 1 print("\nRunning benchmarks...\n") tick_results = run_tick_benchmark() shadowcast_results = run_shadowcast_benchmark() if not tick_results: print("\nFATAL: tick benchmark failed -- no baseline generated") return 1 # Assemble baseline git_info = get_git_info() baseline = { "timestamp": datetime.now(timezone.utc).isoformat(), "git": git_info, "tick_timing": tick_results.get("tick_timing", {}), "entities": tick_results.get("entities", {}), "memory": tick_results.get("memory", {}), } if shadowcast_results: baseline["shadowcast"] = shadowcast_results # Report tt = baseline["tick_timing"] print(f"\n--- Results ---") print(f"Git: {git_info['commit']} ({git_info['branch']})") print(f"Tick timing ({tt.get('measured_ticks', '?')} ticks, " f"{tt.get('warmup_ticks', '?')} warmup):") print(f" min: {tt.get('min_us', '?')}us") print(f" mean: {tt.get('mean_us', '?')}us") print(f" p95: {tt.get('p95_us', '?')}us") print(f" max: {tt.get('max_us', '?')}us") ent = baseline["entities"] print(f"Entities: avg {ent.get('avg_per_snapshot', '?')}, " f"max {ent.get('max_per_snapshot', '?')}") mem = baseline["memory"] rss = mem.get("rss_kb") if rss: print(f"Memory: {rss} KB RSS ({rss / 1024:.1f} MB)") if shadowcast_results: n = len(shadowcast_results.get("configs", [])) print(f"Shadowcast: {n} configurations benchmarked") # Budget check p95 = tt.get("p95_us", 0) if p95 > TICK_BUDGET_US: print(f"\nBUDGET EXCEEDED: p95 {p95}us > {TICK_BUDGET_US}us (D-026)") else: budget_pct = p95 / TICK_BUDGET_US * 100 if TICK_BUDGET_US else 0 print(f"\nBudget: {budget_pct:.1f}% of {TICK_BUDGET_US}us tick budget (D-026)") # Compare with previous baseline if it exists if BASELINE_FILE.exists(): with open(BASELINE_FILE) as f: old_baseline = json.load(f) old_commit = old_baseline.get("git", {}).get("commit", "?") print(f"\n--- Comparison vs {old_commit} ---") regressions, improvements = compare_baselines(old_baseline, baseline) for r in regressions: print(f" REGRESSION: {r}") for i in improvements: print(f" IMPROVEMENT: {i}") if not regressions and not improvements: print(" No significant changes.") if compare_mode and regressions: print(f"\n{len(regressions)} regression(s) detected.") return 1 elif compare_mode: print(f"\nERROR: --compare requires a saved baseline at {BASELINE_FILE.relative_to(ROOT)}") print("Run `make perf-baseline` first to create one.") return 1 if compare_mode: return 0 # Save baseline (strip per-tick array — too noisy for git diffs) PERF_DIR.mkdir(parents=True, exist_ok=True) committed = json.loads(json.dumps(baseline)) committed["tick_timing"].pop("all_us", None) with open(BASELINE_FILE, "w") as f: json.dump(committed, f, indent=2) f.write("\n") print(f"\nBaseline written to {BASELINE_FILE.relative_to(ROOT)}") return 0 if __name__ == "__main__": sys.exit(main())