feat(ci): add performance baseline tooling (#499)
Adds `make perf-baseline` — boots the full server plugin stack with real content, measures 50 ticks (5 warmup), captures per-tick timing, entity counts, and process RSS. Includes shadowcast benchmarks. Saves structured JSON to tests/perf/baseline.json for regression detection. Supports --compare mode (>20% threshold). First baseline: mean 366µs, p95 526µs (0.5% of D-026 100ms budget). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Generated
+1
-1
@@ -978,7 +978,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "settled-reach-server"
|
||||
version = "0.1.0"
|
||||
version = "0.1.9"
|
||||
dependencies = [
|
||||
"bevy_app",
|
||||
"bevy_ecs",
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
//! Performance benchmarks: tick timing and memory usage
|
||||
//!
|
||||
//! Run with: cargo test --release --test perf_bench -- --ignored --nocapture
|
||||
//! Output: PERF_RESULT:{json} lines for tooling/perf-baseline to parse.
|
||||
//!
|
||||
//! Tick budget target: 100ms (D-026)
|
||||
|
||||
use bevy_app::prelude::*;
|
||||
use std::io::{BufReader, BufWriter};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::path::PathBuf;
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use settled_reach_server::bridge::framing::{read_framed, write_framed};
|
||||
use settled_reach_server::bridge::tcp::TcpBridge;
|
||||
use settled_reach_server::bridge::types::*;
|
||||
use settled_reach_server::bridge::{BridgePlugin, BridgeResource};
|
||||
use settled_reach_server::content::{ContentConfig, ContentPlugin};
|
||||
use settled_reach_server::knowledge::registry::EntityRegistry;
|
||||
use settled_reach_server::knowledge::KnowledgeGraph;
|
||||
use settled_reach_server::perception::cognitive_delay::CognitiveDelay;
|
||||
use settled_reach_server::perception::vision_cone::Facing;
|
||||
use settled_reach_server::simulation::interaction::NearbyInteractionBuffer;
|
||||
use settled_reach_server::simulation::listening::ListeningFocus;
|
||||
use settled_reach_server::simulation::monologue::{
|
||||
MonologueBuffer, MonologueState, SprintAnomalyQueue,
|
||||
};
|
||||
use settled_reach_server::simulation::movement::{PlayerCharacter, TilePosition, WalkabilityMap};
|
||||
use settled_reach_server::simulation::stance::{MovementProfile, PlayerMoveCooldown};
|
||||
use settled_reach_server::simulation::SimulationPlugin;
|
||||
|
||||
const WARMUP_TICKS: usize = 5;
|
||||
const MEASURE_TICKS: usize = 50;
|
||||
const TOTAL_TICKS: usize = WARMUP_TICKS + MEASURE_TICKS;
|
||||
|
||||
fn content_root() -> PathBuf {
|
||||
let manifest_dir = env!("CARGO_MANIFEST_DIR");
|
||||
PathBuf::from(manifest_dir).join("../content")
|
||||
}
|
||||
|
||||
fn read_rss_kb() -> Option<u64> {
|
||||
std::fs::read_to_string("/proc/self/status")
|
||||
.ok()
|
||||
.and_then(|s| {
|
||||
s.lines()
|
||||
.find(|l| l.starts_with("VmRSS:"))
|
||||
.and_then(|l| l.split_whitespace().nth(1))
|
||||
.and_then(|v| v.parse().ok())
|
||||
})
|
||||
}
|
||||
|
||||
/// Full plugin stack tick benchmark with real content.
|
||||
///
|
||||
/// Boots the server with production content, runs WARMUP_TICKS to stabilize,
|
||||
/// then measures MEASURE_TICKS of app.update() wall-clock time. Reports entity
|
||||
/// counts from observer snapshots and process RSS.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn perf_tick_timing() {
|
||||
let root = content_root();
|
||||
if !root.join("content.yaml").exists() {
|
||||
eprintln!("Skipping: content directory not found at {:?}", root);
|
||||
return;
|
||||
}
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener");
|
||||
let server_addr = listener.local_addr().expect("get local addr");
|
||||
|
||||
// Server thread: full plugin stack with real content, timed ticks
|
||||
let server_root = root.clone();
|
||||
let server_handle = thread::spawn(move || -> Vec<Duration> {
|
||||
let bridge = TcpBridge::accept_on(listener).expect("accept connection");
|
||||
|
||||
let mut app = App::new();
|
||||
app.add_plugins(SimulationPlugin);
|
||||
app.add_plugins(BridgePlugin);
|
||||
app.add_plugins(settled_reach_server::knowledge::KnowledgePlugin);
|
||||
app.add_plugins(settled_reach_server::npc::NpcPlugin);
|
||||
app.insert_resource(ContentConfig {
|
||||
content_root: server_root,
|
||||
..Default::default()
|
||||
});
|
||||
app.add_plugins(ContentPlugin);
|
||||
app.insert_resource(BridgeResource::new(bridge));
|
||||
app.insert_resource(WalkabilityMap::new(32, 32, 1));
|
||||
|
||||
let profile = MovementProfile::smuggler();
|
||||
let mut registry = EntityRegistry::new(0);
|
||||
let player = app
|
||||
.world_mut()
|
||||
.spawn((
|
||||
PlayerCharacter,
|
||||
TilePosition::new(16, 16, 0),
|
||||
Facing::default(),
|
||||
KnowledgeGraph::new(),
|
||||
NearbyInteractionBuffer::default(),
|
||||
MonologueState::default(),
|
||||
MonologueBuffer::default(),
|
||||
SprintAnomalyQueue::default(),
|
||||
CognitiveDelay::default(),
|
||||
ListeningFocus::new(TilePosition::new(16, 16, 0)),
|
||||
profile,
|
||||
profile.initial_stance(),
|
||||
PlayerMoveCooldown::default(),
|
||||
))
|
||||
.id();
|
||||
registry.register(player);
|
||||
app.insert_resource(registry);
|
||||
|
||||
let mut timings = Vec::with_capacity(TOTAL_TICKS);
|
||||
for _ in 0..TOTAL_TICKS {
|
||||
let start = Instant::now();
|
||||
app.update();
|
||||
timings.push(start.elapsed());
|
||||
}
|
||||
timings
|
||||
});
|
||||
|
||||
// Client: pump protocol — read snapshots, send empty inputs
|
||||
let stream = TcpStream::connect(server_addr).expect("client connect");
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(30)))
|
||||
.expect("set read timeout");
|
||||
let mut reader = BufReader::new(stream.try_clone().expect("clone for reader"));
|
||||
let mut writer = BufWriter::new(stream);
|
||||
|
||||
let mut entity_counts: Vec<usize> = Vec::with_capacity(TOTAL_TICKS);
|
||||
for tick in 0..TOTAL_TICKS {
|
||||
// Server may close connection after its last tick — handle gracefully
|
||||
let payload = match read_framed(&mut reader) {
|
||||
Ok(Some(p)) => p,
|
||||
Ok(None) | Err(_) => break,
|
||||
};
|
||||
|
||||
let snapshot: ObserverSnapshot = rmp_serde::from_slice(&payload)
|
||||
.unwrap_or_else(|e| panic!("deserialization error at tick {}: {}", tick, e));
|
||||
|
||||
entity_counts.push(snapshot.entities.len());
|
||||
|
||||
let empty: Vec<PlayerInput> = vec![];
|
||||
let input_payload = rmp_serde::to_vec(&empty).expect("serialize empty input");
|
||||
if write_framed(&mut writer, &input_payload).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Must have received enough data past warmup for meaningful results
|
||||
assert!(
|
||||
entity_counts.len() > WARMUP_TICKS,
|
||||
"Only received {} snapshots, need at least {} (warmup) + 1",
|
||||
entity_counts.len(),
|
||||
WARMUP_TICKS
|
||||
);
|
||||
|
||||
drop(reader);
|
||||
drop(writer);
|
||||
|
||||
let timings = server_handle
|
||||
.join()
|
||||
.expect("server thread panicked during tick benchmark");
|
||||
|
||||
// Analyze measured ticks (skip warmup)
|
||||
let measured_us: Vec<u64> = timings
|
||||
.iter()
|
||||
.skip(WARMUP_TICKS)
|
||||
.map(|d| d.as_micros() as u64)
|
||||
.collect();
|
||||
|
||||
let min = *measured_us.iter().min().unwrap();
|
||||
let max = *measured_us.iter().max().unwrap();
|
||||
let sum: u64 = measured_us.iter().sum();
|
||||
let mean = sum / measured_us.len() as u64;
|
||||
|
||||
let mut sorted = measured_us.clone();
|
||||
sorted.sort();
|
||||
let p95_idx = ((sorted.len() as f64 * 0.95).ceil() as usize).saturating_sub(1);
|
||||
let p95 = sorted[p95_idx.min(sorted.len() - 1)];
|
||||
|
||||
let entity_counts_measured: Vec<usize> =
|
||||
entity_counts.iter().skip(WARMUP_TICKS).copied().collect();
|
||||
let avg_entities =
|
||||
entity_counts_measured.iter().sum::<usize>() / entity_counts_measured.len().max(1);
|
||||
let max_entities = entity_counts_measured.iter().max().copied().unwrap_or(0);
|
||||
|
||||
let rss_kb = read_rss_kb();
|
||||
|
||||
let result = serde_json::json!({
|
||||
"tick_timing": {
|
||||
"warmup_ticks": WARMUP_TICKS,
|
||||
"measured_ticks": measured_us.len(),
|
||||
"min_us": min,
|
||||
"max_us": max,
|
||||
"mean_us": mean,
|
||||
"p95_us": p95,
|
||||
"all_us": measured_us,
|
||||
},
|
||||
"entities": {
|
||||
"avg_per_snapshot": avg_entities,
|
||||
"max_per_snapshot": max_entities,
|
||||
},
|
||||
"memory": {
|
||||
"rss_kb": rss_kb,
|
||||
},
|
||||
});
|
||||
|
||||
println!(
|
||||
"PERF_RESULT:{}",
|
||||
serde_json::to_string(&result).unwrap()
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user