feat(simulation): error handling and recovery — panic supervision, state hash, structured errors (#85)
Protocol v17: add state_hash (desync detection) and sim_errors (structured error reporting) to ObserverSnapshot. Add SimError, SimErrorKind, SimErrorBuffer types. Wrap main loop app.update() in catch_unwind — on panic, send a final SimError snapshot before exit. Report recoverable deserialization errors to client via SimErrorBuffer. Compute per-tick state hash from player position + NPC count + tick. Update all test fixtures and golden files for protocol v17. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+105
-1
@@ -176,11 +176,43 @@ fn main() {
|
||||
// Targets ~20 ticks/sec (2 game-minutes/sec). The TCP bridge uses
|
||||
// non-blocking reads, so without throttling this loop would spin.
|
||||
// Remaining frame budget is available for NPC AI and pathfinding.
|
||||
//
|
||||
// Panic supervision (#85): each tick is wrapped in catch_unwind.
|
||||
// On panic, the server sends a structured SimError to the client
|
||||
// before shutting down, rather than an abrupt disconnect.
|
||||
let target_frame_time = std::time::Duration::from_millis(50);
|
||||
loop {
|
||||
let frame_start = std::time::Instant::now();
|
||||
|
||||
app.update();
|
||||
// Wrap app.update() in catch_unwind to handle system panics (#85).
|
||||
// AssertUnwindSafe is required because App is not UnwindSafe.
|
||||
let tick_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
app.update();
|
||||
}));
|
||||
|
||||
match tick_result {
|
||||
Ok(()) => {}
|
||||
Err(panic_payload) => {
|
||||
// Extract panic message for error reporting
|
||||
let panic_msg = if let Some(s) = panic_payload.downcast_ref::<&str>() {
|
||||
s.to_string()
|
||||
} else if let Some(s) = panic_payload.downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic".to_string()
|
||||
};
|
||||
|
||||
tracing::error!("Simulation panic caught: {}", panic_msg);
|
||||
|
||||
// Attempt to send a final SimError snapshot to the client.
|
||||
// Best-effort: if the bridge is unavailable, we just log and exit.
|
||||
send_panic_error(&app, &panic_msg);
|
||||
|
||||
tracing::error!("Server shutting down after panic");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !app.world().resource::<ServerRunning>().0 {
|
||||
break;
|
||||
}
|
||||
@@ -200,6 +232,78 @@ fn main() {
|
||||
tracing::info!("Simulation server shutting down");
|
||||
}
|
||||
|
||||
/// Best-effort: send a final SimError snapshot to the client on panic (#85).
|
||||
///
|
||||
/// Builds a minimal ObserverSnapshot with the panic error and sends it
|
||||
/// through the bridge. If the bridge is unavailable or sending fails,
|
||||
/// the error is logged but not fatal (we're already crashing).
|
||||
fn send_panic_error(app: &App, panic_msg: &str) {
|
||||
use settled_reach_server::bridge::types::*;
|
||||
use settled_reach_server::simulation::time::{DayPhase, TickRate};
|
||||
|
||||
let world = app.world();
|
||||
|
||||
// Try to read current tick from SimulationTime
|
||||
let tick = world
|
||||
.get_resource::<settled_reach_server::simulation::time::SimulationTime>()
|
||||
.map(|t| t.tick)
|
||||
.unwrap_or(0);
|
||||
|
||||
let bridge = match world.get_resource::<BridgeResource>() {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
tracing::error!("Cannot send panic error: no BridgeResource");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Build a minimal snapshot carrying the panic error
|
||||
let snapshot = ObserverSnapshot {
|
||||
version: PROTOCOL_VERSION,
|
||||
tick,
|
||||
game_time: GameTime {
|
||||
day: 0,
|
||||
time_of_day: 0,
|
||||
day_phase: DayPhase::Morning,
|
||||
tick_rate: TickRate::Paused,
|
||||
},
|
||||
player_facing: FacingDirection::North,
|
||||
player_stance: MovementStance::default(),
|
||||
player_inventory: vec![],
|
||||
entities: vec![],
|
||||
visible_tiles: vec![],
|
||||
nearby_interactions: vec![],
|
||||
current_monologue: None,
|
||||
pending_recognitions: vec![],
|
||||
dialogue_response: None,
|
||||
blocked_entities: vec![],
|
||||
scan_events: vec![],
|
||||
sound_events: vec![],
|
||||
conversation_events: vec![],
|
||||
conversation_ended: vec![],
|
||||
follow_state: None,
|
||||
character_pressure: None,
|
||||
rng_seed: None,
|
||||
poi_list: vec![],
|
||||
examine_result: None,
|
||||
player_knowledge: None,
|
||||
save_result: None,
|
||||
triangle_crisis_events: vec![],
|
||||
state_hash: None,
|
||||
sim_errors: vec![SimError {
|
||||
kind: SimErrorKind::Panic,
|
||||
message: format!("Simulation panic: {}", panic_msg),
|
||||
tick,
|
||||
}],
|
||||
};
|
||||
|
||||
if let Err(e) = bridge.send_snapshot(&snapshot) {
|
||||
tracing::error!("Failed to send panic error to client: {}", e);
|
||||
} else {
|
||||
tracing::info!("Sent panic SimError to client at tick {}", tick);
|
||||
}
|
||||
}
|
||||
|
||||
/// Print bevy_ecs schedule graph and exit.
|
||||
/// Invoked by --dump-schedule CLI flag (#346).
|
||||
///
|
||||
|
||||
Reference in New Issue
Block a user