feat(simulation): error handling and recovery — panic supervision, state hash, structured errors (#85)

Protocol v17: add state_hash (desync detection) and sim_errors
(structured error reporting) to ObserverSnapshot. Add SimError,
SimErrorKind, SimErrorBuffer types. Wrap main loop app.update() in
catch_unwind — on panic, send a final SimError snapshot before exit.
Report recoverable deserialization errors to client via SimErrorBuffer.
Compute per-tick state hash from player position + NPC count + tick.
Update all test fixtures and golden files for protocol v17.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-27 17:46:27 +01:00
co-authored by Claude Opus 4.6
parent c2cae1f618
commit d31ac1cc70
23 changed files with 557 additions and 7 deletions
+105 -1
View File
@@ -176,11 +176,43 @@ fn main() {
// Targets ~20 ticks/sec (2 game-minutes/sec). The TCP bridge uses
// non-blocking reads, so without throttling this loop would spin.
// Remaining frame budget is available for NPC AI and pathfinding.
//
// Panic supervision (#85): each tick is wrapped in catch_unwind.
// On panic, the server sends a structured SimError to the client
// before shutting down, rather than an abrupt disconnect.
let target_frame_time = std::time::Duration::from_millis(50);
loop {
let frame_start = std::time::Instant::now();
app.update();
// Wrap app.update() in catch_unwind to handle system panics (#85).
// AssertUnwindSafe is required because App is not UnwindSafe.
let tick_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
app.update();
}));
match tick_result {
Ok(()) => {}
Err(panic_payload) => {
// Extract panic message for error reporting
let panic_msg = if let Some(s) = panic_payload.downcast_ref::<&str>() {
s.to_string()
} else if let Some(s) = panic_payload.downcast_ref::<String>() {
s.clone()
} else {
"unknown panic".to_string()
};
tracing::error!("Simulation panic caught: {}", panic_msg);
// Attempt to send a final SimError snapshot to the client.
// Best-effort: if the bridge is unavailable, we just log and exit.
send_panic_error(&app, &panic_msg);
tracing::error!("Server shutting down after panic");
break;
}
}
if !app.world().resource::<ServerRunning>().0 {
break;
}
@@ -200,6 +232,78 @@ fn main() {
tracing::info!("Simulation server shutting down");
}
/// Best-effort: send a final SimError snapshot to the client on panic (#85).
///
/// Builds a minimal ObserverSnapshot with the panic error and sends it
/// through the bridge. If the bridge is unavailable or sending fails,
/// the error is logged but not fatal (we're already crashing).
fn send_panic_error(app: &App, panic_msg: &str) {
use settled_reach_server::bridge::types::*;
use settled_reach_server::simulation::time::{DayPhase, TickRate};
let world = app.world();
// Try to read current tick from SimulationTime
let tick = world
.get_resource::<settled_reach_server::simulation::time::SimulationTime>()
.map(|t| t.tick)
.unwrap_or(0);
let bridge = match world.get_resource::<BridgeResource>() {
Some(b) => b,
None => {
tracing::error!("Cannot send panic error: no BridgeResource");
return;
}
};
// Build a minimal snapshot carrying the panic error
let snapshot = ObserverSnapshot {
version: PROTOCOL_VERSION,
tick,
game_time: GameTime {
day: 0,
time_of_day: 0,
day_phase: DayPhase::Morning,
tick_rate: TickRate::Paused,
},
player_facing: FacingDirection::North,
player_stance: MovementStance::default(),
player_inventory: vec![],
entities: vec![],
visible_tiles: vec![],
nearby_interactions: vec![],
current_monologue: None,
pending_recognitions: vec![],
dialogue_response: None,
blocked_entities: vec![],
scan_events: vec![],
sound_events: vec![],
conversation_events: vec![],
conversation_ended: vec![],
follow_state: None,
character_pressure: None,
rng_seed: None,
poi_list: vec![],
examine_result: None,
player_knowledge: None,
save_result: None,
triangle_crisis_events: vec![],
state_hash: None,
sim_errors: vec![SimError {
kind: SimErrorKind::Panic,
message: format!("Simulation panic: {}", panic_msg),
tick,
}],
};
if let Err(e) = bridge.send_snapshot(&snapshot) {
tracing::error!("Failed to send panic error to client: {}", e);
} else {
tracing::info!("Sent panic SimError to client at tick {}", tick);
}
}
/// Print bevy_ecs schedule graph and exit.
/// Invoked by --dump-schedule CLI flag (#346).
///