feat(simulation): error handling and recovery — panic supervision, state hash, structured errors (#85)
Protocol v17: add state_hash (desync detection) and sim_errors (structured error reporting) to ObserverSnapshot. Add SimError, SimErrorKind, SimErrorBuffer types. Wrap main loop app.update() in catch_unwind — on panic, send a final SimError snapshot before exit. Report recoverable deserialization errors to client via SimErrorBuffer. Compute per-tick state hash from player position + NPC count + tick. Update all test fixtures and golden files for protocol v17. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -91,14 +91,20 @@ impl Default for HandshakeState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive inputs from bridge and push to InputQueue
|
||||
/// Receive inputs from bridge and push to InputQueue.
|
||||
/// Protocol errors (malformed input) are recoverable: the frame is skipped
|
||||
/// and a SimError is pushed to the SimErrorBuffer for client reporting (#85).
|
||||
pub fn receive_bridge_inputs(
|
||||
bridge: Option<Res<BridgeResource>>,
|
||||
mut input_queue: ResMut<crate::simulation::input::InputQueue>,
|
||||
mut running: ResMut<ServerRunning>,
|
||||
handshake: Res<HandshakeState>,
|
||||
mut error_buffer: ResMut<SimErrorBuffer>,
|
||||
time: Option<Res<crate::simulation::time::SimulationTime>>,
|
||||
) {
|
||||
let Some(bridge) = bridge else { return };
|
||||
let current_tick = time.as_ref().map(|t| t.tick).unwrap_or(0);
|
||||
|
||||
match bridge.receive_inputs() {
|
||||
Ok(inputs) => {
|
||||
if !inputs.is_empty() && *handshake == HandshakeState::Pending {
|
||||
@@ -134,8 +140,22 @@ pub fn receive_bridge_inputs(
|
||||
running.0 = false;
|
||||
}
|
||||
Err(BridgeError::DeserializationWithDump(ref msg)) => {
|
||||
// Recoverable: skip this frame's input, don't shut down
|
||||
// Recoverable: skip this frame's input, report to client (#85)
|
||||
tracing::error!("Skipping malformed input frame: {}", msg);
|
||||
error_buffer.push(SimError {
|
||||
kind: SimErrorKind::ProtocolError,
|
||||
message: format!("Malformed input frame: {}", msg),
|
||||
tick: current_tick,
|
||||
});
|
||||
}
|
||||
Err(ref e @ BridgeError::Deserialization(_)) => {
|
||||
// Recoverable deserialization error without dump
|
||||
tracing::error!("Skipping malformed input: {}", e);
|
||||
error_buffer.push(SimError {
|
||||
kind: SimErrorKind::ProtocolError,
|
||||
message: format!("Deserialization error: {}", e),
|
||||
tick: current_tick,
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Bridge receive error: {}", e);
|
||||
@@ -191,6 +211,7 @@ impl Plugin for BridgePlugin {
|
||||
app.init_resource::<SnapshotBuffer>()
|
||||
.init_resource::<ServerRunning>()
|
||||
.init_resource::<HandshakeState>()
|
||||
.init_resource::<SimErrorBuffer>()
|
||||
.init_resource::<crate::perception::query::VisibilityGeometry>()
|
||||
.init_resource::<crate::perception::query::ActivePerceptionMode>()
|
||||
.add_systems(
|
||||
|
||||
@@ -314,6 +314,8 @@ mod tests {
|
||||
player_knowledge: None,
|
||||
save_result: None,
|
||||
triangle_crisis_events: vec![],
|
||||
state_hash: None,
|
||||
sim_errors: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,6 +452,8 @@ mod tests {
|
||||
player_knowledge: None,
|
||||
save_result: None,
|
||||
triangle_crisis_events: vec![],
|
||||
state_hash: None,
|
||||
sim_errors: vec![],
|
||||
};
|
||||
let text = format_snapshot_text(&snap);
|
||||
assert!(text.contains("Tick 0"));
|
||||
|
||||
@@ -17,7 +17,7 @@ pub use crate::simulation::time::{DayPhase, TickRate};
|
||||
/// negotiation is unnecessary. Client should reject snapshots with version !=
|
||||
/// PROTOCOL_VERSION. New fields use #[serde(default)] only during the migration
|
||||
/// period, then the default is removed once both sides are updated.
|
||||
pub const PROTOCOL_VERSION: u8 = 16;
|
||||
pub const PROTOCOL_VERSION: u8 = 17;
|
||||
|
||||
/// Handshake message sent as the very first framed message after connection (#555).
|
||||
/// Client reads this before entering the normal tick loop and validates
|
||||
@@ -51,10 +51,12 @@ pub struct HandshakeMessage {
|
||||
/// player_knowledge (#264, partial KG dump for journal/knowledge panel).
|
||||
/// v15 adds: save_result (#553, save/load operation result for client confirmation).
|
||||
/// v16 adds: triangle_crisis_events (#250, D-087 triangle escalation for future client rendering).
|
||||
/// v17 adds: state_hash (#85, desync detection — fast hash of player pos + NPC count + tick),
|
||||
/// sim_errors (#85, structured error reporting to client).
|
||||
/// Future fields: ambient sound events, HUD state (D-020 expansion).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ObserverSnapshot {
|
||||
/// Protocol version for forward compatibility. Current: 16.
|
||||
/// Protocol version for forward compatibility. Current: 17.
|
||||
pub version: u8,
|
||||
/// Simulation tick when this snapshot was produced
|
||||
pub tick: u64,
|
||||
@@ -157,6 +159,19 @@ pub struct ObserverSnapshot {
|
||||
/// a narrative event or HUD indicator. Empty when no crises occur.
|
||||
#[serde(default)]
|
||||
pub triangle_crisis_events: Vec<TriangleCrisisEventWire>,
|
||||
/// Fast hash of key mutable state for desync detection (#85).
|
||||
/// Hash inputs: player position, NPC count, tick number.
|
||||
/// Client compares against its own computed hash — mismatch indicates
|
||||
/// client and server state have diverged. No auto-recovery in v0.1;
|
||||
/// client logs mismatches for debugging.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub state_hash: Option<u64>,
|
||||
/// Simulation errors reported this tick (#85).
|
||||
/// Non-fatal errors (protocol errors, desync) are collected during
|
||||
/// the tick and sent to the client for logging/display.
|
||||
/// Empty in normal operation. Client may display a warning toast.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub sim_errors: Vec<SimError>,
|
||||
}
|
||||
|
||||
/// Game time data for client display (D-031)
|
||||
@@ -714,6 +729,56 @@ impl From<crate::content::template::TriangleCrisisEvent> for TriangleCrisisEvent
|
||||
}
|
||||
}
|
||||
|
||||
/// Structured simulation error for client reporting (#85).
|
||||
///
|
||||
/// Sent inside `ObserverSnapshot.sim_errors` for recoverable errors
|
||||
/// (protocol errors, desync warnings). For fatal errors (panics),
|
||||
/// a final snapshot is sent with the error before the server exits.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SimError {
|
||||
/// Error category for client-side handling.
|
||||
pub kind: SimErrorKind,
|
||||
/// Human-readable error description.
|
||||
pub message: String,
|
||||
/// Tick when the error occurred (0 if unavailable).
|
||||
pub tick: u64,
|
||||
}
|
||||
|
||||
/// Categories of simulation errors (#85).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum SimErrorKind {
|
||||
/// Simulation system panic — fatal, server will exit after sending this.
|
||||
Panic,
|
||||
/// Protocol/deserialization error — recoverable, server continues.
|
||||
ProtocolError,
|
||||
/// Client-server state hash mismatch — informational, no auto-recovery.
|
||||
DesyncDetected,
|
||||
}
|
||||
|
||||
/// Buffer for collecting simulation errors during a tick (#85).
|
||||
/// Drained by `compute_observer_snapshot` into `ObserverSnapshot.sim_errors`.
|
||||
#[derive(Resource, Debug, Default)]
|
||||
pub struct SimErrorBuffer {
|
||||
errors: Vec<SimError>,
|
||||
}
|
||||
|
||||
impl SimErrorBuffer {
|
||||
/// Push a new error into the buffer.
|
||||
pub fn push(&mut self, error: SimError) {
|
||||
self.errors.push(error);
|
||||
}
|
||||
|
||||
/// Drain all buffered errors, returning them and clearing the buffer.
|
||||
pub fn drain(&mut self) -> Vec<SimError> {
|
||||
std::mem::take(&mut self.errors)
|
||||
}
|
||||
|
||||
/// Check if there are pending errors.
|
||||
pub fn has_errors(&self) -> bool {
|
||||
!self.errors.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot buffer resource for staging outgoing ObserverSnapshots
|
||||
#[derive(Resource, Debug, Default)]
|
||||
pub struct SnapshotBuffer {
|
||||
|
||||
Reference in New Issue
Block a user