From 1a4fd578ccb1ea0c2d1af7b2c8ed7f24eaa50a0c Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Fri, 13 Mar 2026 09:11:44 +0100 Subject: [PATCH] =?UTF-8?q?feat(simulation):=20voice=20pipeline=20integrat?= =?UTF-8?q?ion=20=E2=80=94=20Factual=20bypass,=20observer=20wiring,=20tell?= =?UTF-8?q?=20iteration=20(#650=20#652=20#651)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ContentType::Factual bypasses LLM for fact-bearing lines (numbers, denials). Observer enrichment systems rewrite dialogue/conversation text with voiced versions before snapshot assembly. Friendly and RoutineDeviation tells get concrete surface-pattern examples. Co-Authored-By: Claude Opus 4.6 --- server/src/voice/cache.rs | 2 + server/src/voice/integration.rs | 220 ++++++++++++++++++++++++++++ server/src/voice/lookup.rs | 53 ++++++- server/src/voice/mod.rs | 1 + server/src/voice/prompt_builder.rs | 41 +++++- server/src/voice/worker.rs | 9 +- server/tests/voice_pipeline.rs | 1 + server/tests/voice_quality_batch.rs | 1 + 8 files changed, 316 insertions(+), 12 deletions(-) create mode 100644 server/src/voice/integration.rs diff --git a/server/src/voice/cache.rs b/server/src/voice/cache.rs index 8aa2499f9..05f30a01c 100644 --- a/server/src/voice/cache.rs +++ b/server/src/voice/cache.rs @@ -220,6 +220,7 @@ impl Serialize for ContentType { match self { ContentType::Dialogue => serializer.serialize_u8(0), ContentType::Behavior => serializer.serialize_u8(1), + ContentType::Factual => serializer.serialize_u8(2), } } } @@ -230,6 +231,7 @@ impl<'de> Deserialize<'de> for ContentType { match v { 0 => Ok(ContentType::Dialogue), 1 => Ok(ContentType::Behavior), + 2 => Ok(ContentType::Factual), _ => Err(serde::de::Error::custom("invalid ContentType")), } } diff --git a/server/src/voice/integration.rs b/server/src/voice/integration.rs new file mode 100644 index 000000000..d474e08cd --- /dev/null +++ b/server/src/voice/integration.rs @@ -0,0 +1,220 @@ +//! Observer integration for the voice pipeline (D-138, Phase 3). +//! +//! Two enrichment systems run before `compute_observer_snapshot` and rewrite +//! NPC text in the dialogue and conversation buffers with voiced variants +//! looked up from the cache. Cache miss → base text (never blocks). +//! +//! ## System ordering +//! +//! ```text +//! process_talk_interaction ─┐ +//! run_npc_conversations ─┤─► voice_enrich_dialogue_response ─┐ +//! └─► voice_enrich_conversation_events ─► compute_observer_snapshot +//! ``` +//! +//! ## Content index derivation +//! +//! The voice cache key uses a `content_index: u16` to distinguish individual +//! lines for the same NPC. For dialogue lines the index is derived via +//! FNV-1a hash of `line_id`, matching the derivation used when requests +//! are submitted to the worker queue for pre-voicing. + +use bevy_ecs::prelude::*; + +use crate::knowledge::{EntityRegistry, StableId}; +use crate::npc::tell_state::DerivedTellState; +use crate::npc::{Npc, NpcVoiceProfile}; +use crate::simulation::conversation::ConversationEventBuffer; +use crate::simulation::dialogue::DialogueResponseBuffer; +use crate::simulation::movement::{PlayerCharacter, TilePosition}; +use crate::simulation::zone::ZoneMap; +use crate::voice::lookup::{voiced_behavior, VoiceCacheResource}; +use crate::voice::prompt_builder::ContentType; + +// --------------------------------------------------------------------------- +// Content index derivation +// --------------------------------------------------------------------------- + +/// Derive a deterministic content_index (u16) from a string line identifier. +/// +/// Uses FNV-1a (64-bit) truncated to u16 — same algorithm used by the worker +/// queue when submitting pre-voicing requests, ensuring cache key consistency. +pub fn content_index_from_line_id(line_id: &str) -> u16 { + let mut hash: u64 = 0xcbf29ce484222325; + for b in line_id.bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + hash as u16 +} + +// --------------------------------------------------------------------------- +// Player zone resolution +// --------------------------------------------------------------------------- + +fn player_zone_id( + player_query: &Query<&TilePosition, With>, + zone_map: Option<&ZoneMap>, +) -> u32 { + let Ok(pos) = player_query.single() else { + return 0; + }; + zone_map + .and_then(|zm| zm.zone_at(pos.x, pos.y, pos.z)) + .map(|id| id as u32) + .unwrap_or(0) +} + +// --------------------------------------------------------------------------- +// Systems +// --------------------------------------------------------------------------- + +/// Enrich the dialogue response buffer with a voiced line from the cache. +/// +/// Must run after `process_talk_interaction` and before +/// `compute_observer_snapshot`. No-op when the voice cache resource is absent. +/// +/// The system looks up the NPC's voice culture and current tell state, then +/// calls `voiced_behavior()` to retrieve a cached voiced line. On cache miss +/// the buffer text is unchanged (base text serves as fallback). +#[allow(clippy::type_complexity)] +pub fn voice_enrich_dialogue_response( + voice_cache: Option>, + registry: Res, + zone_map: Option>, + player_query: Query<&TilePosition, With>, + npc_voice_query: Query<(&NpcVoiceProfile, Option<&DerivedTellState>), With>, + mut dialogue_buffer: Query<&mut DialogueResponseBuffer, With>, +) { + let Some(voice_cache) = voice_cache else { + return; + }; + + let Ok(mut buffer) = dialogue_buffer.single_mut() else { + return; + }; + + let Some(ref mut response) = buffer.response else { + return; + }; + + let zone_id = player_zone_id(&player_query, zone_map.as_deref()); + + let speaker_entity = registry.to_entity(&StableId(response.speaker_entity_id)); + let Some(entity) = speaker_entity else { + return; + }; + + let Ok((voice_profile, tell_state_opt)) = npc_voice_query.get(entity) else { + return; + }; + + let tell_state = tell_state_opt.and_then(|t| t.category); + let content_index = content_index_from_line_id(&response.line_id); + + let voiced = voiced_behavior( + &voice_cache.cache, + zone_id, + &voice_profile.culture_id, + response.speaker_entity_id, + ContentType::Dialogue, + content_index, + tell_state, + &response.text, + false, + ); + + response.text = voiced; +} + +/// Enrich the conversation event buffer with voiced lines from the cache. +/// +/// Must run after `run_npc_conversations` and before +/// `compute_observer_snapshot`. No-op when the voice cache resource is absent. +/// +/// Each event in the buffer is processed: the pre-occlusion base text is +/// not available at this point (occlusion has already been applied), so the +/// `occluded_line` is treated as the base text for the voice lookup. +/// This means the voice register wraps the already-occluded line. +#[allow(clippy::type_complexity)] +pub fn voice_enrich_conversation_events( + voice_cache: Option>, + registry: Res, + zone_map: Option>, + player_query: Query<&TilePosition, With>, + npc_voice_query: Query<(&NpcVoiceProfile, Option<&DerivedTellState>), With>, + mut conversation_buffer: Query<&mut ConversationEventBuffer, With>, +) { + let Some(voice_cache) = voice_cache else { + return; + }; + + let Ok(mut buffer) = conversation_buffer.single_mut() else { + return; + }; + + if buffer.events.is_empty() { + return; + } + + let zone_id = player_zone_id(&player_query, zone_map.as_deref()); + + for event in &mut buffer.events { + let speaker_entity = registry.to_entity(&StableId(event.speaker_id)); + let Some(entity) = speaker_entity else { + continue; + }; + + let Ok((voice_profile, tell_state_opt)) = npc_voice_query.get(entity) else { + continue; + }; + + let tell_state = tell_state_opt.and_then(|t| t.category); + // Use speaker_id as the content derivation input — conversation lines + // don't have a stable line_id, so use a hash of the line text itself. + let content_index = content_index_from_line_id(&event.occluded_line); + + let voiced = voiced_behavior( + &voice_cache.cache, + zone_id, + &voice_profile.culture_id, + event.speaker_id, + ContentType::Dialogue, + content_index, + tell_state, + &event.occluded_line, + false, + ); + + event.occluded_line = voiced; + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_index_is_deterministic() { + let a = content_index_from_line_id("dock_d_001"); + let b = content_index_from_line_id("dock_d_001"); + assert_eq!(a, b); + } + + #[test] + fn content_index_differs_for_different_ids() { + let a = content_index_from_line_id("dock_d_001"); + let b = content_index_from_line_id("dock_d_002"); + assert_ne!(a, b); + } + + #[test] + fn content_index_empty_string() { + // Should not panic + let _ = content_index_from_line_id(""); + } +} diff --git a/server/src/voice/lookup.rs b/server/src/voice/lookup.rs index a260e2963..1ce576d39 100644 --- a/server/src/voice/lookup.rs +++ b/server/src/voice/lookup.rs @@ -1,17 +1,25 @@ -//! Voice cache lookup for behavior text (D-138, Phase 3 stub). +//! Voice cache lookup for behavior text (D-138, Phase 3). //! //! Provides the integration point between the voice cache and any system -//! that serves NPC text to the client. Not yet wired into a runtime system — -//! `observable_behaviors` on `NpcBlueprint` is generator output only. -//! -//! Wire `voiced_behavior()` into the behavior-serving path once it exists. +//! that serves NPC text to the client. Wired into the observer pipeline +//! via `voice::integration` enrichment systems. +use bevy_ecs::prelude::*; use std::sync::{Arc, Mutex}; use crate::npc::tell_state::TellCategory; use crate::voice::cache::{CacheKey, VoiceCacheStore}; use crate::voice::prompt_builder::ContentType; +/// ECS resource wrapping the voice cache store. +/// +/// Optional — systems that use it gracefully degrade to base text when absent. +/// Populated at startup by the voice subsystem when the pipeline is enabled. +#[derive(Resource)] +pub struct VoiceCacheResource { + pub cache: Arc>, +} + /// Look up a voiced behavior from cache, falling back to base text. /// /// Tell behaviors (from `NpcBlueprint.tell_behaviors`) are NEVER re-voiced — @@ -33,6 +41,11 @@ pub fn voiced_behavior( return base_text.to_string(); } + // Factual content is always base text — no LLM, no cache lookup. + if content_type == ContentType::Factual { + return base_text.to_string(); + } + let key = CacheKey { culture_id: culture_id.to_string(), npc_stable_id, @@ -99,6 +112,36 @@ mod tests { assert_eq!(result, "Base line."); } + #[test] + fn factual_content_always_passthrough() { + let cache = test_cache(); + // Store a voiced version under the Factual key — it must never be returned. + let key = CacheKey { + culture_id: "krenn".into(), + npc_stable_id: 42, + content_type: ContentType::Factual, + content_index: 0, + tell_state: None, + }; + cache + .lock() + .unwrap() + .store(100, key, "Voiced factual (should be ignored).".into()); + + let result = voiced_behavior( + &cache, + 100, + "krenn", + 42, + ContentType::Factual, + 0, + None, + "14 crates in bay seven.", + false, + ); + assert_eq!(result, "14 crates in bay seven."); + } + #[test] fn tell_behavior_always_passthrough() { let cache = test_cache(); diff --git a/server/src/voice/mod.rs b/server/src/voice/mod.rs index 9db30546c..6d1088519 100644 --- a/server/src/voice/mod.rs +++ b/server/src/voice/mod.rs @@ -14,6 +14,7 @@ pub mod cache; pub mod hardware; +pub mod integration; pub mod lookup; pub mod prompt_builder; pub mod queue; diff --git a/server/src/voice/prompt_builder.rs b/server/src/voice/prompt_builder.rs index 4667fdc8a..8583c6747 100644 --- a/server/src/voice/prompt_builder.rs +++ b/server/src/voice/prompt_builder.rs @@ -22,6 +22,13 @@ pub enum ContentType { Dialogue, /// Observable behavior description — re-voiced with "Describe". Behavior, + /// Fact-bearing line — numbers, causal chains, denials. + /// + /// Factual lines bypass the LLM entirely and are served as base text. + /// Spike 2 showed 2B models corrupt quantitative content ("14 crates in + /// bay seven" → "fourteen crates are missing") and invert denials. + /// The simulation is the truth layer — the LLM only handles register. + Factual, } /// Result of prompt building, including which injections fired. @@ -57,6 +64,15 @@ OUTPUT CONSTRAINTS:\n\ /// "make sentences shorter" instruction causes destructive compression that /// strips facts. Long-Angry instead preserves the full claim and focuses /// intensity on one sentence. +/// +/// ## Iteration history for Friendly and RoutineDeviation +/// +/// Spike 2 showed these two tells produce output indistinguishable from neutral +/// on Gemma 2B. Tracked against a 3-iteration budget (#651): +/// +/// - **Iteration 1** (Sprint 26): Replaced abstract instructions with concrete +/// surface-pattern examples. Friendly: closing aside pattern. RoutineDeviation: +/// self-correction/echo pattern. Both more imitable for a 2B model. fn tell_injector(category: TellCategory) -> &'static str { match category { TellCategory::Nervous => { @@ -70,9 +86,12 @@ fn tell_injector(category: TellCategory) -> &'static str { Do not say they seem angry." } TellCategory::Friendly => { - "TONE: Add one small extra detail or aside that wasn't strictly necessary. \ - Example: \"Inspection's tomorrow — should be fine, though.\" \ - Do not add compliments or forced warmth." + // Iteration 1: replaced abstract "add detail" with a concrete closing-aside + // pattern. 2B models need a recognisable syntactic target, not a description + // of intent. "actually" / "so that's something" are learnable short tokens. + "TONE: End with a brief unprompted aside — a phrase the person didn't need to say. \ + Example: \"Parts are in, so that's something.\" or \"Shift's been quiet, actually.\" \ + Do not use praise or warm adjectives. One short beat after the main point." } TellCategory::Guarded => { "TONE: Use formal, precise words. Answer exactly what was asked, nothing extra. \ @@ -80,9 +99,12 @@ fn tell_injector(category: TellCategory) -> &'static str { Do not say they seem guarded." } TellCategory::RoutineDeviation => { - "TONE: Start the sentence on topic, then add a brief unfinished thought about something else. \ - Example: \"Pressure's fine. I was going to — anyway, it's logged.\" \ - Do not explain what they were thinking about." + // Iteration 1: replaced the two-step interrupted-thought pattern (too complex + // for 2B) with a simpler self-correction / word-echo pattern. The model only + // needs to repeat a key word or phrase — one concrete surface action. + "TONE: Repeat a key word or phrase, as if catching mid-thought. \ + Example: \"Logged it. Got it logged, yeah.\" or \"Pressure's — pressure's holding.\" \ + Do not explain. Just the echo." } } } @@ -269,6 +291,11 @@ pub fn build_prompt( "Describe the following action as a third-person observer. \ Preserve all individual actions in sequence. Do not extract conclusions" } + ContentType::Factual => { + // Factual lines must be intercepted before reaching build_prompt. + // If this branch is hit, the caller has a bug. + unreachable!("ContentType::Factual must not reach build_prompt — bypass at worker") + } }; parts.push(String::new()); parts.push(format!("TASK: {}.", task_verb)); @@ -370,6 +397,7 @@ mod tests { TellCategory::Friendly, ], }], + behavior_modifiers: vec![], } } @@ -399,6 +427,7 @@ mod tests { voice_persona: None, voice_examples: vec![], occasional_injections: vec![], + behavior_modifiers: vec![], } } diff --git a/server/src/voice/worker.rs b/server/src/voice/worker.rs index 9dd9b2da5..e6983614e 100644 --- a/server/src/voice/worker.rs +++ b/server/src/voice/worker.rs @@ -16,7 +16,7 @@ use crossbeam_channel::Receiver; use crate::npc::blueprint::CultureProfile; use crate::voice::cache::{CacheKey, VoiceCacheStore}; -use crate::voice::prompt_builder; +use crate::voice::prompt_builder::{self, ContentType}; use crate::voice::queue::VoiceRequest; /// Minimum token count for a valid response. Below this, retry once. @@ -326,6 +326,13 @@ fn process_request( request: &VoiceRequest, ctx: &WorkerContext, ) { + // Factual lines bypass the LLM entirely — serve base text directly. + // 2B models corrupt numbers and invert denials (Spike 2 finding, D-138). + if request.content_type == ContentType::Factual { + cache_base_text(request, ctx); + return; + } + let culture = match ctx.cultures.get(&request.culture_id) { Some(c) => c, None => { diff --git a/server/tests/voice_pipeline.rs b/server/tests/voice_pipeline.rs index a810fb998..ebf2dee8b 100644 --- a/server/tests/voice_pipeline.rs +++ b/server/tests/voice_pipeline.rs @@ -127,6 +127,7 @@ fn krenn_culture() -> CultureProfile { TellCategory::Friendly, ], }], + behavior_modifiers: vec![], } } diff --git a/server/tests/voice_quality_batch.rs b/server/tests/voice_quality_batch.rs index 36fa0802a..2eb0eb86c 100644 --- a/server/tests/voice_quality_batch.rs +++ b/server/tests/voice_quality_batch.rs @@ -119,6 +119,7 @@ fn krenn_culture() -> CultureProfile { TellCategory::Friendly, ], }], + behavior_modifiers: vec![], } }