//! Voice quality batch test (D-138, Spike 2). //! //! Runs a comprehensive set of edge-case prompts through the voice pipeline //! and writes results to `.tmp/voice-test/quality-batch.txt` for human review. //! //! Run: make test-voice-quality //! Or: cd server && cargo test --test voice_quality_batch -- --nocapture --ignored use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use settled_reach_server::npc::blueprint::{ CulturalValues, CultureProfile, NamingConventions, OccasionalInjection, SpeechPatterns, VoiceExample, }; use settled_reach_server::npc::tell_state::TellCategory; use settled_reach_server::npc::PersonalityTrait; use settled_reach_server::voice::cache::{CacheKey, VoiceCacheStore}; use settled_reach_server::voice::prompt_builder::ContentType; use settled_reach_server::voice::queue::{Priority, VoiceQueue, VoiceRequest}; use settled_reach_server::voice::worker::{VoiceProcessConfig, WorkerPool}; // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- const SR_VOICE_BIN: &str = "sr-voice/target/release/sr-voice"; const SR_VOICE_MODEL: &str = "models/gemma2.gguf"; const OUTPUT_DIR: &str = ".tmp/voice-test"; fn output_dir() -> PathBuf { let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); PathBuf::from(manifest).join("..").join(OUTPUT_DIR) } fn voice_config() -> VoiceProcessConfig { let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into()); let manifest = PathBuf::from(manifest); let bin_path = manifest.join(SR_VOICE_BIN); let model_path = manifest.join(SR_VOICE_MODEL); assert!( bin_path.exists(), "sr-voice binary not found: {} — run `make build-sr-voice`", bin_path.display() ); assert!( model_path.exists(), "Model not found: {}", model_path.display() ); VoiceProcessConfig { binary_path: bin_path.to_string_lossy().into(), model_path: model_path.to_string_lossy().into(), threads: 2, ctx_size: 512, } } // --------------------------------------------------------------------------- // Culture profiles // --------------------------------------------------------------------------- fn van_maanens_star_culture() -> CultureProfile { CultureProfile { id: "van-maanens-star".into(), name: "Van Maanen's Star Culture".into(), description: "Working-class pragmatic".into(), naming: NamingConventions { style: "compact".into(), given_names: vec!["Kael".into(), "Dren".into(), "Sira".into()], family_names: vec!["Davan".into(), "Voss".into()], family_name_used_socially: false, }, speech: SpeechPatterns { register: "direct".into(), filler_words: vec!["look".into()], greetings: vec!["hey".into()], farewells: vec!["shift's calling".into()], exclamations: vec!["void take it".into()], }, values: CulturalValues { description: "Pragmatic".into(), favored_traits: vec![PersonalityTrait::Bold], disfavored_traits: vec![PersonalityTrait::Reclusive], }, voice_persona: Some( "PERSONA: You are a Van Maanen's Star station worker.\n\ 1. Be direct. No pleasantries.\n\ 2. You're working-class and pragmatic." .into(), ), voice_examples: vec![ VoiceExample { input: "declines to answer a question".into(), output: "Look, that's not mine to say.".into(), }, VoiceExample { input: "confirms a task is complete".into(), output: "Done. Logged it.".into(), }, ], occasional_injections: vec![OccasionalInjection { kind: "oath".into(), clause: "Use an oath like \"void take it\" when something is surprising or frustrating." .into(), example: Some(VoiceExample { input: "discovers a critical part is missing".into(), output: "Void take it. The coupling's not here.".into(), }), frequency: 0.25, suppress_on_tells: vec![ TellCategory::Guarded, TellCategory::RoutineDeviation, TellCategory::Friendly, ], }], behavior_modifiers: vec![], } } // --------------------------------------------------------------------------- // Test cases // --------------------------------------------------------------------------- struct TestCase { label: &'static str, category: &'static str, base_text: &'static str, content_type: ContentType, tell_state: Option, seed: u64, } fn test_cases() -> Vec { vec![ // === CATEGORY: Length tiers === TestCase { label: "ultra-short (2 words)", category: "length", base_text: "It's broken.", content_type: ContentType::Dialogue, tell_state: None, seed: 100, }, TestCase { label: "short (5 words)", category: "length", base_text: "The coupling is beyond repair.", content_type: ContentType::Dialogue, tell_state: None, seed: 101, }, TestCase { label: "medium (12 words)", category: "length", base_text: "The pressure readings have been unstable all week and nobody filed a report.", content_type: ContentType::Dialogue, tell_state: None, seed: 102, }, TestCase { label: "long (28 words)", category: "length", base_text: "I was checking the manifests from last quarter and it looks like three shipments \ never arrived at the depot, which means someone either lost them or diverted \ them deliberately.", content_type: ContentType::Dialogue, tell_state: None, seed: 103, }, // === CATEGORY: Tell states on medium content === TestCase { label: "medium + Nervous", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Nervous), seed: 200, }, TestCase { label: "medium + Angry", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Angry), seed: 200, }, TestCase { label: "medium + Friendly", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Friendly), seed: 200, }, TestCase { label: "medium + Guarded", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Guarded), seed: 200, }, TestCase { label: "medium + RoutineDeviation", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::RoutineDeviation), seed: 200, }, TestCase { label: "medium + neutral (baseline)", category: "tell-medium", base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.", content_type: ContentType::Dialogue, tell_state: None, seed: 200, }, // === CATEGORY: Tell states on short content (should NOT differentiate) === TestCase { label: "short + Nervous (expect same as neutral)", category: "tell-short", base_text: "Inspection's next week.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Nervous), seed: 300, }, TestCase { label: "short + Angry (expect same as neutral)", category: "tell-short", base_text: "Inspection's next week.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Angry), seed: 300, }, TestCase { label: "short + neutral (baseline)", category: "tell-short", base_text: "Inspection's next week.", content_type: ContentType::Dialogue, tell_state: None, seed: 300, }, // === CATEGORY: Epistemic markers === TestCase { label: "\"I heard\" — must preserve", category: "epistemic", base_text: "I heard the night crew had to stop the line twice because the coupling was faulty.", content_type: ContentType::Dialogue, tell_state: None, seed: 400, }, TestCase { label: "\"someone told me\" — must preserve", category: "epistemic", base_text: "Someone told me the foreman is transferring out next cycle.", content_type: ContentType::Dialogue, tell_state: None, seed: 401, }, TestCase { label: "\"apparently\" — must preserve", category: "epistemic", base_text: "Apparently three containers went missing from the last shipment and nobody noticed until the audit.", content_type: ContentType::Dialogue, tell_state: None, seed: 402, }, TestCase { label: "\"I think\" + \"might have\" — two markers", category: "epistemic", base_text: "I think the wiring might have been rerouted during the last maintenance cycle without anyone logging it.", content_type: ContentType::Dialogue, tell_state: None, seed: 403, }, TestCase { label: "\"supposedly\" — hedged rumor", category: "epistemic", base_text: "Supposedly the company is cutting the night shift entirely after the audit results come back.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Nervous), seed: 404, }, // === CATEGORY: Occasional injection (oath) === // seed 7 fires oath for van-maanens-star (frequency 0.25, ChaCha8) TestCase { label: "oath should fire (seed 7)", category: "injection", base_text: "The replacement parts never showed up and now we're a full shift behind schedule.", content_type: ContentType::Dialogue, tell_state: None, seed: 7, }, TestCase { label: "oath suppressed by Guarded tell", category: "injection", base_text: "The replacement parts never showed up and now we're a full shift behind schedule.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Guarded), seed: 7, }, TestCase { label: "oath suppressed by Friendly tell", category: "injection", base_text: "The replacement parts never showed up and now we're a full shift behind schedule.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Friendly), seed: 7, }, // === CATEGORY: Behavior descriptions === TestCase { label: "behavior — simple action", category: "behavior", base_text: "checks the pressure gauge and writes in a logbook", content_type: ContentType::Behavior, tell_state: None, seed: 500, }, TestCase { label: "behavior — multi-step action", category: "behavior", base_text: "opens a maintenance panel, inspects the wiring, shakes head, then reseals the panel without making changes", content_type: ContentType::Behavior, tell_state: None, seed: 501, }, TestCase { label: "behavior — subtle body language", category: "behavior", base_text: "glances toward the corridor, hesitates, then continues working", content_type: ContentType::Behavior, tell_state: Some(TellCategory::Nervous), seed: 502, }, TestCase { label: "behavior — routine", category: "behavior", base_text: "sweeps the floor of the cargo bay in a slow, methodical pattern", content_type: ContentType::Behavior, tell_state: None, seed: 503, }, // === CATEGORY: Edge cases === TestCase { label: "named entities — must preserve names", category: "edge", base_text: "Kael told me that Dren Voss moved the shipment to Bay Seven without logging it.", content_type: ContentType::Dialogue, tell_state: None, seed: 600, }, TestCase { label: "technology terms — insert, span gate", category: "edge", base_text: "The insert feed has been glitching since they updated the span gate routing tables last cycle.", content_type: ContentType::Dialogue, tell_state: None, seed: 601, }, TestCase { label: "emotional content + no tell (flat delivery)", category: "edge", base_text: "My partner didn't come home last night and nobody on the station will tell me what happened.", content_type: ContentType::Dialogue, tell_state: None, seed: 602, }, TestCase { label: "Friendly tell + bad news (contradiction test)", category: "edge", base_text: "The contract fell through and we're going to lose half the crew by end of quarter.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Friendly), seed: 603, }, TestCase { label: "question form — NPC asking", category: "edge", base_text: "Have you seen the shift roster for next week? I can't find it anywhere.", content_type: ContentType::Dialogue, tell_state: None, seed: 604, }, TestCase { label: "imperative — NPC giving instruction", category: "edge", base_text: "Check the seals on bay four before you clock out. Last person forgot and we had a pressure drop.", content_type: ContentType::Dialogue, tell_state: None, seed: 605, }, TestCase { label: "long + Angry — emotional pressure on length", category: "edge", base_text: "I've filed three reports about the ventilation in section nine and every single time the response \ comes back saying there's no budget, while they just spent half a million credits refitting the \ executive lounge on deck two.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Angry), seed: 606, }, TestCase { label: "epistemic + tell — compound modifiers", category: "edge", base_text: "I heard the safety team might have flagged section twelve but I think somebody pulled the report \ before it went up the chain.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Nervous), seed: 607, }, // === CATEGORY: Paula's targeted tests (round 3) === // #32 without Angry — does neutral preserve more facts? TestCase { label: "long + neutral (compare to #32 Angry)", category: "paula", base_text: "I've filed three reports about the ventilation in section nine and every single time the response \ comes back saying there's no budget, while they just spent half a million credits refitting the \ executive lounge on deck two.", content_type: ContentType::Dialogue, tell_state: None, seed: 606, }, // Specific numbers — must preserve quantities TestCase { label: "specific numbers — 14 crates, bay 7", category: "paula", base_text: "We counted 14 crates in bay seven but the manifest says 18, so four are missing somewhere between \ the loading dock and here.", content_type: ContentType::Dialogue, tell_state: None, seed: 700, }, // Causal chain — A caused B caused C TestCase { label: "causal chain — must preserve cause-effect", category: "paula", base_text: "The pressure seal on bulkhead nine failed because Torren skipped the inspection, which caused \ the atmosphere leak that put three people in medical.", content_type: ContentType::Dialogue, tell_state: None, seed: 701, }, // Multiple named entities in a relationship TestCase { label: "named entities — Sira, Dren, relationship", category: "paula", base_text: "Sira told Dren about the missing cargo but Dren went straight to the shift lead instead of \ reporting it through proper channels.", content_type: ContentType::Dialogue, tell_state: None, seed: 702, }, // Specific time reference — must not shift tense TestCase { label: "past event — must stay past tense", category: "paula", base_text: "Last Thursday the cooling system failed for six hours and we lost two full batches of \ pharmaceutical stock worth about forty thousand credits.", content_type: ContentType::Dialogue, tell_state: None, seed: 703, }, // Contradiction/denial — NPC denying something TestCase { label: "denial — must preserve what is denied", category: "paula", base_text: "I wasn't anywhere near section twelve that night and whoever said they saw me there is either \ confused or lying.", content_type: ContentType::Dialogue, tell_state: Some(TellCategory::Guarded), seed: 704, }, ] } // --------------------------------------------------------------------------- // Test runner // --------------------------------------------------------------------------- #[test] #[ignore] // Run explicitly: cargo test --test voice_quality_batch -- --ignored --nocapture fn voice_quality_batch() { let out = output_dir(); let cache_dir = out.join("quality-cache"); let _ = std::fs::remove_dir_all(&cache_dir); std::fs::create_dir_all(&cache_dir).expect("failed to create cache dir"); let config = voice_config(); let mut cultures = BTreeMap::new(); cultures.insert("van-maanens-star".to_string(), van_maanens_star_culture()); let cultures = Arc::new(cultures); let cases = test_cases(); let total = cases.len(); let cache = Arc::new(Mutex::new(VoiceCacheStore::new( cache_dir.clone(), 99999, "quality-v1".into(), "quality-i1".into(), ))); let queue = VoiceQueue::new(); let paused = queue.paused_flag(); let mut pool = WorkerPool::spawn( 2, &config, queue.receiver(), Arc::clone(&cache), Arc::clone(&cultures), paused, ); eprintln!("Quality batch: {} test cases, 2 workers", total); // Submit all requests — use zone_id = npc_stable_id = index for easy lookup for (i, case) in cases.iter().enumerate() { let req = VoiceRequest { priority: Priority::High, npc_stable_id: i as u64, zone_id: 1, // all same zone for cache lookup culture_id: "van-maanens-star".into(), base_text: case.base_text.into(), content_type: case.content_type, content_index: case.tell_state.map(|t| t as u16).unwrap_or(0), tell_state: case.tell_state, seed: case.seed, }; assert!(queue.submit(req), "queue rejected request {}", i); } eprintln!("Submitted {} requests", total); // Wait for all to complete let start = Instant::now(); let max_wait = Duration::from_secs(300); // 5 min for ~35 prompts at ~4s each loop { std::thread::sleep(Duration::from_millis(500)); let pending = queue.pending_count(); let active = pool.active_workers(); if pending == 0 && active == 0 { break; } if start.elapsed() > max_wait { eprintln!( "TIMEOUT after {}s (pending={}, active={})", max_wait.as_secs(), pending, active ); break; } } let elapsed = start.elapsed(); eprintln!("Pipeline drained in {:.1}s", elapsed.as_secs_f64()); pool.shutdown(); // Collect results let mut cache_guard = cache.lock().unwrap(); let zone_cache = cache_guard.zone_cache(1); let mut output = String::new(); output.push_str("Voice Quality Batch Results (Spike 2)\n"); output.push_str("=====================================\n"); output.push_str(&format!( "Model: Gemma 2B Q4_K_M | Workers: 2 | Time: {:.1}s\n", elapsed.as_secs_f64() )); output.push_str(&format!( "Test cases: {} | Cache entries: {}\n\n", total, zone_cache.len() )); let mut current_category = ""; for (i, case) in cases.iter().enumerate() { if case.category != current_category { current_category = case.category; let sep = "=".repeat(60); output.push_str(&format!( "\n{}\n== CATEGORY: {} ==\n{}\n\n", sep, current_category, sep )); } let key = CacheKey { culture_id: "van-maanens-star".into(), npc_stable_id: i as u64, content_type: case.content_type, content_index: case.tell_state.map(|t| t as u16).unwrap_or(0), tell_state: case.tell_state, }; let voiced = zone_cache .entries .get(&key) .map(|s| s.as_str()) .unwrap_or("[MISSING — not cached]"); output.push_str(&format!("--- #{}: {} ---\n", i + 1, case.label)); output.push_str(&format!( " tell: {:?} | seed: {} | type: {:?}\n", case.tell_state, case.seed, case.content_type )); output.push_str(&format!(" BASE: {}\n", case.base_text)); output.push_str(&format!(" VOICED: {}\n\n", voiced)); } drop(cache_guard); cache .lock() .unwrap() .save_all() .expect("failed to save cache"); let results_path = out.join("quality-batch.txt"); std::fs::write(&results_path, &output).expect("failed to write results"); eprintln!("Results written to {}", results_path.display()); eprintln!("\n{}", output); // Basic sanity: we should have cached something for every test case let mut cache_guard = cache.lock().unwrap(); let zone_cache = cache_guard.zone_cache(1); assert!( zone_cache.len() >= total - 1, // allow 1 miss for flaky edge cases "Expected at least {} cache entries, got {}", total - 1, zone_cache.len() ); }