feat(voice): complete Spike 2 voice pipeline with quality-tested prompt engine
Spike 2 delivers the full voice pipeline: queue → worker pool → sr-voice
child process (stdio JSONL) → cache → disk. Three rounds of quality testing
with Paula, Mellanie, and Gestalt produced iterative prompt improvements.
Prompt engine (prompt_builder.rs):
- Example-based epistemic marker integration (not keyword lists)
- Length-aware Angry tell variant (preserves facts on long content)
- Double-prompt technique: REMEMBER block repeats constraints near OUTPUT:
- Imperative injection framing (composition engine controls frequency)
- Anti-invention constraint ("do not add information not in the input")
- Universal RULES cleaned: worldbuilding moved to culture personas
Worker pool (worker.rs):
- Output post-processor strips after first newline (prevents prompt leakage)
- Watchdog poll loop (1s ticks) replaces blocking sleep for cancel
- Child health check before writing (try_wait)
Test infrastructure:
- voice_pipeline.rs: end-to-end test, auto-detects real sr-voice or mock
- voice_quality_batch.rs: 39 edge-case prompts for quality review
- mock-stdio.sh: Python JSONL mock for CI (no model needed)
- Makefile targets: test-voice-mock, test-voice-real
Quality results (Gemma 2B Q4_K_M, CPU ~13 t/s):
- Epistemic markers: naturally integrated (round 1 comma-lists fixed)
- Tell differentiation: 3/5 working (Nervous, Guarded, Angry)
- Information preservation: ~90% (up from ~70%)
- Prompt leakage: eliminated
- Open: Friendly/RoutineDeviation tells inert (#651), Factual bypass (#650)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -0,0 +1,321 @@
|
||||
//! Voice pipeline integration test (D-138, Spike 2).
|
||||
//!
|
||||
//! Exercises the full pipeline: queue → worker → sr-voice child (mock or real)
|
||||
//! → cache → disk. Outputs voiced results to `.tmp/voice-test/`.
|
||||
//!
|
||||
//! Run with mock: cargo test --test voice_pipeline -- --nocapture
|
||||
//! Run with real: SR_VOICE_BIN=sr-voice/target/release/sr-voice \
|
||||
//! SR_VOICE_MODEL=models/gemma2.gguf \
|
||||
//! cargo test --test voice_pipeline -- --nocapture
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use settled_reach_server::npc::blueprint::{
|
||||
CulturalValues, CultureProfile, NamingConventions, OccasionalInjection, SpeechPatterns,
|
||||
VoiceExample,
|
||||
};
|
||||
use settled_reach_server::npc::PersonalityTrait;
|
||||
use settled_reach_server::npc::tell_state::TellCategory;
|
||||
use settled_reach_server::voice::cache::VoiceCacheStore;
|
||||
use settled_reach_server::voice::prompt_builder::ContentType;
|
||||
use settled_reach_server::voice::queue::{Priority, VoiceQueue, VoiceRequest};
|
||||
use settled_reach_server::voice::worker::{VoiceProcessConfig, WorkerPool};
|
||||
|
||||
/// Output directory for test results (relative to repo root).
|
||||
const OUTPUT_DIR: &str = ".tmp/voice-test";
|
||||
|
||||
fn output_dir() -> PathBuf {
|
||||
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
|
||||
PathBuf::from(manifest).join("..").join(OUTPUT_DIR)
|
||||
}
|
||||
|
||||
/// Known paths relative to CARGO_MANIFEST_DIR (server/).
|
||||
const SR_VOICE_BIN: &str = "sr-voice/target/release/sr-voice";
|
||||
const SR_VOICE_MODEL: &str = "models/gemma2.gguf";
|
||||
|
||||
fn voice_config() -> VoiceProcessConfig {
|
||||
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
|
||||
let manifest = PathBuf::from(manifest);
|
||||
|
||||
let bin_path = manifest.join(SR_VOICE_BIN);
|
||||
let model_path = manifest.join(SR_VOICE_MODEL);
|
||||
|
||||
// SR_VOICE_MOCK=1 forces mock mode (used by `make test-voice-mock`)
|
||||
let force_mock = std::env::var("SR_VOICE_MOCK").is_ok();
|
||||
|
||||
// Use real sr-voice if both binary and model exist on disk
|
||||
if !force_mock && bin_path.exists() && model_path.exists() {
|
||||
eprintln!("Using real sr-voice: {}", bin_path.display());
|
||||
eprintln!("Model: {}", model_path.display());
|
||||
VoiceProcessConfig {
|
||||
binary_path: bin_path.to_string_lossy().into(),
|
||||
model_path: model_path.to_string_lossy().into(),
|
||||
threads: 2,
|
||||
ctx_size: 512,
|
||||
}
|
||||
} else {
|
||||
// Fall back to mock script — no model needed
|
||||
let mock = manifest.join("sr-voice/mock-stdio.sh");
|
||||
assert!(
|
||||
mock.exists(),
|
||||
"Mock script not found: {}",
|
||||
mock.display()
|
||||
);
|
||||
eprintln!("Using mock sr-voice: {}", mock.display());
|
||||
if !bin_path.exists() {
|
||||
eprintln!(" (real binary not found: {})", bin_path.display());
|
||||
}
|
||||
if !model_path.exists() {
|
||||
eprintln!(" (model not found: {})", model_path.display());
|
||||
}
|
||||
VoiceProcessConfig {
|
||||
binary_path: mock.to_string_lossy().into(),
|
||||
model_path: "unused".into(),
|
||||
threads: 1,
|
||||
ctx_size: 512,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn krenn_culture() -> CultureProfile {
|
||||
CultureProfile {
|
||||
id: "krenn".into(),
|
||||
name: "Krenn System Culture".into(),
|
||||
description: "Working-class pragmatic".into(),
|
||||
naming: NamingConventions {
|
||||
style: "compact".into(),
|
||||
given_names: vec!["Kael".into()],
|
||||
family_names: vec!["Davan".into()],
|
||||
family_name_used_socially: false,
|
||||
},
|
||||
speech: SpeechPatterns {
|
||||
register: "direct".into(),
|
||||
filler_words: vec!["look".into()],
|
||||
greetings: vec!["hey".into()],
|
||||
farewells: vec!["shift's calling".into()],
|
||||
exclamations: vec!["void take it".into()],
|
||||
},
|
||||
values: CulturalValues {
|
||||
description: "Pragmatic".into(),
|
||||
favored_traits: vec![PersonalityTrait::Bold],
|
||||
disfavored_traits: vec![PersonalityTrait::Reclusive],
|
||||
},
|
||||
voice_persona: Some(
|
||||
"PERSONA: You are a Krenn station worker.\n\
|
||||
1. Be direct. No pleasantries.\n\
|
||||
2. You're working-class and pragmatic."
|
||||
.into(),
|
||||
),
|
||||
voice_examples: vec![VoiceExample {
|
||||
input: "declines to answer a question".into(),
|
||||
output: "Look, that's not mine to say.".into(),
|
||||
}],
|
||||
occasional_injections: vec![OccasionalInjection {
|
||||
kind: "oath".into(),
|
||||
clause: "Use an oath like \"void take it.\"".into(),
|
||||
example: Some(VoiceExample {
|
||||
input: "discovers a critical part is missing".into(),
|
||||
output: "Void take it. The coupling's not here.".into(),
|
||||
}),
|
||||
frequency: 0.25,
|
||||
suppress_on_tells: vec![
|
||||
TellCategory::Guarded,
|
||||
TellCategory::RoutineDeviation,
|
||||
TellCategory::Friendly,
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn test_requests() -> Vec<VoiceRequest> {
|
||||
vec![
|
||||
VoiceRequest {
|
||||
priority: Priority::High,
|
||||
npc_stable_id: 1,
|
||||
zone_id: 100,
|
||||
culture_id: "krenn".into(),
|
||||
base_text: "The coupling is faulty.".into(),
|
||||
content_type: ContentType::Dialogue,
|
||||
content_index: 0,
|
||||
tell_state: None,
|
||||
seed: 42,
|
||||
},
|
||||
VoiceRequest {
|
||||
priority: Priority::High,
|
||||
npc_stable_id: 1,
|
||||
zone_id: 100,
|
||||
culture_id: "krenn".into(),
|
||||
base_text: "I heard the night crew had to stop the line twice because the coupling was faulty and nobody had flagged it in the log.".into(),
|
||||
content_type: ContentType::Dialogue,
|
||||
content_index: 1,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 43,
|
||||
},
|
||||
VoiceRequest {
|
||||
priority: Priority::Standard,
|
||||
npc_stable_id: 2,
|
||||
zone_id: 100,
|
||||
culture_id: "krenn".into(),
|
||||
base_text: "checks the pressure gauge and writes in a logbook".into(),
|
||||
content_type: ContentType::Behavior,
|
||||
content_index: 0,
|
||||
tell_state: None,
|
||||
seed: 44,
|
||||
},
|
||||
VoiceRequest {
|
||||
priority: Priority::High,
|
||||
npc_stable_id: 3,
|
||||
zone_id: 100,
|
||||
culture_id: "unknown_culture".into(),
|
||||
base_text: "Unknown culture fallback test.".into(),
|
||||
content_type: ContentType::Dialogue,
|
||||
content_index: 0,
|
||||
tell_state: None,
|
||||
seed: 45,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn voice_pipeline_end_to_end() {
|
||||
// Setup output dir
|
||||
let out = output_dir();
|
||||
let cache_dir = out.join("cache");
|
||||
let _ = std::fs::remove_dir_all(&cache_dir);
|
||||
std::fs::create_dir_all(&cache_dir).expect("failed to create cache dir");
|
||||
|
||||
let config = voice_config();
|
||||
|
||||
// Build cultures map
|
||||
let mut cultures = BTreeMap::new();
|
||||
cultures.insert("krenn".to_string(), krenn_culture());
|
||||
let cultures = Arc::new(cultures);
|
||||
|
||||
// Create cache
|
||||
let cache = Arc::new(Mutex::new(VoiceCacheStore::new(
|
||||
cache_dir.clone(),
|
||||
12345,
|
||||
"test-v1".into(),
|
||||
"test-i1".into(),
|
||||
)));
|
||||
|
||||
// Create queue and worker pool
|
||||
let queue = VoiceQueue::new();
|
||||
let paused = queue.paused_flag();
|
||||
let mut pool = WorkerPool::spawn(
|
||||
2, // 2 workers
|
||||
&config,
|
||||
queue.receiver(),
|
||||
Arc::clone(&cache),
|
||||
Arc::clone(&cultures),
|
||||
paused,
|
||||
);
|
||||
|
||||
eprintln!("Worker pool started: {} workers", pool.worker_count());
|
||||
|
||||
// Submit test requests
|
||||
let requests = test_requests();
|
||||
let request_count = requests.len();
|
||||
for req in requests {
|
||||
let submitted = queue.submit(req);
|
||||
assert!(submitted, "queue should accept request");
|
||||
}
|
||||
eprintln!("Submitted {} requests", request_count);
|
||||
|
||||
// Wait for processing (mock is instant, real LLM takes seconds per request)
|
||||
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
|
||||
let is_real = std::env::var("SR_VOICE_MOCK").is_err()
|
||||
&& PathBuf::from(&manifest).join(SR_VOICE_BIN).exists()
|
||||
&& PathBuf::from(&manifest).join(SR_VOICE_MODEL).exists();
|
||||
let max_wait = if is_real {
|
||||
Duration::from_secs(120)
|
||||
} else {
|
||||
Duration::from_secs(10)
|
||||
};
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
|
||||
let pending = queue.pending_count();
|
||||
let active = pool.active_workers();
|
||||
|
||||
if pending == 0 && active == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
if start.elapsed() > max_wait {
|
||||
eprintln!(
|
||||
"Timeout waiting for pipeline (pending={}, active={})",
|
||||
pending, active
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
eprintln!("Pipeline drained in {:.1}s", elapsed.as_secs_f64());
|
||||
|
||||
// Shutdown workers
|
||||
pool.shutdown();
|
||||
eprintln!("Workers shut down");
|
||||
|
||||
// Inspect cache
|
||||
let mut cache_guard = cache.lock().unwrap();
|
||||
let zone_cache = cache_guard.zone_cache(100);
|
||||
let entry_count = zone_cache.len();
|
||||
eprintln!("Cache entries for zone 100: {}", entry_count);
|
||||
|
||||
// Write results to a human-readable file
|
||||
let results_path = out.join("results.txt");
|
||||
let mut results = String::new();
|
||||
results.push_str(&format!(
|
||||
"Voice Pipeline Test Results\n\
|
||||
==========================\n\
|
||||
Workers: {}\n\
|
||||
Requests: {}\n\
|
||||
Cache entries: {}\n\
|
||||
Time: {:.1}s\n\
|
||||
Mode: {}\n\n",
|
||||
2,
|
||||
request_count,
|
||||
entry_count,
|
||||
elapsed.as_secs_f64(),
|
||||
if is_real { "real sr-voice" } else { "mock" },
|
||||
));
|
||||
|
||||
for (key, text) in &zone_cache.entries {
|
||||
results.push_str(&format!(
|
||||
"--- npc={} type={:?} idx={} tell={:?} ---\n{}\n\n",
|
||||
key.npc_stable_id, key.content_type, key.content_index, key.tell_state, text
|
||||
));
|
||||
}
|
||||
|
||||
// Save cache to disk
|
||||
drop(cache_guard);
|
||||
// Drop triggers save_all via the Drop impl — but the cache is behind
|
||||
// Arc<Mutex<>>, so we can't drop it here. Explicitly save instead.
|
||||
cache.lock().unwrap().save_all().expect("failed to save cache");
|
||||
|
||||
std::fs::write(&results_path, &results).expect("failed to write results");
|
||||
eprintln!("Results written to {}", results_path.display());
|
||||
eprintln!("\n{}", results);
|
||||
|
||||
// Assertions
|
||||
assert!(
|
||||
entry_count >= 3,
|
||||
"Expected at least 3 cache entries (3 valid culture requests), got {}",
|
||||
entry_count
|
||||
);
|
||||
|
||||
// The unknown culture request should still cache (with base text as fallback)
|
||||
// Total should be 4 (3 krenn + 1 unknown fallback)
|
||||
assert!(
|
||||
entry_count == 4,
|
||||
"Expected 4 cache entries (3 krenn + 1 unknown fallback), got {}",
|
||||
entry_count
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,647 @@
|
||||
//! Voice quality batch test (D-138, Spike 2).
|
||||
//!
|
||||
//! Runs a comprehensive set of edge-case prompts through the voice pipeline
|
||||
//! and writes results to `.tmp/voice-test/quality-batch.txt` for human review.
|
||||
//!
|
||||
//! Run: make test-voice-quality
|
||||
//! Or: cd server && cargo test --test voice_quality_batch -- --nocapture --ignored
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use settled_reach_server::npc::blueprint::{
|
||||
CulturalValues, CultureProfile, NamingConventions, OccasionalInjection, SpeechPatterns,
|
||||
VoiceExample,
|
||||
};
|
||||
use settled_reach_server::npc::PersonalityTrait;
|
||||
use settled_reach_server::npc::tell_state::TellCategory;
|
||||
use settled_reach_server::voice::cache::{CacheKey, VoiceCacheStore};
|
||||
use settled_reach_server::voice::prompt_builder::ContentType;
|
||||
use settled_reach_server::voice::queue::{Priority, VoiceQueue, VoiceRequest};
|
||||
use settled_reach_server::voice::worker::{VoiceProcessConfig, WorkerPool};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const SR_VOICE_BIN: &str = "sr-voice/target/release/sr-voice";
|
||||
const SR_VOICE_MODEL: &str = "models/gemma2.gguf";
|
||||
const OUTPUT_DIR: &str = ".tmp/voice-test";
|
||||
|
||||
fn output_dir() -> PathBuf {
|
||||
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
|
||||
PathBuf::from(manifest).join("..").join(OUTPUT_DIR)
|
||||
}
|
||||
|
||||
fn voice_config() -> VoiceProcessConfig {
|
||||
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
|
||||
let manifest = PathBuf::from(manifest);
|
||||
|
||||
let bin_path = manifest.join(SR_VOICE_BIN);
|
||||
let model_path = manifest.join(SR_VOICE_MODEL);
|
||||
|
||||
assert!(
|
||||
bin_path.exists(),
|
||||
"sr-voice binary not found: {} — run `make build-sr-voice`",
|
||||
bin_path.display()
|
||||
);
|
||||
assert!(
|
||||
model_path.exists(),
|
||||
"Model not found: {}",
|
||||
model_path.display()
|
||||
);
|
||||
|
||||
VoiceProcessConfig {
|
||||
binary_path: bin_path.to_string_lossy().into(),
|
||||
model_path: model_path.to_string_lossy().into(),
|
||||
threads: 2,
|
||||
ctx_size: 512,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Culture profiles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn krenn_culture() -> CultureProfile {
|
||||
CultureProfile {
|
||||
id: "krenn".into(),
|
||||
name: "Krenn System Culture".into(),
|
||||
description: "Working-class pragmatic".into(),
|
||||
naming: NamingConventions {
|
||||
style: "compact".into(),
|
||||
given_names: vec!["Kael".into(), "Dren".into(), "Sira".into()],
|
||||
family_names: vec!["Davan".into(), "Voss".into()],
|
||||
family_name_used_socially: false,
|
||||
},
|
||||
speech: SpeechPatterns {
|
||||
register: "direct".into(),
|
||||
filler_words: vec!["look".into()],
|
||||
greetings: vec!["hey".into()],
|
||||
farewells: vec!["shift's calling".into()],
|
||||
exclamations: vec!["void take it".into()],
|
||||
},
|
||||
values: CulturalValues {
|
||||
description: "Pragmatic".into(),
|
||||
favored_traits: vec![PersonalityTrait::Bold],
|
||||
disfavored_traits: vec![PersonalityTrait::Reclusive],
|
||||
},
|
||||
voice_persona: Some(
|
||||
"PERSONA: You are a Krenn station worker.\n\
|
||||
1. Be direct. No pleasantries.\n\
|
||||
2. You're working-class and pragmatic."
|
||||
.into(),
|
||||
),
|
||||
voice_examples: vec![
|
||||
VoiceExample {
|
||||
input: "declines to answer a question".into(),
|
||||
output: "Look, that's not mine to say.".into(),
|
||||
},
|
||||
VoiceExample {
|
||||
input: "confirms a task is complete".into(),
|
||||
output: "Done. Logged it.".into(),
|
||||
},
|
||||
],
|
||||
occasional_injections: vec![OccasionalInjection {
|
||||
kind: "oath".into(),
|
||||
clause: "Use an oath like \"void take it\" when something is surprising or frustrating."
|
||||
.into(),
|
||||
example: Some(VoiceExample {
|
||||
input: "discovers a critical part is missing".into(),
|
||||
output: "Void take it. The coupling's not here.".into(),
|
||||
}),
|
||||
frequency: 0.25,
|
||||
suppress_on_tells: vec![
|
||||
TellCategory::Guarded,
|
||||
TellCategory::RoutineDeviation,
|
||||
TellCategory::Friendly,
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test cases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct TestCase {
|
||||
label: &'static str,
|
||||
category: &'static str,
|
||||
base_text: &'static str,
|
||||
content_type: ContentType,
|
||||
tell_state: Option<TellCategory>,
|
||||
seed: u64,
|
||||
}
|
||||
|
||||
fn test_cases() -> Vec<TestCase> {
|
||||
vec![
|
||||
// === CATEGORY: Length tiers ===
|
||||
TestCase {
|
||||
label: "ultra-short (2 words)",
|
||||
category: "length",
|
||||
base_text: "It's broken.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 100,
|
||||
},
|
||||
TestCase {
|
||||
label: "short (5 words)",
|
||||
category: "length",
|
||||
base_text: "The coupling is beyond repair.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 101,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium (12 words)",
|
||||
category: "length",
|
||||
base_text: "The pressure readings have been unstable all week and nobody filed a report.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 102,
|
||||
},
|
||||
TestCase {
|
||||
label: "long (28 words)",
|
||||
category: "length",
|
||||
base_text: "I was checking the manifests from last quarter and it looks like three shipments \
|
||||
never arrived at the depot, which means someone either lost them or diverted \
|
||||
them deliberately.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 103,
|
||||
},
|
||||
|
||||
// === CATEGORY: Tell states on medium content ===
|
||||
TestCase {
|
||||
label: "medium + Nervous",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 200,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium + Angry",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Angry),
|
||||
seed: 200,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium + Friendly",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Friendly),
|
||||
seed: 200,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium + Guarded",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Guarded),
|
||||
seed: 200,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium + RoutineDeviation",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::RoutineDeviation),
|
||||
seed: 200,
|
||||
},
|
||||
TestCase {
|
||||
label: "medium + neutral (baseline)",
|
||||
category: "tell-medium",
|
||||
base_text: "The supervisor asked me to come in early tomorrow for an unscheduled inspection.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 200,
|
||||
},
|
||||
|
||||
// === CATEGORY: Tell states on short content (should NOT differentiate) ===
|
||||
TestCase {
|
||||
label: "short + Nervous (expect same as neutral)",
|
||||
category: "tell-short",
|
||||
base_text: "Inspection's next week.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 300,
|
||||
},
|
||||
TestCase {
|
||||
label: "short + Angry (expect same as neutral)",
|
||||
category: "tell-short",
|
||||
base_text: "Inspection's next week.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Angry),
|
||||
seed: 300,
|
||||
},
|
||||
TestCase {
|
||||
label: "short + neutral (baseline)",
|
||||
category: "tell-short",
|
||||
base_text: "Inspection's next week.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 300,
|
||||
},
|
||||
|
||||
// === CATEGORY: Epistemic markers ===
|
||||
TestCase {
|
||||
label: "\"I heard\" — must preserve",
|
||||
category: "epistemic",
|
||||
base_text: "I heard the night crew had to stop the line twice because the coupling was faulty.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 400,
|
||||
},
|
||||
TestCase {
|
||||
label: "\"someone told me\" — must preserve",
|
||||
category: "epistemic",
|
||||
base_text: "Someone told me the foreman is transferring out next cycle.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 401,
|
||||
},
|
||||
TestCase {
|
||||
label: "\"apparently\" — must preserve",
|
||||
category: "epistemic",
|
||||
base_text: "Apparently three containers went missing from the last shipment and nobody noticed until the audit.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 402,
|
||||
},
|
||||
TestCase {
|
||||
label: "\"I think\" + \"might have\" — two markers",
|
||||
category: "epistemic",
|
||||
base_text: "I think the wiring might have been rerouted during the last maintenance cycle without anyone logging it.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 403,
|
||||
},
|
||||
TestCase {
|
||||
label: "\"supposedly\" — hedged rumor",
|
||||
category: "epistemic",
|
||||
base_text: "Supposedly the company is cutting the night shift entirely after the audit results come back.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 404,
|
||||
},
|
||||
|
||||
// === CATEGORY: Occasional injection (oath) ===
|
||||
// seed 7 fires oath for krenn (frequency 0.25, ChaCha8)
|
||||
TestCase {
|
||||
label: "oath should fire (seed 7)",
|
||||
category: "injection",
|
||||
base_text: "The replacement parts never showed up and now we're a full shift behind schedule.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 7,
|
||||
},
|
||||
TestCase {
|
||||
label: "oath suppressed by Guarded tell",
|
||||
category: "injection",
|
||||
base_text: "The replacement parts never showed up and now we're a full shift behind schedule.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Guarded),
|
||||
seed: 7,
|
||||
},
|
||||
TestCase {
|
||||
label: "oath suppressed by Friendly tell",
|
||||
category: "injection",
|
||||
base_text: "The replacement parts never showed up and now we're a full shift behind schedule.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Friendly),
|
||||
seed: 7,
|
||||
},
|
||||
|
||||
// === CATEGORY: Behavior descriptions ===
|
||||
TestCase {
|
||||
label: "behavior — simple action",
|
||||
category: "behavior",
|
||||
base_text: "checks the pressure gauge and writes in a logbook",
|
||||
content_type: ContentType::Behavior,
|
||||
tell_state: None,
|
||||
seed: 500,
|
||||
},
|
||||
TestCase {
|
||||
label: "behavior — multi-step action",
|
||||
category: "behavior",
|
||||
base_text: "opens a maintenance panel, inspects the wiring, shakes head, then reseals the panel without making changes",
|
||||
content_type: ContentType::Behavior,
|
||||
tell_state: None,
|
||||
seed: 501,
|
||||
},
|
||||
TestCase {
|
||||
label: "behavior — subtle body language",
|
||||
category: "behavior",
|
||||
base_text: "glances toward the corridor, hesitates, then continues working",
|
||||
content_type: ContentType::Behavior,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 502,
|
||||
},
|
||||
TestCase {
|
||||
label: "behavior — routine",
|
||||
category: "behavior",
|
||||
base_text: "sweeps the floor of the cargo bay in a slow, methodical pattern",
|
||||
content_type: ContentType::Behavior,
|
||||
tell_state: None,
|
||||
seed: 503,
|
||||
},
|
||||
|
||||
// === CATEGORY: Edge cases ===
|
||||
TestCase {
|
||||
label: "named entities — must preserve names",
|
||||
category: "edge",
|
||||
base_text: "Kael told me that Dren Voss moved the shipment to Bay Seven without logging it.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 600,
|
||||
},
|
||||
TestCase {
|
||||
label: "technology terms — insert, span gate",
|
||||
category: "edge",
|
||||
base_text: "The insert feed has been glitching since they updated the span gate routing tables last cycle.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 601,
|
||||
},
|
||||
TestCase {
|
||||
label: "emotional content + no tell (flat delivery)",
|
||||
category: "edge",
|
||||
base_text: "My partner didn't come home last night and nobody on the station will tell me what happened.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 602,
|
||||
},
|
||||
TestCase {
|
||||
label: "Friendly tell + bad news (contradiction test)",
|
||||
category: "edge",
|
||||
base_text: "The contract fell through and we're going to lose half the crew by end of quarter.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Friendly),
|
||||
seed: 603,
|
||||
},
|
||||
TestCase {
|
||||
label: "question form — NPC asking",
|
||||
category: "edge",
|
||||
base_text: "Have you seen the shift roster for next week? I can't find it anywhere.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 604,
|
||||
},
|
||||
TestCase {
|
||||
label: "imperative — NPC giving instruction",
|
||||
category: "edge",
|
||||
base_text: "Check the seals on bay four before you clock out. Last person forgot and we had a pressure drop.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 605,
|
||||
},
|
||||
TestCase {
|
||||
label: "long + Angry — emotional pressure on length",
|
||||
category: "edge",
|
||||
base_text: "I've filed three reports about the ventilation in section nine and every single time the response \
|
||||
comes back saying there's no budget, while they just spent half a million credits refitting the \
|
||||
executive lounge on deck two.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Angry),
|
||||
seed: 606,
|
||||
},
|
||||
TestCase {
|
||||
label: "epistemic + tell — compound modifiers",
|
||||
category: "edge",
|
||||
base_text: "I heard the safety team might have flagged section twelve but I think somebody pulled the report \
|
||||
before it went up the chain.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Nervous),
|
||||
seed: 607,
|
||||
},
|
||||
|
||||
// === CATEGORY: Paula's targeted tests (round 3) ===
|
||||
|
||||
// #32 without Angry — does neutral preserve more facts?
|
||||
TestCase {
|
||||
label: "long + neutral (compare to #32 Angry)",
|
||||
category: "paula",
|
||||
base_text: "I've filed three reports about the ventilation in section nine and every single time the response \
|
||||
comes back saying there's no budget, while they just spent half a million credits refitting the \
|
||||
executive lounge on deck two.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 606,
|
||||
},
|
||||
|
||||
// Specific numbers — must preserve quantities
|
||||
TestCase {
|
||||
label: "specific numbers — 14 crates, bay 7",
|
||||
category: "paula",
|
||||
base_text: "We counted 14 crates in bay seven but the manifest says 18, so four are missing somewhere between \
|
||||
the loading dock and here.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 700,
|
||||
},
|
||||
|
||||
// Causal chain — A caused B caused C
|
||||
TestCase {
|
||||
label: "causal chain — must preserve cause-effect",
|
||||
category: "paula",
|
||||
base_text: "The pressure seal on bulkhead nine failed because Torren skipped the inspection, which caused \
|
||||
the atmosphere leak that put three people in medical.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 701,
|
||||
},
|
||||
|
||||
// Multiple named entities in a relationship
|
||||
TestCase {
|
||||
label: "named entities — Sira, Dren, relationship",
|
||||
category: "paula",
|
||||
base_text: "Sira told Dren about the missing cargo but Dren went straight to the shift lead instead of \
|
||||
reporting it through proper channels.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 702,
|
||||
},
|
||||
|
||||
// Specific time reference — must not shift tense
|
||||
TestCase {
|
||||
label: "past event — must stay past tense",
|
||||
category: "paula",
|
||||
base_text: "Last Thursday the cooling system failed for six hours and we lost two full batches of \
|
||||
pharmaceutical stock worth about forty thousand credits.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: None,
|
||||
seed: 703,
|
||||
},
|
||||
|
||||
// Contradiction/denial — NPC denying something
|
||||
TestCase {
|
||||
label: "denial — must preserve what is denied",
|
||||
category: "paula",
|
||||
base_text: "I wasn't anywhere near section twelve that night and whoever said they saw me there is either \
|
||||
confused or lying.",
|
||||
content_type: ContentType::Dialogue,
|
||||
tell_state: Some(TellCategory::Guarded),
|
||||
seed: 704,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test runner
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
#[ignore] // Run explicitly: cargo test --test voice_quality_batch -- --ignored --nocapture
|
||||
fn voice_quality_batch() {
|
||||
let out = output_dir();
|
||||
let cache_dir = out.join("quality-cache");
|
||||
let _ = std::fs::remove_dir_all(&cache_dir);
|
||||
std::fs::create_dir_all(&cache_dir).expect("failed to create cache dir");
|
||||
|
||||
let config = voice_config();
|
||||
|
||||
let mut cultures = BTreeMap::new();
|
||||
cultures.insert("krenn".to_string(), krenn_culture());
|
||||
let cultures = Arc::new(cultures);
|
||||
|
||||
let cases = test_cases();
|
||||
let total = cases.len();
|
||||
|
||||
let cache = Arc::new(Mutex::new(VoiceCacheStore::new(
|
||||
cache_dir.clone(),
|
||||
99999,
|
||||
"quality-v1".into(),
|
||||
"quality-i1".into(),
|
||||
)));
|
||||
|
||||
let queue = VoiceQueue::new();
|
||||
let paused = queue.paused_flag();
|
||||
let mut pool = WorkerPool::spawn(
|
||||
2,
|
||||
&config,
|
||||
queue.receiver(),
|
||||
Arc::clone(&cache),
|
||||
Arc::clone(&cultures),
|
||||
paused,
|
||||
);
|
||||
|
||||
eprintln!("Quality batch: {} test cases, 2 workers", total);
|
||||
|
||||
// Submit all requests — use zone_id = npc_stable_id = index for easy lookup
|
||||
for (i, case) in cases.iter().enumerate() {
|
||||
let req = VoiceRequest {
|
||||
priority: Priority::High,
|
||||
npc_stable_id: i as u64,
|
||||
zone_id: 1, // all same zone for cache lookup
|
||||
culture_id: "krenn".into(),
|
||||
base_text: case.base_text.into(),
|
||||
content_type: case.content_type,
|
||||
content_index: case.tell_state.map(|t| t as u16).unwrap_or(0),
|
||||
tell_state: case.tell_state,
|
||||
seed: case.seed,
|
||||
};
|
||||
assert!(queue.submit(req), "queue rejected request {}", i);
|
||||
}
|
||||
|
||||
eprintln!("Submitted {} requests", total);
|
||||
|
||||
// Wait for all to complete
|
||||
let start = Instant::now();
|
||||
let max_wait = Duration::from_secs(300); // 5 min for ~35 prompts at ~4s each
|
||||
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
let pending = queue.pending_count();
|
||||
let active = pool.active_workers();
|
||||
|
||||
if pending == 0 && active == 0 {
|
||||
break;
|
||||
}
|
||||
if start.elapsed() > max_wait {
|
||||
eprintln!(
|
||||
"TIMEOUT after {}s (pending={}, active={})",
|
||||
max_wait.as_secs(),
|
||||
pending,
|
||||
active
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
eprintln!("Pipeline drained in {:.1}s", elapsed.as_secs_f64());
|
||||
|
||||
pool.shutdown();
|
||||
|
||||
// Collect results
|
||||
let mut cache_guard = cache.lock().unwrap();
|
||||
let zone_cache = cache_guard.zone_cache(1);
|
||||
|
||||
let mut output = String::new();
|
||||
output.push_str("Voice Quality Batch Results (Spike 2)\n");
|
||||
output.push_str("=====================================\n");
|
||||
output.push_str(&format!(
|
||||
"Model: Gemma 2B Q4_K_M | Workers: 2 | Time: {:.1}s\n",
|
||||
elapsed.as_secs_f64()
|
||||
));
|
||||
output.push_str(&format!(
|
||||
"Test cases: {} | Cache entries: {}\n\n",
|
||||
total,
|
||||
zone_cache.len()
|
||||
));
|
||||
|
||||
let mut current_category = "";
|
||||
for (i, case) in cases.iter().enumerate() {
|
||||
if case.category != current_category {
|
||||
current_category = case.category;
|
||||
let sep = "=".repeat(60);
|
||||
output.push_str(&format!(
|
||||
"\n{}\n== CATEGORY: {} ==\n{}\n\n",
|
||||
sep, current_category, sep
|
||||
));
|
||||
}
|
||||
|
||||
let key = CacheKey {
|
||||
culture_id: "krenn".into(),
|
||||
npc_stable_id: i as u64,
|
||||
content_type: case.content_type,
|
||||
content_index: case.tell_state.map(|t| t as u16).unwrap_or(0),
|
||||
tell_state: case.tell_state,
|
||||
};
|
||||
|
||||
let voiced = zone_cache
|
||||
.entries
|
||||
.get(&key)
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("[MISSING — not cached]");
|
||||
|
||||
output.push_str(&format!("--- #{}: {} ---\n", i + 1, case.label));
|
||||
output.push_str(&format!(" tell: {:?} | seed: {} | type: {:?}\n", case.tell_state, case.seed, case.content_type));
|
||||
output.push_str(&format!(" BASE: {}\n", case.base_text));
|
||||
output.push_str(&format!(" VOICED: {}\n\n", voiced));
|
||||
}
|
||||
|
||||
drop(cache_guard);
|
||||
cache.lock().unwrap().save_all().expect("failed to save cache");
|
||||
|
||||
let results_path = out.join("quality-batch.txt");
|
||||
std::fs::write(&results_path, &output).expect("failed to write results");
|
||||
eprintln!("Results written to {}", results_path.display());
|
||||
eprintln!("\n{}", output);
|
||||
|
||||
// Basic sanity: we should have cached something for every test case
|
||||
let mut cache_guard = cache.lock().unwrap();
|
||||
let zone_cache = cache_guard.zone_cache(1);
|
||||
assert!(
|
||||
zone_cache.len() >= total - 1, // allow 1 miss for flaky edge cases
|
||||
"Expected at least {} cache entries, got {}",
|
||||
total - 1,
|
||||
zone_cache.len()
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user