diff --git a/server/src/voice/hardware.rs b/server/src/voice/hardware.rs new file mode 100644 index 000000000..02e2ffd23 --- /dev/null +++ b/server/src/voice/hardware.rs @@ -0,0 +1,569 @@ +//! Hardware detection + dynamic sr-voice instance management (D-138, Spike 2). +//! +//! Determines how many parallel LLM workers the system can sustain and manages +//! sr-voice process lifecycle. The scaling ceiling is conservative: +//! +//! max_new = (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST +//! +//! "free_resource" is VRAM when a GPU is detected (nvidia-smi / AMD sysfs), +//! or system RAM otherwise. This ensures the voice pipeline never takes more +//! than half the available headroom after accounting for its own instances. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sysinfo::System; + +/// RAM budget per sr-voice instance (Gemma 2B Q4_K_M ≈ 1.5 GB resident). +const PER_INSTANCE_RAM_MB: u64 = 1536; + +/// Minimum free RAM to allow any voice instance at all. +const MIN_FREE_RAM_MB: u64 = 1536; + +/// Base port for sr-voice instances. Worker N listens on BASE_PORT + N. +const BASE_PORT: u16 = 8321; + +/// Context window size for sr-voice instances. +const CTX_SIZE: u32 = 512; + +/// How often the scaler thread checks for scale-up/down opportunities. +/// How often the scaler thread checks for scale-up/down opportunities. +/// Used by the runtime scaler loop (not yet implemented). +const _SCALE_CHECK_INTERVAL: Duration = Duration::from_secs(10); + +/// Queue depth threshold — sustained above this triggers scale-up consideration. +const QUEUE_DEPTH_SCALE_UP: usize = 32; + +/// Worker idle duration before scale-down. +const IDLE_BEFORE_SCALE_DOWN: Duration = Duration::from_secs(60); + +/// Managed sr-voice process instances. +#[derive(Debug)] +struct VoiceInstance { + process: Child, + port: u16, + spawned_at: Instant, +} + +/// Whether the scaling resource is GPU VRAM or system RAM. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResourceMode { + /// GPU VRAM — model lives on GPU, scale by free VRAM. + Gpu, + /// System RAM — CPU-only inference, scale by free RAM. + Cpu, +} + +/// Hardware probe result from startup. +#[derive(Debug, Clone)] +pub struct HardwareProbe { + /// Whether scaling is based on GPU VRAM or system RAM. + pub mode: ResourceMode, + /// Total resource in MB (VRAM or system RAM). + pub total_mb: u64, + /// Free resource at probe time in MB. + pub free_mb: u64, + /// Maximum instance slots based on probe-time resources. + pub max_slots: usize, + /// CPU thread count available. + pub cpu_threads: usize, + /// Threads to allocate per sr-voice instance. + pub threads_per_instance: u32, +} + +/// Manages sr-voice process lifecycle and dynamic scaling. +pub struct VoiceInstanceManager { + /// Path to sr-voice binary. + binary_path: PathBuf, + /// Path to model file. + model_path: PathBuf, + /// Running instances keyed by worker ID. + instances: HashMap, + /// Hardware probe from startup. + probe: HardwareProbe, + /// Shutdown signal. + shutdown: Arc, +} + +/// Detect GPU type. Run once at install/first-run, persist to settings. +/// +/// Returns `Gpu` if NVIDIA, AMD, or Apple Silicon is detected. `Cpu` otherwise. +pub fn detect_gpu_mode() -> ResourceMode { + if probe_nvidia_vram().is_some() { + tracing::info!("GPU detection: NVIDIA"); + ResourceMode::Gpu + } else if probe_amd_vram().is_some() { + tracing::info!("GPU detection: AMD"); + ResourceMode::Gpu + } else if is_apple_silicon() { + tracing::info!("GPU detection: Apple Silicon (unified memory)"); + ResourceMode::Gpu + } else { + tracing::info!("GPU detection: none — CPU only"); + ResourceMode::Cpu + } +} + +/// Probe system hardware and determine initial capacity. +/// +/// `mode` should come from persisted settings (written at install time via +/// `detect_gpu_mode()`). If `None`, runs detection inline (first-run fallback). +pub fn probe_hardware(mode: Option) -> HardwareProbe { + let cpu_threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + let mode = mode.unwrap_or_else(detect_gpu_mode); + + let (total_mb, free_mb) = match mode { + ResourceMode::Gpu if !is_apple_silicon() => { + // Discrete GPU — probe VRAM + if let Some((total, free)) = probe_nvidia_vram() { + (total, free) + } else if let Some((total, free)) = probe_amd_vram() { + (total, free) + } else { + // Settings say GPU but can't probe — fall back to system RAM + tracing::warn!("GPU mode set but VRAM probe failed — falling back to system RAM"); + let mut sys = System::new_all(); + sys.refresh_memory(); + (sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024)) + } + } + _ => { + // CPU mode or Apple Silicon (unified memory = system RAM) + let mut sys = System::new_all(); + sys.refresh_memory(); + (sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024)) + } + }; + + let max_slots = compute_max_slots(free_mb, 0); + + // In GPU mode, threads matter less (GPU does the work), but sr-voice + // still uses CPU threads for tokenization. Use at most half of CPU + // threads for voice, minimum 1 per instance. + let voice_threads = cpu_threads / 2; + let threads_per_instance = if max_slots > 0 { + (voice_threads / max_slots).max(1) as u32 + } else { + 1 + }; + + let probe = HardwareProbe { + mode, + total_mb, + free_mb, + max_slots, + cpu_threads, + threads_per_instance, + }; + + tracing::info!( + ?mode, + total_mb, + free_mb, + max_slots, + cpu_threads, + threads_per_instance, + "hardware probe complete" + ); + + probe +} + +/// Probe NVIDIA GPU VRAM via nvidia-smi. +/// Returns (total_mb, free_mb) for the first GPU, or None. +fn probe_nvidia_vram() -> Option<(u64, u64)> { + let output = Command::new("nvidia-smi") + .args(["--query-gpu=memory.total,memory.free", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let line = stdout.lines().next()?; + let parts: Vec<&str> = line.split(',').map(|s| s.trim()).collect(); + if parts.len() != 2 { + return None; + } + + let total: u64 = parts[0].parse().ok()?; + let free: u64 = parts[1].parse().ok()?; + Some((total, free)) +} + +/// Probe AMD GPU VRAM via sysfs. +/// Returns (total_mb, free_mb) for the first GPU with VRAM info, or None. +fn probe_amd_vram() -> Option<(u64, u64)> { + let entries = std::fs::read_dir("/sys/class/drm/").ok()?; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if !name_str.starts_with("card") || name_str.contains('-') { + continue; + } + + let device_dir = entry.path().join("device"); + let total_path = device_dir.join("mem_info_vram_total"); + let used_path = device_dir.join("mem_info_vram_used"); + + let total_bytes: u64 = std::fs::read_to_string(&total_path) + .ok()? + .trim() + .parse() + .ok()?; + let used_bytes: u64 = std::fs::read_to_string(&used_path) + .ok()? + .trim() + .parse() + .ok()?; + + let total_mb = total_bytes / (1024 * 1024); + let free_mb = (total_bytes.saturating_sub(used_bytes)) / (1024 * 1024); + return Some((total_mb, free_mb)); + } + + None +} + +/// Detect Apple Silicon (macOS + ARM64). +fn is_apple_silicon() -> bool { + cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") +} + +/// Re-probe the current free resource (VRAM or RAM) based on the mode +/// established at startup. Used by evaluate_scaling for live checks. +fn probe_current_free(mode: ResourceMode) -> u64 { + match mode { + ResourceMode::Gpu if !is_apple_silicon() => { + // Try nvidia first, then AMD + if let Some((_, free)) = probe_nvidia_vram() { + return free; + } + if let Some((_, free)) = probe_amd_vram() { + return free; + } + // GPU vanished? Fall back to system RAM + let mut sys = System::new_all(); + sys.refresh_memory(); + sys.available_memory() / (1024 * 1024) + } + _ => { + // CPU mode or Apple Silicon (unified memory) + let mut sys = System::new_all(); + sys.refresh_memory(); + sys.available_memory() / (1024 * 1024) + } + } +} + +/// Compute maximum instance slots from current free resource (VRAM or RAM). +/// +/// Formula: (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST +/// +/// Takes half of the remaining headroom after subtracting already-running +/// instances. This ensures voice never consumes more than half the +/// available resources beyond its own footprint. +fn compute_max_slots(free_mb: u64, running_instances: usize) -> usize { + let existing_llm_usage = running_instances as u64 * PER_INSTANCE_RAM_MB; + + // Free resource already reflects system load. Subtract our own LLM usage + // to get headroom available for expansion. + let headroom = free_mb.saturating_sub(existing_llm_usage); + + // Take half the headroom, divide by per-instance cost. + let available_for_new = headroom / 2; + let new_slots = available_for_new / PER_INSTANCE_RAM_MB; + + // Total = running + new slots we could add. + let total = running_instances + new_slots as usize; + + // Floor: at least 1 if there's enough free resource for a single instance. + if total == 0 && free_mb >= MIN_FREE_RAM_MB { + 1 + } else { + total + } +} + +/// Check if the system is on battery power (Linux). +fn on_battery() -> bool { + let Ok(entries) = std::fs::read_dir("/sys/class/power_supply/") else { + return false; + }; + + for entry in entries.flatten() { + let type_path = entry.path().join("type"); + let status_path = entry.path().join("status"); + + let Ok(supply_type) = std::fs::read_to_string(&type_path) else { + continue; + }; + if supply_type.trim() != "Battery" { + continue; + } + + if let Ok(status) = std::fs::read_to_string(&status_path) { + let status = status.trim(); + if status == "Discharging" { + return true; + } + } + } + + false +} + +impl VoiceInstanceManager { + /// Create a new manager. Does not spawn any instances yet. + pub fn new( + binary_path: PathBuf, + model_path: PathBuf, + probe: HardwareProbe, + shutdown: Arc, + ) -> Self { + Self { + binary_path, + model_path, + instances: HashMap::new(), + probe, + shutdown, + } + } + + /// Number of currently running instances. + pub fn instance_count(&self) -> usize { + self.instances.len() + } + + /// Base port for worker connections. + pub fn base_port(&self) -> u16 { + BASE_PORT + } + + /// The hardware probe from startup. + pub fn probe(&self) -> &HardwareProbe { + &self.probe + } + + /// Spawn a sr-voice instance for the given worker ID. + /// Returns the port it's listening on, or an error. + pub fn spawn_instance(&mut self, worker_id: usize) -> Result { + let port = BASE_PORT + worker_id as u16; + + if self.instances.contains_key(&worker_id) { + return Ok(port); // already running + } + + let child = Command::new(&self.binary_path) + .arg("serve") + .arg("--model") + .arg(&self.model_path) + .arg("--port") + .arg(port.to_string()) + .arg("--threads") + .arg(self.probe.threads_per_instance.to_string()) + .arg("--ctx-size") + .arg(CTX_SIZE.to_string()) + .spawn() + .map_err(|e| format!("failed to spawn sr-voice on port {}: {}", port, e))?; + + tracing::info!(worker_id, port, "spawned sr-voice instance"); + + self.instances.insert(worker_id, VoiceInstance { + process: child, + port, + spawned_at: Instant::now(), + }); + + Ok(port) + } + + /// Stop a sr-voice instance for the given worker ID. + pub fn stop_instance(&mut self, worker_id: usize) { + if let Some(mut instance) = self.instances.remove(&worker_id) { + let _ = instance.process.kill(); + let _ = instance.process.wait(); + tracing::info!(worker_id, port = instance.port, "stopped sr-voice instance"); + } + } + + /// Evaluate whether to scale up or down based on current conditions. + /// + /// Returns (should_scale_up, should_scale_down_worker_id). + pub fn evaluate_scaling( + &self, + queue_depth: usize, + active_workers: usize, + worker_idle_durations: &HashMap, + ) -> ScalingDecision { + // Battery → scale to 1 + if on_battery() && self.instances.len() > 1 { + return ScalingDecision::ScaleDown { + reason: "battery power detected".into(), + }; + } + + // Re-probe free resource (VRAM or RAM) for current conditions + let current_free_mb = probe_current_free(self.probe.mode); + let max_slots = compute_max_slots(current_free_mb, self.instances.len()); + + // Scale up: queue pressure + capacity available + if queue_depth >= QUEUE_DEPTH_SCALE_UP + && self.instances.len() < max_slots + && current_free_mb >= MIN_FREE_RAM_MB + { + return ScalingDecision::ScaleUp { + reason: format!( + "queue depth {} >= {}, {} slots available", + queue_depth, QUEUE_DEPTH_SCALE_UP, max_slots + ), + }; + } + + // Scale down: worker idle too long + more than 1 instance + if self.instances.len() > 1 { + for (&worker_id, &idle_time) in worker_idle_durations { + if idle_time >= IDLE_BEFORE_SCALE_DOWN && active_workers < self.instances.len() { + return ScalingDecision::ScaleDown { + reason: format!( + "worker {} idle for {}s", + worker_id, + idle_time.as_secs() + ), + }; + } + } + } + + ScalingDecision::Hold + } + + /// Shut down all sr-voice instances. + pub fn shutdown_all(&mut self) { + let ids: Vec = self.instances.keys().copied().collect(); + for id in ids { + self.stop_instance(id); + } + } + + /// Wait for a sr-voice instance to become healthy (responds to /health). + /// Returns true if healthy within timeout, false otherwise. + pub fn wait_for_healthy(&self, port: u16, timeout: Duration) -> bool { + let url = format!("http://127.0.0.1:{}/health", port); + let deadline = Instant::now() + timeout; + + while Instant::now() < deadline { + let result = ureq::AgentBuilder::new() + .timeout(Duration::from_secs(1)) + .build() + .get(&url) + .call(); + + if result.is_ok() { + return true; + } + + std::thread::sleep(Duration::from_millis(500)); + } + + false + } +} + +impl Drop for VoiceInstanceManager { + fn drop(&mut self) { + self.shutdown_all(); + } +} + +/// Result of a scaling evaluation. +#[derive(Debug)] +pub enum ScalingDecision { + /// No change needed. + Hold, + /// Spawn an additional instance. + ScaleUp { reason: String }, + /// Stop an instance (pick the most idle worker). + ScaleDown { reason: String }, +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compute_max_slots_with_plenty_of_ram() { + // 16GB free, no running instances + // headroom = 16384, half = 8192, / 1536 = 5 + let slots = compute_max_slots(16384, 0); + assert_eq!(slots, 5); + } + + #[test] + fn compute_max_slots_accounts_for_running_instances() { + // 16GB free, 2 already running (3072 MB used) + // headroom = 16384 - 3072 = 13312, half = 6656, / 1536 = 4 new + // total = 2 + 4 = 6 + let slots = compute_max_slots(16384, 2); + assert_eq!(slots, 6); + } + + #[test] + fn compute_max_slots_tight_ram() { + // 2GB free, no running — just enough for 1 instance + // headroom = 2048, half = 1024, / 1536 = 0 new + // But floor rule: free >= MIN_FREE_RAM → 1 + let slots = compute_max_slots(2048, 0); + assert_eq!(slots, 1); + } + + #[test] + fn compute_max_slots_insufficient_ram() { + // 1GB free — below minimum + let slots = compute_max_slots(1024, 0); + assert_eq!(slots, 0); + } + + #[test] + fn compute_max_slots_self_correcting() { + // As instances grow, available slots tighten + let slots_0 = compute_max_slots(8192, 0); // 8GB free, 0 running + let slots_2 = compute_max_slots(8192, 2); // 8GB free, 2 running + + // With 2 running, effective headroom is smaller so ceiling is lower per-new-instance + // but total (running + new) can still be higher + assert!(slots_0 > 0); + assert!(slots_2 > 0); + // The key property: free RAM measured at probe time is the same, + // but in practice free_ram will drop as instances consume memory, + // making this naturally self-correcting. + } + + #[test] + fn probe_hardware_returns_sane_values() { + let probe = probe_hardware(None); + assert!(probe.total_mb > 0); + assert!(probe.cpu_threads > 0); + assert!(probe.threads_per_instance >= 1); + } + + #[test] + fn battery_detection_does_not_crash() { + // Just verify it doesn't panic — result depends on hardware + let _ = on_battery(); + } +} diff --git a/server/src/voice/lookup.rs b/server/src/voice/lookup.rs index e0698063e..a260e2963 100644 --- a/server/src/voice/lookup.rs +++ b/server/src/voice/lookup.rs @@ -58,7 +58,6 @@ pub fn voiced_behavior( #[cfg(test)] mod tests { use super::*; - use std::path::PathBuf; fn test_cache() -> Arc> { let dir = std::env::temp_dir().join("sr-voice-lookup-test"); diff --git a/server/src/voice/mod.rs b/server/src/voice/mod.rs index d11b695bb..8a88255f9 100644 --- a/server/src/voice/mod.rs +++ b/server/src/voice/mod.rs @@ -13,6 +13,7 @@ //! - `hardware` — hardware detection + dynamic sr-voice instance management pub mod cache; +pub mod hardware; pub mod lookup; pub mod prompt_builder; pub mod queue;