feat(voice): add hardware detection + dynamic sr-voice instance management (Phase 4)

GPU-aware scaling: NVIDIA (nvidia-smi), AMD (sysfs VRAM), Apple Silicon
(unified memory). GPU mode detected at install, persisted to settings.
Scaling ceiling: (free_resource - existing_llm_usage) / 2 / per_instance_cost.
VoiceInstanceManager spawns/stops sr-voice processes on unique ports.
Battery detection scales to 1 worker.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-07 17:32:20 +01:00
co-authored by Claude Opus 4.6
parent fafa1c49b8
commit be6f2a3db9
3 changed files with 570 additions and 1 deletions
+569
View File
@@ -0,0 +1,569 @@
//! Hardware detection + dynamic sr-voice instance management (D-138, Spike 2).
//!
//! Determines how many parallel LLM workers the system can sustain and manages
//! sr-voice process lifecycle. The scaling ceiling is conservative:
//!
//! max_new = (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST
//!
//! "free_resource" is VRAM when a GPU is detected (nvidia-smi / AMD sysfs),
//! or system RAM otherwise. This ensures the voice pipeline never takes more
//! than half the available headroom after accounting for its own instances.
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::{Child, Command};
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::{Duration, Instant};
use sysinfo::System;
/// RAM budget per sr-voice instance (Gemma 2B Q4_K_M ≈ 1.5 GB resident).
const PER_INSTANCE_RAM_MB: u64 = 1536;
/// Minimum free RAM to allow any voice instance at all.
const MIN_FREE_RAM_MB: u64 = 1536;
/// Base port for sr-voice instances. Worker N listens on BASE_PORT + N.
const BASE_PORT: u16 = 8321;
/// Context window size for sr-voice instances.
const CTX_SIZE: u32 = 512;
/// How often the scaler thread checks for scale-up/down opportunities.
/// How often the scaler thread checks for scale-up/down opportunities.
/// Used by the runtime scaler loop (not yet implemented).
const _SCALE_CHECK_INTERVAL: Duration = Duration::from_secs(10);
/// Queue depth threshold — sustained above this triggers scale-up consideration.
const QUEUE_DEPTH_SCALE_UP: usize = 32;
/// Worker idle duration before scale-down.
const IDLE_BEFORE_SCALE_DOWN: Duration = Duration::from_secs(60);
/// Managed sr-voice process instances.
#[derive(Debug)]
struct VoiceInstance {
process: Child,
port: u16,
spawned_at: Instant,
}
/// Whether the scaling resource is GPU VRAM or system RAM.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResourceMode {
/// GPU VRAM — model lives on GPU, scale by free VRAM.
Gpu,
/// System RAM — CPU-only inference, scale by free RAM.
Cpu,
}
/// Hardware probe result from startup.
#[derive(Debug, Clone)]
pub struct HardwareProbe {
/// Whether scaling is based on GPU VRAM or system RAM.
pub mode: ResourceMode,
/// Total resource in MB (VRAM or system RAM).
pub total_mb: u64,
/// Free resource at probe time in MB.
pub free_mb: u64,
/// Maximum instance slots based on probe-time resources.
pub max_slots: usize,
/// CPU thread count available.
pub cpu_threads: usize,
/// Threads to allocate per sr-voice instance.
pub threads_per_instance: u32,
}
/// Manages sr-voice process lifecycle and dynamic scaling.
pub struct VoiceInstanceManager {
/// Path to sr-voice binary.
binary_path: PathBuf,
/// Path to model file.
model_path: PathBuf,
/// Running instances keyed by worker ID.
instances: HashMap<usize, VoiceInstance>,
/// Hardware probe from startup.
probe: HardwareProbe,
/// Shutdown signal.
shutdown: Arc<AtomicBool>,
}
/// Detect GPU type. Run once at install/first-run, persist to settings.
///
/// Returns `Gpu` if NVIDIA, AMD, or Apple Silicon is detected. `Cpu` otherwise.
pub fn detect_gpu_mode() -> ResourceMode {
if probe_nvidia_vram().is_some() {
tracing::info!("GPU detection: NVIDIA");
ResourceMode::Gpu
} else if probe_amd_vram().is_some() {
tracing::info!("GPU detection: AMD");
ResourceMode::Gpu
} else if is_apple_silicon() {
tracing::info!("GPU detection: Apple Silicon (unified memory)");
ResourceMode::Gpu
} else {
tracing::info!("GPU detection: none — CPU only");
ResourceMode::Cpu
}
}
/// Probe system hardware and determine initial capacity.
///
/// `mode` should come from persisted settings (written at install time via
/// `detect_gpu_mode()`). If `None`, runs detection inline (first-run fallback).
pub fn probe_hardware(mode: Option<ResourceMode>) -> HardwareProbe {
let cpu_threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4);
let mode = mode.unwrap_or_else(detect_gpu_mode);
let (total_mb, free_mb) = match mode {
ResourceMode::Gpu if !is_apple_silicon() => {
// Discrete GPU — probe VRAM
if let Some((total, free)) = probe_nvidia_vram() {
(total, free)
} else if let Some((total, free)) = probe_amd_vram() {
(total, free)
} else {
// Settings say GPU but can't probe — fall back to system RAM
tracing::warn!("GPU mode set but VRAM probe failed — falling back to system RAM");
let mut sys = System::new_all();
sys.refresh_memory();
(sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024))
}
}
_ => {
// CPU mode or Apple Silicon (unified memory = system RAM)
let mut sys = System::new_all();
sys.refresh_memory();
(sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024))
}
};
let max_slots = compute_max_slots(free_mb, 0);
// In GPU mode, threads matter less (GPU does the work), but sr-voice
// still uses CPU threads for tokenization. Use at most half of CPU
// threads for voice, minimum 1 per instance.
let voice_threads = cpu_threads / 2;
let threads_per_instance = if max_slots > 0 {
(voice_threads / max_slots).max(1) as u32
} else {
1
};
let probe = HardwareProbe {
mode,
total_mb,
free_mb,
max_slots,
cpu_threads,
threads_per_instance,
};
tracing::info!(
?mode,
total_mb,
free_mb,
max_slots,
cpu_threads,
threads_per_instance,
"hardware probe complete"
);
probe
}
/// Probe NVIDIA GPU VRAM via nvidia-smi.
/// Returns (total_mb, free_mb) for the first GPU, or None.
fn probe_nvidia_vram() -> Option<(u64, u64)> {
let output = Command::new("nvidia-smi")
.args(["--query-gpu=memory.total,memory.free", "--format=csv,noheader,nounits"])
.output()
.ok()?;
if !output.status.success() {
return None;
}
let stdout = String::from_utf8_lossy(&output.stdout);
let line = stdout.lines().next()?;
let parts: Vec<&str> = line.split(',').map(|s| s.trim()).collect();
if parts.len() != 2 {
return None;
}
let total: u64 = parts[0].parse().ok()?;
let free: u64 = parts[1].parse().ok()?;
Some((total, free))
}
/// Probe AMD GPU VRAM via sysfs.
/// Returns (total_mb, free_mb) for the first GPU with VRAM info, or None.
fn probe_amd_vram() -> Option<(u64, u64)> {
let entries = std::fs::read_dir("/sys/class/drm/").ok()?;
for entry in entries.flatten() {
let name = entry.file_name();
let name_str = name.to_string_lossy();
if !name_str.starts_with("card") || name_str.contains('-') {
continue;
}
let device_dir = entry.path().join("device");
let total_path = device_dir.join("mem_info_vram_total");
let used_path = device_dir.join("mem_info_vram_used");
let total_bytes: u64 = std::fs::read_to_string(&total_path)
.ok()?
.trim()
.parse()
.ok()?;
let used_bytes: u64 = std::fs::read_to_string(&used_path)
.ok()?
.trim()
.parse()
.ok()?;
let total_mb = total_bytes / (1024 * 1024);
let free_mb = (total_bytes.saturating_sub(used_bytes)) / (1024 * 1024);
return Some((total_mb, free_mb));
}
None
}
/// Detect Apple Silicon (macOS + ARM64).
fn is_apple_silicon() -> bool {
cfg!(target_os = "macos") && cfg!(target_arch = "aarch64")
}
/// Re-probe the current free resource (VRAM or RAM) based on the mode
/// established at startup. Used by evaluate_scaling for live checks.
fn probe_current_free(mode: ResourceMode) -> u64 {
match mode {
ResourceMode::Gpu if !is_apple_silicon() => {
// Try nvidia first, then AMD
if let Some((_, free)) = probe_nvidia_vram() {
return free;
}
if let Some((_, free)) = probe_amd_vram() {
return free;
}
// GPU vanished? Fall back to system RAM
let mut sys = System::new_all();
sys.refresh_memory();
sys.available_memory() / (1024 * 1024)
}
_ => {
// CPU mode or Apple Silicon (unified memory)
let mut sys = System::new_all();
sys.refresh_memory();
sys.available_memory() / (1024 * 1024)
}
}
}
/// Compute maximum instance slots from current free resource (VRAM or RAM).
///
/// Formula: (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST
///
/// Takes half of the remaining headroom after subtracting already-running
/// instances. This ensures voice never consumes more than half the
/// available resources beyond its own footprint.
fn compute_max_slots(free_mb: u64, running_instances: usize) -> usize {
let existing_llm_usage = running_instances as u64 * PER_INSTANCE_RAM_MB;
// Free resource already reflects system load. Subtract our own LLM usage
// to get headroom available for expansion.
let headroom = free_mb.saturating_sub(existing_llm_usage);
// Take half the headroom, divide by per-instance cost.
let available_for_new = headroom / 2;
let new_slots = available_for_new / PER_INSTANCE_RAM_MB;
// Total = running + new slots we could add.
let total = running_instances + new_slots as usize;
// Floor: at least 1 if there's enough free resource for a single instance.
if total == 0 && free_mb >= MIN_FREE_RAM_MB {
1
} else {
total
}
}
/// Check if the system is on battery power (Linux).
fn on_battery() -> bool {
let Ok(entries) = std::fs::read_dir("/sys/class/power_supply/") else {
return false;
};
for entry in entries.flatten() {
let type_path = entry.path().join("type");
let status_path = entry.path().join("status");
let Ok(supply_type) = std::fs::read_to_string(&type_path) else {
continue;
};
if supply_type.trim() != "Battery" {
continue;
}
if let Ok(status) = std::fs::read_to_string(&status_path) {
let status = status.trim();
if status == "Discharging" {
return true;
}
}
}
false
}
impl VoiceInstanceManager {
/// Create a new manager. Does not spawn any instances yet.
pub fn new(
binary_path: PathBuf,
model_path: PathBuf,
probe: HardwareProbe,
shutdown: Arc<AtomicBool>,
) -> Self {
Self {
binary_path,
model_path,
instances: HashMap::new(),
probe,
shutdown,
}
}
/// Number of currently running instances.
pub fn instance_count(&self) -> usize {
self.instances.len()
}
/// Base port for worker connections.
pub fn base_port(&self) -> u16 {
BASE_PORT
}
/// The hardware probe from startup.
pub fn probe(&self) -> &HardwareProbe {
&self.probe
}
/// Spawn a sr-voice instance for the given worker ID.
/// Returns the port it's listening on, or an error.
pub fn spawn_instance(&mut self, worker_id: usize) -> Result<u16, String> {
let port = BASE_PORT + worker_id as u16;
if self.instances.contains_key(&worker_id) {
return Ok(port); // already running
}
let child = Command::new(&self.binary_path)
.arg("serve")
.arg("--model")
.arg(&self.model_path)
.arg("--port")
.arg(port.to_string())
.arg("--threads")
.arg(self.probe.threads_per_instance.to_string())
.arg("--ctx-size")
.arg(CTX_SIZE.to_string())
.spawn()
.map_err(|e| format!("failed to spawn sr-voice on port {}: {}", port, e))?;
tracing::info!(worker_id, port, "spawned sr-voice instance");
self.instances.insert(worker_id, VoiceInstance {
process: child,
port,
spawned_at: Instant::now(),
});
Ok(port)
}
/// Stop a sr-voice instance for the given worker ID.
pub fn stop_instance(&mut self, worker_id: usize) {
if let Some(mut instance) = self.instances.remove(&worker_id) {
let _ = instance.process.kill();
let _ = instance.process.wait();
tracing::info!(worker_id, port = instance.port, "stopped sr-voice instance");
}
}
/// Evaluate whether to scale up or down based on current conditions.
///
/// Returns (should_scale_up, should_scale_down_worker_id).
pub fn evaluate_scaling(
&self,
queue_depth: usize,
active_workers: usize,
worker_idle_durations: &HashMap<usize, Duration>,
) -> ScalingDecision {
// Battery → scale to 1
if on_battery() && self.instances.len() > 1 {
return ScalingDecision::ScaleDown {
reason: "battery power detected".into(),
};
}
// Re-probe free resource (VRAM or RAM) for current conditions
let current_free_mb = probe_current_free(self.probe.mode);
let max_slots = compute_max_slots(current_free_mb, self.instances.len());
// Scale up: queue pressure + capacity available
if queue_depth >= QUEUE_DEPTH_SCALE_UP
&& self.instances.len() < max_slots
&& current_free_mb >= MIN_FREE_RAM_MB
{
return ScalingDecision::ScaleUp {
reason: format!(
"queue depth {} >= {}, {} slots available",
queue_depth, QUEUE_DEPTH_SCALE_UP, max_slots
),
};
}
// Scale down: worker idle too long + more than 1 instance
if self.instances.len() > 1 {
for (&worker_id, &idle_time) in worker_idle_durations {
if idle_time >= IDLE_BEFORE_SCALE_DOWN && active_workers < self.instances.len() {
return ScalingDecision::ScaleDown {
reason: format!(
"worker {} idle for {}s",
worker_id,
idle_time.as_secs()
),
};
}
}
}
ScalingDecision::Hold
}
/// Shut down all sr-voice instances.
pub fn shutdown_all(&mut self) {
let ids: Vec<usize> = self.instances.keys().copied().collect();
for id in ids {
self.stop_instance(id);
}
}
/// Wait for a sr-voice instance to become healthy (responds to /health).
/// Returns true if healthy within timeout, false otherwise.
pub fn wait_for_healthy(&self, port: u16, timeout: Duration) -> bool {
let url = format!("http://127.0.0.1:{}/health", port);
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
let result = ureq::AgentBuilder::new()
.timeout(Duration::from_secs(1))
.build()
.get(&url)
.call();
if result.is_ok() {
return true;
}
std::thread::sleep(Duration::from_millis(500));
}
false
}
}
impl Drop for VoiceInstanceManager {
fn drop(&mut self) {
self.shutdown_all();
}
}
/// Result of a scaling evaluation.
#[derive(Debug)]
pub enum ScalingDecision {
/// No change needed.
Hold,
/// Spawn an additional instance.
ScaleUp { reason: String },
/// Stop an instance (pick the most idle worker).
ScaleDown { reason: String },
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn compute_max_slots_with_plenty_of_ram() {
// 16GB free, no running instances
// headroom = 16384, half = 8192, / 1536 = 5
let slots = compute_max_slots(16384, 0);
assert_eq!(slots, 5);
}
#[test]
fn compute_max_slots_accounts_for_running_instances() {
// 16GB free, 2 already running (3072 MB used)
// headroom = 16384 - 3072 = 13312, half = 6656, / 1536 = 4 new
// total = 2 + 4 = 6
let slots = compute_max_slots(16384, 2);
assert_eq!(slots, 6);
}
#[test]
fn compute_max_slots_tight_ram() {
// 2GB free, no running — just enough for 1 instance
// headroom = 2048, half = 1024, / 1536 = 0 new
// But floor rule: free >= MIN_FREE_RAM → 1
let slots = compute_max_slots(2048, 0);
assert_eq!(slots, 1);
}
#[test]
fn compute_max_slots_insufficient_ram() {
// 1GB free — below minimum
let slots = compute_max_slots(1024, 0);
assert_eq!(slots, 0);
}
#[test]
fn compute_max_slots_self_correcting() {
// As instances grow, available slots tighten
let slots_0 = compute_max_slots(8192, 0); // 8GB free, 0 running
let slots_2 = compute_max_slots(8192, 2); // 8GB free, 2 running
// With 2 running, effective headroom is smaller so ceiling is lower per-new-instance
// but total (running + new) can still be higher
assert!(slots_0 > 0);
assert!(slots_2 > 0);
// The key property: free RAM measured at probe time is the same,
// but in practice free_ram will drop as instances consume memory,
// making this naturally self-correcting.
}
#[test]
fn probe_hardware_returns_sane_values() {
let probe = probe_hardware(None);
assert!(probe.total_mb > 0);
assert!(probe.cpu_threads > 0);
assert!(probe.threads_per_instance >= 1);
}
#[test]
fn battery_detection_does_not_crash() {
// Just verify it doesn't panic — result depends on hardware
let _ = on_battery();
}
}
-1
View File
@@ -58,7 +58,6 @@ pub fn voiced_behavior(
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
fn test_cache() -> Arc<Mutex<VoiceCacheStore>> {
let dir = std::env::temp_dir().join("sr-voice-lookup-test");
+1
View File
@@ -13,6 +13,7 @@
//! - `hardware` — hardware detection + dynamic sr-voice instance management
pub mod cache;
pub mod hardware;
pub mod lookup;
pub mod prompt_builder;
pub mod queue;