settled-reach/server/src/voice/hardware.rs

//! Hardware detection + dynamic scaling decisions (D-138, Spike 2).
//!
//! Determines how many parallel LLM workers the system can sustain.
//! The scaling ceiling is conservative:
//!
//!   max_new = (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST
//!
//! "free_resource" is VRAM when a GPU is detected (nvidia-smi / AMD sysfs),
//! or system RAM otherwise. This ensures the voice pipeline never takes more
//! than half the available headroom after accounting for its own instances.
//!
//! Process lifecycle is handled by `WorkerPool` / `VoicePipe` in `worker.rs`.
//! This module only probes hardware and advises on scaling — it does not
//! spawn or stop sr-voice processes.

use std::process::Command;
use std::time::Duration;

use sysinfo::System;

use crate::voice::worker::VoiceProcessConfig;

/// RAM/VRAM budget per sr-voice instance (Gemma 2B Q4_K_M ≈ 1.5 GB resident).
const PER_INSTANCE_RAM_MB: u64 = 1536;

/// Minimum free resource to allow any voice instance at all.
const MIN_FREE_RAM_MB: u64 = 1536;

/// Context window size for sr-voice instances.
const CTX_SIZE: u32 = 512;

/// Queue depth threshold — sustained above this triggers scale-up consideration.
const QUEUE_DEPTH_SCALE_UP: usize = 32;

/// Worker idle duration before scale-down.
const IDLE_BEFORE_SCALE_DOWN: Duration = Duration::from_secs(60);

/// Whether the scaling resource is GPU VRAM or system RAM.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResourceMode {
    /// GPU VRAM — model lives on GPU, scale by free VRAM.
    Gpu,
    /// System RAM — CPU-only inference, scale by free RAM.
    Cpu,
}

/// Hardware probe result from startup.
#[derive(Debug, Clone)]
pub struct HardwareProbe {
    /// Whether scaling is based on GPU VRAM or system RAM.
    pub mode: ResourceMode,
    /// Total resource in MB (VRAM or system RAM).
    pub total_mb: u64,
    /// Free resource at probe time in MB.
    pub free_mb: u64,
    /// Maximum instance slots based on probe-time resources.
    pub max_slots: usize,
    /// CPU thread count available.
    pub cpu_threads: usize,
    /// Threads to allocate per sr-voice instance.
    pub threads_per_instance: u32,
}

impl HardwareProbe {
    /// Build a `VoiceProcessConfig` from probe results and user paths.
    pub fn voice_config(&self, binary_path: String, model_path: String) -> VoiceProcessConfig {
        VoiceProcessConfig {
            binary_path,
            model_path,
            threads: self.threads_per_instance,
            ctx_size: CTX_SIZE,
        }
    }
}

/// Detect GPU type. Run once at install/first-run, persist to settings.
///
/// Returns `Gpu` if NVIDIA, AMD, or Apple Silicon is detected. `Cpu` otherwise.
pub fn detect_gpu_mode() -> ResourceMode {
    if probe_nvidia_vram().is_some() {
        tracing::info!("GPU detection: NVIDIA");
        ResourceMode::Gpu
    } else if probe_amd_vram().is_some() {
        tracing::info!("GPU detection: AMD");
        ResourceMode::Gpu
    } else if is_apple_silicon() {
        tracing::info!("GPU detection: Apple Silicon (unified memory)");
        ResourceMode::Gpu
    } else {
        tracing::info!("GPU detection: none — CPU only");
        ResourceMode::Cpu
    }
}

/// Probe system hardware and determine initial capacity.
///
/// `mode` should come from persisted settings (written at install time via
/// `detect_gpu_mode()`). If `None`, runs detection inline (first-run fallback).
pub fn probe_hardware(mode: Option<ResourceMode>) -> HardwareProbe {
    let cpu_threads = std::thread::available_parallelism()
        .map(|n| n.get())
        .unwrap_or(4);

    let mode = mode.unwrap_or_else(detect_gpu_mode);

    let (total_mb, free_mb) = match mode {
        ResourceMode::Gpu if !is_apple_silicon() => {
            // Discrete GPU — probe VRAM
            if let Some((total, free)) = probe_nvidia_vram() {
                (total, free)
            } else if let Some((total, free)) = probe_amd_vram() {
                (total, free)
            } else {
                // Settings say GPU but can't probe — fall back to system RAM
                tracing::warn!("GPU mode set but VRAM probe failed — falling back to system RAM");
                let mut sys = System::new_all();
                sys.refresh_memory();
                (sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024))
            }
        }
        _ => {
            // CPU mode or Apple Silicon (unified memory = system RAM)
            let mut sys = System::new_all();
            sys.refresh_memory();
            (sys.total_memory() / (1024 * 1024), sys.available_memory() / (1024 * 1024))
        }
    };

    let max_slots = compute_max_slots(free_mb, 0);

    // In GPU mode, threads matter less (GPU does the work), but sr-voice
    // still uses CPU threads for tokenization. Use at most half of CPU
    // threads for voice, minimum 1 per instance.
    let voice_threads = cpu_threads / 2;
    let threads_per_instance = if max_slots > 0 {
        (voice_threads / max_slots).max(1) as u32
    } else {
        1
    };

    let probe = HardwareProbe {
        mode,
        total_mb,
        free_mb,
        max_slots,
        cpu_threads,
        threads_per_instance,
    };

    tracing::info!(
        ?mode,
        total_mb,
        free_mb,
        max_slots,
        cpu_threads,
        threads_per_instance,
        "hardware probe complete"
    );

    probe
}

/// Evaluate whether the worker pool should scale up or down.
///
/// Called periodically by the voice pipeline coordinator. Does not take
/// action itself — returns a `ScalingDecision` for the caller to act on.
pub fn evaluate_scaling(
    mode: ResourceMode,
    running_workers: usize,
    queue_depth: usize,
    active_workers: usize,
    max_idle_duration: Option<Duration>,
) -> ScalingDecision {
    // Battery → scale to 1
    if on_battery() && running_workers > 1 {
        return ScalingDecision::ScaleDown {
            reason: "battery power detected".into(),
        };
    }

    // Re-probe free resource for current conditions
    let current_free_mb = probe_current_free(mode);
    let max_slots = compute_max_slots(current_free_mb, running_workers);

    // Scale up: queue pressure + capacity available
    if queue_depth >= QUEUE_DEPTH_SCALE_UP
        && running_workers < max_slots
        && current_free_mb >= MIN_FREE_RAM_MB
    {
        return ScalingDecision::ScaleUp {
            reason: format!(
                "queue depth {} >= {}, {} slots available",
                queue_depth, QUEUE_DEPTH_SCALE_UP, max_slots
            ),
        };
    }

    // Scale down: worker idle too long + more than 1 worker
    if running_workers > 1 {
        if let Some(idle) = max_idle_duration {
            if idle >= IDLE_BEFORE_SCALE_DOWN && active_workers < running_workers {
                return ScalingDecision::ScaleDown {
                    reason: format!("worker idle for {}s", idle.as_secs()),
                };
            }
        }
    }

    ScalingDecision::Hold
}

/// Result of a scaling evaluation.
#[derive(Debug)]
pub enum ScalingDecision {
    /// No change needed.
    Hold,
    /// Spawn an additional worker (caller decides which).
    ScaleUp { reason: String },
    /// Stop an idle worker (caller picks the most idle).
    ScaleDown { reason: String },
}

// ---------------------------------------------------------------------------
// GPU / resource probing
// ---------------------------------------------------------------------------

/// Probe NVIDIA GPU VRAM via nvidia-smi.
/// Returns (total_mb, free_mb) for the first GPU, or None.
fn probe_nvidia_vram() -> Option<(u64, u64)> {
    let output = Command::new("nvidia-smi")
        .args(["--query-gpu=memory.total,memory.free", "--format=csv,noheader,nounits"])
        .output()
        .ok()?;

    if !output.status.success() {
        return None;
    }

    let stdout = String::from_utf8_lossy(&output.stdout);
    let line = stdout.lines().next()?;
    let parts: Vec<&str> = line.split(',').map(|s| s.trim()).collect();
    if parts.len() != 2 {
        return None;
    }

    let total: u64 = parts[0].parse().ok()?;
    let free: u64 = parts[1].parse().ok()?;
    Some((total, free))
}

/// Probe AMD GPU VRAM via sysfs.
/// Returns (total_mb, free_mb) for the first GPU with VRAM info, or None.
fn probe_amd_vram() -> Option<(u64, u64)> {
    let entries = std::fs::read_dir("/sys/class/drm/").ok()?;

    for entry in entries.flatten() {
        let name = entry.file_name();
        let name_str = name.to_string_lossy();
        if !name_str.starts_with("card") || name_str.contains('-') {
            continue;
        }

        let device_dir = entry.path().join("device");
        let total_path = device_dir.join("mem_info_vram_total");
        let used_path = device_dir.join("mem_info_vram_used");

        let total_bytes: u64 = std::fs::read_to_string(&total_path)
            .ok()?
            .trim()
            .parse()
            .ok()?;
        let used_bytes: u64 = std::fs::read_to_string(&used_path)
            .ok()?
            .trim()
            .parse()
            .ok()?;

        let total_mb = total_bytes / (1024 * 1024);
        let free_mb = (total_bytes.saturating_sub(used_bytes)) / (1024 * 1024);
        return Some((total_mb, free_mb));
    }

    None
}

/// Detect Apple Silicon (macOS + ARM64).
fn is_apple_silicon() -> bool {
    cfg!(target_os = "macos") && cfg!(target_arch = "aarch64")
}

/// Re-probe the current free resource (VRAM or RAM) based on the mode
/// established at startup. Used by evaluate_scaling for live checks.
fn probe_current_free(mode: ResourceMode) -> u64 {
    match mode {
        ResourceMode::Gpu if !is_apple_silicon() => {
            // Try nvidia first, then AMD
            if let Some((_, free)) = probe_nvidia_vram() {
                return free;
            }
            if let Some((_, free)) = probe_amd_vram() {
                return free;
            }
            // GPU vanished? Fall back to system RAM
            let mut sys = System::new_all();
            sys.refresh_memory();
            sys.available_memory() / (1024 * 1024)
        }
        _ => {
            // CPU mode or Apple Silicon (unified memory)
            let mut sys = System::new_all();
            sys.refresh_memory();
            sys.available_memory() / (1024 * 1024)
        }
    }
}

/// Compute maximum instance slots from current free resource (VRAM or RAM).
///
/// Formula: (free_resource - existing_llm_usage) / 2 / PER_INSTANCE_COST
///
/// Takes half of the remaining headroom after subtracting already-running
/// instances. This ensures voice never consumes more than half the
/// available resources beyond its own footprint.
fn compute_max_slots(free_mb: u64, running_instances: usize) -> usize {
    let existing_llm_usage = running_instances as u64 * PER_INSTANCE_RAM_MB;

    // Free resource already reflects system load. Subtract our own LLM usage
    // to get headroom available for expansion.
    let headroom = free_mb.saturating_sub(existing_llm_usage);

    // Take half the headroom, divide by per-instance cost.
    let available_for_new = headroom / 2;
    let new_slots = available_for_new / PER_INSTANCE_RAM_MB;

    // Total = running + new slots we could add.
    let total = running_instances + new_slots as usize;

    // Floor: at least 1 if there's enough free resource for a single instance.
    if total == 0 && free_mb >= MIN_FREE_RAM_MB {
        1
    } else {
        total
    }
}

/// Check if the system is on battery power.
///
/// Linux: reads `/sys/class/power_supply/` sysfs entries.
/// macOS: runs `pmset -g batt` and checks for "Battery Power".
/// Other platforms: returns false (assumes AC power).
fn on_battery() -> bool {
    #[cfg(target_os = "linux")]
    {
        let Ok(entries) = std::fs::read_dir("/sys/class/power_supply/") else {
            return false;
        };

        for entry in entries.flatten() {
            let type_path = entry.path().join("type");
            let status_path = entry.path().join("status");

            let Ok(supply_type) = std::fs::read_to_string(&type_path) else {
                continue;
            };
            if supply_type.trim() != "Battery" {
                continue;
            }

            if let Ok(status) = std::fs::read_to_string(&status_path) {
                if status.trim() == "Discharging" {
                    return true;
                }
            }
        }

        false
    }

    #[cfg(target_os = "macos")]
    {
        Command::new("pmset")
            .args(["-g", "batt"])
            .output()
            .ok()
            .map(|o| String::from_utf8_lossy(&o.stdout).contains("Battery Power"))
            .unwrap_or(false)
    }

    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
    {
        false
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn compute_max_slots_with_plenty_of_ram() {
        // 16GB free, no running instances
        // headroom = 16384, half = 8192, / 1536 = 5
        let slots = compute_max_slots(16384, 0);
        assert_eq!(slots, 5);
    }

    #[test]
    fn compute_max_slots_accounts_for_running_instances() {
        // 16GB free, 2 already running (3072 MB used)
        // headroom = 16384 - 3072 = 13312, half = 6656, / 1536 = 4 new
        // total = 2 + 4 = 6
        let slots = compute_max_slots(16384, 2);
        assert_eq!(slots, 6);
    }

    #[test]
    fn compute_max_slots_tight_ram() {
        // 2GB free, no running — just enough for 1 instance
        // headroom = 2048, half = 1024, / 1536 = 0 new
        // But floor rule: free >= MIN_FREE_RAM → 1
        let slots = compute_max_slots(2048, 0);
        assert_eq!(slots, 1);
    }

    #[test]
    fn compute_max_slots_insufficient_ram() {
        // 1GB free — below minimum
        let slots = compute_max_slots(1024, 0);
        assert_eq!(slots, 0);
    }

    #[test]
    fn compute_max_slots_self_correcting() {
        // As instances grow, available slots tighten
        let slots_0 = compute_max_slots(8192, 0); // 8GB free, 0 running
        let slots_2 = compute_max_slots(8192, 2); // 8GB free, 2 running

        // Both should be non-zero with 8GB
        assert!(slots_0 > 0);
        assert!(slots_2 > 0);
        // The key property: free RAM measured at probe time is the same,
        // but in practice free_ram will drop as instances consume memory,
        // making this naturally self-correcting.
    }

    #[test]
    fn probe_hardware_returns_sane_values() {
        let probe = probe_hardware(None);
        assert!(probe.total_mb > 0);
        assert!(probe.cpu_threads > 0);
        assert!(probe.threads_per_instance >= 1);
    }

    #[test]
    fn voice_config_from_probe() {
        let probe = HardwareProbe {
            mode: ResourceMode::Cpu,
            total_mb: 16384,
            free_mb: 8192,
            max_slots: 2,
            cpu_threads: 8,
            threads_per_instance: 2,
        };
        let config = probe.voice_config("/usr/bin/sr-voice".into(), "/models/gemma.gguf".into());
        assert_eq!(config.threads, 2);
        assert_eq!(config.ctx_size, CTX_SIZE);
    }

    #[test]
    fn battery_detection_does_not_crash() {
        // Just verify it doesn't panic — result depends on hardware
        let _ = on_battery();
    }

    #[test]
    fn evaluate_scaling_hold_when_calm() {
        let decision = evaluate_scaling(
            ResourceMode::Cpu,
            2,   // running
            5,   // queue depth (low)
            1,   // active
            None,
        );
        assert!(matches!(decision, ScalingDecision::Hold));
    }
}