diff --git a/server/sr-voice/.gitignore b/server/sr-voice/.gitignore new file mode 100644 index 000000000..5ac889a78 --- /dev/null +++ b/server/sr-voice/.gitignore @@ -0,0 +1,4 @@ +# Built binaries — platform-specific, rebuilt via distrobox + cargo. +# Per #850, release binaries will ship as CI artifacts, not in the repo. +/bin/ +/target/ diff --git a/server/sr-voice/src/inference.rs b/server/sr-voice/src/inference.rs index ee1729575..e95ccdcbd 100644 --- a/server/sr-voice/src/inference.rs +++ b/server/sr-voice/src/inference.rs @@ -39,11 +39,19 @@ pub struct InferenceEngine { impl InferenceEngine { /// Load a GGUF model from disk. + /// + /// Offloads all layers to the GPU via ROCm. The binary is built with + /// llama-cpp-rs + ROCm support (see Makefile `build-sr-voice` target), + /// but `LlamaModelParams::default()` sets `n_gpu_layers = 0`, which + /// runs the entire model on CPU at ~10× lower throughput. Setting + /// `n_gpu_layers` to a large sentinel value (999) asks llama.cpp to + /// offload every layer the model has; it clamps to the real count. + /// For Gemma 2 2B (27 layers) this fully GPU-offloads the model. pub fn load(config: &InferenceConfig) -> Result { let backend = LlamaBackend::init().map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?; - let model_params = LlamaModelParams::default(); + let model_params = LlamaModelParams::default().with_n_gpu_layers(999); let model = LlamaModel::load_from_file( &backend, Path::new(&config.model_path),