From cf8a4e37d874c8d1188efbc845830e781cd59424 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 15 Apr 2026 23:11:14 +0200 Subject: [PATCH] fix(engine): offload Gemma to GPU by setting n_gpu_layers=999 (#833) LlamaModelParams::default() sets n_gpu_layers=0, so even with --features rocm the model ran entirely on CPU at ~19 t/s. Setting n_gpu_layers to a large sentinel value asks llama.cpp to offload every layer the model has; llama.cpp clamps to the real count (27 for Gemma 2 2B). Observed throughput jumps from 19 t/s to 74 t/s on an RX 9070 once the ROCm binary is also compiled for gfx1201 (see tooling commit). Also adds server/sr-voice/.gitignore so locally-built binaries don't sneak into the worktree. Release binaries ship out-of-tree per #850. Co-Authored-By: Claude Opus 4.6 (1M context) --- server/sr-voice/.gitignore | 4 ++++ server/sr-voice/src/inference.rs | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 server/sr-voice/.gitignore diff --git a/server/sr-voice/.gitignore b/server/sr-voice/.gitignore new file mode 100644 index 000000000..5ac889a78 --- /dev/null +++ b/server/sr-voice/.gitignore @@ -0,0 +1,4 @@ +# Built binaries — platform-specific, rebuilt via distrobox + cargo. +# Per #850, release binaries will ship as CI artifacts, not in the repo. +/bin/ +/target/ diff --git a/server/sr-voice/src/inference.rs b/server/sr-voice/src/inference.rs index ee1729575..e95ccdcbd 100644 --- a/server/sr-voice/src/inference.rs +++ b/server/sr-voice/src/inference.rs @@ -39,11 +39,19 @@ pub struct InferenceEngine { impl InferenceEngine { /// Load a GGUF model from disk. + /// + /// Offloads all layers to the GPU via ROCm. The binary is built with + /// llama-cpp-rs + ROCm support (see Makefile `build-sr-voice` target), + /// but `LlamaModelParams::default()` sets `n_gpu_layers = 0`, which + /// runs the entire model on CPU at ~10× lower throughput. Setting + /// `n_gpu_layers` to a large sentinel value (999) asks llama.cpp to + /// offload every layer the model has; it clamps to the real count. + /// For Gemma 2 2B (27 layers) this fully GPU-offloads the model. pub fn load(config: &InferenceConfig) -> Result { let backend = LlamaBackend::init().map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?; - let model_params = LlamaModelParams::default(); + let model_params = LlamaModelParams::default().with_n_gpu_layers(999); let model = LlamaModel::load_from_file( &backend, Path::new(&config.model_path),