fix(cookbook): diagnose vLLM serves on an NVIDIA driver that is too old (#6727) (#6728)

- Add a diagnosis for "NVIDIA driver on your system is too old" and
  "CUDA driver version is insufficient for CUDA runtime version", the
  errors PyPI's CUDA 13 torch/vLLM builds hit on drivers older than 580.
- Remove the nested _diagnose_serve_output copy in cookbook_routes.py.
  It shadowed the helper and had drifted, so /api/cookbook/tasks/status
  never used the helper's newer diagnoses.
- Capture the tmux pane with -J in the status poll so lines wrapped at
  the pane width do not split error text mid-word.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

routes/cookbook_helpers.py

@@ -1491,6 +1491,20 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "Port is already in use.",
             [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}],
         ),
+        (
+            # PyPI torch/vLLM wheels now target CUDA 13, which needs driver 580+.
+            # torch: "The NVIDIA driver on your system is too old (found version 12080)";
+            # vLLM kernels: "CUDA driver version is insufficient for CUDA runtime version".
+            r"NVIDIA driver on your system is too old|CUDA driver version is insufficient",
+            "The installed PyTorch/vLLM build needs a newer NVIDIA driver than this server has.",
+            [
+                {
+                    "label": "install builds for the driver's CUDA version, e.g. vLLM's +cu129 wheel from its GitHub release with uv pip install … --torch-backend cu129",
+                    "op": "manual",
+                },
+                {"label": "or update the NVIDIA driver (CUDA 13 builds need driver 580 or newer)", "op": "manual"},
+            ],
+        ),
         (
             r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid",
             "No GPUs are visible to the serve process.",

routes/cookbook_routes.py

@@ -468,175 +468,6 @@ def setup_cookbook_routes() -> APIRouter:
                     task["payload"].pop("hf_token", None)
         return state
 
-    def _diagnose_serve_output(text: str) -> dict | None:
-        """Server-side mirror of the Cookbook UI's common serve diagnoses.
-
-        The browser uses cookbook-diagnosis.js for clickable fixes. This gives
-        the agent/tool path the same structured signal so it can retry with an
-        adjusted command instead of guessing from raw tmux output.
-        """
-        if not text:
-            return None
-        tail = text[-6000:]
-        patterns = [
-            (
-                r"No available memory for the cache blocks|Available KV cache memory:.*-",
-                "No GPU memory left for KV cache after loading model.",
-                [
-                    {"label": "retry with GPU memory utilization 0.95", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.95"},
-                    {"label": "retry with context 2048", "op": "replace", "flag": "--max-model-len", "value": "2048"},
-                ],
-            ),
-            (
-                r"CUDA out of memory|torch\.cuda\.OutOfMemoryError|CUDA error: out of memory|warming up sampler|max_num_seqs.*gpu_memory_utilization",
-                "GPU ran out of memory during startup or warmup.",
-                [
-                    {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"},
-                    {"label": "retry with GPU memory utilization 0.80", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.80"},
-                    {"label": "retry with --enforce-eager", "op": "append", "arg": "--enforce-eager"},
-                ],
-            ),
-            (
-                r"not divisib|must be divisible|attention heads.*divisible",
-                "Tensor parallel size is incompatible with the model.",
-                [
-                    {"label": "retry with tensor parallel size 1", "op": "replace", "flag": "--tensor-parallel-size", "value": "1"},
-                    {"label": "retry with tensor parallel size 2", "op": "replace", "flag": "--tensor-parallel-size", "value": "2"},
-                ],
-            ),
-            (
-                r"KV cache.*too (small|large)|max_model_len.*exceeds|maximum.*context",
-                "Context length is too large for available GPU memory.",
-                [
-                    {"label": "retry with context 8192", "op": "replace", "flag": "--max-model-len", "value": "8192"},
-                    {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"},
-                ],
-            ),
-            (
-                r"enable-auto-tool-choice requires --tool-call-parser",
-                "Auto tool choice requires an explicit tool call parser.",
-                [{"label": "retry with Hermes tool parser", "op": "append", "arg": "--tool-call-parser hermes"}],
-            ),
-            (
-                r"Please pass.*trust.remote.code=True|contains custom code which must be executed to correctly load|does not recognize this architecture|model type.*but Transformers does not",
-                "Model requires custom code or newer model support.",
-                [{"label": "retry with --trust-remote-code", "op": "append", "arg": "--trust-remote-code"}],
-            ),
-            (
-                r"Either a revision or a version must be specified|transformers\.integrations\.hub_kernels|kernels/layer",
-                "vLLM/Transformers kernel package mismatch.",
-                [{"label": "update vLLM, Transformers, and kernels on this server", "op": "dependency", "package": "vllm transformers kernels"}],
-            ),
-            (
-                r"Address already in use|bind.*address.*in use",
-                "Port is already in use.",
-                [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}],
-            ),
-            (
-                r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid",
-                "No GPUs are visible to the serve process.",
-                [{"label": "clear Cookbook GPU selection or choose available GPUs", "op": "settings", "field": "gpus", "value": ""}],
-            ),
-            (
-                r"Failed to infer device type|NVML Shared Library Not Found|No module named 'amdsmi'|platform is not available",
-                "vLLM could not find a supported GPU (CUDA or ROCm). "
-                "This machine may have integrated or unsupported graphics only.",
-                [
-                    {"label": "switch to llama.cpp (CPU/Metal, works without a discrete GPU)", "op": "manual"},
-                    {"label": "switch to Ollama (CPU/Metal, works without a discrete GPU)", "op": "manual"},
-                ],
-            ),
-            (
-                r"vllm.*command not found|No module named vllm|ERROR: vLLM is not installed",
-                "vLLM is not installed or not in PATH on this server.",
-                [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
-            ),
-            (
-                r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
-                r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
-                r"Could not load any common_ops library|"
-                r"Please ensure sgl_kernel is properly installed",
-                "SGLang native kernel/runtime is missing or mismatched on this server.",
-                [
-                    {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
-                    {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
-                ],
-            ),
-            (
-                r"sglang.*command not found|No module named sglang|SGLang is not installed",
-                "SGLang is not installed or not in PATH on this server.",
-                [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
-            ),
-            (
-                r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
-                "MLX LM is not installed on this server.",
-                [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
-            ),
-            (
-                r"OmniGen2Pipeline|module diffusers has no attribute .*Pipeline|custom_pipeline=.*failed",
-                "This image model uses a custom Diffusers pipeline that the launch environment does not know yet.",
-                [{"label": "update Diffusers image dependencies", "op": "dependency", "package": "diffusers transformers accelerate"}],
-            ),
-            (
-                r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
-                "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
-                [
-                    {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
-                    {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
-                ],
-            ),
-            # System build deps come BEFORE the generic llama.cpp catch-all
-            # so cmake / build-essential / git missing → a specific OS-package
-            # remediation instead of "install llama-cpp-python[server]" (which
-            # itself fails to compile when cmake is absent).
-            (
-                r"cmake: command not found|cmake.*not found.*[Cc]ould not",
-                "cmake is required to build llama.cpp from source but isn't installed on this server.",
-                [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}],
-            ),
-            (
-                r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler",
-                "A C/C++ compiler (build-essential) is required to build llama.cpp from source.",
-                [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
-            ),
-            (
-                r"^git: command not found",
-                "git is required to clone the llama.cpp source tree.",
-                [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
-            ),
-            (
-                r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'",
-                "llama.cpp / llama-cpp-python dependencies are missing.",
-                [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}],
-            ),
-            (
-                r"No GGUF found on this host|no \.gguf file|No GGUF file found",
-                "No GGUF file found for this model on this host. The llama.cpp backend needs a .gguf file.",
-                [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}],
-            ),
-            (
-                r"No module named 'torch'|No module named torch|No module named 'torchvision'|No module named torchvision|No module named 'diffusers'|No module named diffusers|No module named 'scipy'|No module named scipy|install scipy if you want to use beta sigmas|requires the Torchvision library",
-                "Diffusion serving requires PyTorch, Torchvision, Diffusers, Accelerate, and SciPy.",
-                [{"label": "install Diffusers image deps in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch] torchvision accelerate scipy python-multipart"}],
-            ),
-            (
-                r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review",
-                "Model access is gated or unauthorized.",
-                [{"label": "set HF token and request model access on HuggingFace", "op": "manual"}],
-            ),
-        ]
-        for pattern, message, suggestions in patterns:
-            if re.search(pattern, tail, re.I):
-                return {"message": message, "suggestions": suggestions}
-        if re.search(r"Traceback \(most recent call last\)", tail, re.I) and not re.search(
-            r"Application startup complete|GET /v1/|Uvicorn running on", tail, re.I
-        ):
-            return {
-                "message": "Python traceback detected during serve startup.",
-                "suggestions": [{"label": "inspect traceback and retry with adjusted backend/settings", "op": "manual"}],
-            }
-        return None
-
     def _state_for_client(state):
         """Return cookbook state without raw secrets for browser clients."""
         _strip_task_secrets(state)
@@ -4446,7 +4277,9 @@ def setup_cookbook_routes() -> APIRouter:
                 # fills the visible tail. Without this, output_tail ends up
                 # as just "Locale: C / Ubuntu_Odysseus ❯" and the agent
                 # can't diagnose the actual error.
-                capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-500")]
+                # -J joins lines tmux wrapped at the pane width, so an error
+                # split mid-word still matches the serve diagnosis patterns.
+                capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-J", "-S", "-500")]
             elif IS_WINDOWS:
                 # LOCAL Windows task: launched as a detached process (no tmux).
                 # Liveness comes from the <session>.pid file, output from the
@@ -4455,7 +4288,7 @@ def setup_cookbook_routes() -> APIRouter:
                 capture_cmd = None
             else:
                 check_cmd = ["tmux", "has-session", "-t", session_id]
-                capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-500"]
+                capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-J", "-S", "-500"]
 
             local_win_task = (not remote) and IS_WINDOWS
 

tests/test_cookbook_error_feedback.py

@@ -70,3 +70,63 @@ def test_no_gguf_found_pattern():
     result = _diagnose_serve_output(out)
     assert result is not None
     assert "GGUF" in result["message"]
+
+
+def test_cookbook_routes_uses_shared_diagnosis_helper():
+    # /api/cookbook/tasks/status must use the maintained helper. A nested copy in
+    # cookbook_routes.py shadowed the import and silently drifted (missing the
+    # ModelOpt, mflux and MLX bridge diagnoses).
+    import ast
+    from pathlib import Path
+
+    source = (Path(__file__).resolve().parents[1] / "routes" / "cookbook_routes.py").read_text(encoding="utf-8")
+    local_defs = [
+        node.lineno
+        for node in ast.walk(ast.parse(source))
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "_diagnose_serve_output"
+    ]
+    assert local_defs == []
+
+
+def test_modelopt_lm_head_checkpoint_returns_diagnosis():
+    out = "ValueError: There is no module or parameter named 'lm_head.input_scale' in Qwen3ForCausalLM"
+    result = _diagnose_serve_output(out)
+    assert result is not None
+    assert "ModelOpt" in result["message"]
+
+
+def test_nvidia_driver_too_old_for_torch_returns_diagnosis():
+    # vLLM 0.31 from PyPI ships torch cu130; on driver 570 (CUDA 12.8) the engine dies here.
+    out = (
+        "(EngineCore pid=1) ERROR [core.py:1433]     self.driver_worker.init_device()\n"
+        "(EngineCore pid=1) ERROR [core.py:1433] RuntimeError: The NVIDIA driver on your system is too old "
+        "(found version 12080). Please update your GPU driver by downloading and installing a new version\n"
+        "Traceback (most recent call last):\n"
+        "RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}\n"
+    )
+    result = _diagnose_serve_output(out)
+    assert result is not None
+    assert "driver" in result["message"].lower()
+    assert any("cu129" in s["label"] for s in result["suggestions"])
+
+
+def test_cuda_runtime_newer_than_driver_returns_diagnosis():
+    # torch switched to cu128 but the vLLM wheel's own kernels still need CUDA 13.
+    out = (
+        "RuntimeError: get_cuda_view_from_cpu_tensor, /workspace/csrc/libtorch_stable/cuda_view.cu:38, "
+        "cudaHostGetDevicePointer failed: CUDA driver version is insufficient for CUDA runtime version\n"
+        "Traceback (most recent call last):\n"
+    )
+    result = _diagnose_serve_output(out)
+    assert result is not None
+    assert "driver" in result["message"].lower()
+
+
+def test_tasks_status_capture_joins_wrapped_tmux_lines():
+    # tmux wraps at the pane width (often mid-word), which hid errors like
+    # "insufficie|nt for CUDA runtime version" from the diagnosis patterns.
+    from pathlib import Path
+
+    source = (Path(__file__).resolve().parents[1] / "routes" / "cookbook_routes.py").read_text(encoding="utf-8")
+    assert '_remote_tmux_command("capture-pane", "-t", session_id, "-p", "-J", "-S", "-500")' in source
+    assert '["tmux", "capture-pane", "-t", session_id, "-p", "-J", "-S", "-500"]' in source