diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index 0face7fd8..e45881687 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -1491,6 +1491,20 @@ def _diagnose_serve_output(text: str) -> dict | None: "Port is already in use.", [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}], ), + ( + # PyPI torch/vLLM wheels now target CUDA 13, which needs driver 580+. + # torch: "The NVIDIA driver on your system is too old (found version 12080)"; + # vLLM kernels: "CUDA driver version is insufficient for CUDA runtime version". + r"NVIDIA driver on your system is too old|CUDA driver version is insufficient", + "The installed PyTorch/vLLM build needs a newer NVIDIA driver than this server has.", + [ + { + "label": "install builds for the driver's CUDA version, e.g. vLLM's +cu129 wheel from its GitHub release with uv pip install … --torch-backend cu129", + "op": "manual", + }, + {"label": "or update the NVIDIA driver (CUDA 13 builds need driver 580 or newer)", "op": "manual"}, + ], + ), ( r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid", "No GPUs are visible to the serve process.", diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 5f83983ea..89f892dd7 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -468,175 +468,6 @@ def setup_cookbook_routes() -> APIRouter: task["payload"].pop("hf_token", None) return state - def _diagnose_serve_output(text: str) -> dict | None: - """Server-side mirror of the Cookbook UI's common serve diagnoses. - - The browser uses cookbook-diagnosis.js for clickable fixes. This gives - the agent/tool path the same structured signal so it can retry with an - adjusted command instead of guessing from raw tmux output. - """ - if not text: - return None - tail = text[-6000:] - patterns = [ - ( - r"No available memory for the cache blocks|Available KV cache memory:.*-", - "No GPU memory left for KV cache after loading model.", - [ - {"label": "retry with GPU memory utilization 0.95", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.95"}, - {"label": "retry with context 2048", "op": "replace", "flag": "--max-model-len", "value": "2048"}, - ], - ), - ( - r"CUDA out of memory|torch\.cuda\.OutOfMemoryError|CUDA error: out of memory|warming up sampler|max_num_seqs.*gpu_memory_utilization", - "GPU ran out of memory during startup or warmup.", - [ - {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"}, - {"label": "retry with GPU memory utilization 0.80", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.80"}, - {"label": "retry with --enforce-eager", "op": "append", "arg": "--enforce-eager"}, - ], - ), - ( - r"not divisib|must be divisible|attention heads.*divisible", - "Tensor parallel size is incompatible with the model.", - [ - {"label": "retry with tensor parallel size 1", "op": "replace", "flag": "--tensor-parallel-size", "value": "1"}, - {"label": "retry with tensor parallel size 2", "op": "replace", "flag": "--tensor-parallel-size", "value": "2"}, - ], - ), - ( - r"KV cache.*too (small|large)|max_model_len.*exceeds|maximum.*context", - "Context length is too large for available GPU memory.", - [ - {"label": "retry with context 8192", "op": "replace", "flag": "--max-model-len", "value": "8192"}, - {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"}, - ], - ), - ( - r"enable-auto-tool-choice requires --tool-call-parser", - "Auto tool choice requires an explicit tool call parser.", - [{"label": "retry with Hermes tool parser", "op": "append", "arg": "--tool-call-parser hermes"}], - ), - ( - r"Please pass.*trust.remote.code=True|contains custom code which must be executed to correctly load|does not recognize this architecture|model type.*but Transformers does not", - "Model requires custom code or newer model support.", - [{"label": "retry with --trust-remote-code", "op": "append", "arg": "--trust-remote-code"}], - ), - ( - r"Either a revision or a version must be specified|transformers\.integrations\.hub_kernels|kernels/layer", - "vLLM/Transformers kernel package mismatch.", - [{"label": "update vLLM, Transformers, and kernels on this server", "op": "dependency", "package": "vllm transformers kernels"}], - ), - ( - r"Address already in use|bind.*address.*in use", - "Port is already in use.", - [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}], - ), - ( - r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid", - "No GPUs are visible to the serve process.", - [{"label": "clear Cookbook GPU selection or choose available GPUs", "op": "settings", "field": "gpus", "value": ""}], - ), - ( - r"Failed to infer device type|NVML Shared Library Not Found|No module named 'amdsmi'|platform is not available", - "vLLM could not find a supported GPU (CUDA or ROCm). " - "This machine may have integrated or unsupported graphics only.", - [ - {"label": "switch to llama.cpp (CPU/Metal, works without a discrete GPU)", "op": "manual"}, - {"label": "switch to Ollama (CPU/Metal, works without a discrete GPU)", "op": "manual"}, - ], - ), - ( - r"vllm.*command not found|No module named vllm|ERROR: vLLM is not installed", - "vLLM is not installed or not in PATH on this server.", - [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}], - ), - ( - r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|" - r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|" - r"Could not load any common_ops library|" - r"Please ensure sgl_kernel is properly installed", - "SGLang native kernel/runtime is missing or mismatched on this server.", - [ - {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"}, - {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"}, - ], - ), - ( - r"sglang.*command not found|No module named sglang|SGLang is not installed", - "SGLang is not installed or not in PATH on this server.", - [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}], - ), - ( - r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed", - "MLX LM is not installed on this server.", - [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}], - ), - ( - r"OmniGen2Pipeline|module diffusers has no attribute .*Pipeline|custom_pipeline=.*failed", - "This image model uses a custom Diffusers pipeline that the launch environment does not know yet.", - [{"label": "update Diffusers image dependencies", "op": "dependency", "package": "diffusers transformers accelerate"}], - ), - ( - r"Unable to quantize model of type |QuantizedSwitchLinear", - "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.", - [ - {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"}, - {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"}, - ], - ), - # System build deps come BEFORE the generic llama.cpp catch-all - # so cmake / build-essential / git missing → a specific OS-package - # remediation instead of "install llama-cpp-python[server]" (which - # itself fails to compile when cmake is absent). - ( - r"cmake: command not found|cmake.*not found.*[Cc]ould not", - "cmake is required to build llama.cpp from source but isn't installed on this server.", - [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}], - ), - ( - r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler", - "A C/C++ compiler (build-essential) is required to build llama.cpp from source.", - [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], - ), - ( - r"^git: command not found", - "git is required to clone the llama.cpp source tree.", - [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], - ), - ( - r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'", - "llama.cpp / llama-cpp-python dependencies are missing.", - [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}], - ), - ( - r"No GGUF found on this host|no \.gguf file|No GGUF file found", - "No GGUF file found for this model on this host. The llama.cpp backend needs a .gguf file.", - [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}], - ), - ( - r"No module named 'torch'|No module named torch|No module named 'torchvision'|No module named torchvision|No module named 'diffusers'|No module named diffusers|No module named 'scipy'|No module named scipy|install scipy if you want to use beta sigmas|requires the Torchvision library", - "Diffusion serving requires PyTorch, Torchvision, Diffusers, Accelerate, and SciPy.", - [{"label": "install Diffusers image deps in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch] torchvision accelerate scipy python-multipart"}], - ), - ( - r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review", - "Model access is gated or unauthorized.", - [{"label": "set HF token and request model access on HuggingFace", "op": "manual"}], - ), - ] - for pattern, message, suggestions in patterns: - if re.search(pattern, tail, re.I): - return {"message": message, "suggestions": suggestions} - if re.search(r"Traceback \(most recent call last\)", tail, re.I) and not re.search( - r"Application startup complete|GET /v1/|Uvicorn running on", tail, re.I - ): - return { - "message": "Python traceback detected during serve startup.", - "suggestions": [{"label": "inspect traceback and retry with adjusted backend/settings", "op": "manual"}], - } - return None - def _state_for_client(state): """Return cookbook state without raw secrets for browser clients.""" _strip_task_secrets(state) @@ -4446,7 +4277,9 @@ def setup_cookbook_routes() -> APIRouter: # fills the visible tail. Without this, output_tail ends up # as just "Locale: C / Ubuntu_Odysseus ❯" and the agent # can't diagnose the actual error. - capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-500")] + # -J joins lines tmux wrapped at the pane width, so an error + # split mid-word still matches the serve diagnosis patterns. + capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-J", "-S", "-500")] elif IS_WINDOWS: # LOCAL Windows task: launched as a detached process (no tmux). # Liveness comes from the .pid file, output from the @@ -4455,7 +4288,7 @@ def setup_cookbook_routes() -> APIRouter: capture_cmd = None else: check_cmd = ["tmux", "has-session", "-t", session_id] - capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-500"] + capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-J", "-S", "-500"] local_win_task = (not remote) and IS_WINDOWS diff --git a/tests/test_cookbook_error_feedback.py b/tests/test_cookbook_error_feedback.py index 1eb88716d..b3d7f5331 100644 --- a/tests/test_cookbook_error_feedback.py +++ b/tests/test_cookbook_error_feedback.py @@ -70,3 +70,63 @@ def test_no_gguf_found_pattern(): result = _diagnose_serve_output(out) assert result is not None assert "GGUF" in result["message"] + + +def test_cookbook_routes_uses_shared_diagnosis_helper(): + # /api/cookbook/tasks/status must use the maintained helper. A nested copy in + # cookbook_routes.py shadowed the import and silently drifted (missing the + # ModelOpt, mflux and MLX bridge diagnoses). + import ast + from pathlib import Path + + source = (Path(__file__).resolve().parents[1] / "routes" / "cookbook_routes.py").read_text(encoding="utf-8") + local_defs = [ + node.lineno + for node in ast.walk(ast.parse(source)) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "_diagnose_serve_output" + ] + assert local_defs == [] + + +def test_modelopt_lm_head_checkpoint_returns_diagnosis(): + out = "ValueError: There is no module or parameter named 'lm_head.input_scale' in Qwen3ForCausalLM" + result = _diagnose_serve_output(out) + assert result is not None + assert "ModelOpt" in result["message"] + + +def test_nvidia_driver_too_old_for_torch_returns_diagnosis(): + # vLLM 0.31 from PyPI ships torch cu130; on driver 570 (CUDA 12.8) the engine dies here. + out = ( + "(EngineCore pid=1) ERROR [core.py:1433] self.driver_worker.init_device()\n" + "(EngineCore pid=1) ERROR [core.py:1433] RuntimeError: The NVIDIA driver on your system is too old " + "(found version 12080). Please update your GPU driver by downloading and installing a new version\n" + "Traceback (most recent call last):\n" + "RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}\n" + ) + result = _diagnose_serve_output(out) + assert result is not None + assert "driver" in result["message"].lower() + assert any("cu129" in s["label"] for s in result["suggestions"]) + + +def test_cuda_runtime_newer_than_driver_returns_diagnosis(): + # torch switched to cu128 but the vLLM wheel's own kernels still need CUDA 13. + out = ( + "RuntimeError: get_cuda_view_from_cpu_tensor, /workspace/csrc/libtorch_stable/cuda_view.cu:38, " + "cudaHostGetDevicePointer failed: CUDA driver version is insufficient for CUDA runtime version\n" + "Traceback (most recent call last):\n" + ) + result = _diagnose_serve_output(out) + assert result is not None + assert "driver" in result["message"].lower() + + +def test_tasks_status_capture_joins_wrapped_tmux_lines(): + # tmux wraps at the pane width (often mid-word), which hid errors like + # "insufficie|nt for CUDA runtime version" from the diagnosis patterns. + from pathlib import Path + + source = (Path(__file__).resolve().parents[1] / "routes" / "cookbook_routes.py").read_text(encoding="utf-8") + assert '_remote_tmux_command("capture-pane", "-t", session_id, "-p", "-J", "-S", "-500")' in source + assert '["tmux", "capture-pane", "-t", session_id, "-p", "-J", "-S", "-500"]' in source