mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-02 19:22:22 +01:00
Squash Odysseus development history
This commit is contained in:
pewdiepie-archdaemon
+66955
-18700
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+495
-82
@@ -9,7 +9,7 @@ from services.hwfit.models import (
|
||||
GPU_BANDWIDTH = {
|
||||
"5090": 1792, "5080": 960, "5070 ti": 896, "5070": 672, "5060 ti": 448, "5060": 256,
|
||||
"4090": 1008, "4080 super": 736, "4080": 717, "4070 ti super": 672, "4070 ti": 504, "4070 super": 504, "4070": 504, "4060 ti": 288, "4060": 272,
|
||||
"3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360,
|
||||
"3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360, "3050 ti": 192, "3050": 224,
|
||||
"2080 ti": 616, "2080 super": 496, "2080": 448, "2070 super": 448, "2070": 448, "2060 super": 448, "2060": 336,
|
||||
"1660 ti": 288, "1660 super": 336, "1660": 192, "1650 super": 192, "1650": 128,
|
||||
"h100 sxm": 3350, "h100": 2039, "h200": 4800, "a100 sxm": 2039, "a100": 1555,
|
||||
@@ -18,13 +18,38 @@ GPU_BANDWIDTH = {
|
||||
"7900 xtx": 960, "7900 xt": 800, "7900 gre": 576, "7800 xt": 624, "7700 xt": 432, "7600": 288,
|
||||
"6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224,
|
||||
"mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229,
|
||||
"9070 xt": 624, "9070": 488,
|
||||
"9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322,
|
||||
# NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory,
|
||||
# not Apple Silicon, so it lives in the generic GPU table — the Apple-only
|
||||
# lookup never matches it (its name carries no "apple").
|
||||
"gb10": 273,
|
||||
}
|
||||
|
||||
# Pre-sort keys by length descending for correct substring matching
|
||||
_BW_KEYS_SORTED = sorted(GPU_BANDWIDTH.keys(), key=len, reverse=True)
|
||||
|
||||
FALLBACK_K = {"cuda": 220, "rocm": 180, "cpu_x86": 70, "cpu_arm": 90}
|
||||
# Apple Silicon unified-memory bandwidth (GB/s). For chip families with both
|
||||
# binned and full variants under the same "Apple Mx Max" brand string, prefer
|
||||
# GPU core count when hardware detection provides it; otherwise fall back to the
|
||||
# conservative tier so speed estimates do not over-promise.
|
||||
APPLE_BANDWIDTH_FIXED = {
|
||||
"m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68,
|
||||
"m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100,
|
||||
"m3 ultra": 800, "m3 pro": 150, "m3": 100,
|
||||
"m4 pro": 273, "m4": 120,
|
||||
"m5 pro": 307, "m5": 153,
|
||||
}
|
||||
APPLE_BANDWIDTH_BY_CORES = {
|
||||
"m3 max": {30: 300, 40: 400},
|
||||
"m4 max": {32: 410, 40: 546},
|
||||
"m5 max": {32: 460, 40: 614},
|
||||
}
|
||||
_APPLE_FIXED_KEYS_SORTED = sorted(APPLE_BANDWIDTH_FIXED.keys(), key=len, reverse=True)
|
||||
_APPLE_VARIANT_KEYS_SORTED = sorted(APPLE_BANDWIDTH_BY_CORES.keys(), key=len, reverse=True)
|
||||
|
||||
# metal: backstop for Apple Silicon chips not in the explicit tables above
|
||||
# (e.g. a future M6) — use a conservative generic estimate when unknown.
|
||||
FALLBACK_K = {"cuda": 220, "rocm": 180, "metal": 150, "cpu_x86": 70, "cpu_arm": 90}
|
||||
|
||||
USE_CASE_WEIGHTS = {
|
||||
"general": (0.45, 0.30, 0.15, 0.10),
|
||||
@@ -49,37 +74,159 @@ CONTEXT_TARGET = {
|
||||
}
|
||||
|
||||
|
||||
def _lookup_bandwidth(gpu_name):
|
||||
if not gpu_name:
|
||||
def _lookup_apple_bandwidth(system):
|
||||
gpu_name = system.get("gpu_name")
|
||||
if not isinstance(gpu_name, str) or not gpu_name:
|
||||
return None
|
||||
gn = gpu_name.lower()
|
||||
|
||||
# Guard against false matches on non-Apple GPUs whose names contain
|
||||
# "m3"/"m4"/"m5" (e.g. NVIDIA Quadro M4 000).
|
||||
if "apple" not in gn:
|
||||
return None
|
||||
|
||||
raw_cores = system.get("gpu_cores")
|
||||
try:
|
||||
gpu_cores = int(raw_cores) if raw_cores is not None else None
|
||||
except (TypeError, ValueError):
|
||||
gpu_cores = None
|
||||
|
||||
for key in _APPLE_VARIANT_KEYS_SORTED:
|
||||
if key not in gn:
|
||||
continue
|
||||
if gpu_cores in APPLE_BANDWIDTH_BY_CORES[key]:
|
||||
return APPLE_BANDWIDTH_BY_CORES[key][gpu_cores]
|
||||
return min(APPLE_BANDWIDTH_BY_CORES[key].values())
|
||||
|
||||
for key in _APPLE_FIXED_KEYS_SORTED:
|
||||
if key in gn:
|
||||
return APPLE_BANDWIDTH_FIXED[key]
|
||||
return None
|
||||
|
||||
|
||||
def _lookup_bandwidth(system):
|
||||
if isinstance(system, dict):
|
||||
gpu_name = system.get("gpu_name")
|
||||
else:
|
||||
gpu_name = system
|
||||
|
||||
if not isinstance(gpu_name, str) or not gpu_name:
|
||||
return None
|
||||
|
||||
# Apple tiers live only in the Apple-specific table now (#2564), so route
|
||||
# BOTH dict and bare-string callers through it. A bare string carries no
|
||||
# gpu_cores, so the helper falls back to the conservative (lowest) tier for
|
||||
# that model -- before #2564 the generic table answered string lookups, and
|
||||
# dropping that made _lookup_bandwidth("Apple M3 Max") return None.
|
||||
apple_input = system if isinstance(system, dict) else {"gpu_name": gpu_name}
|
||||
bw = _lookup_apple_bandwidth(apple_input)
|
||||
if bw is not None:
|
||||
return bw
|
||||
|
||||
gn = gpu_name.lower()
|
||||
for key in _BW_KEYS_SORTED:
|
||||
if key in gn:
|
||||
return GPU_BANDWIDTH[key]
|
||||
return None
|
||||
|
||||
|
||||
def _estimate_speed(model, quant, run_mode, system):
|
||||
"""Estimate tok/s. Uses active params for MoE (only active experts run per token)."""
|
||||
def _canonical_cpu_backend(system):
|
||||
"""Return the canonical CPU backend for cpu_only speed estimation.
|
||||
|
||||
Normalizes CPU-architecture aliases separately from the GPU backend, and
|
||||
overrides GPU-only backends (CUDA/ROCm/Metal) so they do not inherit a
|
||||
discrete-GPU fallback constant when the model is actually running on CPU.
|
||||
"""
|
||||
backend = (system.get("backend") or "").lower().strip()
|
||||
cpu_arch = (system.get("cpu_arch") or "").lower().strip()
|
||||
cpu_name = (system.get("cpu_name") or "").lower()
|
||||
gpu_name = (system.get("gpu_name") or "").lower()
|
||||
|
||||
# Already-canonical CPU backends
|
||||
if backend in ("cpu_x86", "cpu_arm"):
|
||||
return backend
|
||||
|
||||
# Raw CPU-architecture aliases. Treat plain "arm" as 32-bit ARM, not the
|
||||
# ARM64-class CPU fallback used for Apple Silicon/aarch64 machines.
|
||||
if backend in ("x86_64", "amd64", "i386", "i686"):
|
||||
return "cpu_x86"
|
||||
if backend in ("arm64", "aarch64"):
|
||||
return "cpu_arm"
|
||||
|
||||
# Prefer an explicit CPU architecture field when present
|
||||
if cpu_arch:
|
||||
if cpu_arch in ("x86_64", "amd64", "x86", "i386", "i686"):
|
||||
return "cpu_x86"
|
||||
if cpu_arch in ("arm64", "aarch64"):
|
||||
return "cpu_arm"
|
||||
|
||||
# Apple Silicon enters ranking as backend="metal"; its CPU path is ARM.
|
||||
if backend in ("metal", "mps", "apple") or "apple" in cpu_name or "apple" in gpu_name:
|
||||
return "cpu_arm"
|
||||
|
||||
# Conservative default for CUDA/ROCm/discrete GPU backends and unknowns.
|
||||
return "cpu_x86"
|
||||
|
||||
|
||||
def _is_mlx_model(model, native_q=None):
|
||||
name = (model.get("name") or "").lower()
|
||||
provider = (model.get("provider") or "").lower()
|
||||
fmt = (model.get("format") or "").lower()
|
||||
q = (native_q if native_q is not None else _native_quant(model)).lower()
|
||||
return (
|
||||
q.startswith("mlx-")
|
||||
or provider == "mlx-community"
|
||||
or fmt == "mlx"
|
||||
or name.startswith("mlx-community/")
|
||||
)
|
||||
|
||||
|
||||
def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
|
||||
"""Estimate tok/s. Uses active params for MoE (only active experts run per token).
|
||||
|
||||
offload_frac (0..1): fraction of the model's weights that spill to system RAM
|
||||
(CPU) because they don't fit VRAM. Generation reads every active weight per
|
||||
token, so when part lives in CPU RAM the per-token time is dominated by the
|
||||
slow path. We model effective bandwidth as a blend of GPU VRAM bandwidth and
|
||||
system-RAM bandwidth weighted by what's where — far more accurate than a flat
|
||||
"halve it" for partial offload, which under/over-shoots depending on amount.
|
||||
Calibrated against a measured RX 9060 XT: DeepSeek-Coder-V2-Lite Q4_K_M with
|
||||
light offload → ~59 t/s est vs 59.8 measured.
|
||||
"""
|
||||
pb = _active_params_b(model)
|
||||
is_moe = model.get("is_moe", False)
|
||||
bw = _lookup_bandwidth(system.get("gpu_name"))
|
||||
bw = _lookup_bandwidth(system)
|
||||
backend = system.get("backend", "cpu_x86")
|
||||
|
||||
# CPU-only inference must never inherit a GPU backend's fallback constant,
|
||||
# even if the detected system happens to report a CUDA/Metal/ROCm backend.
|
||||
if run_mode == "cpu_only":
|
||||
backend = _canonical_cpu_backend(system)
|
||||
|
||||
if bw and run_mode in ("gpu", "cpu_offload"):
|
||||
bpp = QUANT_BYTES_PER_PARAM.get(quant, 0.5)
|
||||
model_gb = pb * bpp
|
||||
if model_gb <= 0:
|
||||
return 0.0
|
||||
efficiency = 0.55
|
||||
raw_tps = (bw / model_gb) * efficiency
|
||||
if run_mode == "cpu_offload":
|
||||
mode_factor = 0.5
|
||||
elif is_moe:
|
||||
mode_factor = 0.8
|
||||
else:
|
||||
mode_factor = 1.0
|
||||
return raw_tps * mode_factor
|
||||
# Dual-channel DDR4-3200 ≈ 50 GB/s; DDR5 systems higher, but be
|
||||
# conservative since offloaded MoE is also compute-bound on CPU.
|
||||
cpu_bw = 55.0
|
||||
frac = min(max(offload_frac, 0.0), 1.0)
|
||||
# If we don't know the fraction (legacy callers pass 0 with
|
||||
# cpu_offload), assume a meaningful spill so we don't overestimate.
|
||||
if frac <= 0.0:
|
||||
frac = 0.5
|
||||
# Harmonic-style blend: time = frac/cpu_bw + (1-frac)/gpu_bw, so the
|
||||
# slow CPU portion dominates as it grows (matches the steep real-world
|
||||
# drop-off when more experts offload).
|
||||
eff_bw = 1.0 / (frac / cpu_bw + (1.0 - frac) / bw)
|
||||
raw_tps = (eff_bw / model_gb) * efficiency
|
||||
return raw_tps * (0.8 if is_moe else 1.0)
|
||||
# Fully on GPU.
|
||||
raw_tps = (bw / model_gb) * efficiency
|
||||
return raw_tps * (0.8 if is_moe else 1.0)
|
||||
|
||||
k = FALLBACK_K.get(backend, 70)
|
||||
if pb <= 0:
|
||||
@@ -88,6 +235,27 @@ def _estimate_speed(model, quant, run_mode, system):
|
||||
return k / pb * sm
|
||||
|
||||
|
||||
def _architecture_bonus(model):
|
||||
name = (model.get("name") or "").lower()
|
||||
arch = (model.get("architecture") or "").lower()
|
||||
text = f"{name} {arch}"
|
||||
|
||||
# Keep this intentionally small: hardware fit and speed still matter, but
|
||||
# current model families should not be scored the same as older Qwen2/LLama
|
||||
# era entries just because the parameter count is similar.
|
||||
if "qwen3.6" in text or "qwen3_6" in text:
|
||||
return 9
|
||||
if "qwen3.5" in text or "qwen3_5" in text:
|
||||
return 8
|
||||
if "qwen3-next" in text or "qwen3_next" in text:
|
||||
return 6
|
||||
if "qwen3" in text or arch.startswith("qwen3"):
|
||||
return 4
|
||||
if "qwen2.5" in text or "qwen2_5" in text:
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def _quality_score(model, quant, use_case):
|
||||
pb = params_b(model)
|
||||
if pb < 1:
|
||||
@@ -117,13 +285,21 @@ def _quality_score(model, quant, use_case):
|
||||
if "gemma" in name_lower:
|
||||
base += 1
|
||||
|
||||
base += _architecture_bonus(model)
|
||||
base += QUANT_QUALITY_PENALTY.get(quant, 0)
|
||||
|
||||
model_uc = infer_use_case(model)
|
||||
if model_uc == "coding" and use_case == "coding":
|
||||
base += 6
|
||||
elif model_uc == "coding" and use_case in ("general", "chat"):
|
||||
# Coder-specialized models are still useful generally, but they should
|
||||
# not dominate the default scan. If the user wants code, the Coding
|
||||
# filter gives them the boost above.
|
||||
base -= 10
|
||||
if model_uc == "reasoning" and use_case == "reasoning" and pb >= 13:
|
||||
base += 5
|
||||
elif model_uc == "reasoning" and use_case == "chat":
|
||||
base -= 4
|
||||
if model_uc == "multimodal" and use_case == "multimodal":
|
||||
base += 6
|
||||
|
||||
@@ -150,6 +326,22 @@ def _fit_score(required, available):
|
||||
return 50
|
||||
|
||||
|
||||
def _is_unified_memory_system(system):
|
||||
backend = (system.get("backend") or "").lower()
|
||||
return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
|
||||
|
||||
|
||||
def _fit_level_for_budget(required_gb, budget_gb):
|
||||
if not required_gb or not budget_gb or required_gb > budget_gb:
|
||||
return "too_tight"
|
||||
ratio = required_gb / budget_gb
|
||||
if ratio <= 0.50:
|
||||
return "perfect"
|
||||
if ratio <= 0.78:
|
||||
return "good"
|
||||
return "marginal"
|
||||
|
||||
|
||||
def _context_score(ctx, use_case):
|
||||
target = CONTEXT_TARGET.get(use_case, 4096)
|
||||
if ctx >= target:
|
||||
@@ -186,9 +378,9 @@ def _quant_bits(q):
|
||||
Returns 0 when unknown (caller treats unknown as "don't filter")."""
|
||||
qu = (q or "").upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
# GGUF k-quants + float formats
|
||||
if qu.startswith("Q8") or "FP8" in qu:
|
||||
if qu.startswith("Q8") or "FP8" in qu or "INT8" in qu or qu.startswith("W8"):
|
||||
return 8
|
||||
if qu.startswith("Q4") or qu.startswith("IQ4"):
|
||||
if qu.startswith("Q4") or qu.startswith("IQ4") or "FP4" in qu or "NF4" in qu or "INT4" in qu or qu.startswith("W4"):
|
||||
return 4
|
||||
if qu.startswith("Q2") or qu.startswith("IQ2"):
|
||||
return 2
|
||||
@@ -200,7 +392,7 @@ def _quant_bits(q):
|
||||
return 6
|
||||
if qu.startswith("F16") or qu.startswith("BF16") or qu.startswith("F32"):
|
||||
return 16
|
||||
# Prequantized formats: pull the bit-width digit (AWQ4 / AWQ4BIT / GPTQ8 / 4BIT / INT8 …)
|
||||
# Prequantized formats: pull the bit-width digit (AWQ4 / AWQ4BIT / GPTQ8 / 4BIT / INT8 ...)
|
||||
m = re.search(r"(?:AWQ|GPTQ|MLX|EXL2|BNB|INT|W)(\d{1,2})", qu) or re.search(r"(\d{1,2})BIT", qu)
|
||||
if m:
|
||||
b = int(m.group(1))
|
||||
@@ -209,12 +401,40 @@ def _quant_bits(q):
|
||||
return 0
|
||||
|
||||
|
||||
def analyze_model(model, system, target_quant=None):
|
||||
def _native_quant(model):
|
||||
native_quant = model.get("quantization", "Q4_K_M")
|
||||
name = (model.get("name") or "").lower()
|
||||
fmt = (model.get("format") or "").lower()
|
||||
text = f"{name} {fmt}"
|
||||
if "nvfp4" in text:
|
||||
return "NVFP4"
|
||||
if re.search(r"(^|[-_/])fp8($|[-_/\s])", text):
|
||||
return "FP8"
|
||||
if "gptq" in text:
|
||||
m = re.search(r"(?:gptq|int|w)(?:[-_]?)(\d{1,2})(?:bit)?", text)
|
||||
# Canonical catalog label is "GPTQ-Int4"/"GPTQ-Int8" (see models.py
|
||||
# QUANT_BPP / QUANT_QUALITY_PENALTY keys); "GPTQ-4bit" misses both
|
||||
# maps, so BPP and the quality penalty silently fall to defaults.
|
||||
return f"GPTQ-Int{m.group(1)}" if m else "GPTQ-Int4"
|
||||
if "awq" in text:
|
||||
m = re.search(r"(?:awq|int|w)(?:[-_]?)(\d{1,2})(?:bit)?", text)
|
||||
# Catalog keys are "AWQ-4bit"/"AWQ-8bit"; bare "AWQ" misses the maps.
|
||||
return f"AWQ-{m.group(1)}bit" if m else "AWQ-4bit"
|
||||
if "mlx" in text:
|
||||
m = re.search(r"mlx[-_]?(\d{1,2})bit", text)
|
||||
return f"mlx-{m.group(1)}bit" if m else native_quant
|
||||
if not (model.get("is_gguf") or model.get("gguf_sources")) and re.search(r"(^|[-_/])(?:int)?8bit($|[-_/\s])", text):
|
||||
return "INT8"
|
||||
return native_quant
|
||||
|
||||
|
||||
def analyze_model(model, system, target_quant=None, scoring_use_case=None, target_context=None):
|
||||
pb = params_b(model)
|
||||
if pb <= 0:
|
||||
return None
|
||||
|
||||
use_case = infer_use_case(model)
|
||||
model_use_case = infer_use_case(model)
|
||||
score_use_case = scoring_use_case or "general"
|
||||
has_gpu = system.get("has_gpu", False)
|
||||
gpu_vram = (system.get("gpu_vram_gb") or 0) if has_gpu else 0
|
||||
gpu_count = system.get("gpu_count", 1) or 1
|
||||
@@ -228,9 +448,14 @@ def analyze_model(model, system, target_quant=None):
|
||||
gpu_only = bool(system.get("gpu_only")) and has_gpu and gpu_vram > 0
|
||||
eff_ram = 0 if gpu_only else available_ram
|
||||
is_moe = model.get("is_moe", False)
|
||||
ctx = model.get("context_length", 4096) or 4096
|
||||
model_ctx = model.get("context_length", 4096) or 4096
|
||||
try:
|
||||
target_context = int(target_context or 0)
|
||||
except (TypeError, ValueError):
|
||||
target_context = 0
|
||||
ctx = min(model_ctx, target_context) if target_context > 0 else model_ctx
|
||||
|
||||
native_quant = model.get("quantization", "Q4_K_M")
|
||||
native_quant = _native_quant(model)
|
||||
preq = is_prequantized(model)
|
||||
|
||||
# GGUF models can't be sharded across GPUs — use single GPU VRAM
|
||||
@@ -246,13 +471,22 @@ def analyze_model(model, system, target_quant=None):
|
||||
else:
|
||||
effective_vram = gpu_vram
|
||||
|
||||
native_gpu_only = preq and not native_quant.startswith("mlx-")
|
||||
|
||||
# Determine which quant to evaluate at
|
||||
native_quant_prefixes = (
|
||||
"AWQ-", "GPTQ-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
|
||||
"INT4", "INT8", "W4A16", "W8A8", "W8A16",
|
||||
)
|
||||
|
||||
if preq:
|
||||
# AWQ/GPTQ/FP8/MLX come at a fixed bit-width. If the user picked a
|
||||
# specific quant tier (e.g. Q8 → 8-bit), only keep prequant models whose
|
||||
# native bit-width matches — otherwise selecting Q8 would still surface
|
||||
# AWQ-4bit models, mixing 4- and 8-bit in one view.
|
||||
# Native HF/vLLM quantized repos come at a fixed format. If the user
|
||||
# picked a GGUF quant tier (Q4/Q8/etc.), do not treat same-bit
|
||||
# AWQ/GPTQ/FP8/FP4 builds as equivalent; those formats are separate
|
||||
# serving paths and only appear when explicitly selected or unfiltered.
|
||||
if target_quant:
|
||||
if not any(target_quant.startswith(p) for p in native_quant_prefixes):
|
||||
return None
|
||||
_tb, _nb = _quant_bits(target_quant), _quant_bits(native_quant)
|
||||
if _tb and _nb and _tb != _nb:
|
||||
return None
|
||||
@@ -260,20 +494,25 @@ def analyze_model(model, system, target_quant=None):
|
||||
elif target_quant:
|
||||
# User picked a specific quant
|
||||
quant_to_try = target_quant
|
||||
elif gpu_count >= 2:
|
||||
# Multi-GPU box: vLLM/SGLang can't serve GGUF Q* quants (those are
|
||||
# llama.cpp-only). Default non-prequantized models to BF16 so the row
|
||||
# is meaningful on a multi-GPU rig. If BF16 doesn't fit, the model
|
||||
# surfaces as too_tight — better than showing a Q4 row the user
|
||||
# can't actually serve with vLLM on >1 GPU.
|
||||
quant_to_try = "BF16"
|
||||
else:
|
||||
# Default: Q4_K_M (user's stated preference)
|
||||
# Default: Q4_K_M (user's stated preference) — kept for single-GPU
|
||||
# and RAM modes where llama.cpp serving is the natural path.
|
||||
quant_to_try = "Q4_K_M"
|
||||
|
||||
result = _try_quant_at(model, quant_to_try, ctx, effective_vram, eff_ram)
|
||||
# Multi-GPU filter: skip the row if the resolved quant is a GGUF tier
|
||||
# (Q*/IQ-prefixed) — vLLM/SGLang can't serve those, so showing them on
|
||||
# a 2+ GPU rig just clutters the list with unservable candidates.
|
||||
if gpu_count >= 2 and quant_to_try and not target_quant and quant_to_try.upper().startswith(("Q2", "Q3", "Q4", "Q5", "Q6", "Q8", "IQ")):
|
||||
return None
|
||||
|
||||
# If target quant doesn't fit and it's not pre-quantized, try lower quants
|
||||
if result is None and not preq and target_quant:
|
||||
from services.hwfit.models import QUANT_HIERARCHY
|
||||
idx = QUANT_HIERARCHY.index(target_quant) if target_quant in QUANT_HIERARCHY else -1
|
||||
for q in QUANT_HIERARCHY[idx + 1:]:
|
||||
result = _try_quant_at(model, q, ctx, effective_vram, eff_ram)
|
||||
if result:
|
||||
break
|
||||
result = _try_quant_at(model, quant_to_try, ctx, effective_vram, 0 if native_gpu_only else eff_ram)
|
||||
|
||||
if result is None:
|
||||
# Model doesn't fit on the user's current hardware. Surface it
|
||||
@@ -289,7 +528,7 @@ def analyze_model(model, system, target_quant=None):
|
||||
"parameter_count": model.get("parameter_count"),
|
||||
"params_b": round(pb, 1),
|
||||
"is_moe": is_moe,
|
||||
"use_case": use_case,
|
||||
"use_case": model_use_case,
|
||||
"fit_level": "too_tight",
|
||||
"run_mode": "no_fit",
|
||||
"quant": quant_to_try,
|
||||
@@ -299,36 +538,63 @@ def analyze_model(model, system, target_quant=None):
|
||||
"score": 0,
|
||||
"scores": {"quality": 0, "speed": 0, "fit": 0, "context": 0},
|
||||
"gguf_sources": model.get("gguf_sources", []),
|
||||
"context_length": model.get("context_length", 4096),
|
||||
"context_length": model_ctx,
|
||||
"target_context": target_context or None,
|
||||
}
|
||||
|
||||
run_mode, quant, fit_ctx, required_gb = result
|
||||
|
||||
# Determine fit level
|
||||
budget = effective_vram if run_mode == "gpu" else available_ram
|
||||
unified_memory = _is_unified_memory_system(system)
|
||||
total_ram = system.get("total_ram_gb") or available_ram
|
||||
unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
|
||||
budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
|
||||
if required_gb > budget:
|
||||
return None
|
||||
if run_mode == "gpu":
|
||||
rec = model.get("recommended_ram_gb") or required_gb
|
||||
if rec <= gpu_vram:
|
||||
fit_level = "perfect"
|
||||
elif gpu_vram >= required_gb * 1.2:
|
||||
fit_level = "good"
|
||||
if unified_memory:
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
# GPU-only fit must leave real allocator/KV/runtime headroom. The
|
||||
# old check used recommended_ram_gb (or required_gb as a fallback),
|
||||
# which made any model that barely fit VRAM read as "perfect".
|
||||
# On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
|
||||
# runnable, but not a comfortable perfect fit.
|
||||
if gpu_vram >= required_gb * 1.50:
|
||||
fit_level = "perfect"
|
||||
elif gpu_vram >= required_gb * 1.2:
|
||||
fit_level = "good"
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
elif run_mode == "cpu_offload":
|
||||
fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
if fit_level == "perfect":
|
||||
fit_level = "good"
|
||||
else:
|
||||
fit_level = "marginal"
|
||||
fit_level = _fit_level_for_budget(required_gb, budget)
|
||||
if fit_level == "too_tight":
|
||||
fit_level = "marginal"
|
||||
|
||||
tps = _estimate_speed(model, quant, run_mode, system)
|
||||
# Rows that comfortably fit in a huge RAM/unified-memory pool should not all
|
||||
# look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
|
||||
if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
|
||||
fit_level = "good"
|
||||
if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
|
||||
fit_level = "perfect"
|
||||
|
||||
q_score = _quality_score(model, quant, use_case)
|
||||
s_score = _speed_score(tps, use_case)
|
||||
# Fraction of the model that spills to CPU RAM (drives the offload speed
|
||||
# model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
|
||||
offload_frac = 0.0
|
||||
if run_mode == "cpu_offload" and required_gb > 0 and effective_vram > 0:
|
||||
offload_frac = max(0.0, (required_gb - effective_vram) / required_gb)
|
||||
tps = _estimate_speed(model, quant, run_mode, system, offload_frac=offload_frac)
|
||||
|
||||
q_score = _quality_score(model, quant, score_use_case)
|
||||
s_score = _speed_score(tps, score_use_case)
|
||||
f_score = _fit_score(required_gb, budget)
|
||||
c_score = _context_score(fit_ctx, use_case)
|
||||
c_score = _context_score(fit_ctx, score_use_case)
|
||||
|
||||
wq, ws, wf, wc = USE_CASE_WEIGHTS.get(use_case, (0.45, 0.30, 0.15, 0.10))
|
||||
wq, ws, wf, wc = USE_CASE_WEIGHTS.get(score_use_case, (0.45, 0.30, 0.15, 0.10))
|
||||
composite = q_score * wq + s_score * ws + f_score * wf + c_score * wc
|
||||
|
||||
return {
|
||||
@@ -337,7 +603,7 @@ def analyze_model(model, system, target_quant=None):
|
||||
"parameter_count": model.get("parameter_count"),
|
||||
"params_b": round(pb, 1),
|
||||
"is_moe": is_moe,
|
||||
"use_case": use_case,
|
||||
"use_case": model_use_case,
|
||||
"fit_level": fit_level,
|
||||
"run_mode": run_mode,
|
||||
"quant": quant,
|
||||
@@ -352,21 +618,101 @@ def analyze_model(model, system, target_quant=None):
|
||||
"context": round(c_score, 1),
|
||||
},
|
||||
"gguf_sources": model.get("gguf_sources", []),
|
||||
"context_length": model.get("context_length", 4096),
|
||||
"context_length": model_ctx,
|
||||
"release_date": model.get("release_date", ""),
|
||||
"target_context": target_context or None,
|
||||
}
|
||||
|
||||
|
||||
def _version_key(name):
|
||||
"""Parse the model's version number from its display name so equal-score
|
||||
rows can break ties in favor of the newer release (e.g. M2.7 > M2.5).
|
||||
Returns a float; 0.0 for names with no recognizable version. The regex
|
||||
grabs the FIRST 'word-with-digits' pattern after a hyphen/underscore,
|
||||
so e.g. 'MiniMax-M2.7' -> 2.7, 'Qwen3.6-35B' -> 3.6, 'M2' -> 2.0."""
|
||||
import re as _re
|
||||
if not name:
|
||||
return 0.0
|
||||
# Match the version-marker word: a letter followed by a number with
|
||||
# optional decimal, e.g. M2.7, V4, Pro3. Take the first hit; ignore
|
||||
# "B" param-count suffixes (Qwen3-235B should yield 3, not 235).
|
||||
for m in _re.finditer(r"[A-Za-z](\d+(?:\.\d+)?)(?![A-Za-z])", name):
|
||||
val = m.group(1)
|
||||
# Skip param-count tokens (e.g. "235B" gives "235" but the next
|
||||
# char would be "B" — already excluded by the negative lookahead).
|
||||
try:
|
||||
f = float(val)
|
||||
except ValueError:
|
||||
continue
|
||||
# Heuristic: bare integers >= 100 are almost certainly param counts
|
||||
# (1B/3B/8B/70B/235B…), not version numbers. Skip them.
|
||||
if "." not in val and f >= 100:
|
||||
continue
|
||||
return f
|
||||
return 0.0
|
||||
|
||||
|
||||
SORT_KEYS = {
|
||||
"score": lambda r: r["score"],
|
||||
# Score sort with version-aware tiebreaker — when two rows tie on
|
||||
# composite score (a common case for the SAME base model in different
|
||||
# versions, e.g. MiniMax-M2.5 vs M2.7 both at the same FP8 budget),
|
||||
# prefer the newer version. Without this, ties resolved to whatever
|
||||
# order they came out of the registry, which let older releases land
|
||||
# above newer ones in user-facing lists.
|
||||
"score": lambda r: (r["score"], _version_key(r.get("name") or "")),
|
||||
"speed": lambda r: r["speed_tps"],
|
||||
"vram": lambda r: r["required_gb"],
|
||||
"params": lambda r: r["params_b"],
|
||||
"context": lambda r: r["context"],
|
||||
# Newest first. release_date is an ISO-ish string ("2026-05-30"); plain
|
||||
# string sort is chronological. Missing dates sort last (empty < any date,
|
||||
# and we sort reverse=True for newest, so "" lands at the bottom).
|
||||
"newest": lambda r: r.get("release_date") or "",
|
||||
}
|
||||
|
||||
|
||||
def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None):
|
||||
"""Rank all models against detected hardware. Returns sorted list of fit results."""
|
||||
def _search_blob(*parts):
|
||||
text = " ".join(str(p or "") for p in parts).lower()
|
||||
compact = re.sub(r"[^a-z0-9]+", "", text)
|
||||
spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
|
||||
return f"{text} {spaced} {compact}"
|
||||
|
||||
|
||||
def _matches_search(model, search):
|
||||
terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
|
||||
if not terms:
|
||||
return True
|
||||
blob = _search_blob(
|
||||
model.get("name"),
|
||||
model.get("provider"),
|
||||
model.get("architecture"),
|
||||
model.get("quantization"),
|
||||
model.get("format"),
|
||||
model.get("parameter_count"),
|
||||
)
|
||||
for term in terms:
|
||||
norm = re.sub(r"[^a-z0-9]+", "", term)
|
||||
if term not in blob and (not norm or norm not in blob):
|
||||
if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
|
||||
try:
|
||||
wanted = float(term.rstrip("b"))
|
||||
actual = params_b(model)
|
||||
except (TypeError, ValueError):
|
||||
actual = 0
|
||||
if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
|
||||
"""Rank all models against detected hardware. Returns sorted list of fit results.
|
||||
|
||||
fit_only: when True, drop rows whose fit_level is "too_tight" (model doesn't
|
||||
actually fit on the chosen budget). When False (default), every model is
|
||||
shown — sorting by Param means highest-param PERIOD, even ones that won't
|
||||
run, so the user can see the truth.
|
||||
"""
|
||||
models = get_models()
|
||||
results = []
|
||||
|
||||
@@ -402,44 +748,104 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
|
||||
"is_image_gen": True,
|
||||
"capabilities": im.get("capabilities", []),
|
||||
"description": im.get("description", ""),
|
||||
"dependency_package": im.get("dependency_package", ""),
|
||||
})
|
||||
if use_case == "image_gen":
|
||||
sort_fn = SORT_KEYS.get(sort, SORT_KEYS["score"])
|
||||
results.sort(key=sort_fn, reverse=(sort != "vram"))
|
||||
results.sort(key=sort_fn, reverse=True) # see main path below
|
||||
return results[:limit]
|
||||
|
||||
# If user picked a prequantized format (AWQ/FP8/GPTQ), filter to only those models
|
||||
filter_native = quant and any(quant.startswith(p) for p in ("AWQ-", "GPTQ-", "FP8"))
|
||||
# If user picked a native prequantized format, filter to only those models.
|
||||
filter_native = quant and any(quant.startswith(p) for p in (
|
||||
"AWQ-", "GPTQ-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
|
||||
"INT4", "INT8", "W4A16", "W8A8", "W8A16",
|
||||
))
|
||||
|
||||
# MLX-quantized models only run on Apple Silicon (Metal). Exclude them on
|
||||
# every other backend (CUDA / ROCm / CPU) so Linux/Windows users don't see
|
||||
# unrunnable suggestions.
|
||||
system_backend = (system.get("backend") or "").lower()
|
||||
apple_silicon = system_backend in ("mps", "metal", "apple")
|
||||
rocm = system_backend == "rocm"
|
||||
is_windows = system.get("platform") == "windows"
|
||||
|
||||
# Consumer AMD Radeon (RDNA, gfx10/11/12): the practical local serving path
|
||||
# is GGUF via llama.cpp. vLLM/SGLang on ROCm are validated for datacenter
|
||||
# Instinct (CDNA, gfx9xx) but are unreliable on consumer RDNA — AWQ kernels
|
||||
# are largely unsupported there and FP8 needs out-of-tree patches. So treat
|
||||
# consumer RDNA like Apple Silicon (GGUF-only) and leave CDNA untouched.
|
||||
# Unknown family (no rocminfo) is left untouched to avoid hiding models from
|
||||
# a possibly-capable Instinct box on a misdetect.
|
||||
gpu_family = (system.get("gpu_family") or "").lower()
|
||||
consumer_amd = system_backend == "rocm" and gpu_family == "rdna"
|
||||
|
||||
for m in models:
|
||||
native_q = m.get("quantization", "")
|
||||
native_q = _native_quant(m)
|
||||
is_mlx = _is_mlx_model(m, native_q)
|
||||
|
||||
# Drop MLX models on non-Apple hardware
|
||||
if not apple_silicon and native_q.startswith("mlx-"):
|
||||
# MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
|
||||
# but it is first-class on Metal where mlx_lm.server can serve it.
|
||||
if is_mlx and not apple_silicon:
|
||||
continue
|
||||
|
||||
# Format filter: AWQ tab → only AWQ models, FP8 tab → only FP8 models
|
||||
# ROCm support for vLLM/SGLang quantized safetensors is too brittle to
|
||||
# recommend blindly in the default scan. Keep AWQ/GPTQ/FP8 discoverable
|
||||
# only when the user explicitly picks that format from the quant filter;
|
||||
# otherwise prefer GGUF/Q* entries that Odysseus can route through
|
||||
# llama.cpp/Ollama without pretending "fits VRAM" means "servable".
|
||||
if rocm and is_prequantized(m) and not filter_native:
|
||||
continue
|
||||
|
||||
# On Apple Silicon the only serving engines are llama.cpp and Ollama,
|
||||
# both GGUF-only (vLLM/SGLang are CUDA/ROCm and don't run on macOS). So
|
||||
# a model is Metal-servable ONLY if it ships a real GGUF. Drop everything
|
||||
# else — raw safetensors repos (which the catalog still tags with a
|
||||
# default GGUF quant) and vLLM-only AWQ/GPTQ/FP8 builds alike. Without
|
||||
# this the Cookbook recommends models the Mac can't run; on CUDA these
|
||||
# stay visible because vLLM serves safetensors directly.
|
||||
#
|
||||
# Consumer AMD (RDNA) is the same story: GGUF via llama.cpp is the
|
||||
# servable path, so a model needs a real GGUF to be recommended.
|
||||
# Otherwise the Cookbook rates vLLM-only AWQ/GPTQ builds "GOOD" on a
|
||||
# Radeon that can't actually serve them.
|
||||
#
|
||||
# Windows is the same: Odysseus only supports llama.cpp on Windows,
|
||||
# which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
|
||||
# models without a GGUF source are unservable there.
|
||||
if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
|
||||
continue
|
||||
|
||||
# Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
|
||||
if filter_native:
|
||||
if quant == "FP8" and native_q != "FP8":
|
||||
continue
|
||||
if quant == "FP4" and native_q not in ("FP4", "NVFP4", "MXFP4", "NF4"):
|
||||
continue
|
||||
if quant.startswith("AWQ") and not native_q.startswith("AWQ"):
|
||||
continue
|
||||
if quant.startswith("GPTQ") and not native_q.startswith("GPTQ"):
|
||||
continue
|
||||
|
||||
if search:
|
||||
name = m.get("name", "").lower()
|
||||
provider = m.get("provider", "").lower()
|
||||
if search.lower() not in name and search.lower() not in provider:
|
||||
if quant.startswith("NVFP4") and not native_q.startswith("NVFP4"):
|
||||
continue
|
||||
if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
|
||||
continue
|
||||
|
||||
result = analyze_model(m, system, target_quant=quant)
|
||||
if search and not _matches_search(m, search):
|
||||
continue
|
||||
|
||||
model_quant = quant
|
||||
# UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
|
||||
# CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
|
||||
# safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
|
||||
# AWQ row, analyze_model correctly rejects it as the wrong serving
|
||||
# format, but the result is confusing: highlighting Quant/Q4 hides the
|
||||
# exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
|
||||
# native AWQ rows only on accelerator servers that can serve them.
|
||||
if (
|
||||
quant == "Q4_K_M"
|
||||
and not (apple_silicon or consumer_amd or is_windows)
|
||||
and native_q == "AWQ-4bit"
|
||||
):
|
||||
model_quant = native_q
|
||||
|
||||
result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
|
||||
if result is None:
|
||||
continue
|
||||
|
||||
@@ -450,14 +856,21 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
|
||||
|
||||
results.append(result)
|
||||
|
||||
# Pick the visible SET by best fit (score) first, so it stays the same no
|
||||
# matter which column the user sorts by — otherwise sorting by params would
|
||||
# truncate to the N biggest models (huge ones that don't even fit) while
|
||||
# sorting by vram showed the N smallest. Only AFTER choosing the set do we
|
||||
# order it by the requested column.
|
||||
results.sort(key=SORT_KEYS["score"], reverse=True)
|
||||
results = results[:limit]
|
||||
# Pick the visible SET by the REQUESTED column. Per-user feedback: sorting
|
||||
# by Param should show the highest-param models PERIOD, not just those that
|
||||
# already fit. Same for every other column. Models that don't fit are still
|
||||
# in the list with their fit_level marking the constraint, so the user can
|
||||
# see the truth instead of a quietly-truncated view. Score sort is unchanged
|
||||
# (it's the default ranking and naturally pushes non-fits to the bottom).
|
||||
if fit_only:
|
||||
# Hide rows that definitely don't fit (the "too_tight" badge) — user
|
||||
# explicitly asked for a Fit-only view.
|
||||
results = [r for r in results if r.get("fit_level") != "too_tight"]
|
||||
sort_fn = SORT_KEYS.get(sort, SORT_KEYS["score"])
|
||||
# vram ascending (smallest first), everything else descending (biggest first)
|
||||
results.sort(key=sort_fn, reverse=(sort != "vram"))
|
||||
# Always sort descending then truncate top-N so each column shows the
|
||||
# global highest by that metric. Before, vram was special-cased
|
||||
# ascending → truncate kept the 50 SMALLEST models and "highest VRAM"
|
||||
# could never appear, breaking the column-click toggle.
|
||||
results.sort(key=sort_fn, reverse=True)
|
||||
results = results[:limit]
|
||||
return results
|
||||
|
||||
+518
-68
@@ -1,9 +1,21 @@
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import shlex
|
||||
|
||||
CACHE_TTL = 1800 # 30 min — hardware rarely changes; use the Rescan button to force a re-probe
|
||||
from core.platform_compat import (
|
||||
NVIDIA_PATH_CANDIDATES,
|
||||
SSH_PATH_OVERRIDE,
|
||||
run_ssh_command,
|
||||
)
|
||||
|
||||
CACHE_TTL = 24 * 3600 # 24 h — hardware probes are user-initiated via the Rescan button; bumped
|
||||
# from 30 min so changing filters doesn't keep re-probing the rig every
|
||||
# half-hour during a long session.
|
||||
|
||||
|
||||
_remote_host = None # set by detect_system(host=...)
|
||||
@@ -17,16 +29,17 @@ def _run(cmd):
|
||||
if _remote_host:
|
||||
# Run command on remote host via SSH
|
||||
if isinstance(cmd, list):
|
||||
cmd_str = " ".join(cmd)
|
||||
cmd_str = shlex.join(str(c) for c in cmd)
|
||||
else:
|
||||
cmd_str = cmd
|
||||
ssh_cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=no"]
|
||||
if _remote_port and _remote_port != "22":
|
||||
ssh_cmd += ["-p", _remote_port]
|
||||
ssh_cmd += [_remote_host, cmd_str]
|
||||
r = subprocess.run(
|
||||
ssh_cmd,
|
||||
capture_output=True, text=True, timeout=15,
|
||||
r = run_ssh_command(
|
||||
_remote_host,
|
||||
_remote_port,
|
||||
cmd_str,
|
||||
timeout=15,
|
||||
connect_timeout=5,
|
||||
strict_host_key_checking=False,
|
||||
text=True,
|
||||
)
|
||||
else:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
@@ -72,21 +85,29 @@ def _detect_nvidia():
|
||||
global _last_gpu_error
|
||||
_last_gpu_error = None
|
||||
out = _run(["nvidia-smi", "--query-gpu=memory.total,name", "--format=csv,noheader,nounits"])
|
||||
# Remote fallback: a non-interactive SSH shell often has a minimal PATH
|
||||
# that omits where nvidia-smi lives (/usr/bin, /usr/local/cuda/bin), so the
|
||||
# first call silently returns nothing → "No GPU" on hosts that DO have GPUs.
|
||||
# Fallback: a non-interactive shell (or WSL) often has a minimal PATH
|
||||
# that omits where nvidia-smi lives (/usr/bin, /usr/local/cuda/bin,
|
||||
# /usr/lib/wsl/lib), so the first call silently returns nothing →
|
||||
# "No GPU" on machines that DO have GPUs.
|
||||
# Retry through a login shell with the common CUDA bin dirs on PATH.
|
||||
if not out and _remote_host:
|
||||
out = _run(
|
||||
"bash -lc 'export PATH=\"$PATH:/usr/bin:/usr/local/bin:/usr/local/cuda/bin\"; "
|
||||
f"bash -lc '{SSH_PATH_OVERRIDE}"
|
||||
"nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits'"
|
||||
)
|
||||
# Last resort: call nvidia-smi by absolute path. Some hosts have a login
|
||||
# shell that isn't bash (or a profile that errors), so the bash -lc retry
|
||||
# above still comes back empty even though the binary is right there.
|
||||
if not out and _remote_host:
|
||||
for _p in ("/usr/bin/nvidia-smi", "/usr/local/bin/nvidia-smi", "/usr/local/cuda/bin/nvidia-smi"):
|
||||
out = _run(f"{_p} --query-gpu=memory.total,name --format=csv,noheader,nounits")
|
||||
# Also handles WSL where nvidia-smi lives at /usr/lib/wsl/lib/ — a path
|
||||
# that may not be in the server process's PATH.
|
||||
if not out:
|
||||
for _p in NVIDIA_PATH_CANDIDATES:
|
||||
# Use list form so subprocess.run (local) resolves the absolute path
|
||||
# correctly instead of treating the whole string as an executable name.
|
||||
if _remote_host:
|
||||
out = _run(f"{_p} --query-gpu=memory.total,name --format=csv,noheader,nounits")
|
||||
else:
|
||||
out = _run([_p, "--query-gpu=memory.total,name", "--format=csv,noheader,nounits"])
|
||||
if out:
|
||||
break
|
||||
if not out:
|
||||
@@ -103,6 +124,8 @@ def _detect_nvidia():
|
||||
return None
|
||||
|
||||
gpus = []
|
||||
# Devices nvidia-smi lists with a real name but a non-numeric memory.total.
|
||||
unified = []
|
||||
# nvidia-smi lists GPUs in index order (0,1,2,...), so the row position is
|
||||
# the CUDA device index we'd pass to CUDA_VISIBLE_DEVICES.
|
||||
for idx, line in enumerate(out.strip().split("\n")):
|
||||
@@ -112,9 +135,32 @@ def _detect_nvidia():
|
||||
vram_mb = float(parts[0])
|
||||
gpus.append({"index": idx, "name": parts[1], "vram_gb": vram_mb / 1024.0})
|
||||
except ValueError:
|
||||
# Grace Blackwell GB10 / DGX Spark and other unified-memory
|
||||
# NVIDIA parts report memory.total as "[N/A]"/"Not Supported"
|
||||
# because the GPU shares the system LPDDR pool instead of
|
||||
# carrying discrete VRAM. Don't drop the device — remember it so
|
||||
# we report a unified-memory GPU below rather than "No GPU" (#1340).
|
||||
if parts[1]:
|
||||
unified.append({"index": idx, "name": parts[1]})
|
||||
continue
|
||||
|
||||
if not gpus:
|
||||
if unified:
|
||||
# Unified-memory CUDA box: report the GPU backed by system RAM so the
|
||||
# Cookbook recommends models and serving works. The pool is shared
|
||||
# (not per-GPU discrete VRAM), so report the RAM total once.
|
||||
ram_gb = round(_get_ram_gb(), 1)
|
||||
gpus = [{"index": g["index"], "name": g["name"], "vram_gb": ram_gb} for g in unified]
|
||||
return {
|
||||
"gpu_name": gpus[0]["name"],
|
||||
"gpu_vram_gb": ram_gb,
|
||||
"gpu_count": len(gpus),
|
||||
"gpus": gpus,
|
||||
"gpu_groups": _group_gpus(gpus),
|
||||
"homogeneous": True,
|
||||
"backend": "cuda",
|
||||
"unified_memory": True,
|
||||
}
|
||||
return None
|
||||
total_vram = sum(g["vram_gb"] for g in gpus)
|
||||
groups = _group_gpus(gpus)
|
||||
@@ -129,6 +175,33 @@ def _detect_nvidia():
|
||||
}
|
||||
|
||||
|
||||
def classify_amd_gfx(gfx):
|
||||
"""Map an AMD ISA target (e.g. "gfx1200") to (gfx, family).
|
||||
|
||||
family is one of:
|
||||
"rdna" — consumer Radeon RX (gfx10xx RDNA1/2, gfx11xx RDNA3, gfx12xx RDNA4)
|
||||
"cdna" — datacenter Instinct (gfx908 MI100, gfx90a MI200, gfx94x/95x MI300+)
|
||||
"gcn" — older GCN/Vega (gfx900/906)
|
||||
"unknown" — empty/unrecognized; callers must treat conservatively
|
||||
|
||||
This drives the serving decision: vLLM/SGLang on ROCm are validated on CDNA
|
||||
but fragile on consumer RDNA (AWQ kernels largely unsupported, FP8 needs
|
||||
out-of-tree patches), so RDNA is steered to GGUF/llama.cpp.
|
||||
"""
|
||||
gfx = (gfx or "").lower().strip()
|
||||
m = re.fullmatch(r"gfx(\d+[a-f]?)", gfx)
|
||||
if not m:
|
||||
return "", "unknown"
|
||||
digits = m.group(1)
|
||||
if digits[:2] in ("10", "11", "12"):
|
||||
return gfx, "rdna"
|
||||
if digits in ("908", "90a") or digits[:2] in ("94", "95"):
|
||||
return gfx, "cdna"
|
||||
if digits[:1] == "9":
|
||||
return gfx, "gcn"
|
||||
return gfx, "unknown"
|
||||
|
||||
|
||||
def _detect_amd():
|
||||
"""Detect AMD GPUs. Handles both discrete cards (with mem_info_vram_total)
|
||||
and APUs / unified-memory SoCs like Strix Halo (which expose
|
||||
@@ -138,7 +211,7 @@ def _detect_amd():
|
||||
val = _run(["cat", path])
|
||||
return val.strip() if val else None
|
||||
try:
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
return f.read().strip()
|
||||
except Exception:
|
||||
return None
|
||||
@@ -154,6 +227,17 @@ def _detect_amd():
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def _amd_arch():
|
||||
"""Best-effort AMD GPU ISA + family from rocminfo.
|
||||
|
||||
rocminfo is the source of truth; its GPU agents report a `Name: gfxNNNN`
|
||||
line (CPU agents report a brand string, not a gfx target), so the first
|
||||
gfx match is the GPU ISA. Returns (gfx, family) — see classify_amd_gfx.
|
||||
"""
|
||||
info = _run(["rocminfo"]) or _run(["/opt/rocm/bin/rocminfo"]) or ""
|
||||
m = re.search(r"gfx\d+[a-f]?", info)
|
||||
return classify_amd_gfx(m.group(0) if m else "")
|
||||
|
||||
try:
|
||||
cards = []
|
||||
is_apu = False
|
||||
@@ -186,6 +270,7 @@ def _detect_amd():
|
||||
return None
|
||||
total_vram = sum(c["vram_gb"] for c in cards)
|
||||
groups = _group_gpus(cards)
|
||||
gfx, family = _amd_arch()
|
||||
# NOTE: for APUs with BIOS UMA carveout (e.g. Strix Halo), vis_vram_total
|
||||
# is the real usable GPU memory — it's physically backed but reserved
|
||||
# by BIOS so it doesn't appear in /proc/meminfo. Don't cap it at system
|
||||
@@ -197,19 +282,146 @@ def _detect_amd():
|
||||
"gpus": cards,
|
||||
"gpu_groups": groups,
|
||||
"homogeneous": len(groups) <= 1,
|
||||
"backend": "rocm",
|
||||
# Pick the actual runtime label: ROCm/HIP only when its
|
||||
# toolchain is installed, otherwise Vulkan if vulkaninfo is
|
||||
# present (mesa RADV works fine on RDNA/CDNA when ROCm
|
||||
# packages are absent — see Strix Halo where ROCm support
|
||||
# is still backporting). Reporting "rocm" on a Vulkan-only
|
||||
# host misleads downstream env-var pinning
|
||||
# (HIP_VISIBLE_DEVICES is a no-op there).
|
||||
"backend": (
|
||||
"rocm" if (_run(["which", "rocminfo"]) or _run(["which", "hipconfig"]))
|
||||
else ("vulkan" if _run(["which", "vulkaninfo"]) else "rocm")
|
||||
),
|
||||
"unified_memory": is_apu,
|
||||
# AMD ISA/family so downstream can tell datacenter Instinct (CDNA,
|
||||
# where vLLM/SGLang run AWQ/GPTQ reliably) from consumer Radeon
|
||||
# (RDNA, where the practical path is GGUF via llama.cpp). Empty/
|
||||
# "unknown" when rocminfo isn't available — callers must treat
|
||||
# unknown conservatively, not assume vLLM works.
|
||||
"gpu_arch": gfx,
|
||||
"gpu_family": family,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _detect_apple_silicon():
|
||||
"""Detect Apple Silicon (M-series) GPUs.
|
||||
|
||||
Macs have no discrete VRAM — the GPU shares the system's unified memory.
|
||||
We report a fraction of total RAM as the usable GPU budget (matching macOS's
|
||||
default Metal working-set limit) so the Cookbook recommends models that
|
||||
actually run on the GPU instead of classifying the machine as CPU-only.
|
||||
|
||||
backend="metal" is what services.hwfit.fit and the serve-command generation
|
||||
key off of (they already understand MLX / llama.cpp-Metal). Works locally
|
||||
(platform.system()=="Darwin") and over SSH (uname -s == Darwin).
|
||||
"""
|
||||
# Gate to macOS — locally via platform, remotely via uname.
|
||||
if _remote_host:
|
||||
if "darwin" not in (_run(["uname", "-s"]) or "").lower():
|
||||
return None
|
||||
arch = (_run(["uname", "-m"]) or "").lower()
|
||||
else:
|
||||
if platform.system() != "Darwin":
|
||||
return None
|
||||
arch = platform.machine().lower()
|
||||
|
||||
# Only Apple Silicon (arm64) has a Metal GPU worth serving LLMs on; Intel
|
||||
# Macs fall through to the CPU path.
|
||||
if _canonical_cpu_arch(arch) != "arm64":
|
||||
return None
|
||||
|
||||
# Chip name, e.g. "Apple M4 Max" — carries the Pro/Max/Ultra variant that
|
||||
# the fit bandwidth table keys off of.
|
||||
brand = (_run(["sysctl", "-n", "machdep.cpu.brand_string"]) or "Apple Silicon").strip()
|
||||
|
||||
# Total unified memory in bytes.
|
||||
memsize = _run(["sysctl", "-n", "hw.memsize"])
|
||||
try:
|
||||
total_gb = int(memsize) / (1024**3) if memsize else 0.0
|
||||
except ValueError:
|
||||
total_gb = 0.0
|
||||
if total_gb <= 0:
|
||||
return None
|
||||
|
||||
def _parse_apple_gpu_cores(text):
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
data = None
|
||||
if isinstance(data, dict):
|
||||
for gpu in data.get("SPDisplaysDataType") or []:
|
||||
if not isinstance(gpu, dict):
|
||||
continue
|
||||
model = str(gpu.get("sppci_model") or gpu.get("_name") or "")
|
||||
if "apple" not in model.lower():
|
||||
continue
|
||||
cores = gpu.get("sppci_cores")
|
||||
try:
|
||||
return int(str(cores).strip())
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
m = re.search(r"Total Number of Cores:\s*(\d+)", text)
|
||||
if m:
|
||||
try:
|
||||
return int(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType", "-json"]))
|
||||
if gpu_cores is None:
|
||||
gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType"]))
|
||||
|
||||
# Usable GPU budget. macOS lets Metal use most of unified memory, but the
|
||||
# default working-set limit scales with RAM: small machines have to keep
|
||||
# more back for the OS + app. These fractions track Apple's
|
||||
# recommendedMaxWorkingSetSize defaults across the lineup. Honour an
|
||||
# explicit override if the user raised it with
|
||||
# `sudo sysctl iogpu.wired_limit_mb=…`.
|
||||
if total_gb <= 16:
|
||||
frac = 0.67
|
||||
elif total_gb <= 64:
|
||||
frac = 0.75
|
||||
else:
|
||||
frac = 0.80
|
||||
vram_gb = round(total_gb * frac, 1)
|
||||
wired = _run(["sysctl", "-n", "iogpu.wired_limit_mb"])
|
||||
try:
|
||||
wired_mb = int(wired) if wired else 0
|
||||
if wired_mb > 0:
|
||||
vram_gb = round(wired_mb / 1024.0, 1)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
gpu = {"index": 0, "name": brand, "vram_gb": vram_gb}
|
||||
info = {
|
||||
"gpu_name": brand,
|
||||
"gpu_vram_gb": vram_gb,
|
||||
"gpu_count": 1,
|
||||
"gpus": [gpu],
|
||||
"gpu_groups": _group_gpus([gpu]),
|
||||
"homogeneous": True,
|
||||
"backend": "metal",
|
||||
# Unified memory: the "VRAM" above is carved out of system RAM, not a
|
||||
# separate pool — downstream fit logic uses this to avoid double-budgeting.
|
||||
"unified_memory": True,
|
||||
}
|
||||
if gpu_cores is not None:
|
||||
info["gpu_cores"] = gpu_cores
|
||||
return info
|
||||
|
||||
|
||||
def _read_file(path):
|
||||
"""Read a file, locally or via SSH."""
|
||||
if _remote_host:
|
||||
return _run(["cat", path])
|
||||
try:
|
||||
with open(path) as f:
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
@@ -238,7 +450,9 @@ def _get_ram_gb():
|
||||
if "MemTotal" in meminfo:
|
||||
return meminfo["MemTotal"] / (1024**2)
|
||||
|
||||
if not _remote_host:
|
||||
# os.sysconf only exists on Unix; on Windows it's absent (AttributeError)
|
||||
# and these constants aren't defined — guard so this never raises there.
|
||||
if not _remote_host and hasattr(os, "sysconf") and "SC_PHYS_PAGES" in getattr(os, "sysconf_names", {}):
|
||||
try:
|
||||
pages = os.sysconf("SC_PHYS_PAGES")
|
||||
page_size = os.sysconf("SC_PAGE_SIZE")
|
||||
@@ -246,6 +460,15 @@ def _get_ram_gb():
|
||||
return (pages * page_size) / (1024**3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# macOS has no /proc/meminfo — fall back to sysctl (works locally and over
|
||||
# SSH to a remote Mac, where the sysconf path above isn't taken).
|
||||
memsize = _run(["sysctl", "-n", "hw.memsize"])
|
||||
if memsize:
|
||||
try:
|
||||
return int(memsize.strip()) / (1024**3)
|
||||
except ValueError:
|
||||
pass
|
||||
return 0.0
|
||||
|
||||
|
||||
@@ -263,6 +486,12 @@ def _get_cpu_name():
|
||||
if line.startswith("model name"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
|
||||
# macOS has no /proc/cpuinfo — sysctl gives the chip name (e.g. "Apple M4").
|
||||
# Harmlessly returns nothing on Linux, so it's safe to try unconditionally.
|
||||
brand = _run(["sysctl", "-n", "machdep.cpu.brand_string"])
|
||||
if brand and brand.strip():
|
||||
return brand.strip()
|
||||
|
||||
if not _remote_host:
|
||||
return platform.processor() or "unknown"
|
||||
return "unknown"
|
||||
@@ -270,7 +499,8 @@ def _get_cpu_name():
|
||||
|
||||
def _get_cpu_count():
|
||||
if _remote_host:
|
||||
out = _run(["nproc"])
|
||||
# nproc on Linux; hw.ncpu via sysctl on a remote Mac (no nproc there).
|
||||
out = _run(["nproc"]) or _run(["sysctl", "-n", "hw.ncpu"])
|
||||
if out:
|
||||
try:
|
||||
return int(out.strip())
|
||||
@@ -283,60 +513,156 @@ def _get_cpu_count():
|
||||
return os.cpu_count() or 1
|
||||
|
||||
|
||||
def _canonical_cpu_arch(value):
|
||||
arch = str(value or "").lower().strip().replace("-", "_")
|
||||
if arch in ("x86_64", "amd64", "x64"):
|
||||
return "x86_64"
|
||||
if arch in ("i386", "i686", "x86"):
|
||||
return "x86"
|
||||
if arch in ("arm64", "aarch64"):
|
||||
return "arm64"
|
||||
if arch == "arm" or arch.startswith("armv"):
|
||||
return "arm"
|
||||
return arch
|
||||
|
||||
|
||||
def _get_cpu_arch():
|
||||
if _remote_host:
|
||||
return _canonical_cpu_arch(_run(["uname", "-m"]) or "")
|
||||
return _canonical_cpu_arch(platform.machine())
|
||||
|
||||
|
||||
def _powershell_exe():
|
||||
"""Pick the best PowerShell executable for LOCAL execution: prefer pwsh
|
||||
(PowerShell 7+), fall back to Windows PowerShell 5.1. Returns an absolute
|
||||
path so we don't depend on a particular PATH ordering."""
|
||||
return shutil.which("pwsh") or shutil.which("powershell") or "powershell"
|
||||
|
||||
def _powershell_encoded_for_ssh(script: str):
|
||||
"""Run a PowerShell script on a remote Windows host over SSH.
|
||||
|
||||
Nested quotes in powershell -Command break when passed through Windows
|
||||
OpenSSH's cmd wrapper; -EncodedCommand avoids that.
|
||||
"""
|
||||
import base64
|
||||
encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii")
|
||||
return _run(f"powershell -NoProfile -EncodedCommand {encoded}")
|
||||
|
||||
|
||||
def _probe_remote_platform():
|
||||
"""Best-effort OS detection over SSH when the caller didn't pass platform."""
|
||||
out = _run("echo %OS%")
|
||||
if out and "Windows_NT" in out:
|
||||
return "windows"
|
||||
uname = (_run(["uname", "-s"]) or "").strip().lower()
|
||||
if uname == "darwin":
|
||||
# Mac uses the linux detection path (_detect_apple_silicon over SSH).
|
||||
return "linux"
|
||||
if uname == "linux":
|
||||
out = _run("test -d /data/data/com.termux && echo termux || echo linux")
|
||||
if out and "termux" in out:
|
||||
return "termux"
|
||||
return "linux"
|
||||
|
||||
|
||||
def _detect_windows():
|
||||
"""Detect Windows hardware in a single SSH call using PowerShell."""
|
||||
"""Detect Windows hardware via PowerShell/WMI.
|
||||
|
||||
Works for BOTH local (host="") and remote (SSH) detection:
|
||||
* remote -> `_run` ships the string to the host over SSH.
|
||||
* local -> `_run` executes a list argv directly (no shell quoting hell).
|
||||
"""
|
||||
# Single PowerShell command that gathers all hardware info at once
|
||||
ps_cmd = (
|
||||
"$r = @{}; "
|
||||
"$os = Get-CimInstance Win32_OperatingSystem; "
|
||||
"$r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1); "
|
||||
"$r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1); "
|
||||
"$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1; "
|
||||
"$r.cpu_name = $cpu.Name; "
|
||||
"$r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum; "
|
||||
"$r.arch = $cpu.AddressWidth; "
|
||||
"""
|
||||
$r = @{}
|
||||
$os = Get-CimInstance Win32_OperatingSystem
|
||||
$r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1)
|
||||
$r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1)
|
||||
$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1
|
||||
$r.cpu_name = $cpu.Name
|
||||
$r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
|
||||
$r.arch = $cpu.AddressWidth
|
||||
$r.cpu_arch = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE }
|
||||
# GPU detection via nvidia-smi (fastest) or WMI fallback
|
||||
"try { "
|
||||
" $nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null; "
|
||||
" if ($LASTEXITCODE -eq 0 -and $nv) { "
|
||||
" $gpus = @(); "
|
||||
" foreach ($line in $nv -split \"`n\") { "
|
||||
" $p = $line -split ','; "
|
||||
" if ($p.Count -ge 2) { $gpus += @{name=$p[1].Trim(); vram_mb=[double]$p[0].Trim()} } "
|
||||
" }; "
|
||||
" $r.gpu_name = $gpus[0].name; "
|
||||
" $r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1); "
|
||||
" $r.gpu_count = $gpus.Count; "
|
||||
" $r.gpu_backend = 'cuda'; "
|
||||
" } "
|
||||
"} catch {}; "
|
||||
"if (-not $r.gpu_name) { "
|
||||
" $wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1; "
|
||||
" if ($wmiGpu) { "
|
||||
" $r.gpu_name = $wmiGpu.Name; "
|
||||
" $r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1); "
|
||||
" $r.gpu_count = 1; "
|
||||
" $r.gpu_backend = 'cpu_x86'; " # WMI doesn't tell us CUDA/ROCm
|
||||
" } "
|
||||
"}; "
|
||||
"$r | ConvertTo-Json -Compress"
|
||||
try {
|
||||
$nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and $nv) {
|
||||
$gpus = @()
|
||||
foreach ($line in $nv -split "`n") {
|
||||
$p = $line -split ','
|
||||
if ($p.Count -ge 2) { $gpus += [pscustomobject]@{name = $p[1].Trim(); vram_mb = [double]$p[0].Trim() } }
|
||||
}
|
||||
$r.gpu_name = $gpus[0].name
|
||||
$r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1)
|
||||
$r.gpu_count = $gpus.Count
|
||||
$r.gpu_backend = 'cuda'
|
||||
}
|
||||
}
|
||||
catch {}
|
||||
if (-not $r.gpu_name) {
|
||||
$wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1
|
||||
$GPUDriverKey = "HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0*"
|
||||
$GPUDeviceID = $wmiGpu.PNPDeviceID.Split('&')[0..1] -join '&'
|
||||
$VRAMfromRegistry = Get-ItemProperty -Path $GPUDriverKey |
|
||||
Where-Object { $_.MatchingDeviceId -like "${GPUDeviceID}*" } |
|
||||
# Sometimes there happen to be multiple driver classes for the same gpu.
|
||||
Select-Object -ExpandProperty HardwareInformation.qwMemorySize -ErrorAction SilentlyContinue -First 1
|
||||
if ($wmiGpu) {
|
||||
$r.gpu_name = $wmiGpu.Name
|
||||
# Edge case: driver is broken, otherwise $wmiGpu.AdapterRAM is redundant
|
||||
if ($VRAMfromRegistry -ge $wmiGpu.AdapterRAM) {
|
||||
$r.gpu_vram_gb = [math]::Round($VRAMfromRegistry / 1073741824, 1)
|
||||
}
|
||||
else {
|
||||
$r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1)
|
||||
}
|
||||
$r.gpu_count = 1
|
||||
# WMI doesn't tell us CUDA/ROCm
|
||||
$r.gpu_backend = 'cpu_x86';
|
||||
}
|
||||
}
|
||||
$r | ConvertTo-Json -Compress
|
||||
"""
|
||||
)
|
||||
out = _run(f'powershell -Command "{ps_cmd}"')
|
||||
if _remote_host:
|
||||
# Remote: use -EncodedCommand so OpenSSH/cmd quoting does not break the script.
|
||||
out = _powershell_encoded_for_ssh(ps_cmd.strip())
|
||||
else:
|
||||
# Local: pass a LIST argv straight to subprocess so the OS hands ps_cmd
|
||||
# to PowerShell verbatim — no fragile string-level quote escaping. Prefer
|
||||
# pwsh (PS7), else Windows PowerShell 5.1.
|
||||
out = _run([_powershell_exe(), "-NoProfile", "-NonInteractive", "-Command", ps_cmd])
|
||||
if not out:
|
||||
return None
|
||||
import json as _json
|
||||
try:
|
||||
d = _json.loads(out)
|
||||
# PowerShell's Measure-Object .Sum / .Count come back as JSON numbers and
|
||||
# decode to float; the Linux path returns plain ints for these — coerce
|
||||
# so the dict shape (and downstream int math) matches across platforms.
|
||||
def _as_int(v, default):
|
||||
try:
|
||||
return int(v)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
_cpu_name = (d.get("cpu_name") or "unknown")
|
||||
if isinstance(_cpu_name, str):
|
||||
_cpu_name = _cpu_name.strip() or "unknown"
|
||||
result = {
|
||||
"total_ram_gb": d.get("ram_gb", 0),
|
||||
"available_ram_gb": d.get("avail_gb", 0),
|
||||
"cpu_cores": d.get("cpu_cores", 1),
|
||||
"cpu_name": d.get("cpu_name", "unknown"),
|
||||
"cpu_cores": _as_int(d.get("cpu_cores"), 1),
|
||||
"cpu_name": _cpu_name,
|
||||
"cpu_arch": _canonical_cpu_arch(d.get("cpu_arch")),
|
||||
"has_gpu": bool(d.get("gpu_name")),
|
||||
"gpu_name": d.get("gpu_name"),
|
||||
"gpu_vram_gb": d.get("gpu_vram_gb"),
|
||||
"gpu_count": d.get("gpu_count", 0),
|
||||
"gpu_count": _as_int(d.get("gpu_count"), 0),
|
||||
"backend": d.get("gpu_backend", "cpu_x86"),
|
||||
"homogeneous": True,
|
||||
"gpu_error": None,
|
||||
"platform": "windows",
|
||||
}
|
||||
# PowerShell only reports aggregate GPU info, not per-card detail, so we
|
||||
# can't tell a mixed box from a uniform one here — assume one homogeneous
|
||||
@@ -363,6 +689,106 @@ def _detect_windows():
|
||||
_cache_by_host = {} # host -> (timestamp, result)
|
||||
|
||||
|
||||
def _cache_key(host: str, ssh_port: str, platform_name: str):
|
||||
"""Build a stable cache key that isolates remote SSH context.
|
||||
|
||||
Same host aliases can have different hardware due to visibility, forwarding etc.
|
||||
To avoid using the wrong cached hardware info, include the SSH port and platform in the cache key.
|
||||
"""
|
||||
return (
|
||||
host or "_local",
|
||||
str(ssh_port or ""),
|
||||
str(platform_name or "").lower(),
|
||||
)
|
||||
|
||||
|
||||
def _is_containerized():
|
||||
"""Best-effort check for whether the local Odysseus process is running in a container."""
|
||||
if _remote_host:
|
||||
return False
|
||||
|
||||
if os.path.exists("/.dockerenv"):
|
||||
return True
|
||||
|
||||
try:
|
||||
with open("/proc/1/cgroup", encoding="utf-8", errors="replace") as f:
|
||||
text = f.read().lower()
|
||||
return any(marker in text for marker in ("docker", "containerd", "kubepods"))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _hardware_visibility_warning(result):
|
||||
"""Return a non-blocking UX warning when detected hardware may only be container-visible."""
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
|
||||
if result.get("manual_hardware"):
|
||||
return None
|
||||
|
||||
if not result.get("containerized"):
|
||||
return None
|
||||
|
||||
if result.get("gpu_error"):
|
||||
return None
|
||||
|
||||
if not result.get("has_gpu"):
|
||||
return {
|
||||
"code": "container_no_gpu_visible",
|
||||
"severity": "warning",
|
||||
"title": "No GPU visible inside Docker",
|
||||
"message": (
|
||||
"Cookbook is scanning hardware from inside the Odysseus container. "
|
||||
"If your host has a GPU, Docker may not be exposing it to the container, "
|
||||
"so model recommendations may be CPU-only or too conservative."
|
||||
),
|
||||
"actions": [
|
||||
"manual_hardware",
|
||||
"rescan",
|
||||
"copy_diagnostics",
|
||||
],
|
||||
}
|
||||
|
||||
total_ram = result.get("total_ram_gb") or 0
|
||||
if total_ram and total_ram <= 8:
|
||||
return {
|
||||
"code": "container_low_ram_visible",
|
||||
"severity": "info",
|
||||
"title": "Container-visible RAM may be lower than host RAM",
|
||||
"message": (
|
||||
"Cookbook is seeing the RAM available inside the container. "
|
||||
"If your host has more memory, validate host RAM separately or use Manual Hardware."
|
||||
),
|
||||
"actions": [
|
||||
"manual_hardware",
|
||||
"rescan",
|
||||
"copy_diagnostics",
|
||||
],
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _attach_probe_context(result, host=""):
|
||||
"""Attach probe-scope metadata and optional hardware visibility warning."""
|
||||
if not isinstance(result, dict) or result.get("error"):
|
||||
return result
|
||||
|
||||
is_remote = bool(host)
|
||||
containerized = False if is_remote else _is_containerized()
|
||||
|
||||
result["probe_scope"] = "remote" if is_remote else ("container" if containerized else "native")
|
||||
result["containerized"] = containerized
|
||||
|
||||
warning = _hardware_visibility_warning(result)
|
||||
if warning:
|
||||
result["hardware_visibility_warning"] = warning
|
||||
else:
|
||||
result.pop("hardware_visibility_warning", None)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
"""Detect system hardware: RAM, CPU, GPU. Cached per host (hardware rarely
|
||||
changes, and probing a remote host over SSH is slow). Pass fresh=True to
|
||||
@@ -372,7 +798,14 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
"""
|
||||
global _remote_host, _remote_port, _remote_platform
|
||||
|
||||
cache_key = host or "_local"
|
||||
if host and not platform:
|
||||
_remote_host = host
|
||||
_remote_port = ssh_port or None
|
||||
platform = _probe_remote_platform()
|
||||
_remote_host = None
|
||||
_remote_port = None
|
||||
|
||||
cache_key = _cache_key(host, ssh_port, platform)
|
||||
now = time.time()
|
||||
if not fresh and cache_key in _cache_by_host:
|
||||
ts, cached = _cache_by_host[cache_key]
|
||||
@@ -387,17 +820,31 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
if _remote_platform == "windows" and _remote_host:
|
||||
result = _detect_windows()
|
||||
if result:
|
||||
result = _attach_probe_context(result, host=host)
|
||||
_remote_host = None
|
||||
_remote_platform = None
|
||||
_cache_by_host[cache_key] = (now, result)
|
||||
return result
|
||||
# If Windows detection failed, return error
|
||||
result = {"error": f"Cannot connect to {host}", "host": host}
|
||||
# SSH may work while the PowerShell hardware probe still fails.
|
||||
result = {"error": f"Windows hardware probe failed for {host}", "host": host}
|
||||
_remote_host = None
|
||||
_remote_platform = None
|
||||
_cache_by_host[cache_key] = (now, result)
|
||||
return result
|
||||
|
||||
# Local Windows: the Linux /proc + /sys + os.sysconf path returns 0 GB RAM,
|
||||
# "unknown" CPU and no GPU on Windows (and os.sysconf doesn't even exist),
|
||||
# so detect locally via PowerShell/WMI instead. _detect_windows() runs the
|
||||
# same probe used for remote Windows, but _run() executes it locally.
|
||||
if not _remote_host and os.name == "nt":
|
||||
result = _detect_windows()
|
||||
if result:
|
||||
result = _attach_probe_context(result, host=host)
|
||||
_cache_by_host[cache_key] = (now, result)
|
||||
return result
|
||||
# PowerShell probe failed entirely — fall through to the generic path
|
||||
# below so we at least return a well-shaped dict rather than crashing.
|
||||
|
||||
# Linux/Termux: existing multi-command detection
|
||||
total_ram = round(_get_ram_gb(), 1)
|
||||
# If remote host returns 0 RAM, connection likely failed
|
||||
@@ -410,8 +857,9 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
available_ram = round(_get_available_ram_gb(), 1)
|
||||
cpu_cores = _get_cpu_count()
|
||||
cpu_name = _get_cpu_name()
|
||||
cpu_arch = _get_cpu_arch()
|
||||
|
||||
gpu_info = _detect_nvidia() or _detect_amd()
|
||||
gpu_info = _detect_apple_silicon() or _detect_nvidia() or _detect_amd()
|
||||
|
||||
if gpu_info:
|
||||
result = {
|
||||
@@ -419,27 +867,28 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
"available_ram_gb": available_ram,
|
||||
"cpu_cores": cpu_cores,
|
||||
"cpu_name": cpu_name,
|
||||
"cpu_arch": cpu_arch,
|
||||
"has_gpu": True,
|
||||
"gpu_name": gpu_info["gpu_name"],
|
||||
"gpu_vram_gb": gpu_info["gpu_vram_gb"],
|
||||
"gpu_count": gpu_info["gpu_count"],
|
||||
"gpu_cores": gpu_info.get("gpu_cores"),
|
||||
"gpus": gpu_info.get("gpus", []),
|
||||
"gpu_groups": gpu_info.get("gpu_groups", []),
|
||||
"homogeneous": gpu_info.get("homogeneous", True),
|
||||
"backend": gpu_info["backend"],
|
||||
# Apple Silicon / AMD APUs share system RAM with the GPU — carry the
|
||||
# flag through so callers can tell unified from discrete VRAM.
|
||||
"unified_memory": gpu_info.get("unified_memory", False),
|
||||
}
|
||||
else:
|
||||
if _remote_host:
|
||||
arch_out = _run(["uname", "-m"]) or ""
|
||||
else:
|
||||
import platform as _platform
|
||||
arch_out = _platform.machine().lower()
|
||||
backend = "cpu_arm" if "aarch64" in arch_out or "arm" in arch_out else "cpu_x86"
|
||||
backend = "cpu_arm" if cpu_arch == "arm64" else "cpu_x86"
|
||||
result = {
|
||||
"total_ram_gb": total_ram,
|
||||
"available_ram_gb": available_ram,
|
||||
"cpu_cores": cpu_cores,
|
||||
"cpu_name": cpu_name,
|
||||
"cpu_arch": cpu_arch,
|
||||
"has_gpu": False,
|
||||
"gpu_name": None,
|
||||
"gpu_vram_gb": None,
|
||||
@@ -451,6 +900,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
|
||||
"gpu_error": _last_gpu_error,
|
||||
}
|
||||
|
||||
result = _attach_probe_context(result, host=host)
|
||||
_remote_host = None
|
||||
_remote_platform = None
|
||||
_cache_by_host[cache_key] = (now, result)
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
|
||||
from src.constants import DATA_DIR
|
||||
|
||||
|
||||
HF_COLLECTIONS_URL = "https://huggingface.co/api/collections"
|
||||
HW_FIT_CACHE_DIR = Path(DATA_DIR) / "hwfit"
|
||||
MLX_COMMUNITY_CACHE = HW_FIT_CACHE_DIR / "mlx_community_models.json"
|
||||
HF_COLLECTION_MODELS_CACHE = HW_FIT_CACHE_DIR / "hf_collection_models.json"
|
||||
HF_COLLECTION_TTL_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
HF_COLLECTION_SOURCES = (
|
||||
{
|
||||
"key": "mlx_community",
|
||||
"owner": "mlx-community",
|
||||
"provider": "mlx-community",
|
||||
"repo_prefix": "mlx-community/",
|
||||
"mlx_only": True,
|
||||
},
|
||||
{
|
||||
"key": "zai_org",
|
||||
"owner": "zai-org",
|
||||
"provider": "zai-org",
|
||||
},
|
||||
{
|
||||
"key": "deepseek_ai",
|
||||
"owner": "deepseek-ai",
|
||||
"provider": "deepseek-ai",
|
||||
},
|
||||
{
|
||||
"key": "minimax_ai",
|
||||
"owner": "MiniMaxAI",
|
||||
"provider": "MiniMaxAI",
|
||||
},
|
||||
{
|
||||
"key": "qwen",
|
||||
"owner": "Qwen",
|
||||
"provider": "Qwen",
|
||||
},
|
||||
{
|
||||
"key": "stepfun_ai",
|
||||
"owner": "stepfun-ai",
|
||||
"provider": "stepfun-ai",
|
||||
},
|
||||
{
|
||||
"key": "google",
|
||||
"owner": "google",
|
||||
"provider": "google",
|
||||
},
|
||||
{
|
||||
"key": "openai",
|
||||
"owner": "openai",
|
||||
"provider": "openai",
|
||||
},
|
||||
{
|
||||
"key": "mistralai",
|
||||
"owner": "mistralai",
|
||||
"provider": "mistralai",
|
||||
},
|
||||
{
|
||||
"key": "meta_llama",
|
||||
"owner": "meta-llama",
|
||||
"provider": "meta-llama",
|
||||
},
|
||||
{
|
||||
"key": "nousresearch",
|
||||
"owner": "NousResearch",
|
||||
"provider": "NousResearch",
|
||||
},
|
||||
{
|
||||
"key": "moonshotai",
|
||||
"owner": "moonshotai",
|
||||
"provider": "moonshotai",
|
||||
},
|
||||
{
|
||||
"key": "mllama",
|
||||
"owner": "mllama",
|
||||
"provider": "mllama",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _format_params(raw):
|
||||
try:
|
||||
n = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
n = 0
|
||||
if n <= 0:
|
||||
return "", 0
|
||||
if n >= 1_000_000_000_000:
|
||||
return f"{n / 1_000_000_000_000:.3g}T", n
|
||||
if n >= 1_000_000_000:
|
||||
return f"{n / 1_000_000_000:.4g}B", n
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.4g}M", n
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.4g}K", n
|
||||
return str(n), n
|
||||
|
||||
|
||||
def _parse_params_from_name(repo_id):
|
||||
name = (repo_id or "").rsplit("/", 1)[-1]
|
||||
active = None
|
||||
m_active = re.search(r"[-_][Aa](\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name)
|
||||
if m_active:
|
||||
active = int(float(m_active.group(1)) * 1_000_000_000)
|
||||
name = name[: m_active.start()] + name[m_active.end() :]
|
||||
total = None
|
||||
for m in re.finditer(r"(\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name):
|
||||
total = int(float(m.group(1)) * 1_000_000_000)
|
||||
break
|
||||
if total is None:
|
||||
for m in re.finditer(r"(\d+(?:\.\d+)?)[Mm](?![a-zA-Z])", name):
|
||||
total = int(float(m.group(1)) * 1_000_000)
|
||||
break
|
||||
return total or 0, active
|
||||
|
||||
|
||||
def _infer_quant(repo_id, source):
|
||||
name = (repo_id or "").rsplit("/", 1)[-1].lower()
|
||||
if source.get("mlx_only"):
|
||||
if "8bit" in name or "8-bit" in name:
|
||||
return "mlx-8bit"
|
||||
if "6bit" in name or "6-bit" in name:
|
||||
return "mlx-6bit"
|
||||
if "5bit" in name or "5-bit" in name:
|
||||
return "mlx-5bit"
|
||||
if "3bit" in name or "3-bit" in name:
|
||||
return "mlx-3bit"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", name):
|
||||
return "BF16"
|
||||
return "mlx-4bit"
|
||||
if "awq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
|
||||
return "AWQ-8bit"
|
||||
if "awq" in name or "4bit" in name or "4-bit" in name:
|
||||
return "AWQ-4bit"
|
||||
if "gptq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
|
||||
return "GPTQ-Int8"
|
||||
if "gptq" in name:
|
||||
return "GPTQ-Int4"
|
||||
if "mxfp4" in name or "nvfp4" in name or re.search(r"(^|[-_/])fp4($|[-_/])", name):
|
||||
return "FP4-MoE-Mixed"
|
||||
if "mxfp8" in name or re.search(r"(^|[-_/])fp8($|[-_/])", name):
|
||||
return "FP8-Mixed"
|
||||
if "gguf" in name or "q4_k" in name or "q4-k" in name:
|
||||
return "Q4_K_M"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", name):
|
||||
return "BF16"
|
||||
return "BF16"
|
||||
|
||||
|
||||
def _quant_bytes_per_param(quant):
|
||||
return {
|
||||
"BF16": 2.2,
|
||||
"FP8": 1.15,
|
||||
"FP8-Mixed": 1.15,
|
||||
"FP4-MoE-Mixed": 0.62,
|
||||
"AWQ-4bit": 0.62,
|
||||
"AWQ-8bit": 1.15,
|
||||
"GPTQ-Int4": 0.62,
|
||||
"GPTQ-Int8": 1.15,
|
||||
"Q4_K_M": 0.62,
|
||||
"mlx-8bit": 1.25,
|
||||
"mlx-6bit": 0.95,
|
||||
"mlx-5bit": 0.82,
|
||||
"mlx-4bit": 0.70,
|
||||
"mlx-3bit": 0.55,
|
||||
}.get(quant, 2.2)
|
||||
|
||||
|
||||
def _infer_context(repo_id, pipeline_tag):
|
||||
text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
|
||||
if any(k in text for k in ("whisper", "asr", "speech-recognition", "tts", "audio", "image", "video", "diffusion")):
|
||||
return 4096
|
||||
if any(k in text for k in ("glm-5.2", "deepseek-v4", "minimax-m3")):
|
||||
return 1_000_000
|
||||
if any(k in text for k in ("qwen3", "glm", "deepseek", "minimax")):
|
||||
return 32768
|
||||
return 32768
|
||||
|
||||
|
||||
def _infer_use_case(repo_id, pipeline_tag):
|
||||
text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
|
||||
if any(k in text for k in ("whisper", "asr", "speech-recognition", "transcrib")):
|
||||
return "stt"
|
||||
if any(k in text for k in ("tts", "text-to-speech", "kokoro", "audio")):
|
||||
return "tts"
|
||||
if any(k in text for k in ("image-text", "vision", "vlm", "vl-", "ocr", "multimodal")):
|
||||
return "multimodal"
|
||||
if any(k in text for k in ("code", "coder")):
|
||||
return "coding"
|
||||
if any(k in text for k in ("reason", "thinking", "thinker", "r1")):
|
||||
return "reasoning"
|
||||
return "general"
|
||||
|
||||
|
||||
def _entry_from_collection_item(collection, item, source):
|
||||
repo_id = item.get("id") or ""
|
||||
if item.get("type") != "model" or not repo_id:
|
||||
return None
|
||||
repo_prefix = source.get("repo_prefix")
|
||||
if repo_prefix and not repo_id.startswith(repo_prefix):
|
||||
return None
|
||||
raw_params = item.get("numParameters") or 0
|
||||
active = None
|
||||
if not raw_params:
|
||||
raw_params, active = _parse_params_from_name(repo_id)
|
||||
param_label, raw_params = _format_params(raw_params)
|
||||
if not raw_params:
|
||||
return None
|
||||
|
||||
quant = _infer_quant(repo_id, source)
|
||||
pipeline_tag = item.get("pipeline_tag") or ""
|
||||
min_ram = round((raw_params / 1_000_000_000) * _quant_bytes_per_param(quant) + 0.8, 1)
|
||||
last_modified = item.get("lastModified") or collection.get("lastUpdated") or ""
|
||||
release_date = ""
|
||||
if last_modified:
|
||||
try:
|
||||
release_date = parsedate_to_datetime(last_modified).date().isoformat()
|
||||
except Exception:
|
||||
release_date = str(last_modified)[:10]
|
||||
|
||||
entry = {
|
||||
"name": repo_id,
|
||||
"provider": source.get("provider") or repo_id.split("/", 1)[0],
|
||||
"parameter_count": param_label,
|
||||
"parameters_raw": raw_params,
|
||||
"min_ram_gb": min_ram,
|
||||
"recommended_ram_gb": round(min_ram * 1.3 + 0.5, 1),
|
||||
"min_vram_gb": 0.0 if source.get("mlx_only") else min_ram,
|
||||
"quantization": quant,
|
||||
"context_length": _infer_context(repo_id, pipeline_tag),
|
||||
"use_case": _infer_use_case(repo_id, pipeline_tag),
|
||||
"capabilities": ["mlx"] if source.get("mlx_only") else ["vllm", "sglang"],
|
||||
"pipeline_tag": pipeline_tag,
|
||||
"architecture": "",
|
||||
"hf_downloads": int(item.get("downloads") or 0),
|
||||
"hf_likes": int(item.get("likes") or 0),
|
||||
"release_date": release_date,
|
||||
"format": "mlx" if source.get("mlx_only") else "safetensors",
|
||||
"collection": collection.get("title") or "",
|
||||
"description": collection.get("description") or "",
|
||||
"_discovered": True,
|
||||
"_source": "hf_collections",
|
||||
"_source_owner": source.get("owner") or "",
|
||||
}
|
||||
if source.get("mlx_only"):
|
||||
entry["mlx_only"] = True
|
||||
if quant == "Q4_K_M":
|
||||
entry["is_gguf"] = True
|
||||
entry["format"] = "gguf"
|
||||
entry["capabilities"] = ["llama.cpp"]
|
||||
if active:
|
||||
entry["is_moe"] = True
|
||||
entry["active_parameters"] = active
|
||||
return entry
|
||||
|
||||
|
||||
def _next_link(header):
|
||||
if not header:
|
||||
return None
|
||||
m = re.search(r'<([^>]+)>;\s*rel="next"', header)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def fetch_collection_models(source, timeout=20, max_pages=20):
|
||||
params = urllib.parse.urlencode({
|
||||
"owner": source["owner"],
|
||||
"limit": "100",
|
||||
"expand": "true",
|
||||
})
|
||||
url = f"{HF_COLLECTIONS_URL}?{params}"
|
||||
models = {}
|
||||
pages = 0
|
||||
while url and pages < max_pages:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "odysseus-hwfit/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
payload = json.load(resp)
|
||||
url = _next_link(resp.headers.get("Link"))
|
||||
pages += 1
|
||||
if not isinstance(payload, list):
|
||||
break
|
||||
for collection in payload:
|
||||
if not isinstance(collection, dict):
|
||||
continue
|
||||
for item in collection.get("items") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
entry = _entry_from_collection_item(collection, item, source)
|
||||
if entry and entry["name"] not in models:
|
||||
models[entry["name"]] = entry
|
||||
rows = list(models.values())
|
||||
rows.sort(key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""), reverse=True)
|
||||
return rows
|
||||
|
||||
|
||||
def _load_cache(path):
|
||||
try:
|
||||
with path.open(encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
rows = data.get("models") if isinstance(data, dict) else data
|
||||
return rows if isinstance(rows, list) else []
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_cache(path, source, rows):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"source": source,
|
||||
"fetched_at": int(time.time()),
|
||||
"count": len(rows),
|
||||
"models": rows,
|
||||
}
|
||||
tmp = path.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def load_cached_mlx_community_models():
|
||||
return _load_cache(MLX_COMMUNITY_CACHE)
|
||||
|
||||
|
||||
def load_cached_hf_collection_models():
|
||||
return _load_cache(HF_COLLECTION_MODELS_CACHE)
|
||||
|
||||
|
||||
def _cache_fresh(path):
|
||||
try:
|
||||
return (time.time() - path.stat().st_mtime) < HF_COLLECTION_TTL_SECONDS
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def refresh_mlx_community_cache(force=False):
|
||||
if not force and _cache_fresh(MLX_COMMUNITY_CACHE):
|
||||
return load_cached_mlx_community_models()
|
||||
source = next(s for s in HF_COLLECTION_SOURCES if s["key"] == "mlx_community")
|
||||
rows = fetch_collection_models(source)
|
||||
_write_cache(MLX_COMMUNITY_CACHE, "https://huggingface.co/mlx-community/collections", rows)
|
||||
return rows
|
||||
|
||||
|
||||
def refresh_hf_collection_models_cache(force=False):
|
||||
if not force and _cache_fresh(HF_COLLECTION_MODELS_CACHE):
|
||||
return load_cached_hf_collection_models()
|
||||
rows_by_name = {}
|
||||
for source in HF_COLLECTION_SOURCES:
|
||||
if source["key"] == "mlx_community":
|
||||
continue
|
||||
try:
|
||||
for row in fetch_collection_models(source):
|
||||
rows_by_name.setdefault(row["name"], row)
|
||||
except Exception:
|
||||
# Keep partial refreshes useful. A temporary DNS/provider issue for
|
||||
# one brand should not invalidate the other cached collection rows.
|
||||
continue
|
||||
rows = sorted(
|
||||
rows_by_name.values(),
|
||||
key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
if rows:
|
||||
_write_cache(HF_COLLECTION_MODELS_CACHE, "https://huggingface.co/collections", rows)
|
||||
return rows
|
||||
return load_cached_hf_collection_models()
|
||||
+379
-276
@@ -1,278 +1,369 @@
|
||||
"""Image generation model registry and VRAM fitting for Cookbook."""
|
||||
|
||||
# Curated registry of image generation models supported by diffusers.
|
||||
# ONLY verified HuggingFace repo IDs.
|
||||
# VRAM estimates are for inference (single image generation).
|
||||
IMAGE_MODEL_REGISTRY = [
|
||||
# ── Z-Image (Alibaba Tongyi) ──
|
||||
{
|
||||
"id": "Tongyi-MAI/Z-Image-Turbo",
|
||||
"name": "Z-Image Turbo",
|
||||
"provider": "Tongyi",
|
||||
"params_b": 6.0,
|
||||
"vram_bf16": 19.0,
|
||||
"vram_fp8": 10.0,
|
||||
"vram_q4": 6.0,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {
|
||||
"FP8": "drbaph/Z-Image-Turbo-FP8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "6B distilled, 8-step. Sub-second on H800. Apache 2.0.",
|
||||
"quality": 92,
|
||||
"speed": 95,
|
||||
"released": "2025-12",
|
||||
},
|
||||
{
|
||||
"id": "Tongyi-MAI/Z-Image",
|
||||
"name": "Z-Image",
|
||||
"provider": "Tongyi",
|
||||
"params_b": 6.0,
|
||||
"vram_bf16": 19.0,
|
||||
"vram_fp8": 10.0,
|
||||
"vram_q4": 6.0,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {
|
||||
"FP8": "drbaph/Z-Image-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "Full undistilled model. Highest creative freedom. Apache 2.0.",
|
||||
"quality": 93,
|
||||
"speed": 70,
|
||||
"released": "2025-12",
|
||||
},
|
||||
# ── Qwen Image ──
|
||||
{
|
||||
"id": "Qwen/Qwen-Image-2512",
|
||||
"name": "Qwen Image 2512",
|
||||
"provider": "Qwen",
|
||||
"params_b": 20.0,
|
||||
"vram_bf16": 42.0,
|
||||
"vram_fp8": 22.0,
|
||||
"vram_q4": 14.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["text-to-image", "text-rendering"],
|
||||
"description": "Dec 2025 update. Better humans, finer detail, strong text. Apache 2.0.",
|
||||
"quality": 95,
|
||||
"speed": 50,
|
||||
"released": "2025-12",
|
||||
},
|
||||
{
|
||||
"id": "Qwen/Qwen-Image",
|
||||
"name": "Qwen Image",
|
||||
"provider": "Qwen",
|
||||
"params_b": 20.0,
|
||||
"vram_bf16": 42.0,
|
||||
"vram_fp8": 22.0,
|
||||
"vram_q4": 14.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["text-to-image", "text-rendering"],
|
||||
"description": "20B foundation. Best text rendering in images. Apache 2.0.",
|
||||
"quality": 94,
|
||||
"speed": 50,
|
||||
"released": "2025-08",
|
||||
},
|
||||
{
|
||||
"id": "Qwen/Qwen-Image-Edit-2511",
|
||||
"name": "Qwen Image Edit",
|
||||
"provider": "Qwen",
|
||||
"params_b": 20.0,
|
||||
"vram_bf16": 42.0,
|
||||
"vram_fp8": 22.0,
|
||||
"vram_q4": 14.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["image-editing", "inpainting"],
|
||||
"description": "Dedicated editing. Style transfer, object removal. Apache 2.0.",
|
||||
"quality": 92,
|
||||
"speed": 50,
|
||||
"released": "2025-11",
|
||||
},
|
||||
# ── Stable Diffusion (dedicated inpainting) ──
|
||||
{
|
||||
"id": "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
|
||||
"name": "SDXL Inpainting",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 3.5,
|
||||
"vram_bf16": 12.0,
|
||||
"vram_fp8": 8.0,
|
||||
"vram_q4": 6.0,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["inpainting", "image-editing"],
|
||||
"description": "SDXL fine-tuned for inpainting (9-channel UNet). Best SD-family fill quality; fits a 24GB card comfortably.",
|
||||
"quality": 86,
|
||||
"speed": 68,
|
||||
"released": "2023-11",
|
||||
},
|
||||
{
|
||||
"id": "stable-diffusion-v1-5/stable-diffusion-inpainting",
|
||||
"name": "SD 1.5 Inpainting",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 1.1,
|
||||
"vram_bf16": 4.0,
|
||||
"vram_fp8": 3.0,
|
||||
"vram_q4": 2.5,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["inpainting"],
|
||||
"description": "Classic SD 1.5 inpaint. Very light and fast; lower fidelity than SDXL.",
|
||||
"quality": 70,
|
||||
"speed": 92,
|
||||
"released": "2022-10",
|
||||
},
|
||||
# ── FLUX ──
|
||||
{
|
||||
"id": "black-forest-labs/FLUX.1-dev",
|
||||
"name": "FLUX.1 Dev",
|
||||
"provider": "Black Forest Labs",
|
||||
"params_b": 12.0,
|
||||
"vram_bf16": 33.0,
|
||||
"vram_fp8": 17.0,
|
||||
"vram_q4": 10.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {
|
||||
"FP8": "diffusers/FLUX.1-dev-torchao-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "High quality, detailed. Popular community model. Non-commercial.",
|
||||
"quality": 92,
|
||||
"speed": 55,
|
||||
"released": "2024-08",
|
||||
},
|
||||
{
|
||||
"id": "black-forest-labs/FLUX.1-schnell",
|
||||
"name": "FLUX.1 Schnell",
|
||||
"provider": "Black Forest Labs",
|
||||
"params_b": 12.0,
|
||||
"vram_bf16": 33.0,
|
||||
"vram_fp8": 17.0,
|
||||
"vram_q4": 10.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {
|
||||
"FP8": "Kijai/flux-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "Fast 4-step variant. Apache 2.0 license.",
|
||||
"quality": 85,
|
||||
"speed": 90,
|
||||
"released": "2024-08",
|
||||
},
|
||||
# ── Stable Diffusion ──
|
||||
{
|
||||
"id": "stabilityai/stable-diffusion-3.5-medium",
|
||||
"name": "SD 3.5 Medium",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 2.5,
|
||||
"vram_bf16": 12.0,
|
||||
"vram_fp8": 7.0,
|
||||
"vram_q4": None,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {
|
||||
"FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "2.5B lightweight, fast. Fits almost any GPU.",
|
||||
"quality": 75,
|
||||
"speed": 95,
|
||||
"released": "2024-10",
|
||||
},
|
||||
{
|
||||
"id": "stabilityai/stable-diffusion-3.5-large",
|
||||
"name": "SD 3.5 Large",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 8.1,
|
||||
"vram_bf16": 22.0,
|
||||
"vram_fp8": 12.0,
|
||||
"vram_q4": None,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {
|
||||
"FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "8B high quality. Good balance of speed and quality.",
|
||||
"quality": 85,
|
||||
"speed": 70,
|
||||
"released": "2024-10",
|
||||
},
|
||||
{
|
||||
"id": "stabilityai/stable-diffusion-3.5-large-turbo",
|
||||
"name": "SD 3.5 Large Turbo",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 8.1,
|
||||
"vram_bf16": 22.0,
|
||||
"vram_fp8": 12.0,
|
||||
"vram_q4": None,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {
|
||||
"FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
|
||||
},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "Distilled for few-step inference. Fastest large SD.",
|
||||
"quality": 80,
|
||||
"speed": 92,
|
||||
"released": "2024-10",
|
||||
},
|
||||
{
|
||||
"id": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"name": "SDXL",
|
||||
"provider": "Stability AI",
|
||||
"params_b": 3.5,
|
||||
"vram_bf16": 10.0,
|
||||
"vram_fp8": 6.0,
|
||||
"vram_q4": None,
|
||||
"default_quant": "BF16",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["text-to-image"],
|
||||
"description": "Classic workhorse. Huge LoRA ecosystem. Fits 8GB+.",
|
||||
"quality": 72,
|
||||
"speed": 90,
|
||||
"released": "2023-07",
|
||||
},
|
||||
# ── Hunyuan ──
|
||||
{
|
||||
"id": "tencent/HunyuanImage-3.0",
|
||||
"name": "HunyuanImage 3.0",
|
||||
"provider": "Tencent",
|
||||
"params_b": 13.0,
|
||||
"vram_bf16": 30.0,
|
||||
"vram_fp8": 16.0,
|
||||
"vram_q4": 9.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {
|
||||
"Q4": "wikeeyang/Hunyuan-Image-30-Qint4",
|
||||
"NF4": "EricRollei/HunyuanImage-3.0-Instruct-NF4",
|
||||
},
|
||||
"capabilities": ["text-to-image", "text-rendering"],
|
||||
"description": "Strong text rendering. Bilingual Chinese/English. 13B activated per token.",
|
||||
"quality": 88,
|
||||
"speed": 60,
|
||||
"released": "2025-09",
|
||||
},
|
||||
{
|
||||
"id": "tencent/HunyuanImage-3.0-Instruct-Distil",
|
||||
"name": "HunyuanImage 3.0 Distil",
|
||||
"provider": "Tencent",
|
||||
"params_b": 13.0,
|
||||
"vram_bf16": 30.0,
|
||||
"vram_fp8": 16.0,
|
||||
"vram_q4": 9.0,
|
||||
"default_quant": "FP8",
|
||||
"quant_repos": {},
|
||||
"capabilities": ["text-to-image", "text-rendering"],
|
||||
"description": "Distilled variant, fewer steps. Faster with comparable quality.",
|
||||
"quality": 85,
|
||||
"speed": 80,
|
||||
"released": "2026-01",
|
||||
},
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.constants import DATA_DIR
|
||||
|
||||
# Image models are discovered from HuggingFace collections/search and local cache.
|
||||
# Keep this empty: source-coded repo IDs become hidden recommendations.
|
||||
IMAGE_MODEL_REGISTRY: list[dict[str, Any]] = []
|
||||
|
||||
HF_IMAGE_COLLECTIONS = [
|
||||
"stabilityai/image",
|
||||
"stabilityai/stable-diffusion-35",
|
||||
"black-forest-labs/flux2",
|
||||
]
|
||||
|
||||
HF_MLX_IMAGE_COLLECTIONS = [
|
||||
"mlx-community/flux2-klein-mlx",
|
||||
"mlx-community/inpainting-mlx",
|
||||
"mlx-community/ddcolor-mlx",
|
||||
"mlx-community/boogu-image-01-mlx",
|
||||
]
|
||||
|
||||
HF_MLX_IMAGE_REPO_SEEDS: list[str] = []
|
||||
HF_IMAGE_REPO_SEEDS: list[str] = []
|
||||
|
||||
_HF_COLLECTION_CACHE = {"ts": 0.0, "models": []}
|
||||
_HF_COLLECTION_TTL = 30 * 60
|
||||
_IMAGE_COLLECTION_DISK_CACHE = Path(DATA_DIR) / "hwfit" / "image_collection_models.json"
|
||||
_IMAGE_COLLECTION_DISK_TTL = 24 * 3600
|
||||
_HF_VARIANT_CACHE: dict[str, dict[str, str]] = {}
|
||||
_HF_SEARCH_DISABLED_UNTIL = 0.0
|
||||
|
||||
|
||||
def _repo_display_name(repo_id: str) -> str:
|
||||
name = str(repo_id or "").split("/")[-1]
|
||||
return name.replace("-", " ").replace("_", " ").strip() or repo_id
|
||||
|
||||
|
||||
def _provider_from_repo(repo_id: str) -> str:
|
||||
owner = str(repo_id or "").split("/", 1)[0].lower()
|
||||
return {
|
||||
"stabilityai": "Stability AI",
|
||||
"black-forest-labs": "Black Forest Labs",
|
||||
"tongyi-mai": "Tongyi",
|
||||
"qwen": "Qwen",
|
||||
"mlx-community": "mlx-community",
|
||||
}.get(owner, owner.replace("-", " ").title() if owner else "HuggingFace")
|
||||
|
||||
|
||||
def _infer_capabilities(item: dict[str, Any], repo_id: str) -> list[str]:
|
||||
tasks = set()
|
||||
pipeline = str(item.get("pipeline_tag") or "").strip().lower()
|
||||
if pipeline:
|
||||
tasks.add(pipeline)
|
||||
for provider in item.get("availableInferenceProviders") or []:
|
||||
if isinstance(provider, dict) and provider.get("task"):
|
||||
tasks.add(str(provider["task"]).strip().lower())
|
||||
text = f"{repo_id} {' '.join(tasks)}".lower()
|
||||
caps = []
|
||||
if "image-to-image" in tasks or "edit" in text or "inpaint" in text:
|
||||
caps.append("image-editing")
|
||||
if "inpaint" in text:
|
||||
caps.append("inpainting")
|
||||
if "text-to-image" in tasks or not caps:
|
||||
caps.append("text-to-image")
|
||||
return caps
|
||||
|
||||
|
||||
def _estimate_image_model(repo_id: str) -> dict[str, Any]:
|
||||
text = str(repo_id or "").lower()
|
||||
params_b = 8.0
|
||||
param_match = re.search(r"(?<![\d.])(\d+(?:\.\d+)?)\s*b(?:\b|[-_])", text)
|
||||
if param_match:
|
||||
params_b = max(0.01, float(param_match.group(1)))
|
||||
if any(k in text for k in ("mi-gan", "big-lama", "lama-")):
|
||||
return {"params_b": 0.01, "bf16": 1.0, "fp8": 0.7, "q4": 0.5, "quality": 65, "speed": 98, "quant": "BF16"}
|
||||
quant = "BF16"
|
||||
if any(k in text for k in ("4bit", "q4", "nf4")):
|
||||
quant = "Q4"
|
||||
elif "fp8" in text or "8bit" in text:
|
||||
quant = "FP8"
|
||||
bf16 = max(1.0, round(params_b * 2.6 + 3.0, 1))
|
||||
fp8 = max(0.7, round(params_b * 1.35 + 2.0, 1))
|
||||
q4 = max(0.5, round(params_b * 0.8 + 1.5, 1))
|
||||
speed = max(35, min(95, int(98 - params_b * 3)))
|
||||
quality = max(60, min(88, int(70 + min(params_b, 18) * 0.8)))
|
||||
return {"params_b": params_b, "bf16": bf16, "fp8": fp8, "q4": q4, "quality": quality, "speed": speed, "quant": quant}
|
||||
|
||||
|
||||
def _params_b_from_item(item: dict[str, Any]) -> float | None:
|
||||
raw = item.get("numParameters")
|
||||
if isinstance(raw, (int, float)) and raw > 0:
|
||||
return max(0.01, round(float(raw) / 1_000_000_000.0, 3))
|
||||
return None
|
||||
|
||||
|
||||
def _mlx_quantize_estimate(repo_id: str, est: dict[str, Any]) -> dict[str, Any]:
|
||||
text = str(repo_id or "").lower()
|
||||
out = dict(est)
|
||||
if "3bit" in text or "4bit" in text or "q4" in text:
|
||||
out["quant"] = "Q4"
|
||||
out["bf16"] = None
|
||||
out["fp8"] = None
|
||||
elif "8bit" in text:
|
||||
out["quant"] = "FP8"
|
||||
out["bf16"] = None
|
||||
elif "6bit" in text or "5bit" in text:
|
||||
out["quant"] = "Q4"
|
||||
out["bf16"] = None
|
||||
out["fp8"] = out.get("fp8") or out.get("q4")
|
||||
elif "bf16" in text or "fp16" in text:
|
||||
out["quant"] = "BF16"
|
||||
out["fp8"] = None
|
||||
out["q4"] = None
|
||||
return out
|
||||
|
||||
|
||||
def _collection_item_to_model(item: dict[str, Any], collection_title: str = "", mlx_only: bool = False) -> dict[str, Any] | None:
|
||||
repo_id = str(item.get("id") or "").strip()
|
||||
if "/" not in repo_id:
|
||||
return None
|
||||
typ = str(item.get("type") or item.get("itemType") or "model").lower()
|
||||
if typ not in {"", "model"}:
|
||||
return None
|
||||
est = _estimate_image_model(repo_id)
|
||||
item_params_b = _params_b_from_item(item)
|
||||
if item_params_b is not None:
|
||||
est = {
|
||||
**est,
|
||||
"params_b": item_params_b,
|
||||
"bf16": max(0.5, round(item_params_b * 2.4 + 0.8, 1)),
|
||||
"fp8": max(0.5, round(item_params_b * 1.3 + 0.5, 1)),
|
||||
"q4": max(0.4, round(item_params_b * 0.8 + 0.4, 1)),
|
||||
}
|
||||
if mlx_only:
|
||||
est = _mlx_quantize_estimate(repo_id, est)
|
||||
caps = _infer_capabilities(item, repo_id)
|
||||
gated = item.get("gated")
|
||||
desc_bits = []
|
||||
if collection_title:
|
||||
desc_bits.append(f"HF collection: {collection_title}.")
|
||||
if gated:
|
||||
desc_bits.append("Gated on HuggingFace.")
|
||||
out = {
|
||||
"id": repo_id,
|
||||
"name": _repo_display_name(repo_id),
|
||||
"provider": _provider_from_repo(repo_id),
|
||||
"params_b": est["params_b"],
|
||||
"vram_bf16": est["bf16"],
|
||||
"vram_fp8": est["fp8"],
|
||||
"vram_q4": est["q4"],
|
||||
"default_quant": est["quant"],
|
||||
"quant_repos": {},
|
||||
"capabilities": caps,
|
||||
"description": " ".join(desc_bits).strip() or "Imported from HuggingFace collection.",
|
||||
"quality": est["quality"],
|
||||
"speed": est["speed"],
|
||||
"released": "",
|
||||
}
|
||||
# Optional catalog metadata may identify a non-default runtime package.
|
||||
# Keep this data-driven: the fitter must not infer private/model-specific
|
||||
# dependencies from repository names.
|
||||
dependency_package = item.get("dependency_package") or item.get("runtime_dependency")
|
||||
if isinstance(dependency_package, str) and dependency_package.strip():
|
||||
out["dependency_package"] = dependency_package.strip()
|
||||
if mlx_only:
|
||||
out["mlx_only"] = True
|
||||
out["description"] = (out["description"] + " Apple Silicon / MLX only.").strip()
|
||||
return out
|
||||
|
||||
|
||||
def _fetch_hf_image_collection_models() -> list[dict[str, Any]]:
|
||||
now = time.time()
|
||||
if now - float(_HF_COLLECTION_CACHE.get("ts") or 0) < _HF_COLLECTION_TTL:
|
||||
return list(_HF_COLLECTION_CACHE.get("models") or [])
|
||||
# Reuse the last successful discovery across process restarts. A stale
|
||||
# catalog is preferable to blocking the first image-tab render on several
|
||||
# sequential Hugging Face requests; a later refresh replaces it.
|
||||
if not _HF_COLLECTION_CACHE.get("models"):
|
||||
try:
|
||||
cached = json.loads(_IMAGE_COLLECTION_DISK_CACHE.read_text(encoding="utf-8"))
|
||||
cached_models = cached.get("models") if isinstance(cached, dict) else None
|
||||
cached_ts = float(cached.get("fetched_at") or 0) if isinstance(cached, dict) else 0
|
||||
if isinstance(cached_models, list) and cached_models:
|
||||
_HF_COLLECTION_CACHE["ts"] = cached_ts
|
||||
_HF_COLLECTION_CACHE["models"] = cached_models
|
||||
if now - cached_ts < _IMAGE_COLLECTION_DISK_TTL:
|
||||
return list(cached_models)
|
||||
except (OSError, ValueError, TypeError):
|
||||
pass
|
||||
models: list[dict[str, Any]] = []
|
||||
for slug, mlx_only in [(slug, False) for slug in HF_IMAGE_COLLECTIONS] + [(slug, True) for slug in HF_MLX_IMAGE_COLLECTIONS]:
|
||||
url = f"https://huggingface.co/api/collections/{slug}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Odysseus-Cookbook/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=2.5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8", "replace"))
|
||||
except Exception:
|
||||
continue
|
||||
title = str(data.get("title") or slug)
|
||||
for item in data.get("items") or []:
|
||||
if isinstance(item, dict):
|
||||
model = _collection_item_to_model(item, title, mlx_only=mlx_only)
|
||||
if model:
|
||||
models.append(model)
|
||||
if models:
|
||||
_HF_COLLECTION_CACHE["ts"] = now
|
||||
_HF_COLLECTION_CACHE["models"] = models
|
||||
try:
|
||||
_IMAGE_COLLECTION_DISK_CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = _IMAGE_COLLECTION_DISK_CACHE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps({"fetched_at": now, "models": models}), encoding="utf-8")
|
||||
tmp.replace(_IMAGE_COLLECTION_DISK_CACHE)
|
||||
except OSError:
|
||||
pass
|
||||
return list(models)
|
||||
# Preserve stale results if the network is unavailable. The in-memory
|
||||
# timestamp prevents every subsequent ranking request from retrying it.
|
||||
if _HF_COLLECTION_CACHE.get("models"):
|
||||
_HF_COLLECTION_CACHE["ts"] = now
|
||||
return list(_HF_COLLECTION_CACHE["models"])
|
||||
_HF_COLLECTION_CACHE["ts"] = now
|
||||
return []
|
||||
|
||||
|
||||
def _hf_model_search(query: str, limit: int = 10) -> list[dict[str, Any]]:
|
||||
global _HF_SEARCH_DISABLED_UNTIL
|
||||
now = time.time()
|
||||
if now < _HF_SEARCH_DISABLED_UNTIL:
|
||||
return []
|
||||
url = "https://huggingface.co/api/models?" + urllib.parse.urlencode({
|
||||
"search": query,
|
||||
"limit": str(limit),
|
||||
})
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Odysseus-Cookbook/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=2.5) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8", "replace"))
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
_HF_SEARCH_DISABLED_UNTIL = now + 10 * 60
|
||||
return []
|
||||
|
||||
|
||||
def _variant_score(candidate: dict[str, Any], base_repo: str, want: str) -> float:
|
||||
rid = str(candidate.get("id") or candidate.get("modelId") or "")
|
||||
text = " ".join([
|
||||
rid,
|
||||
str(candidate.get("library_name") or ""),
|
||||
str(candidate.get("pipeline_tag") or ""),
|
||||
" ".join(str(t) for t in candidate.get("tags") or []),
|
||||
]).lower()
|
||||
base = base_repo.lower()
|
||||
base_short = base_repo.rsplit("/", 1)[-1].lower()
|
||||
if want == "gguf" and "gguf" not in text:
|
||||
return -1
|
||||
if want == "fp8" and not any(k in text for k in ("fp8", "nvfp4", "mxfp8", "mxfp4")):
|
||||
return -1
|
||||
score = float(candidate.get("downloads") or 0) / 1000.0 + float(candidate.get("likes") or 0)
|
||||
if f"base_model:{base}" in text or f"base_model:quantized:{base}" in text:
|
||||
score += 10000
|
||||
elif base_short and base_short in rid.lower():
|
||||
score += 1000
|
||||
else:
|
||||
score -= 200
|
||||
if "diffusers" in text:
|
||||
score += 50
|
||||
if str(candidate.get("private")).lower() == "true":
|
||||
score -= 10000
|
||||
return score
|
||||
|
||||
|
||||
def _best_variant_repo(base_repo: str, want: str) -> str:
|
||||
base_short = str(base_repo or "").rsplit("/", 1)[-1]
|
||||
candidates = _hf_model_search(f"{base_short} {want}", limit=12)
|
||||
scored = []
|
||||
for item in candidates:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
rid = str(item.get("id") or item.get("modelId") or "").strip()
|
||||
if "/" not in rid or rid.lower() == base_repo.lower():
|
||||
continue
|
||||
score = _variant_score(item, base_repo, want)
|
||||
if score >= 0:
|
||||
scored.append((score, rid))
|
||||
scored.sort(reverse=True)
|
||||
return scored[0][1] if scored else ""
|
||||
|
||||
|
||||
def _should_discover_variants(repo_id: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _discover_quant_repos(repo_id: str, need_fp8: bool = True, need_gguf: bool = True) -> dict[str, str]:
|
||||
key = str(repo_id or "").strip()
|
||||
if not key:
|
||||
return {}
|
||||
cache_key = f"{key.lower()}|fp8={int(need_fp8)}|gguf={int(need_gguf)}"
|
||||
if cache_key in _HF_VARIANT_CACHE:
|
||||
return dict(_HF_VARIANT_CACHE[cache_key])
|
||||
found: dict[str, str] = {}
|
||||
if need_fp8:
|
||||
fp8 = _best_variant_repo(key, "fp8")
|
||||
if fp8:
|
||||
found["FP8"] = fp8
|
||||
if need_gguf:
|
||||
gguf = _best_variant_repo(key, "gguf")
|
||||
if gguf:
|
||||
# The image-model fitter's smallest bucket is Q4; most HF image GGUF
|
||||
# repos expose Q4/Q5/Q8 files under one repo, so use it as the low-VRAM
|
||||
# download source while preserving the explicit GGUF label for callers.
|
||||
found["Q4"] = gguf
|
||||
found["GGUF"] = gguf
|
||||
_HF_VARIANT_CACHE[cache_key] = found
|
||||
return dict(found)
|
||||
|
||||
|
||||
def _merge_quant_repos(model: dict[str, Any]) -> dict[str, Any]:
|
||||
out = dict(model)
|
||||
existing = dict(out.get("quant_repos") or {})
|
||||
repo_id = str(out.get("id") or "")
|
||||
if _should_discover_variants(repo_id):
|
||||
discovered = _discover_quant_repos(
|
||||
repo_id,
|
||||
need_fp8="FP8" not in existing,
|
||||
need_gguf="Q4" not in existing and "GGUF" not in existing,
|
||||
)
|
||||
for k, v in discovered.items():
|
||||
existing.setdefault(k, v)
|
||||
out["quant_repos"] = existing
|
||||
return out
|
||||
|
||||
|
||||
def get_image_models():
|
||||
"""Return the image model registry."""
|
||||
return IMAGE_MODEL_REGISTRY
|
||||
merged = [_merge_quant_repos(m) for m in IMAGE_MODEL_REGISTRY]
|
||||
seen = {str(m.get("id") or "").lower() for m in merged if isinstance(m, dict)}
|
||||
for model in _fetch_hf_image_collection_models():
|
||||
key = str(model.get("id") or "").lower()
|
||||
if key and key not in seen:
|
||||
merged.append(_merge_quant_repos(model))
|
||||
seen.add(key)
|
||||
return merged
|
||||
|
||||
|
||||
def _is_apple_image_system(system: dict[str, Any]) -> bool:
|
||||
backend = str(system.get("backend") or "").lower()
|
||||
gpu_name = str(system.get("gpu_name") or "").lower()
|
||||
cpu_name = str(system.get("cpu_name") or "").lower()
|
||||
platform = str(system.get("platform") or "").lower()
|
||||
return (
|
||||
bool(system.get("unified_memory"))
|
||||
or backend in {"metal", "mps", "apple"}
|
||||
or "apple" in gpu_name
|
||||
or "apple" in cpu_name
|
||||
or platform == "darwin"
|
||||
)
|
||||
|
||||
|
||||
def rank_image_models(system, search=None, sort="fit"):
|
||||
@@ -280,13 +371,23 @@ def rank_image_models(system, search=None, sort="fit"):
|
||||
|
||||
Returns list of models with fit info (vram needed, fits, recommended quant).
|
||||
"""
|
||||
if not isinstance(system, dict):
|
||||
system = {}
|
||||
gpu_vram = system.get("gpu_vram_gb", 0) or 0
|
||||
has_gpu = system.get("has_gpu", False)
|
||||
ram_gb = system.get("available_ram_gb") or system.get("total_ram_gb") or 0
|
||||
budget_gb = gpu_vram if has_gpu and gpu_vram > 0 else ram_gb
|
||||
budget_kind = "gpu" if has_gpu and gpu_vram > 0 else "ram"
|
||||
apple_system = _is_apple_image_system(system)
|
||||
results = []
|
||||
|
||||
for model in IMAGE_MODEL_REGISTRY:
|
||||
for model in get_image_models():
|
||||
if apple_system and not (model.get("mlx_only") or model.get("apple_ok")):
|
||||
continue
|
||||
if model.get("mlx_only") and not apple_system:
|
||||
continue
|
||||
# Filter by search
|
||||
if search:
|
||||
if isinstance(search, str) and search:
|
||||
s = search.lower()
|
||||
if s not in model["name"].lower() and s not in model["id"].lower() and s not in model.get("description", "").lower():
|
||||
continue
|
||||
@@ -297,11 +398,11 @@ def rank_image_models(system, search=None, sort="fit"):
|
||||
fits = False
|
||||
quant_repo = None
|
||||
|
||||
if has_gpu and gpu_vram > 0:
|
||||
if budget_gb > 0:
|
||||
# Try BF16 first, then FP8, then Q4
|
||||
for q, vram_key in [("BF16", "vram_bf16"), ("FP8", "vram_fp8"), ("Q4", "vram_q4")]:
|
||||
v = model.get(vram_key)
|
||||
if v is not None and v <= gpu_vram * 0.90: # 10% headroom
|
||||
if v is not None and v <= budget_gb * 0.90: # 10% headroom
|
||||
quant = q
|
||||
vram_needed = v
|
||||
fits = True
|
||||
@@ -313,15 +414,15 @@ def rank_image_models(system, search=None, sort="fit"):
|
||||
vram_needed = model.get("vram_bf16", 0)
|
||||
|
||||
# Fit label
|
||||
if not has_gpu:
|
||||
if budget_gb <= 0:
|
||||
fit = "no_gpu"
|
||||
fit_label = "No GPU"
|
||||
elif fits:
|
||||
headroom = gpu_vram - vram_needed
|
||||
if headroom > gpu_vram * 0.3:
|
||||
headroom = budget_gb - vram_needed
|
||||
if headroom > budget_gb * 0.3:
|
||||
fit = "perfect"
|
||||
fit_label = "Perfect"
|
||||
elif headroom > gpu_vram * 0.1:
|
||||
elif headroom > budget_gb * 0.1:
|
||||
fit = "good"
|
||||
fit_label = "Good"
|
||||
else:
|
||||
@@ -353,12 +454,14 @@ def rank_image_models(system, search=None, sort="fit"):
|
||||
"fits": fits,
|
||||
"fit": fit,
|
||||
"fit_label": fit_label,
|
||||
"fit_budget": budget_kind,
|
||||
"quality": model["quality"],
|
||||
"speed": model["speed"],
|
||||
"score": round(score, 1),
|
||||
"capabilities": model["capabilities"],
|
||||
"description": model["description"],
|
||||
"released": model.get("released", ""),
|
||||
"dependency_package": model.get("dependency_package", ""),
|
||||
})
|
||||
|
||||
# Sort
|
||||
|
||||
+181
-15
@@ -6,47 +6,147 @@ QUANT_HIERARCHY = ["Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "Q3_K_M", "Q2_K"]
|
||||
|
||||
QUANT_BPP = {
|
||||
"F32": 4.0, "F16": 2.0, "BF16": 2.0, "FP8": 1.0,
|
||||
"FP4": 0.50, "NVFP4": 0.50, "MXFP4": 0.50, "NF4": 0.50,
|
||||
"INT4": 0.50, "INT8": 1.0, "W4A16": 0.50, "W8A8": 1.0, "W8A16": 1.0,
|
||||
"Q8_0": 1.05, "Q6_K": 0.80, "Q5_K_M": 0.68,
|
||||
"Q4_K_M": 0.58, "Q4_0": 0.58, "Q3_K_M": 0.48, "Q2_K": 0.37,
|
||||
"AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
|
||||
"GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
|
||||
"mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
|
||||
"QAT-INT4": 0.50, "QAT-INT8": 1.0,
|
||||
"mlx-3bit": 0.42, "mlx-4bit": 0.55, "mlx-5bit": 0.65, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
|
||||
# DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
|
||||
# expert dense in FP8, embeddings/LM head in BF16. By weight count the
|
||||
# experts dominate so the effective BPP sits closer to FP4 than FP8.
|
||||
# Empirical: DeepSeek-V4-Flash 284B / 156 GB ≈ 0.55 B/param.
|
||||
"FP4-MoE-Mixed": 0.55,
|
||||
# FP8-Mixed = the *-Base variants (MoE experts also FP8, not FP4).
|
||||
"FP8-Mixed": 1.0,
|
||||
}
|
||||
|
||||
QUANT_SPEED_MULT = {
|
||||
"F16": 0.6, "BF16": 0.6, "FP8": 0.85,
|
||||
"FP4": 1.15, "NVFP4": 1.15, "MXFP4": 1.15, "NF4": 1.10,
|
||||
"INT4": 1.15, "INT8": 0.85, "W4A16": 1.15, "W8A8": 0.85, "W8A16": 0.85,
|
||||
"Q8_0": 0.8, "Q6_K": 0.95, "Q5_K_M": 1.0,
|
||||
"Q4_K_M": 1.15, "Q4_0": 1.15, "Q3_K_M": 1.25, "Q2_K": 1.35,
|
||||
"AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
|
||||
"GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
|
||||
"mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
|
||||
"QAT-INT4": 1.15, "QAT-INT8": 0.85,
|
||||
"mlx-3bit": 1.25, "mlx-4bit": 1.15, "mlx-5bit": 1.05, "mlx-6bit": 1.0, "mlx-8bit": 0.85,
|
||||
"FP4-MoE-Mixed": 1.10, # slightly slower than pure FP4 because of mixed-dtype dispatch
|
||||
"FP8-Mixed": 0.85,
|
||||
}
|
||||
|
||||
QUANT_QUALITY_PENALTY = {
|
||||
"F16": 0.0, "BF16": 0.0, "FP8": 0.0,
|
||||
"FP4": -3.0, "NVFP4": -3.0, "MXFP4": -3.0, "NF4": -4.0,
|
||||
"INT4": -4.0, "INT8": 0.0, "W4A16": -4.0, "W8A8": 0.0, "W8A16": 0.0,
|
||||
"Q8_0": 0.0, "Q6_K": -1.0, "Q5_K_M": -2.0,
|
||||
"Q4_K_M": -5.0, "Q4_0": -5.0, "Q3_K_M": -8.0, "Q2_K": -12.0,
|
||||
"AWQ-4bit": -3.0, "AWQ-8bit": 0.0,
|
||||
"GPTQ-Int4": -3.0, "GPTQ-Int8": 0.0,
|
||||
"mlx-4bit": -4.0, "mlx-8bit": 0.0, "mlx-6bit": -1.0,
|
||||
# Bare "AWQ" and "AWQ-8bit" used to be 0.0 (tied with FP8). In practice
|
||||
# AWQ-anything is a calibrated reconstruction, not raw 8-bit weights —
|
||||
# there's a small but real quality loss vs FP8. Give them a slight
|
||||
# penalty so FP8 wins when both fit. AWQ-4bit stays heavier.
|
||||
"AWQ": -1.0, "AWQ-4bit": -4.0, "AWQ-8bit": -1.0,
|
||||
"GPTQ": -1.0, "GPTQ-Int4": -4.0, "GPTQ-Int8": -1.0,
|
||||
# Quantization-aware training recovers most of the int4 quality loss, so a
|
||||
# QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
|
||||
# (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
|
||||
"QAT-INT4": -1.0, "QAT-INT8": 0.0,
|
||||
"mlx-3bit": -8.0, "mlx-4bit": -4.0, "mlx-5bit": -2.5, "mlx-6bit": -1.5, "mlx-8bit": -0.5,
|
||||
# DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
|
||||
# so the realized quality is much closer to FP8 than to pure FP4 —
|
||||
# the activation-sensitive layers stay high-precision. ~0 penalty.
|
||||
"FP4-MoE-Mixed": -0.5,
|
||||
"FP8-Mixed": 0.0,
|
||||
}
|
||||
|
||||
QUANT_BYTES_PER_PARAM = {
|
||||
"F16": 2.0, "BF16": 2.0, "FP8": 1.0,
|
||||
"FP4": 0.5, "NVFP4": 0.5, "MXFP4": 0.5, "NF4": 0.5,
|
||||
"INT4": 0.5, "INT8": 1.0, "W4A16": 0.5, "W8A8": 1.0, "W8A16": 1.0,
|
||||
"Q8_0": 1.0, "Q6_K": 0.75, "Q5_K_M": 0.625,
|
||||
"Q4_K_M": 0.5, "Q4_0": 0.5, "Q3_K_M": 0.375, "Q2_K": 0.25,
|
||||
"AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
|
||||
"GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
|
||||
"mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
|
||||
"QAT-INT4": 0.5, "QAT-INT8": 1.0,
|
||||
"mlx-3bit": 0.375, "mlx-4bit": 0.5, "mlx-5bit": 0.625, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
|
||||
"FP4-MoE-Mixed": 0.55,
|
||||
"FP8-Mixed": 1.0,
|
||||
}
|
||||
|
||||
# Pre-quantized formats that should NOT go through the GGUF quant hierarchy
|
||||
PREQUANTIZED_PREFIXES = ("AWQ-", "GPTQ-", "mlx-", "FP8")
|
||||
# Pre-quantized formats that should NOT go through the GGUF quant hierarchy.
|
||||
# These are native HF/vLLM-style repos, not llama.cpp GGUF quant tiers.
|
||||
PREQUANTIZED_PREFIXES = (
|
||||
"AWQ-", "GPTQ-", "mlx-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
|
||||
"INT4", "INT8", "W4A16", "W8A8", "W8A16",
|
||||
"FP4-MoE-Mixed", "FP8-Mixed",
|
||||
"QAT-",
|
||||
)
|
||||
|
||||
|
||||
def infer_quantization_from_name(name):
|
||||
n = (name or "").lower()
|
||||
model_name = n.rsplit("/", 1)[-1]
|
||||
if "nvfp4" in n:
|
||||
return "NVFP4"
|
||||
if re.search(r"(^|[-_/])bf16($|[-_/])", model_name):
|
||||
return "BF16"
|
||||
if "mxfp4" in n:
|
||||
return "MXFP4"
|
||||
if re.search(r"(^|[-_/])nf4($|[-_/])", n):
|
||||
return "NF4"
|
||||
if re.search(r"(^|[-_/])fp4($|[-_/])", n):
|
||||
return "FP4"
|
||||
if re.search(r"(^|[-_/])w4a16($|[-_/])", n):
|
||||
return "W4A16"
|
||||
if re.search(r"(^|[-_/])w8a8($|[-_/])", n):
|
||||
return "W8A8"
|
||||
if re.search(r"(^|[-_/])w8a16($|[-_/])", n):
|
||||
return "W8A16"
|
||||
is8 = "8bit" in n or "8-bit" in n or "int8" in n
|
||||
if "awq" in n:
|
||||
return "AWQ-8bit" if is8 else "AWQ-4bit"
|
||||
if "gptq" in n:
|
||||
return "GPTQ-Int8" if is8 else "GPTQ-Int4"
|
||||
if n.startswith("mlx-community/") or "mlx" in model_name:
|
||||
if "3bit" in model_name:
|
||||
return "mlx-3bit"
|
||||
if "5bit" in model_name:
|
||||
return "mlx-5bit"
|
||||
if "6bit" in model_name:
|
||||
return "mlx-6bit"
|
||||
return "mlx-8bit" if is8 else "mlx-4bit"
|
||||
if "fp8" in n:
|
||||
return "FP8"
|
||||
if "int4" in n or "4bit" in n or "4-bit" in n:
|
||||
return "INT4"
|
||||
if "int8" in n or "8bit" in n or "8-bit" in n:
|
||||
return "INT8"
|
||||
return ""
|
||||
|
||||
|
||||
def _normalize_model_entry(model):
|
||||
if not isinstance(model, dict):
|
||||
return model
|
||||
inferred = infer_quantization_from_name(model.get("name", ""))
|
||||
if inferred and (model.get("quantization") in (None, "", "Q4_K_M") or model.get("_discovered")):
|
||||
model["quantization"] = inferred
|
||||
return model
|
||||
|
||||
|
||||
def is_prequantized(model):
|
||||
q = model.get("quantization", "")
|
||||
return any(q.startswith(p) for p in PREQUANTIZED_PREFIXES)
|
||||
name = (model.get("name") or "").lower()
|
||||
fmt = (model.get("format") or "").lower()
|
||||
text = f"{name} {fmt}"
|
||||
return (
|
||||
"nvfp4" in text
|
||||
or re.search(r"(^|[-_/])fp8($|[-_/\s])", text) is not None
|
||||
or (not (model.get("is_gguf") or model.get("gguf_sources")) and re.search(r"(^|[-_/])(?:int)?8bit($|[-_/\s])", text) is not None)
|
||||
or any(x in text for x in ("awq", "gptq", "mlx"))
|
||||
or any(isinstance(q, str) and q.startswith(p) for p in PREQUANTIZED_PREFIXES)
|
||||
)
|
||||
|
||||
|
||||
def params_b(model):
|
||||
@@ -55,11 +155,17 @@ def params_b(model):
|
||||
return raw / 1_000_000_000.0
|
||||
|
||||
pc = model.get("parameter_count", "")
|
||||
if pc:
|
||||
if isinstance(pc, str) and pc:
|
||||
pc = pc.strip().upper()
|
||||
m = re.match(r"^([\d.]+)\s*([BKMGT]?)$", pc)
|
||||
if m:
|
||||
val = float(m.group(1))
|
||||
try:
|
||||
val = float(m.group(1))
|
||||
except ValueError:
|
||||
# Malformed count like "1.5.3B" — [\d.]+ matches but float()
|
||||
# rejects it. One bad catalog row must not abort the whole
|
||||
# ranking pass, so treat it as unknown size.
|
||||
return 0.0
|
||||
suffix = m.group(2)
|
||||
if suffix == "B":
|
||||
return val
|
||||
@@ -161,15 +267,75 @@ def infer_use_case(model):
|
||||
|
||||
_models_cache = None
|
||||
|
||||
def _load_model_file(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
loaded = json.load(f)
|
||||
return loaded if isinstance(loaded, list) else []
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return []
|
||||
|
||||
def reset_model_cache():
|
||||
global _models_cache
|
||||
_models_cache = None
|
||||
|
||||
def refresh_dynamic_catalogs(force=False):
|
||||
"""Refresh API-backed model catalogs and invalidate the merged cache.
|
||||
|
||||
The bundled JSON files remain the offline fallback. Dynamic catalogs live
|
||||
under DATA_DIR so runtime refreshes do not dirty the source tree.
|
||||
"""
|
||||
from services.hwfit.hf_discovery import (
|
||||
refresh_hf_collection_models_cache,
|
||||
refresh_mlx_community_cache,
|
||||
)
|
||||
|
||||
refreshed = {
|
||||
"mlx_community": len(refresh_mlx_community_cache(force=force)),
|
||||
"hf_collections": len(refresh_hf_collection_models_cache(force=force)),
|
||||
}
|
||||
reset_model_cache()
|
||||
return refreshed
|
||||
|
||||
def get_models():
|
||||
global _models_cache
|
||||
if _models_cache is None:
|
||||
data_path = os.path.join(os.path.dirname(__file__), "data", "hf_models.json")
|
||||
static_mlx_path = os.path.join(os.path.dirname(__file__), "data", "mlx_community_models.json")
|
||||
try:
|
||||
with open(data_path) as f:
|
||||
_models_cache = json.load(f)
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
_models_cache = []
|
||||
from services.hwfit.hf_discovery import (
|
||||
load_cached_hf_collection_models,
|
||||
load_cached_mlx_community_models,
|
||||
)
|
||||
dynamic_mlx_models = load_cached_mlx_community_models()
|
||||
dynamic_hf_models = load_cached_hf_collection_models()
|
||||
except Exception:
|
||||
dynamic_mlx_models = []
|
||||
dynamic_hf_models = []
|
||||
seen = set()
|
||||
rows = []
|
||||
def _append_models(models):
|
||||
for model in models:
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
name = model.get("name")
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
rows.append(_normalize_model_entry(model))
|
||||
|
||||
for model in _load_model_file(data_path):
|
||||
if not isinstance(model, dict):
|
||||
continue
|
||||
name = model.get("name")
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
rows.append(_normalize_model_entry(model))
|
||||
_append_models(dynamic_hf_models)
|
||||
_append_models(dynamic_mlx_models)
|
||||
_append_models(_load_model_file(static_mlx_path))
|
||||
_models_cache = rows
|
||||
return _models_cache
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Compute intelligent llama.cpp serve profiles from detected hardware.
|
||||
|
||||
Given a system (VRAM/RAM/arch) and a model, produce 1-4 ready-to-launch
|
||||
profiles — Quality / Balanced / Speed — with concrete llama.cpp flags
|
||||
(n_gpu_layers, n_cpu_moe, cache-type, context). This turns the by-hand tuning
|
||||
(how many MoE layers fit on the GPU, when to spend VRAM on a q8 KV cache vs more
|
||||
context, how much headroom to leave for a vision encoder) into a formula.
|
||||
|
||||
Pure/deterministic — no benchmarking, no I/O. Reuses the same VRAM math as
|
||||
fit.py/models.py so "what the Cookbook recommends" and "what it serves" agree.
|
||||
|
||||
NOTE: token/s figures are NOT computed here — real speed on partial-offload MoE
|
||||
is CPU-bound and not reliably predictable from specs. The UI labels profiles by
|
||||
their tradeoff (Quality/Balanced/Speed), and the VRAM fit (the part that decides
|
||||
whether it even loads) is what's computed from real numbers.
|
||||
"""
|
||||
|
||||
from services.hwfit.models import (
|
||||
QUANT_BPP,
|
||||
params_b,
|
||||
_active_params_b,
|
||||
is_prequantized,
|
||||
)
|
||||
|
||||
# GGUF KV-cache cost per token, in bytes-per-active-billion-param, by cache type.
|
||||
# q4_0 is ~half of q8_0 is ~half of f16. The 8e-6 base in estimate_memory_gb is
|
||||
# the q8_0-ish figure; scale from there.
|
||||
_KV_FACTOR = {"q4_0": 0.5, "q8_0": 1.0, "f16": 2.0}
|
||||
|
||||
# Quant ladder from highest quality/size down. A profile that wants "best quant
|
||||
# that fits fully on GPU" walks this until one fits.
|
||||
_QUANT_LADDER = ["Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "Q3_K_M", "Q2_K"]
|
||||
|
||||
|
||||
def _weights_gb(model, quant, fixed_gb=None):
|
||||
"""VRAM for the full weights. When fixed_gb is given (serving a specific GGUF
|
||||
file already on disk), use its real size — the quant is whatever the file is,
|
||||
not something we get to pick."""
|
||||
if fixed_gb and fixed_gb > 0:
|
||||
return float(fixed_gb)
|
||||
return params_b(model) * QUANT_BPP.get(quant, 0.58)
|
||||
|
||||
|
||||
def _kv_gb(model, ctx, kv_type):
|
||||
"""KV-cache VRAM at a context length and cache type."""
|
||||
kv_params = _active_params_b(model)
|
||||
return 0.000008 * kv_params * ctx * _KV_FACTOR.get(kv_type, 1.0)
|
||||
|
||||
|
||||
def _n_layers(model):
|
||||
"""Best-effort total transformer block count (for n-cpu-moe math)."""
|
||||
for k in ("num_hidden_layers", "n_layers", "num_layers", "block_count"):
|
||||
v = model.get(k)
|
||||
if isinstance(v, (int, float)) and v > 0:
|
||||
return int(v)
|
||||
# Fallback heuristic by size — most MoE/dense LLMs land 28-64 layers.
|
||||
pb = params_b(model)
|
||||
if pb >= 60:
|
||||
return 64
|
||||
if pb >= 25:
|
||||
return 48
|
||||
if pb >= 12:
|
||||
return 40
|
||||
return 32
|
||||
|
||||
|
||||
def _cpu_moe_for_budget(model, quant, kv_gb, vram_budget_gb, fixed_gb=None):
|
||||
"""How many MoE layers must move to CPU so weights+KV fit vram_budget_gb.
|
||||
|
||||
Returns (n_cpu_moe, fits_fully). When the model already fits, n_cpu_moe=0.
|
||||
Each offloaded layer frees roughly weights/n_layers of VRAM. We only model
|
||||
this for MoE (where --n-cpu-moe applies); dense models just report whether
|
||||
they fit at the given n_gpu_layers=999.
|
||||
"""
|
||||
weights = _weights_gb(model, quant, fixed_gb)
|
||||
needed = weights + kv_gb + 0.6 # +0.6 GB runtime/compute buffers
|
||||
if needed <= vram_budget_gb:
|
||||
return 0, True
|
||||
if not model.get("is_moe"):
|
||||
# Dense: no per-expert offload knob; either it fits or it spills via -ngl.
|
||||
return 0, False
|
||||
layers = _n_layers(model)
|
||||
per_layer = weights / max(layers, 1)
|
||||
overflow = needed - vram_budget_gb
|
||||
import math
|
||||
n = math.ceil(overflow / max(per_layer, 1e-6))
|
||||
n = max(0, min(n, layers)) # clamp
|
||||
return n, False
|
||||
|
||||
|
||||
def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=None):
|
||||
"""Return a list of profile dicts for llama.cpp serving of `model` on `system`.
|
||||
|
||||
Each profile: {key, label, quant, n_gpu_layers, n_cpu_moe, cache_type, ctx,
|
||||
est_vram_gb, fits, note}. Empty list if no GGUF path makes
|
||||
sense (caller should fall back to manual flags).
|
||||
|
||||
DOWNLOAD mode (default): the quant isn't chosen yet, so profiles vary it
|
||||
(Quality=Q6, Balanced=Q4, Speed=Q2…) to show download options.
|
||||
|
||||
SERVE mode (serve_weights_gb set): a specific GGUF file already exists on
|
||||
disk — its quant is FIXED. Profiles then keep that quant/size and differ only
|
||||
in the actual serving knobs (n_cpu_moe, KV-cache type, context). serve_quant
|
||||
is the file's quant label (e.g. "Q4_K_M") just for display.
|
||||
"""
|
||||
if not isinstance(system, dict) or not isinstance(model, dict):
|
||||
return []
|
||||
|
||||
vram = float(system.get("gpu_vram_gb") or 0)
|
||||
if vram <= 0:
|
||||
return []
|
||||
|
||||
serve_mode = bool(serve_weights_gb and serve_weights_gb > 0)
|
||||
|
||||
# Never propose more context than the model was trained for — asking llama.cpp
|
||||
# for ctx > n_ctx_train triggers a "training context overflow" and, with a
|
||||
# quantized KV cache, an oversized allocation that can crash the GPU
|
||||
# (radv/amdgpu ErrorDeviceLost). Cap every profile at the model's real limit.
|
||||
model_ctx_max = 0
|
||||
for k in ("context_length", "max_position_embeddings", "n_ctx_train", "context"):
|
||||
v = model.get(k)
|
||||
if isinstance(v, (int, float)) and v > 0:
|
||||
model_ctx_max = int(v)
|
||||
break
|
||||
if model_ctx_max <= 0:
|
||||
model_ctx_max = 131072 # conservative default when the catalog omits it
|
||||
|
||||
# Vision models need headroom for the image encoder (~1 GB on top of weights).
|
||||
is_vision = bool(
|
||||
model.get("is_multimodal") or model.get("vision") or model.get("mmproj")
|
||||
or "vl" in str(model.get("name", "")).lower()
|
||||
)
|
||||
headroom = 1.1 if is_vision else 0.4
|
||||
budget = max(vram - headroom, 1.0)
|
||||
|
||||
# Prequantized (AWQ/GPTQ/FP8) served via GGUF fallback use a fixed ~Q4 quant;
|
||||
# GGUF models can pick their quant. Pick a sensible per-profile quant.
|
||||
fixed_quant = model.get("quantization") if is_prequantized(model) else None
|
||||
|
||||
is_moe = bool(model.get("is_moe"))
|
||||
|
||||
def _pick_quant(prefer, require_full_fit):
|
||||
"""Choose a quant for a profile.
|
||||
|
||||
- fixed_quant (AWQ/GPTQ/FP8 served via GGUF): always that.
|
||||
- require_full_fit=True (Speed): walk DOWN from `prefer` to the best quant
|
||||
whose weights fit fully on the GPU (no offload) — fastest.
|
||||
- require_full_fit=False (Quality on MoE): keep `prefer` even if it must
|
||||
offload experts to CPU; that's the whole point of n-cpu-moe on a card
|
||||
too small to hold the weights. For dense models we can't offload
|
||||
per-expert, so fall back to the largest fully-fitting quant.
|
||||
"""
|
||||
if fixed_quant:
|
||||
return fixed_quant
|
||||
start = _QUANT_LADDER.index(prefer) if prefer in _QUANT_LADDER else 3
|
||||
if require_full_fit or not is_moe:
|
||||
for q in _QUANT_LADDER[start:]:
|
||||
if _weights_gb(model, q) + 0.6 <= budget:
|
||||
return q
|
||||
return _QUANT_LADDER[-1]
|
||||
# MoE quality: keep the preferred (big) quant; offload handles overflow.
|
||||
return prefer
|
||||
|
||||
if serve_mode:
|
||||
# Fixed file on disk — quant can't change. Vary only the serving knobs.
|
||||
fq = serve_quant or model.get("quantization") or "GGUF"
|
||||
specs = [
|
||||
# key, label, prefer_quant, full_fit, kv_type, ctx, note
|
||||
("quality", "Quality", fq, False, "q8_0", 131072,
|
||||
"Sharp q8 KV cache + full context. Best long-context accuracy; offloads MoE layers to CPU if needed."),
|
||||
("balanced", "Balanced", fq, False, "q4_0", 131072,
|
||||
"Compact q4 KV at full context — good speed/quality mix."),
|
||||
("speed", "Speed", fq, False, "q4_0", 32768,
|
||||
"Trimmed context + light KV for the fastest tokens/s."),
|
||||
]
|
||||
else:
|
||||
specs = [
|
||||
# key, label, prefer_quant, full_fit, kv_type, ctx, note
|
||||
("quality", "Quality", "Q6_K", False, "q8_0", 131072,
|
||||
"Biggest quant + sharp q8 KV cache. Best answers; offloads MoE layers to CPU if needed."),
|
||||
("balanced", "Balanced", "Q4_K_M", False, "q4_0", 131072,
|
||||
"Q4 weights + compact q4 KV. Good speed/quality mix at full context."),
|
||||
("speed", "Speed", "Q4_K_M", True, "q4_0", 32768,
|
||||
"Smallest offload + trimmed context for the fastest tokens/s."),
|
||||
]
|
||||
|
||||
profiles = []
|
||||
for key, label, prefer_q, full_fit, kv_type, ctx, note in specs:
|
||||
# In serve mode the quant is fixed (the file's); in download mode we pick.
|
||||
quant = prefer_q if serve_mode else _pick_quant(prefer_q, full_fit)
|
||||
# Shrink context if even the chosen KV won't fit alongside weights.
|
||||
# Start from the smaller of the profile's target and the model's limit.
|
||||
cur_ctx = min(ctx, model_ctx_max)
|
||||
# Floor the context-shrink loop at 8192, but never above the model's own
|
||||
# trained limit. A model with a sub-8192 context (e.g. a 2048-token
|
||||
# SmolLM) starts below 8192, so a hard-coded 8192 guard skipped the loop
|
||||
# entirely and produced NO profile — the serve UI then fell back to
|
||||
# manual flags even though the model fits the GPU trivially.
|
||||
ctx_floor = min(8192, model_ctx_max)
|
||||
while cur_ctx >= ctx_floor:
|
||||
kv = _kv_gb(model, cur_ctx, kv_type)
|
||||
n_cpu_moe, fits = _cpu_moe_for_budget(model, quant, kv, budget, fixed_gb=serve_weights_gb)
|
||||
est = _weights_gb(model, quant, serve_weights_gb) + kv + 0.6
|
||||
# If a non-MoE model can't fit even fully offloaded, try less context.
|
||||
if model.get("is_moe") or fits or cur_ctx <= ctx_floor:
|
||||
profiles.append({
|
||||
"key": key,
|
||||
"label": label,
|
||||
"quant": quant,
|
||||
"n_gpu_layers": 999,
|
||||
"n_cpu_moe": n_cpu_moe,
|
||||
"cache_type": kv_type,
|
||||
"ctx": cur_ctx,
|
||||
# When experts offload, GPU-resident VRAM tops out at the
|
||||
# budget (weights beyond it live in system RAM), so cap the
|
||||
# estimate at `budget`, not the full card — this also leaves
|
||||
# the vision-encoder headroom visible in the number.
|
||||
"est_vram_gb": round(min(est, budget), 1),
|
||||
# For MoE we treat it as fitting via offload; report whether
|
||||
# it fit WITHOUT offload as the "clean" flag.
|
||||
"fits": fits or bool(model.get("is_moe")),
|
||||
"offloads": n_cpu_moe > 0,
|
||||
"note": note,
|
||||
})
|
||||
break
|
||||
cur_ctx //= 2
|
||||
|
||||
# De-dupe identical profiles (e.g. tiny model where all three collapse to the
|
||||
# same all-GPU config) — keep the first/highest-quality label.
|
||||
seen = set()
|
||||
deduped = []
|
||||
for p in profiles:
|
||||
sig = (p["quant"], p["n_cpu_moe"], p["cache_type"], p["ctx"])
|
||||
if sig in seen:
|
||||
continue
|
||||
seen.add(sig)
|
||||
deduped.append(p)
|
||||
return deduped
|
||||
Reference in New Issue
Block a user