Squash Odysseus development history

services/__init__.py

@@ -1,18 +1,43 @@
-# services/__init__.py
-"""
-Service layer — plug-in capabilities for the chat core.
+"""Service-layer exports with lazy loading.
 
-Each service:
-- Does one thing well
-- Exposes a clean async interface
-- Can run in-process or as a standalone HTTP service
+Importing one service, such as ``services.hwfit``, must not initialize every
+other service. The eager exports previously imported search, document,
+research, memory, and shell stacks during any ``services.*`` import, making
+Cookbook hardware/model discovery needlessly slow on a cold process.
 """
 
-from .search import SearchService, SearchResult, SearchResponse
-from .docs import DocsService, DocChunk, IndexResult
-from .research import ResearchService, ResearchResult, ResearchSource
-from .memory import MemoryService, Memory, MemorySearchResult
-from .shell import ShellService, ShellResult
+from importlib import import_module
+
+_LAZY_EXPORTS = {
+    "SearchService": ("search", "SearchService"),
+    "SearchResult": ("search", "SearchResult"),
+    "SearchResponse": ("search", "SearchResponse"),
+    "DocsService": ("docs", "DocsService"),
+    "DocChunk": ("docs", "DocChunk"),
+    "IndexResult": ("docs", "IndexResult"),
+    "ResearchService": ("research", "ResearchService"),
+    "ResearchResult": ("research", "ResearchResult"),
+    "ResearchSource": ("research", "ResearchSource"),
+    "MemoryService": ("memory", "MemoryService"),
+    "Memory": ("memory", "Memory"),
+    "MemorySearchResult": ("memory", "MemorySearchResult"),
+    "ShellService": ("shell", "ShellService"),
+    "ShellResult": ("shell", "ShellResult"),
+}
+
+
+def __getattr__(name):
+    target = _LAZY_EXPORTS.get(name)
+    if target is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module_name, attribute = target
+    value = getattr(import_module(f"{__name__}.{module_name}"), attribute)
+    globals()[name] = value
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(_LAZY_EXPORTS))
 
 __all__ = [
     # Search

services/docs/service.py

@@ -5,6 +5,7 @@ from dataclasses import dataclass
 from typing import List, Dict, Any
 
 from src.rag_manager import RAGManager
+from src.constants import CHROMA_DIR
 
 
 @dataclass
@@ -34,7 +35,7 @@ class DocsService:
         results = await service.query("what is async await?")
     """
 
-    def __init__(self, persist_dir: str = "data/chroma"):
+    def __init__(self, persist_dir: str = CHROMA_DIR):
         self.rag = RAGManager(persist_directory=persist_dir)
 
     async def query(self, query: str, top_k: int = 5) -> List[DocChunk]:
@@ -49,15 +50,46 @@ class DocsService:
             List of DocChunk objects
         """
         results = self.rag.search(query, k=top_k)
-        return [
-            DocChunk(
-                text=r.get("text", r.get("content", "")),
-                source=r.get("source", r.get("metadata", {}).get("source", "unknown")),
-                score=r.get("score", 0.0),
-                metadata=r.get("metadata"),
+        chunks = []
+
+        for result in results:
+            if not isinstance(result, dict):
+                continue
+
+            metadata = result.get("metadata")
+            if not isinstance(metadata, dict):
+                metadata = {}
+
+            text = result.get("document")
+            if text is None:
+                text = result.get("text")
+            if text is None:
+                text = result.get("content")
+            if text is None:
+                text = ""
+
+            source = result.get("source")
+            if source is None:
+                source = metadata.get("source")
+            if source is None:
+                source = "unknown"
+
+            score = result.get("similarity")
+            if score is None:
+                score = result.get("score")
+            if score is None:
+                score = 0.0
+
+            chunks.append(
+                DocChunk(
+                    text=text,
+                    source=source,
+                    score=score,
+                    metadata=metadata,
+                )
             )
-            for r in results
-        ]
+
+        return chunks
 
     async def index(self, directory: str) -> IndexResult:
         """
@@ -71,8 +103,8 @@ class DocsService:
         """
         result = self.rag.index_personal_documents(directory)
         return IndexResult(
-            indexed=result.get("indexed", 0),
-            failed=result.get("failed", 0),
+            indexed=result.get("indexed_count", result.get("indexed", 0)),
+            failed=result.get("failed_count", result.get("failed", 0)),
             errors=result.get("errors", []),
         )
 

services/hwfit/fit.py

@@ -9,7 +9,7 @@ from services.hwfit.models import (
 GPU_BANDWIDTH = {
     "5090": 1792, "5080": 960, "5070 ti": 896, "5070": 672, "5060 ti": 448, "5060": 256,
     "4090": 1008, "4080 super": 736, "4080": 717, "4070 ti super": 672, "4070 ti": 504, "4070 super": 504, "4070": 504, "4060 ti": 288, "4060": 272,
-    "3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360,
+    "3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360, "3050 ti": 192, "3050": 224,
     "2080 ti": 616, "2080 super": 496, "2080": 448, "2070 super": 448, "2070": 448, "2060 super": 448, "2060": 336,
     "1660 ti": 288, "1660 super": 336, "1660": 192, "1650 super": 192, "1650": 128,
     "h100 sxm": 3350, "h100": 2039, "h200": 4800, "a100 sxm": 2039, "a100": 1555,
@@ -18,13 +18,38 @@ GPU_BANDWIDTH = {
     "7900 xtx": 960, "7900 xt": 800, "7900 gre": 576, "7800 xt": 624, "7700 xt": 432, "7600": 288,
     "6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224,
     "mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229,
-    "9070 xt": 624, "9070": 488,
+    "9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322,
+    # NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory,
+    # not Apple Silicon, so it lives in the generic GPU table — the Apple-only
+    # lookup never matches it (its name carries no "apple").
+    "gb10": 273,
 }
 
 # Pre-sort keys by length descending for correct substring matching
 _BW_KEYS_SORTED = sorted(GPU_BANDWIDTH.keys(), key=len, reverse=True)
 
-FALLBACK_K = {"cuda": 220, "rocm": 180, "cpu_x86": 70, "cpu_arm": 90}
+# Apple Silicon unified-memory bandwidth (GB/s). For chip families with both
+# binned and full variants under the same "Apple Mx Max" brand string, prefer
+# GPU core count when hardware detection provides it; otherwise fall back to the
+# conservative tier so speed estimates do not over-promise.
+APPLE_BANDWIDTH_FIXED = {
+    "m1 ultra": 800, "m1 max": 400, "m1 pro": 200, "m1": 68,
+    "m2 ultra": 800, "m2 max": 400, "m2 pro": 200, "m2": 100,
+    "m3 ultra": 800, "m3 pro": 150, "m3": 100,
+    "m4 pro": 273, "m4": 120,
+    "m5 pro": 307, "m5": 153,
+}
+APPLE_BANDWIDTH_BY_CORES = {
+    "m3 max": {30: 300, 40: 400},
+    "m4 max": {32: 410, 40: 546},
+    "m5 max": {32: 460, 40: 614},
+}
+_APPLE_FIXED_KEYS_SORTED = sorted(APPLE_BANDWIDTH_FIXED.keys(), key=len, reverse=True)
+_APPLE_VARIANT_KEYS_SORTED = sorted(APPLE_BANDWIDTH_BY_CORES.keys(), key=len, reverse=True)
+
+# metal: backstop for Apple Silicon chips not in the explicit tables above
+# (e.g. a future M6) — use a conservative generic estimate when unknown.
+FALLBACK_K = {"cuda": 220, "rocm": 180, "metal": 150, "cpu_x86": 70, "cpu_arm": 90}
 
 USE_CASE_WEIGHTS = {
     "general":    (0.45, 0.30, 0.15, 0.10),
@@ -49,37 +74,159 @@ CONTEXT_TARGET = {
 }
 
 
-def _lookup_bandwidth(gpu_name):
-    if not gpu_name:
+def _lookup_apple_bandwidth(system):
+    gpu_name = system.get("gpu_name")
+    if not isinstance(gpu_name, str) or not gpu_name:
         return None
     gn = gpu_name.lower()
+
+    # Guard against false matches on non-Apple GPUs whose names contain
+    # "m3"/"m4"/"m5" (e.g. NVIDIA Quadro M4 000).
+    if "apple" not in gn:
+        return None
+
+    raw_cores = system.get("gpu_cores")
+    try:
+        gpu_cores = int(raw_cores) if raw_cores is not None else None
+    except (TypeError, ValueError):
+        gpu_cores = None
+
+    for key in _APPLE_VARIANT_KEYS_SORTED:
+        if key not in gn:
+            continue
+        if gpu_cores in APPLE_BANDWIDTH_BY_CORES[key]:
+            return APPLE_BANDWIDTH_BY_CORES[key][gpu_cores]
+        return min(APPLE_BANDWIDTH_BY_CORES[key].values())
+
+    for key in _APPLE_FIXED_KEYS_SORTED:
+        if key in gn:
+            return APPLE_BANDWIDTH_FIXED[key]
+    return None
+
+
+def _lookup_bandwidth(system):
+    if isinstance(system, dict):
+        gpu_name = system.get("gpu_name")
+    else:
+        gpu_name = system
+
+    if not isinstance(gpu_name, str) or not gpu_name:
+        return None
+
+    # Apple tiers live only in the Apple-specific table now (#2564), so route
+    # BOTH dict and bare-string callers through it. A bare string carries no
+    # gpu_cores, so the helper falls back to the conservative (lowest) tier for
+    # that model -- before #2564 the generic table answered string lookups, and
+    # dropping that made _lookup_bandwidth("Apple M3 Max") return None.
+    apple_input = system if isinstance(system, dict) else {"gpu_name": gpu_name}
+    bw = _lookup_apple_bandwidth(apple_input)
+    if bw is not None:
+        return bw
+
+    gn = gpu_name.lower()
     for key in _BW_KEYS_SORTED:
         if key in gn:
             return GPU_BANDWIDTH[key]
     return None
 
 
-def _estimate_speed(model, quant, run_mode, system):
-    """Estimate tok/s. Uses active params for MoE (only active experts run per token)."""
+def _canonical_cpu_backend(system):
+    """Return the canonical CPU backend for cpu_only speed estimation.
+
+    Normalizes CPU-architecture aliases separately from the GPU backend, and
+    overrides GPU-only backends (CUDA/ROCm/Metal) so they do not inherit a
+    discrete-GPU fallback constant when the model is actually running on CPU.
+    """
+    backend = (system.get("backend") or "").lower().strip()
+    cpu_arch = (system.get("cpu_arch") or "").lower().strip()
+    cpu_name = (system.get("cpu_name") or "").lower()
+    gpu_name = (system.get("gpu_name") or "").lower()
+
+    # Already-canonical CPU backends
+    if backend in ("cpu_x86", "cpu_arm"):
+        return backend
+
+    # Raw CPU-architecture aliases. Treat plain "arm" as 32-bit ARM, not the
+    # ARM64-class CPU fallback used for Apple Silicon/aarch64 machines.
+    if backend in ("x86_64", "amd64", "i386", "i686"):
+        return "cpu_x86"
+    if backend in ("arm64", "aarch64"):
+        return "cpu_arm"
+
+    # Prefer an explicit CPU architecture field when present
+    if cpu_arch:
+        if cpu_arch in ("x86_64", "amd64", "x86", "i386", "i686"):
+            return "cpu_x86"
+        if cpu_arch in ("arm64", "aarch64"):
+            return "cpu_arm"
+
+    # Apple Silicon enters ranking as backend="metal"; its CPU path is ARM.
+    if backend in ("metal", "mps", "apple") or "apple" in cpu_name or "apple" in gpu_name:
+        return "cpu_arm"
+
+    # Conservative default for CUDA/ROCm/discrete GPU backends and unknowns.
+    return "cpu_x86"
+
+
+def _is_mlx_model(model, native_q=None):
+    name = (model.get("name") or "").lower()
+    provider = (model.get("provider") or "").lower()
+    fmt = (model.get("format") or "").lower()
+    q = (native_q if native_q is not None else _native_quant(model)).lower()
+    return (
+        q.startswith("mlx-")
+        or provider == "mlx-community"
+        or fmt == "mlx"
+        or name.startswith("mlx-community/")
+    )
+
+
+def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
+    """Estimate tok/s. Uses active params for MoE (only active experts run per token).
+
+    offload_frac (0..1): fraction of the model's weights that spill to system RAM
+    (CPU) because they don't fit VRAM. Generation reads every active weight per
+    token, so when part lives in CPU RAM the per-token time is dominated by the
+    slow path. We model effective bandwidth as a blend of GPU VRAM bandwidth and
+    system-RAM bandwidth weighted by what's where — far more accurate than a flat
+    "halve it" for partial offload, which under/over-shoots depending on amount.
+    Calibrated against a measured RX 9060 XT: DeepSeek-Coder-V2-Lite Q4_K_M with
+    light offload → ~59 t/s est vs 59.8 measured.
+    """
     pb = _active_params_b(model)
     is_moe = model.get("is_moe", False)
-    bw = _lookup_bandwidth(system.get("gpu_name"))
+    bw = _lookup_bandwidth(system)
     backend = system.get("backend", "cpu_x86")
 
+    # CPU-only inference must never inherit a GPU backend's fallback constant,
+    # even if the detected system happens to report a CUDA/Metal/ROCm backend.
+    if run_mode == "cpu_only":
+        backend = _canonical_cpu_backend(system)
+
     if bw and run_mode in ("gpu", "cpu_offload"):
         bpp = QUANT_BYTES_PER_PARAM.get(quant, 0.5)
         model_gb = pb * bpp
         if model_gb <= 0:
             return 0.0
         efficiency = 0.55
-        raw_tps = (bw / model_gb) * efficiency
         if run_mode == "cpu_offload":
-            mode_factor = 0.5
-        elif is_moe:
-            mode_factor = 0.8
-        else:
-            mode_factor = 1.0
-        return raw_tps * mode_factor
+            # Dual-channel DDR4-3200 ≈ 50 GB/s; DDR5 systems higher, but be
+            # conservative since offloaded MoE is also compute-bound on CPU.
+            cpu_bw = 55.0
+            frac = min(max(offload_frac, 0.0), 1.0)
+            # If we don't know the fraction (legacy callers pass 0 with
+            # cpu_offload), assume a meaningful spill so we don't overestimate.
+            if frac <= 0.0:
+                frac = 0.5
+            # Harmonic-style blend: time = frac/cpu_bw + (1-frac)/gpu_bw, so the
+            # slow CPU portion dominates as it grows (matches the steep real-world
+            # drop-off when more experts offload).
+            eff_bw = 1.0 / (frac / cpu_bw + (1.0 - frac) / bw)
+            raw_tps = (eff_bw / model_gb) * efficiency
+            return raw_tps * (0.8 if is_moe else 1.0)
+        # Fully on GPU.
+        raw_tps = (bw / model_gb) * efficiency
+        return raw_tps * (0.8 if is_moe else 1.0)
 
     k = FALLBACK_K.get(backend, 70)
     if pb <= 0:
@@ -88,6 +235,27 @@ def _estimate_speed(model, quant, run_mode, system):
     return k / pb * sm
 
 
+def _architecture_bonus(model):
+    name = (model.get("name") or "").lower()
+    arch = (model.get("architecture") or "").lower()
+    text = f"{name} {arch}"
+
+    # Keep this intentionally small: hardware fit and speed still matter, but
+    # current model families should not be scored the same as older Qwen2/LLama
+    # era entries just because the parameter count is similar.
+    if "qwen3.6" in text or "qwen3_6" in text:
+        return 9
+    if "qwen3.5" in text or "qwen3_5" in text:
+        return 8
+    if "qwen3-next" in text or "qwen3_next" in text:
+        return 6
+    if "qwen3" in text or arch.startswith("qwen3"):
+        return 4
+    if "qwen2.5" in text or "qwen2_5" in text:
+        return 2
+    return 0
+
+
 def _quality_score(model, quant, use_case):
     pb = params_b(model)
     if pb < 1:
@@ -117,13 +285,21 @@ def _quality_score(model, quant, use_case):
     if "gemma" in name_lower:
         base += 1
 
+    base += _architecture_bonus(model)
     base += QUANT_QUALITY_PENALTY.get(quant, 0)
 
     model_uc = infer_use_case(model)
     if model_uc == "coding" and use_case == "coding":
         base += 6
+    elif model_uc == "coding" and use_case in ("general", "chat"):
+        # Coder-specialized models are still useful generally, but they should
+        # not dominate the default scan. If the user wants code, the Coding
+        # filter gives them the boost above.
+        base -= 10
     if model_uc == "reasoning" and use_case == "reasoning" and pb >= 13:
         base += 5
+    elif model_uc == "reasoning" and use_case == "chat":
+        base -= 4
     if model_uc == "multimodal" and use_case == "multimodal":
         base += 6
 
@@ -150,6 +326,22 @@ def _fit_score(required, available):
     return 50
 
 
+def _is_unified_memory_system(system):
+    backend = (system.get("backend") or "").lower()
+    return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
+
+
+def _fit_level_for_budget(required_gb, budget_gb):
+    if not required_gb or not budget_gb or required_gb > budget_gb:
+        return "too_tight"
+    ratio = required_gb / budget_gb
+    if ratio <= 0.50:
+        return "perfect"
+    if ratio <= 0.78:
+        return "good"
+    return "marginal"
+
+
 def _context_score(ctx, use_case):
     target = CONTEXT_TARGET.get(use_case, 4096)
     if ctx >= target:
@@ -186,9 +378,9 @@ def _quant_bits(q):
     Returns 0 when unknown (caller treats unknown as "don't filter")."""
     qu = (q or "").upper().replace("-", "").replace("_", "").replace(" ", "")
     # GGUF k-quants + float formats
-    if qu.startswith("Q8") or "FP8" in qu:
+    if qu.startswith("Q8") or "FP8" in qu or "INT8" in qu or qu.startswith("W8"):
         return 8
-    if qu.startswith("Q4") or qu.startswith("IQ4"):
+    if qu.startswith("Q4") or qu.startswith("IQ4") or "FP4" in qu or "NF4" in qu or "INT4" in qu or qu.startswith("W4"):
         return 4
     if qu.startswith("Q2") or qu.startswith("IQ2"):
         return 2
@@ -200,7 +392,7 @@ def _quant_bits(q):
         return 6
     if qu.startswith("F16") or qu.startswith("BF16") or qu.startswith("F32"):
         return 16
-    # Prequantized formats: pull the bit-width digit (AWQ4 / AWQ4BIT / GPTQ8 / 4BIT / INT8 …)
+    # Prequantized formats: pull the bit-width digit (AWQ4 / AWQ4BIT / GPTQ8 / 4BIT / INT8 ...)
     m = re.search(r"(?:AWQ|GPTQ|MLX|EXL2|BNB|INT|W)(\d{1,2})", qu) or re.search(r"(\d{1,2})BIT", qu)
     if m:
         b = int(m.group(1))
@@ -209,12 +401,40 @@ def _quant_bits(q):
     return 0
 
 
-def analyze_model(model, system, target_quant=None):
+def _native_quant(model):
+    native_quant = model.get("quantization", "Q4_K_M")
+    name = (model.get("name") or "").lower()
+    fmt = (model.get("format") or "").lower()
+    text = f"{name} {fmt}"
+    if "nvfp4" in text:
+        return "NVFP4"
+    if re.search(r"(^|[-_/])fp8($|[-_/\s])", text):
+        return "FP8"
+    if "gptq" in text:
+        m = re.search(r"(?:gptq|int|w)(?:[-_]?)(\d{1,2})(?:bit)?", text)
+        # Canonical catalog label is "GPTQ-Int4"/"GPTQ-Int8" (see models.py
+        # QUANT_BPP / QUANT_QUALITY_PENALTY keys); "GPTQ-4bit" misses both
+        # maps, so BPP and the quality penalty silently fall to defaults.
+        return f"GPTQ-Int{m.group(1)}" if m else "GPTQ-Int4"
+    if "awq" in text:
+        m = re.search(r"(?:awq|int|w)(?:[-_]?)(\d{1,2})(?:bit)?", text)
+        # Catalog keys are "AWQ-4bit"/"AWQ-8bit"; bare "AWQ" misses the maps.
+        return f"AWQ-{m.group(1)}bit" if m else "AWQ-4bit"
+    if "mlx" in text:
+        m = re.search(r"mlx[-_]?(\d{1,2})bit", text)
+        return f"mlx-{m.group(1)}bit" if m else native_quant
+    if not (model.get("is_gguf") or model.get("gguf_sources")) and re.search(r"(^|[-_/])(?:int)?8bit($|[-_/\s])", text):
+        return "INT8"
+    return native_quant
+
+
+def analyze_model(model, system, target_quant=None, scoring_use_case=None, target_context=None):
     pb = params_b(model)
     if pb <= 0:
         return None
 
-    use_case = infer_use_case(model)
+    model_use_case = infer_use_case(model)
+    score_use_case = scoring_use_case or "general"
     has_gpu = system.get("has_gpu", False)
     gpu_vram = (system.get("gpu_vram_gb") or 0) if has_gpu else 0
     gpu_count = system.get("gpu_count", 1) or 1
@@ -228,9 +448,14 @@ def analyze_model(model, system, target_quant=None):
     gpu_only = bool(system.get("gpu_only")) and has_gpu and gpu_vram > 0
     eff_ram = 0 if gpu_only else available_ram
     is_moe = model.get("is_moe", False)
-    ctx = model.get("context_length", 4096) or 4096
+    model_ctx = model.get("context_length", 4096) or 4096
+    try:
+        target_context = int(target_context or 0)
+    except (TypeError, ValueError):
+        target_context = 0
+    ctx = min(model_ctx, target_context) if target_context > 0 else model_ctx
 
-    native_quant = model.get("quantization", "Q4_K_M")
+    native_quant = _native_quant(model)
     preq = is_prequantized(model)
 
     # GGUF models can't be sharded across GPUs — use single GPU VRAM
@@ -246,13 +471,22 @@ def analyze_model(model, system, target_quant=None):
     else:
         effective_vram = gpu_vram
 
+    native_gpu_only = preq and not native_quant.startswith("mlx-")
+
     # Determine which quant to evaluate at
+    native_quant_prefixes = (
+        "AWQ-", "GPTQ-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
+        "INT4", "INT8", "W4A16", "W8A8", "W8A16",
+    )
+
     if preq:
-        # AWQ/GPTQ/FP8/MLX come at a fixed bit-width. If the user picked a
-        # specific quant tier (e.g. Q8 → 8-bit), only keep prequant models whose
-        # native bit-width matches — otherwise selecting Q8 would still surface
-        # AWQ-4bit models, mixing 4- and 8-bit in one view.
+        # Native HF/vLLM quantized repos come at a fixed format. If the user
+        # picked a GGUF quant tier (Q4/Q8/etc.), do not treat same-bit
+        # AWQ/GPTQ/FP8/FP4 builds as equivalent; those formats are separate
+        # serving paths and only appear when explicitly selected or unfiltered.
         if target_quant:
+            if not any(target_quant.startswith(p) for p in native_quant_prefixes):
+                return None
             _tb, _nb = _quant_bits(target_quant), _quant_bits(native_quant)
             if _tb and _nb and _tb != _nb:
                 return None
@@ -260,20 +494,25 @@ def analyze_model(model, system, target_quant=None):
     elif target_quant:
         # User picked a specific quant
         quant_to_try = target_quant
+    elif gpu_count >= 2:
+        # Multi-GPU box: vLLM/SGLang can't serve GGUF Q* quants (those are
+        # llama.cpp-only). Default non-prequantized models to BF16 so the row
+        # is meaningful on a multi-GPU rig. If BF16 doesn't fit, the model
+        # surfaces as too_tight — better than showing a Q4 row the user
+        # can't actually serve with vLLM on >1 GPU.
+        quant_to_try = "BF16"
     else:
-        # Default: Q4_K_M (user's stated preference)
+        # Default: Q4_K_M (user's stated preference) — kept for single-GPU
+        # and RAM modes where llama.cpp serving is the natural path.
         quant_to_try = "Q4_K_M"
 
-    result = _try_quant_at(model, quant_to_try, ctx, effective_vram, eff_ram)
+    # Multi-GPU filter: skip the row if the resolved quant is a GGUF tier
+    # (Q*/IQ-prefixed) — vLLM/SGLang can't serve those, so showing them on
+    # a 2+ GPU rig just clutters the list with unservable candidates.
+    if gpu_count >= 2 and quant_to_try and not target_quant and quant_to_try.upper().startswith(("Q2", "Q3", "Q4", "Q5", "Q6", "Q8", "IQ")):
+        return None
 
-    # If target quant doesn't fit and it's not pre-quantized, try lower quants
-    if result is None and not preq and target_quant:
-        from services.hwfit.models import QUANT_HIERARCHY
-        idx = QUANT_HIERARCHY.index(target_quant) if target_quant in QUANT_HIERARCHY else -1
-        for q in QUANT_HIERARCHY[idx + 1:]:
-            result = _try_quant_at(model, q, ctx, effective_vram, eff_ram)
-            if result:
-                break
+    result = _try_quant_at(model, quant_to_try, ctx, effective_vram, 0 if native_gpu_only else eff_ram)
 
     if result is None:
         # Model doesn't fit on the user's current hardware. Surface it
@@ -289,7 +528,7 @@ def analyze_model(model, system, target_quant=None):
             "parameter_count": model.get("parameter_count"),
             "params_b": round(pb, 1),
             "is_moe": is_moe,
-            "use_case": use_case,
+            "use_case": model_use_case,
             "fit_level": "too_tight",
             "run_mode": "no_fit",
             "quant": quant_to_try,
@@ -299,36 +538,63 @@ def analyze_model(model, system, target_quant=None):
             "score": 0,
             "scores": {"quality": 0, "speed": 0, "fit": 0, "context": 0},
             "gguf_sources": model.get("gguf_sources", []),
-            "context_length": model.get("context_length", 4096),
+            "context_length": model_ctx,
+            "target_context": target_context or None,
         }
 
     run_mode, quant, fit_ctx, required_gb = result
 
     # Determine fit level
-    budget = effective_vram if run_mode == "gpu" else available_ram
+    unified_memory = _is_unified_memory_system(system)
+    total_ram = system.get("total_ram_gb") or available_ram
+    unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
+    budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
     if required_gb > budget:
         return None
     if run_mode == "gpu":
-        rec = model.get("recommended_ram_gb") or required_gb
-        if rec <= gpu_vram:
-            fit_level = "perfect"
-        elif gpu_vram >= required_gb * 1.2:
-            fit_level = "good"
+        if unified_memory:
+            fit_level = _fit_level_for_budget(required_gb, budget)
         else:
-            fit_level = "marginal"
+            # GPU-only fit must leave real allocator/KV/runtime headroom. The
+            # old check used recommended_ram_gb (or required_gb as a fallback),
+            # which made any model that barely fit VRAM read as "perfect".
+            # On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
+            # runnable, but not a comfortable perfect fit.
+            if gpu_vram >= required_gb * 1.50:
+                fit_level = "perfect"
+            elif gpu_vram >= required_gb * 1.2:
+                fit_level = "good"
+            else:
+                fit_level = "marginal"
     elif run_mode == "cpu_offload":
-        fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
+        fit_level = _fit_level_for_budget(required_gb, budget)
+        if fit_level == "perfect":
+            fit_level = "good"
     else:
-        fit_level = "marginal"
+        fit_level = _fit_level_for_budget(required_gb, budget)
+        if fit_level == "too_tight":
+            fit_level = "marginal"
 
-    tps = _estimate_speed(model, quant, run_mode, system)
+    # Rows that comfortably fit in a huge RAM/unified-memory pool should not all
+    # look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
+    if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
+        fit_level = "good"
+    if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
+        fit_level = "perfect"
 
-    q_score = _quality_score(model, quant, use_case)
-    s_score = _speed_score(tps, use_case)
+    # Fraction of the model that spills to CPU RAM (drives the offload speed
+    # model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
+    offload_frac = 0.0
+    if run_mode == "cpu_offload" and required_gb > 0 and effective_vram > 0:
+        offload_frac = max(0.0, (required_gb - effective_vram) / required_gb)
+    tps = _estimate_speed(model, quant, run_mode, system, offload_frac=offload_frac)
+
+    q_score = _quality_score(model, quant, score_use_case)
+    s_score = _speed_score(tps, score_use_case)
     f_score = _fit_score(required_gb, budget)
-    c_score = _context_score(fit_ctx, use_case)
+    c_score = _context_score(fit_ctx, score_use_case)
 
-    wq, ws, wf, wc = USE_CASE_WEIGHTS.get(use_case, (0.45, 0.30, 0.15, 0.10))
+    wq, ws, wf, wc = USE_CASE_WEIGHTS.get(score_use_case, (0.45, 0.30, 0.15, 0.10))
     composite = q_score * wq + s_score * ws + f_score * wf + c_score * wc
 
     return {
@@ -337,7 +603,7 @@ def analyze_model(model, system, target_quant=None):
         "parameter_count": model.get("parameter_count"),
         "params_b": round(pb, 1),
         "is_moe": is_moe,
-        "use_case": use_case,
+        "use_case": model_use_case,
         "fit_level": fit_level,
         "run_mode": run_mode,
         "quant": quant,
@@ -352,21 +618,101 @@ def analyze_model(model, system, target_quant=None):
             "context": round(c_score, 1),
         },
         "gguf_sources": model.get("gguf_sources", []),
-        "context_length": model.get("context_length", 4096),
+        "context_length": model_ctx,
+        "release_date": model.get("release_date", ""),
+        "target_context": target_context or None,
     }
 
 
+def _version_key(name):
+    """Parse the model's version number from its display name so equal-score
+    rows can break ties in favor of the newer release (e.g. M2.7 > M2.5).
+    Returns a float; 0.0 for names with no recognizable version. The regex
+    grabs the FIRST 'word-with-digits' pattern after a hyphen/underscore,
+    so e.g. 'MiniMax-M2.7' -> 2.7, 'Qwen3.6-35B' -> 3.6, 'M2' -> 2.0."""
+    import re as _re
+    if not name:
+        return 0.0
+    # Match the version-marker word: a letter followed by a number with
+    # optional decimal, e.g. M2.7, V4, Pro3. Take the first hit; ignore
+    # "B" param-count suffixes (Qwen3-235B should yield 3, not 235).
+    for m in _re.finditer(r"[A-Za-z](\d+(?:\.\d+)?)(?![A-Za-z])", name):
+        val = m.group(1)
+        # Skip param-count tokens (e.g. "235B" gives "235" but the next
+        # char would be "B" — already excluded by the negative lookahead).
+        try:
+            f = float(val)
+        except ValueError:
+            continue
+        # Heuristic: bare integers >= 100 are almost certainly param counts
+        # (1B/3B/8B/70B/235B…), not version numbers. Skip them.
+        if "." not in val and f >= 100:
+            continue
+        return f
+    return 0.0
+
+
 SORT_KEYS = {
-    "score": lambda r: r["score"],
+    # Score sort with version-aware tiebreaker — when two rows tie on
+    # composite score (a common case for the SAME base model in different
+    # versions, e.g. MiniMax-M2.5 vs M2.7 both at the same FP8 budget),
+    # prefer the newer version. Without this, ties resolved to whatever
+    # order they came out of the registry, which let older releases land
+    # above newer ones in user-facing lists.
+    "score": lambda r: (r["score"], _version_key(r.get("name") or "")),
     "speed": lambda r: r["speed_tps"],
     "vram": lambda r: r["required_gb"],
     "params": lambda r: r["params_b"],
     "context": lambda r: r["context"],
+    # Newest first. release_date is an ISO-ish string ("2026-05-30"); plain
+    # string sort is chronological. Missing dates sort last (empty < any date,
+    # and we sort reverse=True for newest, so "" lands at the bottom).
+    "newest": lambda r: r.get("release_date") or "",
 }
 
 
-def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None):
-    """Rank all models against detected hardware. Returns sorted list of fit results."""
+def _search_blob(*parts):
+    text = " ".join(str(p or "") for p in parts).lower()
+    compact = re.sub(r"[^a-z0-9]+", "", text)
+    spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
+    return f"{text} {spaced} {compact}"
+
+
+def _matches_search(model, search):
+    terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
+    if not terms:
+        return True
+    blob = _search_blob(
+        model.get("name"),
+        model.get("provider"),
+        model.get("architecture"),
+        model.get("quantization"),
+        model.get("format"),
+        model.get("parameter_count"),
+    )
+    for term in terms:
+        norm = re.sub(r"[^a-z0-9]+", "", term)
+        if term not in blob and (not norm or norm not in blob):
+            if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
+                try:
+                    wanted = float(term.rstrip("b"))
+                    actual = params_b(model)
+                except (TypeError, ValueError):
+                    actual = 0
+                if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
+                    continue
+            return False
+    return True
+
+
+def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
+    """Rank all models against detected hardware. Returns sorted list of fit results.
+
+    fit_only: when True, drop rows whose fit_level is "too_tight" (model doesn't
+    actually fit on the chosen budget). When False (default), every model is
+    shown — sorting by Param means highest-param PERIOD, even ones that won't
+    run, so the user can see the truth.
+    """
     models = get_models()
     results = []
 
@@ -402,44 +748,104 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
                 "is_image_gen": True,
                 "capabilities": im.get("capabilities", []),
                 "description": im.get("description", ""),
+                "dependency_package": im.get("dependency_package", ""),
             })
         if use_case == "image_gen":
             sort_fn = SORT_KEYS.get(sort, SORT_KEYS["score"])
-            results.sort(key=sort_fn, reverse=(sort != "vram"))
+            results.sort(key=sort_fn, reverse=True)  # see main path below
             return results[:limit]
 
-    # If user picked a prequantized format (AWQ/FP8/GPTQ), filter to only those models
-    filter_native = quant and any(quant.startswith(p) for p in ("AWQ-", "GPTQ-", "FP8"))
+    # If user picked a native prequantized format, filter to only those models.
+    filter_native = quant and any(quant.startswith(p) for p in (
+        "AWQ-", "GPTQ-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
+        "INT4", "INT8", "W4A16", "W8A8", "W8A16",
+    ))
 
-    # MLX-quantized models only run on Apple Silicon (Metal). Exclude them on
-    # every other backend (CUDA / ROCm / CPU) so Linux/Windows users don't see
-    # unrunnable suggestions.
     system_backend = (system.get("backend") or "").lower()
     apple_silicon = system_backend in ("mps", "metal", "apple")
+    rocm = system_backend == "rocm"
+    is_windows = system.get("platform") == "windows"
+
+    # Consumer AMD Radeon (RDNA, gfx10/11/12): the practical local serving path
+    # is GGUF via llama.cpp. vLLM/SGLang on ROCm are validated for datacenter
+    # Instinct (CDNA, gfx9xx) but are unreliable on consumer RDNA — AWQ kernels
+    # are largely unsupported there and FP8 needs out-of-tree patches. So treat
+    # consumer RDNA like Apple Silicon (GGUF-only) and leave CDNA untouched.
+    # Unknown family (no rocminfo) is left untouched to avoid hiding models from
+    # a possibly-capable Instinct box on a misdetect.
+    gpu_family = (system.get("gpu_family") or "").lower()
+    consumer_amd = system_backend == "rocm" and gpu_family == "rdna"
 
     for m in models:
-        native_q = m.get("quantization", "")
+        native_q = _native_quant(m)
+        is_mlx = _is_mlx_model(m, native_q)
 
-        # Drop MLX models on non-Apple hardware
-        if not apple_silicon and native_q.startswith("mlx-"):
+        # MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
+        # but it is first-class on Metal where mlx_lm.server can serve it.
+        if is_mlx and not apple_silicon:
             continue
 
-        # Format filter: AWQ tab → only AWQ models, FP8 tab → only FP8 models
+        # ROCm support for vLLM/SGLang quantized safetensors is too brittle to
+        # recommend blindly in the default scan. Keep AWQ/GPTQ/FP8 discoverable
+        # only when the user explicitly picks that format from the quant filter;
+        # otherwise prefer GGUF/Q* entries that Odysseus can route through
+        # llama.cpp/Ollama without pretending "fits VRAM" means "servable".
+        if rocm and is_prequantized(m) and not filter_native:
+            continue
+
+        # On Apple Silicon the only serving engines are llama.cpp and Ollama,
+        # both GGUF-only (vLLM/SGLang are CUDA/ROCm and don't run on macOS). So
+        # a model is Metal-servable ONLY if it ships a real GGUF. Drop everything
+        # else — raw safetensors repos (which the catalog still tags with a
+        # default GGUF quant) and vLLM-only AWQ/GPTQ/FP8 builds alike. Without
+        # this the Cookbook recommends models the Mac can't run; on CUDA these
+        # stay visible because vLLM serves safetensors directly.
+        #
+        # Consumer AMD (RDNA) is the same story: GGUF via llama.cpp is the
+        # servable path, so a model needs a real GGUF to be recommended.
+        # Otherwise the Cookbook rates vLLM-only AWQ/GPTQ builds "GOOD" on a
+        # Radeon that can't actually serve them.
+        #
+        # Windows is the same: Odysseus only supports llama.cpp on Windows,
+        # which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
+        # models without a GGUF source are unservable there.
+        if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
+            continue
+
+        # Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
         if filter_native:
             if quant == "FP8" and native_q != "FP8":
                 continue
+            if quant == "FP4" and native_q not in ("FP4", "NVFP4", "MXFP4", "NF4"):
+                continue
             if quant.startswith("AWQ") and not native_q.startswith("AWQ"):
                 continue
             if quant.startswith("GPTQ") and not native_q.startswith("GPTQ"):
                 continue
-
-        if search:
-            name = m.get("name", "").lower()
-            provider = m.get("provider", "").lower()
-            if search.lower() not in name and search.lower() not in provider:
+            if quant.startswith("NVFP4") and not native_q.startswith("NVFP4"):
+                continue
+            if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
                 continue
 
-        result = analyze_model(m, system, target_quant=quant)
+        if search and not _matches_search(m, search):
+            continue
+
+        model_quant = quant
+        # UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
+        # CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
+        # safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
+        # AWQ row, analyze_model correctly rejects it as the wrong serving
+        # format, but the result is confusing: highlighting Quant/Q4 hides the
+        # exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
+        # native AWQ rows only on accelerator servers that can serve them.
+        if (
+            quant == "Q4_K_M"
+            and not (apple_silicon or consumer_amd or is_windows)
+            and native_q == "AWQ-4bit"
+        ):
+            model_quant = native_q
+
+        result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
         if result is None:
             continue
 
@@ -450,14 +856,21 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
 
         results.append(result)
 
-    # Pick the visible SET by best fit (score) first, so it stays the same no
-    # matter which column the user sorts by — otherwise sorting by params would
-    # truncate to the N biggest models (huge ones that don't even fit) while
-    # sorting by vram showed the N smallest. Only AFTER choosing the set do we
-    # order it by the requested column.
-    results.sort(key=SORT_KEYS["score"], reverse=True)
-    results = results[:limit]
+    # Pick the visible SET by the REQUESTED column. Per-user feedback: sorting
+    # by Param should show the highest-param models PERIOD, not just those that
+    # already fit. Same for every other column. Models that don't fit are still
+    # in the list with their fit_level marking the constraint, so the user can
+    # see the truth instead of a quietly-truncated view. Score sort is unchanged
+    # (it's the default ranking and naturally pushes non-fits to the bottom).
+    if fit_only:
+        # Hide rows that definitely don't fit (the "too_tight" badge) — user
+        # explicitly asked for a Fit-only view.
+        results = [r for r in results if r.get("fit_level") != "too_tight"]
     sort_fn = SORT_KEYS.get(sort, SORT_KEYS["score"])
-    # vram ascending (smallest first), everything else descending (biggest first)
-    results.sort(key=sort_fn, reverse=(sort != "vram"))
+    # Always sort descending then truncate top-N so each column shows the
+    # global highest by that metric. Before, vram was special-cased
+    # ascending → truncate kept the 50 SMALLEST models and "highest VRAM"
+    # could never appear, breaking the column-click toggle.
+    results.sort(key=sort_fn, reverse=True)
+    results = results[:limit]
     return results

services/hwfit/hardware.py

@@ -1,9 +1,21 @@
+import json
 import os
 import platform
+import re
+import shutil
 import subprocess
 import time
+import shlex
 
-CACHE_TTL = 1800  # 30 min — hardware rarely changes; use the Rescan button to force a re-probe
+from core.platform_compat import (
+    NVIDIA_PATH_CANDIDATES,
+    SSH_PATH_OVERRIDE,
+    run_ssh_command,
+)
+
+CACHE_TTL = 24 * 3600  # 24 h — hardware probes are user-initiated via the Rescan button; bumped
+                       # from 30 min so changing filters doesn't keep re-probing the rig every
+                       # half-hour during a long session.
 
 
 _remote_host = None  # set by detect_system(host=...)
@@ -17,16 +29,17 @@ def _run(cmd):
         if _remote_host:
             # Run command on remote host via SSH
             if isinstance(cmd, list):
-                cmd_str = " ".join(cmd)
+                cmd_str = shlex.join(str(c) for c in cmd)
             else:
                 cmd_str = cmd
-            ssh_cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=no"]
-            if _remote_port and _remote_port != "22":
-                ssh_cmd += ["-p", _remote_port]
-            ssh_cmd += [_remote_host, cmd_str]
-            r = subprocess.run(
-                ssh_cmd,
-                capture_output=True, text=True, timeout=15,
+            r = run_ssh_command(
+                _remote_host,
+                _remote_port,
+                cmd_str,
+                timeout=15,
+                connect_timeout=5,
+                strict_host_key_checking=False,
+                text=True,
             )
         else:
             r = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
@@ -72,21 +85,29 @@ def _detect_nvidia():
     global _last_gpu_error
     _last_gpu_error = None
     out = _run(["nvidia-smi", "--query-gpu=memory.total,name", "--format=csv,noheader,nounits"])
-    # Remote fallback: a non-interactive SSH shell often has a minimal PATH
-    # that omits where nvidia-smi lives (/usr/bin, /usr/local/cuda/bin), so the
-    # first call silently returns nothing → "No GPU" on hosts that DO have GPUs.
+    # Fallback: a non-interactive shell (or WSL) often has a minimal PATH
+    # that omits where nvidia-smi lives (/usr/bin, /usr/local/cuda/bin,
+    # /usr/lib/wsl/lib), so the first call silently returns nothing →
+    # "No GPU" on machines that DO have GPUs.
     # Retry through a login shell with the common CUDA bin dirs on PATH.
     if not out and _remote_host:
         out = _run(
-            "bash -lc 'export PATH=\"$PATH:/usr/bin:/usr/local/bin:/usr/local/cuda/bin\"; "
+            f"bash -lc '{SSH_PATH_OVERRIDE}"
             "nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits'"
         )
     # Last resort: call nvidia-smi by absolute path. Some hosts have a login
     # shell that isn't bash (or a profile that errors), so the bash -lc retry
     # above still comes back empty even though the binary is right there.
-    if not out and _remote_host:
-        for _p in ("/usr/bin/nvidia-smi", "/usr/local/bin/nvidia-smi", "/usr/local/cuda/bin/nvidia-smi"):
-            out = _run(f"{_p} --query-gpu=memory.total,name --format=csv,noheader,nounits")
+    # Also handles WSL where nvidia-smi lives at /usr/lib/wsl/lib/ — a path
+    # that may not be in the server process's PATH.
+    if not out:
+        for _p in NVIDIA_PATH_CANDIDATES:
+            # Use list form so subprocess.run (local) resolves the absolute path
+            # correctly instead of treating the whole string as an executable name.
+            if _remote_host:
+                out = _run(f"{_p} --query-gpu=memory.total,name --format=csv,noheader,nounits")
+            else:
+                out = _run([_p, "--query-gpu=memory.total,name", "--format=csv,noheader,nounits"])
             if out:
                 break
     if not out:
@@ -103,6 +124,8 @@ def _detect_nvidia():
         return None
 
     gpus = []
+    # Devices nvidia-smi lists with a real name but a non-numeric memory.total.
+    unified = []
     # nvidia-smi lists GPUs in index order (0,1,2,...), so the row position is
     # the CUDA device index we'd pass to CUDA_VISIBLE_DEVICES.
     for idx, line in enumerate(out.strip().split("\n")):
@@ -112,9 +135,32 @@ def _detect_nvidia():
                 vram_mb = float(parts[0])
                 gpus.append({"index": idx, "name": parts[1], "vram_gb": vram_mb / 1024.0})
             except ValueError:
+                # Grace Blackwell GB10 / DGX Spark and other unified-memory
+                # NVIDIA parts report memory.total as "[N/A]"/"Not Supported"
+                # because the GPU shares the system LPDDR pool instead of
+                # carrying discrete VRAM. Don't drop the device — remember it so
+                # we report a unified-memory GPU below rather than "No GPU" (#1340).
+                if parts[1]:
+                    unified.append({"index": idx, "name": parts[1]})
                 continue
 
     if not gpus:
+        if unified:
+            # Unified-memory CUDA box: report the GPU backed by system RAM so the
+            # Cookbook recommends models and serving works. The pool is shared
+            # (not per-GPU discrete VRAM), so report the RAM total once.
+            ram_gb = round(_get_ram_gb(), 1)
+            gpus = [{"index": g["index"], "name": g["name"], "vram_gb": ram_gb} for g in unified]
+            return {
+                "gpu_name": gpus[0]["name"],
+                "gpu_vram_gb": ram_gb,
+                "gpu_count": len(gpus),
+                "gpus": gpus,
+                "gpu_groups": _group_gpus(gpus),
+                "homogeneous": True,
+                "backend": "cuda",
+                "unified_memory": True,
+            }
         return None
     total_vram = sum(g["vram_gb"] for g in gpus)
     groups = _group_gpus(gpus)
@@ -129,6 +175,33 @@ def _detect_nvidia():
     }
 
 
+def classify_amd_gfx(gfx):
+    """Map an AMD ISA target (e.g. "gfx1200") to (gfx, family).
+
+    family is one of:
+      "rdna"    — consumer Radeon RX (gfx10xx RDNA1/2, gfx11xx RDNA3, gfx12xx RDNA4)
+      "cdna"    — datacenter Instinct (gfx908 MI100, gfx90a MI200, gfx94x/95x MI300+)
+      "gcn"     — older GCN/Vega (gfx900/906)
+      "unknown" — empty/unrecognized; callers must treat conservatively
+
+    This drives the serving decision: vLLM/SGLang on ROCm are validated on CDNA
+    but fragile on consumer RDNA (AWQ kernels largely unsupported, FP8 needs
+    out-of-tree patches), so RDNA is steered to GGUF/llama.cpp.
+    """
+    gfx = (gfx or "").lower().strip()
+    m = re.fullmatch(r"gfx(\d+[a-f]?)", gfx)
+    if not m:
+        return "", "unknown"
+    digits = m.group(1)
+    if digits[:2] in ("10", "11", "12"):
+        return gfx, "rdna"
+    if digits in ("908", "90a") or digits[:2] in ("94", "95"):
+        return gfx, "cdna"
+    if digits[:1] == "9":
+        return gfx, "gcn"
+    return gfx, "unknown"
+
+
 def _detect_amd():
     """Detect AMD GPUs. Handles both discrete cards (with mem_info_vram_total)
     and APUs / unified-memory SoCs like Strix Halo (which expose
@@ -138,7 +211,7 @@ def _detect_amd():
             val = _run(["cat", path])
             return val.strip() if val else None
         try:
-            with open(path) as f:
+            with open(path, encoding="utf-8", errors="replace") as f:
                 return f.read().strip()
         except Exception:
             return None
@@ -154,6 +227,17 @@ def _detect_amd():
         except Exception:
             return []
 
+    def _amd_arch():
+        """Best-effort AMD GPU ISA + family from rocminfo.
+
+        rocminfo is the source of truth; its GPU agents report a `Name: gfxNNNN`
+        line (CPU agents report a brand string, not a gfx target), so the first
+        gfx match is the GPU ISA. Returns (gfx, family) — see classify_amd_gfx.
+        """
+        info = _run(["rocminfo"]) or _run(["/opt/rocm/bin/rocminfo"]) or ""
+        m = re.search(r"gfx\d+[a-f]?", info)
+        return classify_amd_gfx(m.group(0) if m else "")
+
     try:
         cards = []
         is_apu = False
@@ -186,6 +270,7 @@ def _detect_amd():
             return None
         total_vram = sum(c["vram_gb"] for c in cards)
         groups = _group_gpus(cards)
+        gfx, family = _amd_arch()
         # NOTE: for APUs with BIOS UMA carveout (e.g. Strix Halo), vis_vram_total
         # is the real usable GPU memory — it's physically backed but reserved
         # by BIOS so it doesn't appear in /proc/meminfo. Don't cap it at system
@@ -197,19 +282,146 @@ def _detect_amd():
             "gpus": cards,
             "gpu_groups": groups,
             "homogeneous": len(groups) <= 1,
-            "backend": "rocm",
+            # Pick the actual runtime label: ROCm/HIP only when its
+            # toolchain is installed, otherwise Vulkan if vulkaninfo is
+            # present (mesa RADV works fine on RDNA/CDNA when ROCm
+            # packages are absent — see Strix Halo where ROCm support
+            # is still backporting). Reporting "rocm" on a Vulkan-only
+            # host misleads downstream env-var pinning
+            # (HIP_VISIBLE_DEVICES is a no-op there).
+            "backend": (
+                "rocm" if (_run(["which", "rocminfo"]) or _run(["which", "hipconfig"]))
+                else ("vulkan" if _run(["which", "vulkaninfo"]) else "rocm")
+            ),
             "unified_memory": is_apu,
+            # AMD ISA/family so downstream can tell datacenter Instinct (CDNA,
+            # where vLLM/SGLang run AWQ/GPTQ reliably) from consumer Radeon
+            # (RDNA, where the practical path is GGUF via llama.cpp). Empty/
+            # "unknown" when rocminfo isn't available — callers must treat
+            # unknown conservatively, not assume vLLM works.
+            "gpu_arch": gfx,
+            "gpu_family": family,
         }
     except Exception:
         return None
 
 
+def _detect_apple_silicon():
+    """Detect Apple Silicon (M-series) GPUs.
+
+    Macs have no discrete VRAM — the GPU shares the system's unified memory.
+    We report a fraction of total RAM as the usable GPU budget (matching macOS's
+    default Metal working-set limit) so the Cookbook recommends models that
+    actually run on the GPU instead of classifying the machine as CPU-only.
+
+    backend="metal" is what services.hwfit.fit and the serve-command generation
+    key off of (they already understand MLX / llama.cpp-Metal). Works locally
+    (platform.system()=="Darwin") and over SSH (uname -s == Darwin).
+    """
+    # Gate to macOS — locally via platform, remotely via uname.
+    if _remote_host:
+        if "darwin" not in (_run(["uname", "-s"]) or "").lower():
+            return None
+        arch = (_run(["uname", "-m"]) or "").lower()
+    else:
+        if platform.system() != "Darwin":
+            return None
+        arch = platform.machine().lower()
+
+    # Only Apple Silicon (arm64) has a Metal GPU worth serving LLMs on; Intel
+    # Macs fall through to the CPU path.
+    if _canonical_cpu_arch(arch) != "arm64":
+        return None
+
+    # Chip name, e.g. "Apple M4 Max" — carries the Pro/Max/Ultra variant that
+    # the fit bandwidth table keys off of.
+    brand = (_run(["sysctl", "-n", "machdep.cpu.brand_string"]) or "Apple Silicon").strip()
+
+    # Total unified memory in bytes.
+    memsize = _run(["sysctl", "-n", "hw.memsize"])
+    try:
+        total_gb = int(memsize) / (1024**3) if memsize else 0.0
+    except ValueError:
+        total_gb = 0.0
+    if total_gb <= 0:
+        return None
+
+    def _parse_apple_gpu_cores(text):
+        if not text:
+            return None
+        try:
+            data = json.loads(text)
+        except (TypeError, ValueError, json.JSONDecodeError):
+            data = None
+        if isinstance(data, dict):
+            for gpu in data.get("SPDisplaysDataType") or []:
+                if not isinstance(gpu, dict):
+                    continue
+                model = str(gpu.get("sppci_model") or gpu.get("_name") or "")
+                if "apple" not in model.lower():
+                    continue
+                cores = gpu.get("sppci_cores")
+                try:
+                    return int(str(cores).strip())
+                except (TypeError, ValueError):
+                    continue
+        m = re.search(r"Total Number of Cores:\s*(\d+)", text)
+        if m:
+            try:
+                return int(m.group(1))
+            except ValueError:
+                return None
+        return None
+
+    gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType", "-json"]))
+    if gpu_cores is None:
+        gpu_cores = _parse_apple_gpu_cores(_run(["system_profiler", "SPDisplaysDataType"]))
+
+    # Usable GPU budget. macOS lets Metal use most of unified memory, but the
+    # default working-set limit scales with RAM: small machines have to keep
+    # more back for the OS + app. These fractions track Apple's
+    # recommendedMaxWorkingSetSize defaults across the lineup. Honour an
+    # explicit override if the user raised it with
+    # `sudo sysctl iogpu.wired_limit_mb=…`.
+    if total_gb <= 16:
+        frac = 0.67
+    elif total_gb <= 64:
+        frac = 0.75
+    else:
+        frac = 0.80
+    vram_gb = round(total_gb * frac, 1)
+    wired = _run(["sysctl", "-n", "iogpu.wired_limit_mb"])
+    try:
+        wired_mb = int(wired) if wired else 0
+        if wired_mb > 0:
+            vram_gb = round(wired_mb / 1024.0, 1)
+    except ValueError:
+        pass
+
+    gpu = {"index": 0, "name": brand, "vram_gb": vram_gb}
+    info = {
+        "gpu_name": brand,
+        "gpu_vram_gb": vram_gb,
+        "gpu_count": 1,
+        "gpus": [gpu],
+        "gpu_groups": _group_gpus([gpu]),
+        "homogeneous": True,
+        "backend": "metal",
+        # Unified memory: the "VRAM" above is carved out of system RAM, not a
+        # separate pool — downstream fit logic uses this to avoid double-budgeting.
+        "unified_memory": True,
+    }
+    if gpu_cores is not None:
+        info["gpu_cores"] = gpu_cores
+    return info
+
+
 def _read_file(path):
     """Read a file, locally or via SSH."""
     if _remote_host:
         return _run(["cat", path])
     try:
-        with open(path) as f:
+        with open(path, encoding="utf-8", errors="replace") as f:
             return f.read()
     except Exception:
         return None
@@ -238,7 +450,9 @@ def _get_ram_gb():
     if "MemTotal" in meminfo:
         return meminfo["MemTotal"] / (1024**2)
 
-    if not _remote_host:
+    # os.sysconf only exists on Unix; on Windows it's absent (AttributeError)
+    # and these constants aren't defined — guard so this never raises there.
+    if not _remote_host and hasattr(os, "sysconf") and "SC_PHYS_PAGES" in getattr(os, "sysconf_names", {}):
         try:
             pages = os.sysconf("SC_PHYS_PAGES")
             page_size = os.sysconf("SC_PAGE_SIZE")
@@ -246,6 +460,15 @@ def _get_ram_gb():
                 return (pages * page_size) / (1024**3)
         except Exception:
             pass
+
+    # macOS has no /proc/meminfo — fall back to sysctl (works locally and over
+    # SSH to a remote Mac, where the sysconf path above isn't taken).
+    memsize = _run(["sysctl", "-n", "hw.memsize"])
+    if memsize:
+        try:
+            return int(memsize.strip()) / (1024**3)
+        except ValueError:
+            pass
     return 0.0
 
 
@@ -263,6 +486,12 @@ def _get_cpu_name():
             if line.startswith("model name"):
                 return line.split(":", 1)[1].strip()
 
+    # macOS has no /proc/cpuinfo — sysctl gives the chip name (e.g. "Apple M4").
+    # Harmlessly returns nothing on Linux, so it's safe to try unconditionally.
+    brand = _run(["sysctl", "-n", "machdep.cpu.brand_string"])
+    if brand and brand.strip():
+        return brand.strip()
+
     if not _remote_host:
         return platform.processor() or "unknown"
     return "unknown"
@@ -270,7 +499,8 @@ def _get_cpu_name():
 
 def _get_cpu_count():
     if _remote_host:
-        out = _run(["nproc"])
+        # nproc on Linux; hw.ncpu via sysctl on a remote Mac (no nproc there).
+        out = _run(["nproc"]) or _run(["sysctl", "-n", "hw.ncpu"])
         if out:
             try:
                 return int(out.strip())
@@ -283,60 +513,156 @@ def _get_cpu_count():
     return os.cpu_count() or 1
 
 
+def _canonical_cpu_arch(value):
+    arch = str(value or "").lower().strip().replace("-", "_")
+    if arch in ("x86_64", "amd64", "x64"):
+        return "x86_64"
+    if arch in ("i386", "i686", "x86"):
+        return "x86"
+    if arch in ("arm64", "aarch64"):
+        return "arm64"
+    if arch == "arm" or arch.startswith("armv"):
+        return "arm"
+    return arch
+
+
+def _get_cpu_arch():
+    if _remote_host:
+        return _canonical_cpu_arch(_run(["uname", "-m"]) or "")
+    return _canonical_cpu_arch(platform.machine())
+
+
+def _powershell_exe():
+    """Pick the best PowerShell executable for LOCAL execution: prefer pwsh
+    (PowerShell 7+), fall back to Windows PowerShell 5.1. Returns an absolute
+    path so we don't depend on a particular PATH ordering."""
+    return shutil.which("pwsh") or shutil.which("powershell") or "powershell"
+
+def _powershell_encoded_for_ssh(script: str):
+    """Run a PowerShell script on a remote Windows host over SSH.
+
+    Nested quotes in powershell -Command break when passed through Windows
+    OpenSSH's cmd wrapper; -EncodedCommand avoids that.
+    """
+    import base64
+    encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii")
+    return _run(f"powershell -NoProfile -EncodedCommand {encoded}")
+
+
+def _probe_remote_platform():
+    """Best-effort OS detection over SSH when the caller didn't pass platform."""
+    out = _run("echo %OS%")
+    if out and "Windows_NT" in out:
+        return "windows"
+    uname = (_run(["uname", "-s"]) or "").strip().lower()
+    if uname == "darwin":
+        # Mac uses the linux detection path (_detect_apple_silicon over SSH).
+        return "linux"
+    if uname == "linux":
+        out = _run("test -d /data/data/com.termux && echo termux || echo linux")
+        if out and "termux" in out:
+            return "termux"
+    return "linux"
+
+
 def _detect_windows():
-    """Detect Windows hardware in a single SSH call using PowerShell."""
+    """Detect Windows hardware via PowerShell/WMI.
+
+    Works for BOTH local (host="") and remote (SSH) detection:
+      * remote  -> `_run` ships the string to the host over SSH.
+      * local   -> `_run` executes a list argv directly (no shell quoting hell).
+    """
     # Single PowerShell command that gathers all hardware info at once
     ps_cmd = (
-        "$r = @{}; "
-        "$os = Get-CimInstance Win32_OperatingSystem; "
-        "$r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1); "
-        "$r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1); "
-        "$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1; "
-        "$r.cpu_name = $cpu.Name; "
-        "$r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum; "
-        "$r.arch = $cpu.AddressWidth; "
+        """
+        $r = @{}
+        $os = Get-CimInstance Win32_OperatingSystem
+        $r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1)
+        $r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1)
+        $cpu = Get-CimInstance Win32_Processor | Select-Object -First 1
+        $r.cpu_name = $cpu.Name
+        $r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
+        $r.arch = $cpu.AddressWidth
+        $r.cpu_arch = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE }
         # GPU detection via nvidia-smi (fastest) or WMI fallback
-        "try { "
-        "  $nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null; "
-        "  if ($LASTEXITCODE -eq 0 -and $nv) { "
-        "    $gpus = @(); "
-        "    foreach ($line in $nv -split \"`n\") { "
-        "      $p = $line -split ','; "
-        "      if ($p.Count -ge 2) { $gpus += @{name=$p[1].Trim(); vram_mb=[double]$p[0].Trim()} } "
-        "    }; "
-        "    $r.gpu_name = $gpus[0].name; "
-        "    $r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1); "
-        "    $r.gpu_count = $gpus.Count; "
-        "    $r.gpu_backend = 'cuda'; "
-        "  } "
-        "} catch {}; "
-        "if (-not $r.gpu_name) { "
-        "  $wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1; "
-        "  if ($wmiGpu) { "
-        "    $r.gpu_name = $wmiGpu.Name; "
-        "    $r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1); "
-        "    $r.gpu_count = 1; "
-        "    $r.gpu_backend = 'cpu_x86'; "  # WMI doesn't tell us CUDA/ROCm
-        "  } "
-        "}; "
-        "$r | ConvertTo-Json -Compress"
+        try { 
+            $nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null
+            if ($LASTEXITCODE -eq 0 -and $nv) { 
+                $gpus = @()
+                foreach ($line in $nv -split "`n") { 
+                    $p = $line -split ','
+                    if ($p.Count -ge 2) { $gpus += [pscustomobject]@{name = $p[1].Trim(); vram_mb = [double]$p[0].Trim() } } 
+                }
+                $r.gpu_name = $gpus[0].name
+                $r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1)
+                $r.gpu_count = $gpus.Count
+                $r.gpu_backend = 'cuda'
+            } 
+        }
+        catch {}
+        if (-not $r.gpu_name) { 
+            $wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1
+            $GPUDriverKey = "HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0*"
+            $GPUDeviceID = $wmiGpu.PNPDeviceID.Split('&')[0..1] -join '&'
+            $VRAMfromRegistry = Get-ItemProperty -Path $GPUDriverKey |
+            Where-Object { $_.MatchingDeviceId -like "${GPUDeviceID}*" } |
+            # Sometimes there happen to be multiple driver classes for the same gpu.
+            Select-Object -ExpandProperty HardwareInformation.qwMemorySize -ErrorAction SilentlyContinue -First 1
+            if ($wmiGpu) { 
+                $r.gpu_name = $wmiGpu.Name
+                # Edge case: driver is broken, otherwise $wmiGpu.AdapterRAM is redundant
+                if ($VRAMfromRegistry -ge $wmiGpu.AdapterRAM) {
+                    $r.gpu_vram_gb = [math]::Round($VRAMfromRegistry / 1073741824, 1)
+                }
+                else {
+                    $r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1)
+                }
+                $r.gpu_count = 1
+                # WMI doesn't tell us CUDA/ROCm
+                $r.gpu_backend = 'cpu_x86';
+            } 
+        }
+        $r | ConvertTo-Json -Compress
+    """
     )
-    out = _run(f'powershell -Command "{ps_cmd}"')
+    if _remote_host:
+        # Remote: use -EncodedCommand so OpenSSH/cmd quoting does not break the script.
+        out = _powershell_encoded_for_ssh(ps_cmd.strip())
+    else:
+        # Local: pass a LIST argv straight to subprocess so the OS hands ps_cmd
+        # to PowerShell verbatim — no fragile string-level quote escaping. Prefer
+        # pwsh (PS7), else Windows PowerShell 5.1.
+        out = _run([_powershell_exe(), "-NoProfile", "-NonInteractive", "-Command", ps_cmd])
     if not out:
         return None
     import json as _json
     try:
         d = _json.loads(out)
+        # PowerShell's Measure-Object .Sum / .Count come back as JSON numbers and
+        # decode to float; the Linux path returns plain ints for these — coerce
+        # so the dict shape (and downstream int math) matches across platforms.
+        def _as_int(v, default):
+            try:
+                return int(v)
+            except (TypeError, ValueError):
+                return default
+        _cpu_name = (d.get("cpu_name") or "unknown")
+        if isinstance(_cpu_name, str):
+            _cpu_name = _cpu_name.strip() or "unknown"
         result = {
             "total_ram_gb": d.get("ram_gb", 0),
             "available_ram_gb": d.get("avail_gb", 0),
-            "cpu_cores": d.get("cpu_cores", 1),
-            "cpu_name": d.get("cpu_name", "unknown"),
+            "cpu_cores": _as_int(d.get("cpu_cores"), 1),
+            "cpu_name": _cpu_name,
+            "cpu_arch": _canonical_cpu_arch(d.get("cpu_arch")),
             "has_gpu": bool(d.get("gpu_name")),
             "gpu_name": d.get("gpu_name"),
             "gpu_vram_gb": d.get("gpu_vram_gb"),
-            "gpu_count": d.get("gpu_count", 0),
+            "gpu_count": _as_int(d.get("gpu_count"), 0),
             "backend": d.get("gpu_backend", "cpu_x86"),
+            "homogeneous": True,
+            "gpu_error": None,
+            "platform": "windows",
         }
         # PowerShell only reports aggregate GPU info, not per-card detail, so we
         # can't tell a mixed box from a uniform one here — assume one homogeneous
@@ -363,6 +689,106 @@ def _detect_windows():
 _cache_by_host = {}  # host -> (timestamp, result)
 
 
+def _cache_key(host: str, ssh_port: str, platform_name: str):
+    """Build a stable cache key that isolates remote SSH context.
+
+    Same host aliases can have different hardware due to visibility, forwarding etc.
+    To avoid using the wrong cached hardware info, include the SSH port and platform in the cache key.
+    """
+    return (
+        host or "_local",
+        str(ssh_port or ""),
+        str(platform_name or "").lower(),
+    )
+
+
+def _is_containerized():
+    """Best-effort check for whether the local Odysseus process is running in a container."""
+    if _remote_host:
+        return False
+
+    if os.path.exists("/.dockerenv"):
+        return True
+
+    try:
+        with open("/proc/1/cgroup", encoding="utf-8", errors="replace") as f:
+            text = f.read().lower()
+        return any(marker in text for marker in ("docker", "containerd", "kubepods"))
+    except Exception:
+        return False
+
+
+def _hardware_visibility_warning(result):
+    """Return a non-blocking UX warning when detected hardware may only be container-visible."""
+    if not isinstance(result, dict):
+        return None
+
+    if result.get("manual_hardware"):
+        return None
+
+    if not result.get("containerized"):
+        return None
+
+    if result.get("gpu_error"):
+        return None
+
+    if not result.get("has_gpu"):
+        return {
+            "code": "container_no_gpu_visible",
+            "severity": "warning",
+            "title": "No GPU visible inside Docker",
+            "message": (
+                "Cookbook is scanning hardware from inside the Odysseus container. "
+                "If your host has a GPU, Docker may not be exposing it to the container, "
+                "so model recommendations may be CPU-only or too conservative."
+            ),
+            "actions": [
+                "manual_hardware",
+                "rescan",
+                "copy_diagnostics",
+            ],
+        }
+
+    total_ram = result.get("total_ram_gb") or 0
+    if total_ram and total_ram <= 8:
+        return {
+            "code": "container_low_ram_visible",
+            "severity": "info",
+            "title": "Container-visible RAM may be lower than host RAM",
+            "message": (
+                "Cookbook is seeing the RAM available inside the container. "
+                "If your host has more memory, validate host RAM separately or use Manual Hardware."
+            ),
+            "actions": [
+                "manual_hardware",
+                "rescan",
+                "copy_diagnostics",
+            ],
+        }
+
+    return None
+
+
+def _attach_probe_context(result, host=""):
+    """Attach probe-scope metadata and optional hardware visibility warning."""
+    if not isinstance(result, dict) or result.get("error"):
+        return result
+
+    is_remote = bool(host)
+    containerized = False if is_remote else _is_containerized()
+
+    result["probe_scope"] = "remote" if is_remote else ("container" if containerized else "native")
+    result["containerized"] = containerized
+
+    warning = _hardware_visibility_warning(result)
+    if warning:
+        result["hardware_visibility_warning"] = warning
+    else:
+        result.pop("hardware_visibility_warning", None)
+
+    return result
+
+
 def detect_system(host="", ssh_port="", platform="", fresh=False):
     """Detect system hardware: RAM, CPU, GPU. Cached per host (hardware rarely
     changes, and probing a remote host over SSH is slow). Pass fresh=True to
@@ -372,7 +798,14 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     """
     global _remote_host, _remote_port, _remote_platform
 
-    cache_key = host or "_local"
+    if host and not platform:
+        _remote_host = host
+        _remote_port = ssh_port or None
+        platform = _probe_remote_platform()
+        _remote_host = None
+        _remote_port = None
+
+    cache_key = _cache_key(host, ssh_port, platform)
     now = time.time()
     if not fresh and cache_key in _cache_by_host:
         ts, cached = _cache_by_host[cache_key]
@@ -387,17 +820,31 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     if _remote_platform == "windows" and _remote_host:
         result = _detect_windows()
         if result:
+            result = _attach_probe_context(result, host=host)
             _remote_host = None
             _remote_platform = None
             _cache_by_host[cache_key] = (now, result)
             return result
-        # If Windows detection failed, return error
-        result = {"error": f"Cannot connect to {host}", "host": host}
+        # SSH may work while the PowerShell hardware probe still fails.
+        result = {"error": f"Windows hardware probe failed for {host}", "host": host}
         _remote_host = None
         _remote_platform = None
         _cache_by_host[cache_key] = (now, result)
         return result
 
+    # Local Windows: the Linux /proc + /sys + os.sysconf path returns 0 GB RAM,
+    # "unknown" CPU and no GPU on Windows (and os.sysconf doesn't even exist),
+    # so detect locally via PowerShell/WMI instead. _detect_windows() runs the
+    # same probe used for remote Windows, but _run() executes it locally.
+    if not _remote_host and os.name == "nt":
+        result = _detect_windows()
+        if result:
+            result = _attach_probe_context(result, host=host)
+            _cache_by_host[cache_key] = (now, result)
+            return result
+        # PowerShell probe failed entirely — fall through to the generic path
+        # below so we at least return a well-shaped dict rather than crashing.
+
     # Linux/Termux: existing multi-command detection
     total_ram = round(_get_ram_gb(), 1)
     # If remote host returns 0 RAM, connection likely failed
@@ -410,8 +857,9 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     available_ram = round(_get_available_ram_gb(), 1)
     cpu_cores = _get_cpu_count()
     cpu_name = _get_cpu_name()
+    cpu_arch = _get_cpu_arch()
 
-    gpu_info = _detect_nvidia() or _detect_amd()
+    gpu_info = _detect_apple_silicon() or _detect_nvidia() or _detect_amd()
 
     if gpu_info:
         result = {
@@ -419,27 +867,28 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
             "available_ram_gb": available_ram,
             "cpu_cores": cpu_cores,
             "cpu_name": cpu_name,
+            "cpu_arch": cpu_arch,
             "has_gpu": True,
             "gpu_name": gpu_info["gpu_name"],
             "gpu_vram_gb": gpu_info["gpu_vram_gb"],
             "gpu_count": gpu_info["gpu_count"],
+            "gpu_cores": gpu_info.get("gpu_cores"),
             "gpus": gpu_info.get("gpus", []),
             "gpu_groups": gpu_info.get("gpu_groups", []),
             "homogeneous": gpu_info.get("homogeneous", True),
             "backend": gpu_info["backend"],
+            # Apple Silicon / AMD APUs share system RAM with the GPU — carry the
+            # flag through so callers can tell unified from discrete VRAM.
+            "unified_memory": gpu_info.get("unified_memory", False),
         }
     else:
-        if _remote_host:
-            arch_out = _run(["uname", "-m"]) or ""
-        else:
-            import platform as _platform
-            arch_out = _platform.machine().lower()
-        backend = "cpu_arm" if "aarch64" in arch_out or "arm" in arch_out else "cpu_x86"
+        backend = "cpu_arm" if cpu_arch == "arm64" else "cpu_x86"
         result = {
             "total_ram_gb": total_ram,
             "available_ram_gb": available_ram,
             "cpu_cores": cpu_cores,
             "cpu_name": cpu_name,
+            "cpu_arch": cpu_arch,
             "has_gpu": False,
             "gpu_name": None,
             "gpu_vram_gb": None,
@@ -451,6 +900,7 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
             "gpu_error": _last_gpu_error,
         }
 
+    result = _attach_probe_context(result, host=host)
     _remote_host = None
     _remote_platform = None
     _cache_by_host[cache_key] = (now, result)

services/hwfit/hf_discovery.py

@@ -0,0 +1,374 @@
+import json
+import os
+import re
+import time
+import urllib.parse
+import urllib.request
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+from src.constants import DATA_DIR
+
+
+HF_COLLECTIONS_URL = "https://huggingface.co/api/collections"
+HW_FIT_CACHE_DIR = Path(DATA_DIR) / "hwfit"
+MLX_COMMUNITY_CACHE = HW_FIT_CACHE_DIR / "mlx_community_models.json"
+HF_COLLECTION_MODELS_CACHE = HW_FIT_CACHE_DIR / "hf_collection_models.json"
+HF_COLLECTION_TTL_SECONDS = 24 * 3600
+
+
+HF_COLLECTION_SOURCES = (
+    {
+        "key": "mlx_community",
+        "owner": "mlx-community",
+        "provider": "mlx-community",
+        "repo_prefix": "mlx-community/",
+        "mlx_only": True,
+    },
+    {
+        "key": "zai_org",
+        "owner": "zai-org",
+        "provider": "zai-org",
+    },
+    {
+        "key": "deepseek_ai",
+        "owner": "deepseek-ai",
+        "provider": "deepseek-ai",
+    },
+    {
+        "key": "minimax_ai",
+        "owner": "MiniMaxAI",
+        "provider": "MiniMaxAI",
+    },
+    {
+        "key": "qwen",
+        "owner": "Qwen",
+        "provider": "Qwen",
+    },
+    {
+        "key": "stepfun_ai",
+        "owner": "stepfun-ai",
+        "provider": "stepfun-ai",
+    },
+    {
+        "key": "google",
+        "owner": "google",
+        "provider": "google",
+    },
+    {
+        "key": "openai",
+        "owner": "openai",
+        "provider": "openai",
+    },
+    {
+        "key": "mistralai",
+        "owner": "mistralai",
+        "provider": "mistralai",
+    },
+    {
+        "key": "meta_llama",
+        "owner": "meta-llama",
+        "provider": "meta-llama",
+    },
+    {
+        "key": "nousresearch",
+        "owner": "NousResearch",
+        "provider": "NousResearch",
+    },
+    {
+        "key": "moonshotai",
+        "owner": "moonshotai",
+        "provider": "moonshotai",
+    },
+    {
+        "key": "mllama",
+        "owner": "mllama",
+        "provider": "mllama",
+    },
+)
+
+
+def _format_params(raw):
+    try:
+        n = int(raw or 0)
+    except (TypeError, ValueError):
+        n = 0
+    if n <= 0:
+        return "", 0
+    if n >= 1_000_000_000_000:
+        return f"{n / 1_000_000_000_000:.3g}T", n
+    if n >= 1_000_000_000:
+        return f"{n / 1_000_000_000:.4g}B", n
+    if n >= 1_000_000:
+        return f"{n / 1_000_000:.4g}M", n
+    if n >= 1_000:
+        return f"{n / 1_000:.4g}K", n
+    return str(n), n
+
+
+def _parse_params_from_name(repo_id):
+    name = (repo_id or "").rsplit("/", 1)[-1]
+    active = None
+    m_active = re.search(r"[-_][Aa](\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name)
+    if m_active:
+        active = int(float(m_active.group(1)) * 1_000_000_000)
+        name = name[: m_active.start()] + name[m_active.end() :]
+    total = None
+    for m in re.finditer(r"(\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name):
+        total = int(float(m.group(1)) * 1_000_000_000)
+        break
+    if total is None:
+        for m in re.finditer(r"(\d+(?:\.\d+)?)[Mm](?![a-zA-Z])", name):
+            total = int(float(m.group(1)) * 1_000_000)
+            break
+    return total or 0, active
+
+
+def _infer_quant(repo_id, source):
+    name = (repo_id or "").rsplit("/", 1)[-1].lower()
+    if source.get("mlx_only"):
+        if "8bit" in name or "8-bit" in name:
+            return "mlx-8bit"
+        if "6bit" in name or "6-bit" in name:
+            return "mlx-6bit"
+        if "5bit" in name or "5-bit" in name:
+            return "mlx-5bit"
+        if "3bit" in name or "3-bit" in name:
+            return "mlx-3bit"
+        if re.search(r"(^|[-_/])bf16($|[-_/])", name):
+            return "BF16"
+        return "mlx-4bit"
+    if "awq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
+        return "AWQ-8bit"
+    if "awq" in name or "4bit" in name or "4-bit" in name:
+        return "AWQ-4bit"
+    if "gptq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
+        return "GPTQ-Int8"
+    if "gptq" in name:
+        return "GPTQ-Int4"
+    if "mxfp4" in name or "nvfp4" in name or re.search(r"(^|[-_/])fp4($|[-_/])", name):
+        return "FP4-MoE-Mixed"
+    if "mxfp8" in name or re.search(r"(^|[-_/])fp8($|[-_/])", name):
+        return "FP8-Mixed"
+    if "gguf" in name or "q4_k" in name or "q4-k" in name:
+        return "Q4_K_M"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", name):
+        return "BF16"
+    return "BF16"
+
+
+def _quant_bytes_per_param(quant):
+    return {
+        "BF16": 2.2,
+        "FP8": 1.15,
+        "FP8-Mixed": 1.15,
+        "FP4-MoE-Mixed": 0.62,
+        "AWQ-4bit": 0.62,
+        "AWQ-8bit": 1.15,
+        "GPTQ-Int4": 0.62,
+        "GPTQ-Int8": 1.15,
+        "Q4_K_M": 0.62,
+        "mlx-8bit": 1.25,
+        "mlx-6bit": 0.95,
+        "mlx-5bit": 0.82,
+        "mlx-4bit": 0.70,
+        "mlx-3bit": 0.55,
+    }.get(quant, 2.2)
+
+
+def _infer_context(repo_id, pipeline_tag):
+    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
+    if any(k in text for k in ("whisper", "asr", "speech-recognition", "tts", "audio", "image", "video", "diffusion")):
+        return 4096
+    if any(k in text for k in ("glm-5.2", "deepseek-v4", "minimax-m3")):
+        return 1_000_000
+    if any(k in text for k in ("qwen3", "glm", "deepseek", "minimax")):
+        return 32768
+    return 32768
+
+
+def _infer_use_case(repo_id, pipeline_tag):
+    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
+    if any(k in text for k in ("whisper", "asr", "speech-recognition", "transcrib")):
+        return "stt"
+    if any(k in text for k in ("tts", "text-to-speech", "kokoro", "audio")):
+        return "tts"
+    if any(k in text for k in ("image-text", "vision", "vlm", "vl-", "ocr", "multimodal")):
+        return "multimodal"
+    if any(k in text for k in ("code", "coder")):
+        return "coding"
+    if any(k in text for k in ("reason", "thinking", "thinker", "r1")):
+        return "reasoning"
+    return "general"
+
+
+def _entry_from_collection_item(collection, item, source):
+    repo_id = item.get("id") or ""
+    if item.get("type") != "model" or not repo_id:
+        return None
+    repo_prefix = source.get("repo_prefix")
+    if repo_prefix and not repo_id.startswith(repo_prefix):
+        return None
+    raw_params = item.get("numParameters") or 0
+    active = None
+    if not raw_params:
+        raw_params, active = _parse_params_from_name(repo_id)
+    param_label, raw_params = _format_params(raw_params)
+    if not raw_params:
+        return None
+
+    quant = _infer_quant(repo_id, source)
+    pipeline_tag = item.get("pipeline_tag") or ""
+    min_ram = round((raw_params / 1_000_000_000) * _quant_bytes_per_param(quant) + 0.8, 1)
+    last_modified = item.get("lastModified") or collection.get("lastUpdated") or ""
+    release_date = ""
+    if last_modified:
+        try:
+            release_date = parsedate_to_datetime(last_modified).date().isoformat()
+        except Exception:
+            release_date = str(last_modified)[:10]
+
+    entry = {
+        "name": repo_id,
+        "provider": source.get("provider") or repo_id.split("/", 1)[0],
+        "parameter_count": param_label,
+        "parameters_raw": raw_params,
+        "min_ram_gb": min_ram,
+        "recommended_ram_gb": round(min_ram * 1.3 + 0.5, 1),
+        "min_vram_gb": 0.0 if source.get("mlx_only") else min_ram,
+        "quantization": quant,
+        "context_length": _infer_context(repo_id, pipeline_tag),
+        "use_case": _infer_use_case(repo_id, pipeline_tag),
+        "capabilities": ["mlx"] if source.get("mlx_only") else ["vllm", "sglang"],
+        "pipeline_tag": pipeline_tag,
+        "architecture": "",
+        "hf_downloads": int(item.get("downloads") or 0),
+        "hf_likes": int(item.get("likes") or 0),
+        "release_date": release_date,
+        "format": "mlx" if source.get("mlx_only") else "safetensors",
+        "collection": collection.get("title") or "",
+        "description": collection.get("description") or "",
+        "_discovered": True,
+        "_source": "hf_collections",
+        "_source_owner": source.get("owner") or "",
+    }
+    if source.get("mlx_only"):
+        entry["mlx_only"] = True
+    if quant == "Q4_K_M":
+        entry["is_gguf"] = True
+        entry["format"] = "gguf"
+        entry["capabilities"] = ["llama.cpp"]
+    if active:
+        entry["is_moe"] = True
+        entry["active_parameters"] = active
+    return entry
+
+
+def _next_link(header):
+    if not header:
+        return None
+    m = re.search(r'<([^>]+)>;\s*rel="next"', header)
+    return m.group(1) if m else None
+
+
+def fetch_collection_models(source, timeout=20, max_pages=20):
+    params = urllib.parse.urlencode({
+        "owner": source["owner"],
+        "limit": "100",
+        "expand": "true",
+    })
+    url = f"{HF_COLLECTIONS_URL}?{params}"
+    models = {}
+    pages = 0
+    while url and pages < max_pages:
+        req = urllib.request.Request(url, headers={"User-Agent": "odysseus-hwfit/1.0"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            payload = json.load(resp)
+            url = _next_link(resp.headers.get("Link"))
+        pages += 1
+        if not isinstance(payload, list):
+            break
+        for collection in payload:
+            if not isinstance(collection, dict):
+                continue
+            for item in collection.get("items") or []:
+                if not isinstance(item, dict):
+                    continue
+                entry = _entry_from_collection_item(collection, item, source)
+                if entry and entry["name"] not in models:
+                    models[entry["name"]] = entry
+    rows = list(models.values())
+    rows.sort(key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""), reverse=True)
+    return rows
+
+
+def _load_cache(path):
+    try:
+        with path.open(encoding="utf-8") as f:
+            data = json.load(f)
+        rows = data.get("models") if isinstance(data, dict) else data
+        return rows if isinstance(rows, list) else []
+    except (OSError, ValueError):
+        return []
+
+
+def _write_cache(path, source, rows):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "source": source,
+        "fetched_at": int(time.time()),
+        "count": len(rows),
+        "models": rows,
+    }
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def load_cached_mlx_community_models():
+    return _load_cache(MLX_COMMUNITY_CACHE)
+
+
+def load_cached_hf_collection_models():
+    return _load_cache(HF_COLLECTION_MODELS_CACHE)
+
+
+def _cache_fresh(path):
+    try:
+        return (time.time() - path.stat().st_mtime) < HF_COLLECTION_TTL_SECONDS
+    except OSError:
+        return False
+
+
+def refresh_mlx_community_cache(force=False):
+    if not force and _cache_fresh(MLX_COMMUNITY_CACHE):
+        return load_cached_mlx_community_models()
+    source = next(s for s in HF_COLLECTION_SOURCES if s["key"] == "mlx_community")
+    rows = fetch_collection_models(source)
+    _write_cache(MLX_COMMUNITY_CACHE, "https://huggingface.co/mlx-community/collections", rows)
+    return rows
+
+
+def refresh_hf_collection_models_cache(force=False):
+    if not force and _cache_fresh(HF_COLLECTION_MODELS_CACHE):
+        return load_cached_hf_collection_models()
+    rows_by_name = {}
+    for source in HF_COLLECTION_SOURCES:
+        if source["key"] == "mlx_community":
+            continue
+        try:
+            for row in fetch_collection_models(source):
+                rows_by_name.setdefault(row["name"], row)
+        except Exception:
+            # Keep partial refreshes useful. A temporary DNS/provider issue for
+            # one brand should not invalidate the other cached collection rows.
+            continue
+    rows = sorted(
+        rows_by_name.values(),
+        key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""),
+        reverse=True,
+    )
+    if rows:
+        _write_cache(HF_COLLECTION_MODELS_CACHE, "https://huggingface.co/collections", rows)
+        return rows
+    return load_cached_hf_collection_models()

services/hwfit/image_models.py

@@ -1,278 +1,369 @@
 """Image generation model registry and VRAM fitting for Cookbook."""
 
-# Curated registry of image generation models supported by diffusers.
-# ONLY verified HuggingFace repo IDs.
-# VRAM estimates are for inference (single image generation).
-IMAGE_MODEL_REGISTRY = [
-    # ── Z-Image (Alibaba Tongyi) ──
-    {
-        "id": "Tongyi-MAI/Z-Image-Turbo",
-        "name": "Z-Image Turbo",
-        "provider": "Tongyi",
-        "params_b": 6.0,
-        "vram_bf16": 19.0,
-        "vram_fp8": 10.0,
-        "vram_q4": 6.0,
-        "default_quant": "BF16",
-        "quant_repos": {
-            "FP8": "drbaph/Z-Image-Turbo-FP8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "6B distilled, 8-step. Sub-second on H800. Apache 2.0.",
-        "quality": 92,
-        "speed": 95,
-        "released": "2025-12",
-    },
-    {
-        "id": "Tongyi-MAI/Z-Image",
-        "name": "Z-Image",
-        "provider": "Tongyi",
-        "params_b": 6.0,
-        "vram_bf16": 19.0,
-        "vram_fp8": 10.0,
-        "vram_q4": 6.0,
-        "default_quant": "BF16",
-        "quant_repos": {
-            "FP8": "drbaph/Z-Image-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "Full undistilled model. Highest creative freedom. Apache 2.0.",
-        "quality": 93,
-        "speed": 70,
-        "released": "2025-12",
-    },
-    # ── Qwen Image ──
-    {
-        "id": "Qwen/Qwen-Image-2512",
-        "name": "Qwen Image 2512",
-        "provider": "Qwen",
-        "params_b": 20.0,
-        "vram_bf16": 42.0,
-        "vram_fp8": 22.0,
-        "vram_q4": 14.0,
-        "default_quant": "FP8",
-        "quant_repos": {},
-        "capabilities": ["text-to-image", "text-rendering"],
-        "description": "Dec 2025 update. Better humans, finer detail, strong text. Apache 2.0.",
-        "quality": 95,
-        "speed": 50,
-        "released": "2025-12",
-    },
-    {
-        "id": "Qwen/Qwen-Image",
-        "name": "Qwen Image",
-        "provider": "Qwen",
-        "params_b": 20.0,
-        "vram_bf16": 42.0,
-        "vram_fp8": 22.0,
-        "vram_q4": 14.0,
-        "default_quant": "FP8",
-        "quant_repos": {},
-        "capabilities": ["text-to-image", "text-rendering"],
-        "description": "20B foundation. Best text rendering in images. Apache 2.0.",
-        "quality": 94,
-        "speed": 50,
-        "released": "2025-08",
-    },
-    {
-        "id": "Qwen/Qwen-Image-Edit-2511",
-        "name": "Qwen Image Edit",
-        "provider": "Qwen",
-        "params_b": 20.0,
-        "vram_bf16": 42.0,
-        "vram_fp8": 22.0,
-        "vram_q4": 14.0,
-        "default_quant": "FP8",
-        "quant_repos": {},
-        "capabilities": ["image-editing", "inpainting"],
-        "description": "Dedicated editing. Style transfer, object removal. Apache 2.0.",
-        "quality": 92,
-        "speed": 50,
-        "released": "2025-11",
-    },
-    # ── Stable Diffusion (dedicated inpainting) ──
-    {
-        "id": "diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
-        "name": "SDXL Inpainting",
-        "provider": "Stability AI",
-        "params_b": 3.5,
-        "vram_bf16": 12.0,
-        "vram_fp8": 8.0,
-        "vram_q4": 6.0,
-        "default_quant": "BF16",
-        "quant_repos": {},
-        "capabilities": ["inpainting", "image-editing"],
-        "description": "SDXL fine-tuned for inpainting (9-channel UNet). Best SD-family fill quality; fits a 24GB card comfortably.",
-        "quality": 86,
-        "speed": 68,
-        "released": "2023-11",
-    },
-    {
-        "id": "stable-diffusion-v1-5/stable-diffusion-inpainting",
-        "name": "SD 1.5 Inpainting",
-        "provider": "Stability AI",
-        "params_b": 1.1,
-        "vram_bf16": 4.0,
-        "vram_fp8": 3.0,
-        "vram_q4": 2.5,
-        "default_quant": "BF16",
-        "quant_repos": {},
-        "capabilities": ["inpainting"],
-        "description": "Classic SD 1.5 inpaint. Very light and fast; lower fidelity than SDXL.",
-        "quality": 70,
-        "speed": 92,
-        "released": "2022-10",
-    },
-    # ── FLUX ──
-    {
-        "id": "black-forest-labs/FLUX.1-dev",
-        "name": "FLUX.1 Dev",
-        "provider": "Black Forest Labs",
-        "params_b": 12.0,
-        "vram_bf16": 33.0,
-        "vram_fp8": 17.0,
-        "vram_q4": 10.0,
-        "default_quant": "FP8",
-        "quant_repos": {
-            "FP8": "diffusers/FLUX.1-dev-torchao-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "High quality, detailed. Popular community model. Non-commercial.",
-        "quality": 92,
-        "speed": 55,
-        "released": "2024-08",
-    },
-    {
-        "id": "black-forest-labs/FLUX.1-schnell",
-        "name": "FLUX.1 Schnell",
-        "provider": "Black Forest Labs",
-        "params_b": 12.0,
-        "vram_bf16": 33.0,
-        "vram_fp8": 17.0,
-        "vram_q4": 10.0,
-        "default_quant": "FP8",
-        "quant_repos": {
-            "FP8": "Kijai/flux-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "Fast 4-step variant. Apache 2.0 license.",
-        "quality": 85,
-        "speed": 90,
-        "released": "2024-08",
-    },
-    # ── Stable Diffusion ──
-    {
-        "id": "stabilityai/stable-diffusion-3.5-medium",
-        "name": "SD 3.5 Medium",
-        "provider": "Stability AI",
-        "params_b": 2.5,
-        "vram_bf16": 12.0,
-        "vram_fp8": 7.0,
-        "vram_q4": None,
-        "default_quant": "BF16",
-        "quant_repos": {
-            "FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "2.5B lightweight, fast. Fits almost any GPU.",
-        "quality": 75,
-        "speed": 95,
-        "released": "2024-10",
-    },
-    {
-        "id": "stabilityai/stable-diffusion-3.5-large",
-        "name": "SD 3.5 Large",
-        "provider": "Stability AI",
-        "params_b": 8.1,
-        "vram_bf16": 22.0,
-        "vram_fp8": 12.0,
-        "vram_q4": None,
-        "default_quant": "BF16",
-        "quant_repos": {
-            "FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "8B high quality. Good balance of speed and quality.",
-        "quality": 85,
-        "speed": 70,
-        "released": "2024-10",
-    },
-    {
-        "id": "stabilityai/stable-diffusion-3.5-large-turbo",
-        "name": "SD 3.5 Large Turbo",
-        "provider": "Stability AI",
-        "params_b": 8.1,
-        "vram_bf16": 22.0,
-        "vram_fp8": 12.0,
-        "vram_q4": None,
-        "default_quant": "BF16",
-        "quant_repos": {
-            "FP8": "Comfy-Org/stable-diffusion-3.5-fp8",
-        },
-        "capabilities": ["text-to-image"],
-        "description": "Distilled for few-step inference. Fastest large SD.",
-        "quality": 80,
-        "speed": 92,
-        "released": "2024-10",
-    },
-    {
-        "id": "stabilityai/stable-diffusion-xl-base-1.0",
-        "name": "SDXL",
-        "provider": "Stability AI",
-        "params_b": 3.5,
-        "vram_bf16": 10.0,
-        "vram_fp8": 6.0,
-        "vram_q4": None,
-        "default_quant": "BF16",
-        "quant_repos": {},
-        "capabilities": ["text-to-image"],
-        "description": "Classic workhorse. Huge LoRA ecosystem. Fits 8GB+.",
-        "quality": 72,
-        "speed": 90,
-        "released": "2023-07",
-    },
-    # ── Hunyuan ──
-    {
-        "id": "tencent/HunyuanImage-3.0",
-        "name": "HunyuanImage 3.0",
-        "provider": "Tencent",
-        "params_b": 13.0,
-        "vram_bf16": 30.0,
-        "vram_fp8": 16.0,
-        "vram_q4": 9.0,
-        "default_quant": "FP8",
-        "quant_repos": {
-            "Q4": "wikeeyang/Hunyuan-Image-30-Qint4",
-            "NF4": "EricRollei/HunyuanImage-3.0-Instruct-NF4",
-        },
-        "capabilities": ["text-to-image", "text-rendering"],
-        "description": "Strong text rendering. Bilingual Chinese/English. 13B activated per token.",
-        "quality": 88,
-        "speed": 60,
-        "released": "2025-09",
-    },
-    {
-        "id": "tencent/HunyuanImage-3.0-Instruct-Distil",
-        "name": "HunyuanImage 3.0 Distil",
-        "provider": "Tencent",
-        "params_b": 13.0,
-        "vram_bf16": 30.0,
-        "vram_fp8": 16.0,
-        "vram_q4": 9.0,
-        "default_quant": "FP8",
-        "quant_repos": {},
-        "capabilities": ["text-to-image", "text-rendering"],
-        "description": "Distilled variant, fewer steps. Faster with comparable quality.",
-        "quality": 85,
-        "speed": 80,
-        "released": "2026-01",
-    },
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from src.constants import DATA_DIR
+
+# Image models are discovered from HuggingFace collections/search and local cache.
+# Keep this empty: source-coded repo IDs become hidden recommendations.
+IMAGE_MODEL_REGISTRY: list[dict[str, Any]] = []
+
+HF_IMAGE_COLLECTIONS = [
+    "stabilityai/image",
+    "stabilityai/stable-diffusion-35",
+    "black-forest-labs/flux2",
 ]
 
+HF_MLX_IMAGE_COLLECTIONS = [
+    "mlx-community/flux2-klein-mlx",
+    "mlx-community/inpainting-mlx",
+    "mlx-community/ddcolor-mlx",
+    "mlx-community/boogu-image-01-mlx",
+]
+
+HF_MLX_IMAGE_REPO_SEEDS: list[str] = []
+HF_IMAGE_REPO_SEEDS: list[str] = []
+
+_HF_COLLECTION_CACHE = {"ts": 0.0, "models": []}
+_HF_COLLECTION_TTL = 30 * 60
+_IMAGE_COLLECTION_DISK_CACHE = Path(DATA_DIR) / "hwfit" / "image_collection_models.json"
+_IMAGE_COLLECTION_DISK_TTL = 24 * 3600
+_HF_VARIANT_CACHE: dict[str, dict[str, str]] = {}
+_HF_SEARCH_DISABLED_UNTIL = 0.0
+
+
+def _repo_display_name(repo_id: str) -> str:
+    name = str(repo_id or "").split("/")[-1]
+    return name.replace("-", " ").replace("_", " ").strip() or repo_id
+
+
+def _provider_from_repo(repo_id: str) -> str:
+    owner = str(repo_id or "").split("/", 1)[0].lower()
+    return {
+        "stabilityai": "Stability AI",
+        "black-forest-labs": "Black Forest Labs",
+        "tongyi-mai": "Tongyi",
+        "qwen": "Qwen",
+        "mlx-community": "mlx-community",
+    }.get(owner, owner.replace("-", " ").title() if owner else "HuggingFace")
+
+
+def _infer_capabilities(item: dict[str, Any], repo_id: str) -> list[str]:
+    tasks = set()
+    pipeline = str(item.get("pipeline_tag") or "").strip().lower()
+    if pipeline:
+        tasks.add(pipeline)
+    for provider in item.get("availableInferenceProviders") or []:
+        if isinstance(provider, dict) and provider.get("task"):
+            tasks.add(str(provider["task"]).strip().lower())
+    text = f"{repo_id} {' '.join(tasks)}".lower()
+    caps = []
+    if "image-to-image" in tasks or "edit" in text or "inpaint" in text:
+        caps.append("image-editing")
+    if "inpaint" in text:
+        caps.append("inpainting")
+    if "text-to-image" in tasks or not caps:
+        caps.append("text-to-image")
+    return caps
+
+
+def _estimate_image_model(repo_id: str) -> dict[str, Any]:
+    text = str(repo_id or "").lower()
+    params_b = 8.0
+    param_match = re.search(r"(?<![\d.])(\d+(?:\.\d+)?)\s*b(?:\b|[-_])", text)
+    if param_match:
+        params_b = max(0.01, float(param_match.group(1)))
+    if any(k in text for k in ("mi-gan", "big-lama", "lama-")):
+        return {"params_b": 0.01, "bf16": 1.0, "fp8": 0.7, "q4": 0.5, "quality": 65, "speed": 98, "quant": "BF16"}
+    quant = "BF16"
+    if any(k in text for k in ("4bit", "q4", "nf4")):
+        quant = "Q4"
+    elif "fp8" in text or "8bit" in text:
+        quant = "FP8"
+    bf16 = max(1.0, round(params_b * 2.6 + 3.0, 1))
+    fp8 = max(0.7, round(params_b * 1.35 + 2.0, 1))
+    q4 = max(0.5, round(params_b * 0.8 + 1.5, 1))
+    speed = max(35, min(95, int(98 - params_b * 3)))
+    quality = max(60, min(88, int(70 + min(params_b, 18) * 0.8)))
+    return {"params_b": params_b, "bf16": bf16, "fp8": fp8, "q4": q4, "quality": quality, "speed": speed, "quant": quant}
+
+
+def _params_b_from_item(item: dict[str, Any]) -> float | None:
+    raw = item.get("numParameters")
+    if isinstance(raw, (int, float)) and raw > 0:
+        return max(0.01, round(float(raw) / 1_000_000_000.0, 3))
+    return None
+
+
+def _mlx_quantize_estimate(repo_id: str, est: dict[str, Any]) -> dict[str, Any]:
+    text = str(repo_id or "").lower()
+    out = dict(est)
+    if "3bit" in text or "4bit" in text or "q4" in text:
+        out["quant"] = "Q4"
+        out["bf16"] = None
+        out["fp8"] = None
+    elif "8bit" in text:
+        out["quant"] = "FP8"
+        out["bf16"] = None
+    elif "6bit" in text or "5bit" in text:
+        out["quant"] = "Q4"
+        out["bf16"] = None
+        out["fp8"] = out.get("fp8") or out.get("q4")
+    elif "bf16" in text or "fp16" in text:
+        out["quant"] = "BF16"
+        out["fp8"] = None
+        out["q4"] = None
+    return out
+
+
+def _collection_item_to_model(item: dict[str, Any], collection_title: str = "", mlx_only: bool = False) -> dict[str, Any] | None:
+    repo_id = str(item.get("id") or "").strip()
+    if "/" not in repo_id:
+        return None
+    typ = str(item.get("type") or item.get("itemType") or "model").lower()
+    if typ not in {"", "model"}:
+        return None
+    est = _estimate_image_model(repo_id)
+    item_params_b = _params_b_from_item(item)
+    if item_params_b is not None:
+        est = {
+            **est,
+            "params_b": item_params_b,
+            "bf16": max(0.5, round(item_params_b * 2.4 + 0.8, 1)),
+            "fp8": max(0.5, round(item_params_b * 1.3 + 0.5, 1)),
+            "q4": max(0.4, round(item_params_b * 0.8 + 0.4, 1)),
+        }
+    if mlx_only:
+        est = _mlx_quantize_estimate(repo_id, est)
+    caps = _infer_capabilities(item, repo_id)
+    gated = item.get("gated")
+    desc_bits = []
+    if collection_title:
+        desc_bits.append(f"HF collection: {collection_title}.")
+    if gated:
+        desc_bits.append("Gated on HuggingFace.")
+    out = {
+        "id": repo_id,
+        "name": _repo_display_name(repo_id),
+        "provider": _provider_from_repo(repo_id),
+        "params_b": est["params_b"],
+        "vram_bf16": est["bf16"],
+        "vram_fp8": est["fp8"],
+        "vram_q4": est["q4"],
+        "default_quant": est["quant"],
+        "quant_repos": {},
+        "capabilities": caps,
+        "description": " ".join(desc_bits).strip() or "Imported from HuggingFace collection.",
+        "quality": est["quality"],
+        "speed": est["speed"],
+        "released": "",
+    }
+    # Optional catalog metadata may identify a non-default runtime package.
+    # Keep this data-driven: the fitter must not infer private/model-specific
+    # dependencies from repository names.
+    dependency_package = item.get("dependency_package") or item.get("runtime_dependency")
+    if isinstance(dependency_package, str) and dependency_package.strip():
+        out["dependency_package"] = dependency_package.strip()
+    if mlx_only:
+        out["mlx_only"] = True
+        out["description"] = (out["description"] + " Apple Silicon / MLX only.").strip()
+    return out
+
+
+def _fetch_hf_image_collection_models() -> list[dict[str, Any]]:
+    now = time.time()
+    if now - float(_HF_COLLECTION_CACHE.get("ts") or 0) < _HF_COLLECTION_TTL:
+        return list(_HF_COLLECTION_CACHE.get("models") or [])
+    # Reuse the last successful discovery across process restarts. A stale
+    # catalog is preferable to blocking the first image-tab render on several
+    # sequential Hugging Face requests; a later refresh replaces it.
+    if not _HF_COLLECTION_CACHE.get("models"):
+        try:
+            cached = json.loads(_IMAGE_COLLECTION_DISK_CACHE.read_text(encoding="utf-8"))
+            cached_models = cached.get("models") if isinstance(cached, dict) else None
+            cached_ts = float(cached.get("fetched_at") or 0) if isinstance(cached, dict) else 0
+            if isinstance(cached_models, list) and cached_models:
+                _HF_COLLECTION_CACHE["ts"] = cached_ts
+                _HF_COLLECTION_CACHE["models"] = cached_models
+                if now - cached_ts < _IMAGE_COLLECTION_DISK_TTL:
+                    return list(cached_models)
+        except (OSError, ValueError, TypeError):
+            pass
+    models: list[dict[str, Any]] = []
+    for slug, mlx_only in [(slug, False) for slug in HF_IMAGE_COLLECTIONS] + [(slug, True) for slug in HF_MLX_IMAGE_COLLECTIONS]:
+        url = f"https://huggingface.co/api/collections/{slug}"
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "Odysseus-Cookbook/1.0"})
+            with urllib.request.urlopen(req, timeout=2.5) as resp:
+                data = json.loads(resp.read().decode("utf-8", "replace"))
+        except Exception:
+            continue
+        title = str(data.get("title") or slug)
+        for item in data.get("items") or []:
+            if isinstance(item, dict):
+                model = _collection_item_to_model(item, title, mlx_only=mlx_only)
+                if model:
+                    models.append(model)
+    if models:
+        _HF_COLLECTION_CACHE["ts"] = now
+        _HF_COLLECTION_CACHE["models"] = models
+        try:
+            _IMAGE_COLLECTION_DISK_CACHE.parent.mkdir(parents=True, exist_ok=True)
+            tmp = _IMAGE_COLLECTION_DISK_CACHE.with_suffix(".tmp")
+            tmp.write_text(json.dumps({"fetched_at": now, "models": models}), encoding="utf-8")
+            tmp.replace(_IMAGE_COLLECTION_DISK_CACHE)
+        except OSError:
+            pass
+        return list(models)
+    # Preserve stale results if the network is unavailable. The in-memory
+    # timestamp prevents every subsequent ranking request from retrying it.
+    if _HF_COLLECTION_CACHE.get("models"):
+        _HF_COLLECTION_CACHE["ts"] = now
+        return list(_HF_COLLECTION_CACHE["models"])
+    _HF_COLLECTION_CACHE["ts"] = now
+    return []
+
+
+def _hf_model_search(query: str, limit: int = 10) -> list[dict[str, Any]]:
+    global _HF_SEARCH_DISABLED_UNTIL
+    now = time.time()
+    if now < _HF_SEARCH_DISABLED_UNTIL:
+        return []
+    url = "https://huggingface.co/api/models?" + urllib.parse.urlencode({
+        "search": query,
+        "limit": str(limit),
+    })
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "Odysseus-Cookbook/1.0"})
+        with urllib.request.urlopen(req, timeout=2.5) as resp:
+            data = json.loads(resp.read().decode("utf-8", "replace"))
+        return data if isinstance(data, list) else []
+    except Exception:
+        _HF_SEARCH_DISABLED_UNTIL = now + 10 * 60
+        return []
+
+
+def _variant_score(candidate: dict[str, Any], base_repo: str, want: str) -> float:
+    rid = str(candidate.get("id") or candidate.get("modelId") or "")
+    text = " ".join([
+        rid,
+        str(candidate.get("library_name") or ""),
+        str(candidate.get("pipeline_tag") or ""),
+        " ".join(str(t) for t in candidate.get("tags") or []),
+    ]).lower()
+    base = base_repo.lower()
+    base_short = base_repo.rsplit("/", 1)[-1].lower()
+    if want == "gguf" and "gguf" not in text:
+        return -1
+    if want == "fp8" and not any(k in text for k in ("fp8", "nvfp4", "mxfp8", "mxfp4")):
+        return -1
+    score = float(candidate.get("downloads") or 0) / 1000.0 + float(candidate.get("likes") or 0)
+    if f"base_model:{base}" in text or f"base_model:quantized:{base}" in text:
+        score += 10000
+    elif base_short and base_short in rid.lower():
+        score += 1000
+    else:
+        score -= 200
+    if "diffusers" in text:
+        score += 50
+    if str(candidate.get("private")).lower() == "true":
+        score -= 10000
+    return score
+
+
+def _best_variant_repo(base_repo: str, want: str) -> str:
+    base_short = str(base_repo or "").rsplit("/", 1)[-1]
+    candidates = _hf_model_search(f"{base_short} {want}", limit=12)
+    scored = []
+    for item in candidates:
+        if not isinstance(item, dict):
+            continue
+        rid = str(item.get("id") or item.get("modelId") or "").strip()
+        if "/" not in rid or rid.lower() == base_repo.lower():
+            continue
+        score = _variant_score(item, base_repo, want)
+        if score >= 0:
+            scored.append((score, rid))
+    scored.sort(reverse=True)
+    return scored[0][1] if scored else ""
+
+
+def _should_discover_variants(repo_id: str) -> bool:
+    return False
+
+
+def _discover_quant_repos(repo_id: str, need_fp8: bool = True, need_gguf: bool = True) -> dict[str, str]:
+    key = str(repo_id or "").strip()
+    if not key:
+        return {}
+    cache_key = f"{key.lower()}|fp8={int(need_fp8)}|gguf={int(need_gguf)}"
+    if cache_key in _HF_VARIANT_CACHE:
+        return dict(_HF_VARIANT_CACHE[cache_key])
+    found: dict[str, str] = {}
+    if need_fp8:
+        fp8 = _best_variant_repo(key, "fp8")
+        if fp8:
+            found["FP8"] = fp8
+    if need_gguf:
+        gguf = _best_variant_repo(key, "gguf")
+        if gguf:
+            # The image-model fitter's smallest bucket is Q4; most HF image GGUF
+            # repos expose Q4/Q5/Q8 files under one repo, so use it as the low-VRAM
+            # download source while preserving the explicit GGUF label for callers.
+            found["Q4"] = gguf
+            found["GGUF"] = gguf
+    _HF_VARIANT_CACHE[cache_key] = found
+    return dict(found)
+
+
+def _merge_quant_repos(model: dict[str, Any]) -> dict[str, Any]:
+    out = dict(model)
+    existing = dict(out.get("quant_repos") or {})
+    repo_id = str(out.get("id") or "")
+    if _should_discover_variants(repo_id):
+        discovered = _discover_quant_repos(
+            repo_id,
+            need_fp8="FP8" not in existing,
+            need_gguf="Q4" not in existing and "GGUF" not in existing,
+        )
+        for k, v in discovered.items():
+            existing.setdefault(k, v)
+    out["quant_repos"] = existing
+    return out
+
 
 def get_image_models():
     """Return the image model registry."""
-    return IMAGE_MODEL_REGISTRY
+    merged = [_merge_quant_repos(m) for m in IMAGE_MODEL_REGISTRY]
+    seen = {str(m.get("id") or "").lower() for m in merged if isinstance(m, dict)}
+    for model in _fetch_hf_image_collection_models():
+        key = str(model.get("id") or "").lower()
+        if key and key not in seen:
+            merged.append(_merge_quant_repos(model))
+            seen.add(key)
+    return merged
+
+
+def _is_apple_image_system(system: dict[str, Any]) -> bool:
+    backend = str(system.get("backend") or "").lower()
+    gpu_name = str(system.get("gpu_name") or "").lower()
+    cpu_name = str(system.get("cpu_name") or "").lower()
+    platform = str(system.get("platform") or "").lower()
+    return (
+        bool(system.get("unified_memory"))
+        or backend in {"metal", "mps", "apple"}
+        or "apple" in gpu_name
+        or "apple" in cpu_name
+        or platform == "darwin"
+    )
 
 
 def rank_image_models(system, search=None, sort="fit"):
@@ -280,13 +371,23 @@ def rank_image_models(system, search=None, sort="fit"):
 
     Returns list of models with fit info (vram needed, fits, recommended quant).
     """
+    if not isinstance(system, dict):
+        system = {}
     gpu_vram = system.get("gpu_vram_gb", 0) or 0
     has_gpu = system.get("has_gpu", False)
+    ram_gb = system.get("available_ram_gb") or system.get("total_ram_gb") or 0
+    budget_gb = gpu_vram if has_gpu and gpu_vram > 0 else ram_gb
+    budget_kind = "gpu" if has_gpu and gpu_vram > 0 else "ram"
+    apple_system = _is_apple_image_system(system)
     results = []
 
-    for model in IMAGE_MODEL_REGISTRY:
+    for model in get_image_models():
+        if apple_system and not (model.get("mlx_only") or model.get("apple_ok")):
+            continue
+        if model.get("mlx_only") and not apple_system:
+            continue
         # Filter by search
-        if search:
+        if isinstance(search, str) and search:
             s = search.lower()
             if s not in model["name"].lower() and s not in model["id"].lower() and s not in model.get("description", "").lower():
                 continue
@@ -297,11 +398,11 @@ def rank_image_models(system, search=None, sort="fit"):
         fits = False
         quant_repo = None
 
-        if has_gpu and gpu_vram > 0:
+        if budget_gb > 0:
             # Try BF16 first, then FP8, then Q4
             for q, vram_key in [("BF16", "vram_bf16"), ("FP8", "vram_fp8"), ("Q4", "vram_q4")]:
                 v = model.get(vram_key)
-                if v is not None and v <= gpu_vram * 0.90:  # 10% headroom
+                if v is not None and v <= budget_gb * 0.90:  # 10% headroom
                     quant = q
                     vram_needed = v
                     fits = True
@@ -313,15 +414,15 @@ def rank_image_models(system, search=None, sort="fit"):
                 vram_needed = model.get("vram_bf16", 0)
 
         # Fit label
-        if not has_gpu:
+        if budget_gb <= 0:
             fit = "no_gpu"
             fit_label = "No GPU"
         elif fits:
-            headroom = gpu_vram - vram_needed
-            if headroom > gpu_vram * 0.3:
+            headroom = budget_gb - vram_needed
+            if headroom > budget_gb * 0.3:
                 fit = "perfect"
                 fit_label = "Perfect"
-            elif headroom > gpu_vram * 0.1:
+            elif headroom > budget_gb * 0.1:
                 fit = "good"
                 fit_label = "Good"
             else:
@@ -353,12 +454,14 @@ def rank_image_models(system, search=None, sort="fit"):
             "fits": fits,
             "fit": fit,
             "fit_label": fit_label,
+            "fit_budget": budget_kind,
             "quality": model["quality"],
             "speed": model["speed"],
             "score": round(score, 1),
             "capabilities": model["capabilities"],
             "description": model["description"],
             "released": model.get("released", ""),
+            "dependency_package": model.get("dependency_package", ""),
         })
 
     # Sort

services/hwfit/models.py

@@ -6,47 +6,147 @@ QUANT_HIERARCHY = ["Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "Q3_K_M", "Q2_K"]
 
 QUANT_BPP = {
     "F32": 4.0, "F16": 2.0, "BF16": 2.0, "FP8": 1.0,
+    "FP4": 0.50, "NVFP4": 0.50, "MXFP4": 0.50, "NF4": 0.50,
+    "INT4": 0.50, "INT8": 1.0, "W4A16": 0.50, "W8A8": 1.0, "W8A16": 1.0,
     "Q8_0": 1.05, "Q6_K": 0.80, "Q5_K_M": 0.68,
     "Q4_K_M": 0.58, "Q4_0": 0.58, "Q3_K_M": 0.48, "Q2_K": 0.37,
     "AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
-    "mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
+    "QAT-INT4": 0.50, "QAT-INT8": 1.0,
+    "mlx-3bit": 0.42, "mlx-4bit": 0.55, "mlx-5bit": 0.65, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
+    # DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
+    # expert dense in FP8, embeddings/LM head in BF16. By weight count the
+    # experts dominate so the effective BPP sits closer to FP4 than FP8.
+    # Empirical: DeepSeek-V4-Flash 284B / 156 GB ≈ 0.55 B/param.
+    "FP4-MoE-Mixed": 0.55,
+    # FP8-Mixed = the *-Base variants (MoE experts also FP8, not FP4).
+    "FP8-Mixed": 1.0,
 }
 
 QUANT_SPEED_MULT = {
     "F16": 0.6, "BF16": 0.6, "FP8": 0.85,
+    "FP4": 1.15, "NVFP4": 1.15, "MXFP4": 1.15, "NF4": 1.10,
+    "INT4": 1.15, "INT8": 0.85, "W4A16": 1.15, "W8A8": 0.85, "W8A16": 0.85,
     "Q8_0": 0.8, "Q6_K": 0.95, "Q5_K_M": 1.0,
     "Q4_K_M": 1.15, "Q4_0": 1.15, "Q3_K_M": 1.25, "Q2_K": 1.35,
     "AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
     "GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
-    "mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
+    "QAT-INT4": 1.15, "QAT-INT8": 0.85,
+    "mlx-3bit": 1.25, "mlx-4bit": 1.15, "mlx-5bit": 1.05, "mlx-6bit": 1.0, "mlx-8bit": 0.85,
+    "FP4-MoE-Mixed": 1.10,  # slightly slower than pure FP4 because of mixed-dtype dispatch
+    "FP8-Mixed": 0.85,
 }
 
 QUANT_QUALITY_PENALTY = {
     "F16": 0.0, "BF16": 0.0, "FP8": 0.0,
+    "FP4": -3.0, "NVFP4": -3.0, "MXFP4": -3.0, "NF4": -4.0,
+    "INT4": -4.0, "INT8": 0.0, "W4A16": -4.0, "W8A8": 0.0, "W8A16": 0.0,
     "Q8_0": 0.0, "Q6_K": -1.0, "Q5_K_M": -2.0,
     "Q4_K_M": -5.0, "Q4_0": -5.0, "Q3_K_M": -8.0, "Q2_K": -12.0,
-    "AWQ-4bit": -3.0, "AWQ-8bit": 0.0,
-    "GPTQ-Int4": -3.0, "GPTQ-Int8": 0.0,
-    "mlx-4bit": -4.0, "mlx-8bit": 0.0, "mlx-6bit": -1.0,
+    # Bare "AWQ" and "AWQ-8bit" used to be 0.0 (tied with FP8). In practice
+    # AWQ-anything is a calibrated reconstruction, not raw 8-bit weights —
+    # there's a small but real quality loss vs FP8. Give them a slight
+    # penalty so FP8 wins when both fit. AWQ-4bit stays heavier.
+    "AWQ": -1.0, "AWQ-4bit": -4.0, "AWQ-8bit": -1.0,
+    "GPTQ": -1.0, "GPTQ-Int4": -4.0, "GPTQ-Int8": -1.0,
+    # Quantization-aware training recovers most of the int4 quality loss, so a
+    # QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
+    # (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
+    "QAT-INT4": -1.0, "QAT-INT8": 0.0,
+    "mlx-3bit": -8.0, "mlx-4bit": -4.0, "mlx-5bit": -2.5, "mlx-6bit": -1.5, "mlx-8bit": -0.5,
+    # DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
+    # so the realized quality is much closer to FP8 than to pure FP4 —
+    # the activation-sensitive layers stay high-precision. ~0 penalty.
+    "FP4-MoE-Mixed": -0.5,
+    "FP8-Mixed": 0.0,
 }
 
 QUANT_BYTES_PER_PARAM = {
     "F16": 2.0, "BF16": 2.0, "FP8": 1.0,
+    "FP4": 0.5, "NVFP4": 0.5, "MXFP4": 0.5, "NF4": 0.5,
+    "INT4": 0.5, "INT8": 1.0, "W4A16": 0.5, "W8A8": 1.0, "W8A16": 1.0,
     "Q8_0": 1.0, "Q6_K": 0.75, "Q5_K_M": 0.625,
     "Q4_K_M": 0.5, "Q4_0": 0.5, "Q3_K_M": 0.375, "Q2_K": 0.25,
     "AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
-    "mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
+    "QAT-INT4": 0.5, "QAT-INT8": 1.0,
+    "mlx-3bit": 0.375, "mlx-4bit": 0.5, "mlx-5bit": 0.625, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
+    "FP4-MoE-Mixed": 0.55,
+    "FP8-Mixed": 1.0,
 }
 
-# Pre-quantized formats that should NOT go through the GGUF quant hierarchy
-PREQUANTIZED_PREFIXES = ("AWQ-", "GPTQ-", "mlx-", "FP8")
+# Pre-quantized formats that should NOT go through the GGUF quant hierarchy.
+# These are native HF/vLLM-style repos, not llama.cpp GGUF quant tiers.
+PREQUANTIZED_PREFIXES = (
+    "AWQ-", "GPTQ-", "mlx-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
+    "INT4", "INT8", "W4A16", "W8A8", "W8A16",
+    "FP4-MoE-Mixed", "FP8-Mixed",
+    "QAT-",
+)
+
+
+def infer_quantization_from_name(name):
+    n = (name or "").lower()
+    model_name = n.rsplit("/", 1)[-1]
+    if "nvfp4" in n:
+        return "NVFP4"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", model_name):
+        return "BF16"
+    if "mxfp4" in n:
+        return "MXFP4"
+    if re.search(r"(^|[-_/])nf4($|[-_/])", n):
+        return "NF4"
+    if re.search(r"(^|[-_/])fp4($|[-_/])", n):
+        return "FP4"
+    if re.search(r"(^|[-_/])w4a16($|[-_/])", n):
+        return "W4A16"
+    if re.search(r"(^|[-_/])w8a8($|[-_/])", n):
+        return "W8A8"
+    if re.search(r"(^|[-_/])w8a16($|[-_/])", n):
+        return "W8A16"
+    is8 = "8bit" in n or "8-bit" in n or "int8" in n
+    if "awq" in n:
+        return "AWQ-8bit" if is8 else "AWQ-4bit"
+    if "gptq" in n:
+        return "GPTQ-Int8" if is8 else "GPTQ-Int4"
+    if n.startswith("mlx-community/") or "mlx" in model_name:
+        if "3bit" in model_name:
+            return "mlx-3bit"
+        if "5bit" in model_name:
+            return "mlx-5bit"
+        if "6bit" in model_name:
+            return "mlx-6bit"
+        return "mlx-8bit" if is8 else "mlx-4bit"
+    if "fp8" in n:
+        return "FP8"
+    if "int4" in n or "4bit" in n or "4-bit" in n:
+        return "INT4"
+    if "int8" in n or "8bit" in n or "8-bit" in n:
+        return "INT8"
+    return ""
+
+
+def _normalize_model_entry(model):
+    if not isinstance(model, dict):
+        return model
+    inferred = infer_quantization_from_name(model.get("name", ""))
+    if inferred and (model.get("quantization") in (None, "", "Q4_K_M") or model.get("_discovered")):
+        model["quantization"] = inferred
+    return model
 
 
 def is_prequantized(model):
     q = model.get("quantization", "")
-    return any(q.startswith(p) for p in PREQUANTIZED_PREFIXES)
+    name = (model.get("name") or "").lower()
+    fmt = (model.get("format") or "").lower()
+    text = f"{name} {fmt}"
+    return (
+        "nvfp4" in text
+        or re.search(r"(^|[-_/])fp8($|[-_/\s])", text) is not None
+        or (not (model.get("is_gguf") or model.get("gguf_sources")) and re.search(r"(^|[-_/])(?:int)?8bit($|[-_/\s])", text) is not None)
+        or any(x in text for x in ("awq", "gptq", "mlx"))
+        or any(isinstance(q, str) and q.startswith(p) for p in PREQUANTIZED_PREFIXES)
+    )
 
 
 def params_b(model):
@@ -55,11 +155,17 @@ def params_b(model):
         return raw / 1_000_000_000.0
 
     pc = model.get("parameter_count", "")
-    if pc:
+    if isinstance(pc, str) and pc:
         pc = pc.strip().upper()
         m = re.match(r"^([\d.]+)\s*([BKMGT]?)$", pc)
         if m:
-            val = float(m.group(1))
+            try:
+                val = float(m.group(1))
+            except ValueError:
+                # Malformed count like "1.5.3B" — [\d.]+ matches but float()
+                # rejects it. One bad catalog row must not abort the whole
+                # ranking pass, so treat it as unknown size.
+                return 0.0
             suffix = m.group(2)
             if suffix == "B":
                 return val
@@ -161,15 +267,75 @@ def infer_use_case(model):
 
 _models_cache = None
 
+def _load_model_file(path):
+    try:
+        with open(path, encoding="utf-8") as f:
+            loaded = json.load(f)
+            return loaded if isinstance(loaded, list) else []
+    except (FileNotFoundError, json.JSONDecodeError):
+        return []
+
+def reset_model_cache():
+    global _models_cache
+    _models_cache = None
+
+def refresh_dynamic_catalogs(force=False):
+    """Refresh API-backed model catalogs and invalidate the merged cache.
+
+    The bundled JSON files remain the offline fallback. Dynamic catalogs live
+    under DATA_DIR so runtime refreshes do not dirty the source tree.
+    """
+    from services.hwfit.hf_discovery import (
+        refresh_hf_collection_models_cache,
+        refresh_mlx_community_cache,
+    )
+
+    refreshed = {
+        "mlx_community": len(refresh_mlx_community_cache(force=force)),
+        "hf_collections": len(refresh_hf_collection_models_cache(force=force)),
+    }
+    reset_model_cache()
+    return refreshed
+
 def get_models():
     global _models_cache
     if _models_cache is None:
         data_path = os.path.join(os.path.dirname(__file__), "data", "hf_models.json")
+        static_mlx_path = os.path.join(os.path.dirname(__file__), "data", "mlx_community_models.json")
         try:
-            with open(data_path) as f:
-                _models_cache = json.load(f)
-        except (FileNotFoundError, json.JSONDecodeError):
-            _models_cache = []
+            from services.hwfit.hf_discovery import (
+                load_cached_hf_collection_models,
+                load_cached_mlx_community_models,
+            )
+            dynamic_mlx_models = load_cached_mlx_community_models()
+            dynamic_hf_models = load_cached_hf_collection_models()
+        except Exception:
+            dynamic_mlx_models = []
+            dynamic_hf_models = []
+        seen = set()
+        rows = []
+        def _append_models(models):
+            for model in models:
+                if not isinstance(model, dict):
+                    continue
+                name = model.get("name")
+                if not name or name in seen:
+                    continue
+                seen.add(name)
+                rows.append(_normalize_model_entry(model))
+
+        for model in _load_model_file(data_path):
+            if not isinstance(model, dict):
+                continue
+            name = model.get("name")
+            if not name or name in seen:
+                continue
+            seen.add(name)
+            rows.append(_normalize_model_entry(model))
+        _append_models(dynamic_hf_models)
+        _append_models(dynamic_mlx_models)
+        _append_models(_load_model_file(static_mlx_path))
+        _models_cache = rows
     return _models_cache
 
 

services/hwfit/profiles.py

@@ -0,0 +1,238 @@
+"""Compute intelligent llama.cpp serve profiles from detected hardware.
+
+Given a system (VRAM/RAM/arch) and a model, produce 1-4 ready-to-launch
+profiles — Quality / Balanced / Speed — with concrete llama.cpp flags
+(n_gpu_layers, n_cpu_moe, cache-type, context). This turns the by-hand tuning
+(how many MoE layers fit on the GPU, when to spend VRAM on a q8 KV cache vs more
+context, how much headroom to leave for a vision encoder) into a formula.
+
+Pure/deterministic — no benchmarking, no I/O. Reuses the same VRAM math as
+fit.py/models.py so "what the Cookbook recommends" and "what it serves" agree.
+
+NOTE: token/s figures are NOT computed here — real speed on partial-offload MoE
+is CPU-bound and not reliably predictable from specs. The UI labels profiles by
+their tradeoff (Quality/Balanced/Speed), and the VRAM fit (the part that decides
+whether it even loads) is what's computed from real numbers.
+"""
+
+from services.hwfit.models import (
+    QUANT_BPP,
+    params_b,
+    _active_params_b,
+    is_prequantized,
+)
+
+# GGUF KV-cache cost per token, in bytes-per-active-billion-param, by cache type.
+# q4_0 is ~half of q8_0 is ~half of f16. The 8e-6 base in estimate_memory_gb is
+# the q8_0-ish figure; scale from there.
+_KV_FACTOR = {"q4_0": 0.5, "q8_0": 1.0, "f16": 2.0}
+
+# Quant ladder from highest quality/size down. A profile that wants "best quant
+# that fits fully on GPU" walks this until one fits.
+_QUANT_LADDER = ["Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "Q3_K_M", "Q2_K"]
+
+
+def _weights_gb(model, quant, fixed_gb=None):
+    """VRAM for the full weights. When fixed_gb is given (serving a specific GGUF
+    file already on disk), use its real size — the quant is whatever the file is,
+    not something we get to pick."""
+    if fixed_gb and fixed_gb > 0:
+        return float(fixed_gb)
+    return params_b(model) * QUANT_BPP.get(quant, 0.58)
+
+
+def _kv_gb(model, ctx, kv_type):
+    """KV-cache VRAM at a context length and cache type."""
+    kv_params = _active_params_b(model)
+    return 0.000008 * kv_params * ctx * _KV_FACTOR.get(kv_type, 1.0)
+
+
+def _n_layers(model):
+    """Best-effort total transformer block count (for n-cpu-moe math)."""
+    for k in ("num_hidden_layers", "n_layers", "num_layers", "block_count"):
+        v = model.get(k)
+        if isinstance(v, (int, float)) and v > 0:
+            return int(v)
+    # Fallback heuristic by size — most MoE/dense LLMs land 28-64 layers.
+    pb = params_b(model)
+    if pb >= 60:
+        return 64
+    if pb >= 25:
+        return 48
+    if pb >= 12:
+        return 40
+    return 32
+
+
+def _cpu_moe_for_budget(model, quant, kv_gb, vram_budget_gb, fixed_gb=None):
+    """How many MoE layers must move to CPU so weights+KV fit vram_budget_gb.
+
+    Returns (n_cpu_moe, fits_fully). When the model already fits, n_cpu_moe=0.
+    Each offloaded layer frees roughly weights/n_layers of VRAM. We only model
+    this for MoE (where --n-cpu-moe applies); dense models just report whether
+    they fit at the given n_gpu_layers=999.
+    """
+    weights = _weights_gb(model, quant, fixed_gb)
+    needed = weights + kv_gb + 0.6  # +0.6 GB runtime/compute buffers
+    if needed <= vram_budget_gb:
+        return 0, True
+    if not model.get("is_moe"):
+        # Dense: no per-expert offload knob; either it fits or it spills via -ngl.
+        return 0, False
+    layers = _n_layers(model)
+    per_layer = weights / max(layers, 1)
+    overflow = needed - vram_budget_gb
+    import math
+    n = math.ceil(overflow / max(per_layer, 1e-6))
+    n = max(0, min(n, layers))   # clamp
+    return n, False
+
+
+def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=None):
+    """Return a list of profile dicts for llama.cpp serving of `model` on `system`.
+
+    Each profile: {key, label, quant, n_gpu_layers, n_cpu_moe, cache_type, ctx,
+                   est_vram_gb, fits, note}. Empty list if no GGUF path makes
+    sense (caller should fall back to manual flags).
+
+    DOWNLOAD mode (default): the quant isn't chosen yet, so profiles vary it
+    (Quality=Q6, Balanced=Q4, Speed=Q2…) to show download options.
+
+    SERVE mode (serve_weights_gb set): a specific GGUF file already exists on
+    disk — its quant is FIXED. Profiles then keep that quant/size and differ only
+    in the actual serving knobs (n_cpu_moe, KV-cache type, context). serve_quant
+    is the file's quant label (e.g. "Q4_K_M") just for display.
+    """
+    if not isinstance(system, dict) or not isinstance(model, dict):
+        return []
+
+    vram = float(system.get("gpu_vram_gb") or 0)
+    if vram <= 0:
+        return []
+
+    serve_mode = bool(serve_weights_gb and serve_weights_gb > 0)
+
+    # Never propose more context than the model was trained for — asking llama.cpp
+    # for ctx > n_ctx_train triggers a "training context overflow" and, with a
+    # quantized KV cache, an oversized allocation that can crash the GPU
+    # (radv/amdgpu ErrorDeviceLost). Cap every profile at the model's real limit.
+    model_ctx_max = 0
+    for k in ("context_length", "max_position_embeddings", "n_ctx_train", "context"):
+        v = model.get(k)
+        if isinstance(v, (int, float)) and v > 0:
+            model_ctx_max = int(v)
+            break
+    if model_ctx_max <= 0:
+        model_ctx_max = 131072  # conservative default when the catalog omits it
+
+    # Vision models need headroom for the image encoder (~1 GB on top of weights).
+    is_vision = bool(
+        model.get("is_multimodal") or model.get("vision") or model.get("mmproj")
+        or "vl" in str(model.get("name", "")).lower()
+    )
+    headroom = 1.1 if is_vision else 0.4
+    budget = max(vram - headroom, 1.0)
+
+    # Prequantized (AWQ/GPTQ/FP8) served via GGUF fallback use a fixed ~Q4 quant;
+    # GGUF models can pick their quant. Pick a sensible per-profile quant.
+    fixed_quant = model.get("quantization") if is_prequantized(model) else None
+
+    is_moe = bool(model.get("is_moe"))
+
+    def _pick_quant(prefer, require_full_fit):
+        """Choose a quant for a profile.
+
+        - fixed_quant (AWQ/GPTQ/FP8 served via GGUF): always that.
+        - require_full_fit=True (Speed): walk DOWN from `prefer` to the best quant
+          whose weights fit fully on the GPU (no offload) — fastest.
+        - require_full_fit=False (Quality on MoE): keep `prefer` even if it must
+          offload experts to CPU; that's the whole point of n-cpu-moe on a card
+          too small to hold the weights. For dense models we can't offload
+          per-expert, so fall back to the largest fully-fitting quant.
+        """
+        if fixed_quant:
+            return fixed_quant
+        start = _QUANT_LADDER.index(prefer) if prefer in _QUANT_LADDER else 3
+        if require_full_fit or not is_moe:
+            for q in _QUANT_LADDER[start:]:
+                if _weights_gb(model, q) + 0.6 <= budget:
+                    return q
+            return _QUANT_LADDER[-1]
+        # MoE quality: keep the preferred (big) quant; offload handles overflow.
+        return prefer
+
+    if serve_mode:
+        # Fixed file on disk — quant can't change. Vary only the serving knobs.
+        fq = serve_quant or model.get("quantization") or "GGUF"
+        specs = [
+            # key, label, prefer_quant, full_fit, kv_type, ctx, note
+            ("quality", "Quality", fq, False, "q8_0", 131072,
+             "Sharp q8 KV cache + full context. Best long-context accuracy; offloads MoE layers to CPU if needed."),
+            ("balanced", "Balanced", fq, False, "q4_0", 131072,
+             "Compact q4 KV at full context — good speed/quality mix."),
+            ("speed", "Speed", fq, False, "q4_0", 32768,
+             "Trimmed context + light KV for the fastest tokens/s."),
+        ]
+    else:
+        specs = [
+            # key, label, prefer_quant, full_fit, kv_type, ctx, note
+            ("quality", "Quality", "Q6_K", False, "q8_0", 131072,
+             "Biggest quant + sharp q8 KV cache. Best answers; offloads MoE layers to CPU if needed."),
+            ("balanced", "Balanced", "Q4_K_M", False, "q4_0", 131072,
+             "Q4 weights + compact q4 KV. Good speed/quality mix at full context."),
+            ("speed", "Speed", "Q4_K_M", True, "q4_0", 32768,
+             "Smallest offload + trimmed context for the fastest tokens/s."),
+        ]
+
+    profiles = []
+    for key, label, prefer_q, full_fit, kv_type, ctx, note in specs:
+        # In serve mode the quant is fixed (the file's); in download mode we pick.
+        quant = prefer_q if serve_mode else _pick_quant(prefer_q, full_fit)
+        # Shrink context if even the chosen KV won't fit alongside weights.
+        # Start from the smaller of the profile's target and the model's limit.
+        cur_ctx = min(ctx, model_ctx_max)
+        # Floor the context-shrink loop at 8192, but never above the model's own
+        # trained limit. A model with a sub-8192 context (e.g. a 2048-token
+        # SmolLM) starts below 8192, so a hard-coded 8192 guard skipped the loop
+        # entirely and produced NO profile — the serve UI then fell back to
+        # manual flags even though the model fits the GPU trivially.
+        ctx_floor = min(8192, model_ctx_max)
+        while cur_ctx >= ctx_floor:
+            kv = _kv_gb(model, cur_ctx, kv_type)
+            n_cpu_moe, fits = _cpu_moe_for_budget(model, quant, kv, budget, fixed_gb=serve_weights_gb)
+            est = _weights_gb(model, quant, serve_weights_gb) + kv + 0.6
+            # If a non-MoE model can't fit even fully offloaded, try less context.
+            if model.get("is_moe") or fits or cur_ctx <= ctx_floor:
+                profiles.append({
+                    "key": key,
+                    "label": label,
+                    "quant": quant,
+                    "n_gpu_layers": 999,
+                    "n_cpu_moe": n_cpu_moe,
+                    "cache_type": kv_type,
+                    "ctx": cur_ctx,
+                    # When experts offload, GPU-resident VRAM tops out at the
+                    # budget (weights beyond it live in system RAM), so cap the
+                    # estimate at `budget`, not the full card — this also leaves
+                    # the vision-encoder headroom visible in the number.
+                    "est_vram_gb": round(min(est, budget), 1),
+                    # For MoE we treat it as fitting via offload; report whether
+                    # it fit WITHOUT offload as the "clean" flag.
+                    "fits": fits or bool(model.get("is_moe")),
+                    "offloads": n_cpu_moe > 0,
+                    "note": note,
+                })
+                break
+            cur_ctx //= 2
+
+    # De-dupe identical profiles (e.g. tiny model where all three collapse to the
+    # same all-GPU config) — keep the first/highest-quality label.
+    seen = set()
+    deduped = []
+    for p in profiles:
+        sig = (p["quant"], p["n_cpu_moe"], p["cache_type"], p["ctx"])
+        if sig in seen:
+            continue
+        seen.add(sig)
+        deduped.append(p)
+    return deduped

services/memory/__init__.py

@@ -2,7 +2,7 @@
 """Memory service — persistent memory storage and retrieval."""
 
 from .service import MemoryService, Memory, MemorySearchResult
-from .memory import MemoryManager
+from .memory import MemoryManager, MemoryStoreUnreadable
 from .memory_vector import MemoryVectorStore
 
 __all__ = [
@@ -10,5 +10,6 @@ __all__ = [
     "Memory",
     "MemorySearchResult",
     "MemoryManager",
+    "MemoryStoreUnreadable",
     "MemoryVectorStore",
 ]

services/memory/builtin_skills.py

@@ -0,0 +1,74 @@
+"""Install tracked built-in skills into the shared immutable skill catalog."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable
+
+from .skill_format import Skill
+from .skills import SkillsManager
+
+
+_BUILTIN_ROOT = Path(__file__).resolve().parents[2] / "resources" / "skills"
+_SYNC_FIELDS = (
+    "name",
+    "description",
+    "version",
+    "category",
+    "tags",
+    "status",
+    "confidence",
+    "source",
+    "owner",
+    "when_to_use",
+    "procedure",
+    "pitfalls",
+    "verification",
+    "platforms",
+    "requires_toolsets",
+    "fallback_for_toolsets",
+    "body_extra",
+)
+
+
+def install_builtin_skills(manager: SkillsManager, owners: Iterable[str]) -> int:
+    """Copy missing built-in skills into the ownerless shared catalog.
+
+    Built-ins are explicitly marked and remain ownerless because the on-disk
+    skill path is not owner-qualified. ``SkillsManager.load(owner=...)``
+    exposes only these immutable built-ins in addition to that owner's files.
+    Installation is safe before first-user setup because no owner identity is
+    assigned and unauthenticated requests still cannot access skill routes.
+    """
+    existing = {row.get("name") for row in manager.load_all()}
+    installed = 0
+    paths = sorted(_BUILTIN_ROOT.rglob("SKILL.md")) if _BUILTIN_ROOT.is_dir() else []
+    for path in paths:
+        try:
+            skill = Skill.from_markdown(path.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+        # Tracked procedures ship as trusted application behavior. They are
+        # available immediately and never enter the user's audit queue.
+        skill.status = "published"
+        skill.confidence = 1.0
+        existing_rows = [row for row in manager.load_all() if row.get("name") == skill.name]
+        if existing_rows:
+            row = existing_rows[0]
+            # Built-ins are immutable tracked assets. Synchronize updated
+            # versions/procedures on startup while leaving usage counters in
+            # their sidecar untouched. Older startup code could also stamp the
+            # first admin onto one; normalize that migration at the same time.
+            if row.get("source") == "builtin":
+                skill.owner = ""
+                skill.source = "builtin"
+                desired = skill.to_dict()
+                if any(row.get(field) != desired.get(field) for field in _SYNC_FIELDS):
+                    manager._write_skill(skill)
+            continue
+        skill.owner = ""
+        skill.source = "builtin"
+        manager._write_skill(skill)
+        existing.add(skill.name)
+        installed += 1
+    return installed

services/memory/memory.py

@@ -1,359 +1,20 @@
+"""Compatibility import for the canonical memory manager.
 
-import json
-import logging
-import os
-import time
-import uuid
-import re
-from typing import List, Dict, Tuple
-from datetime import datetime
+Historically this package carried a second copy of ``MemoryManager``. The
+application runtime instantiates ``src.memory.MemoryManager``, so keeping a
+parallel implementation here risks silent drift between import paths.
+"""
 
-logger = logging.getLogger(__name__)
+from src.memory import (
+    MemoryManager,
+    MemoryStoreUnreadable,
+    get_text_similarity,
+    tokenize,
+)
 
-def tokenize(text: str) -> List[str]:
-    """Simple tokenizer that splits on whitespace and removes punctuation."""
-    return [word.strip('.,!?";') for word in text.split()]
-
-def get_text_similarity(text1: str, text2: str) -> float:
-    """Calculate Jaccard similarity between two texts."""
-    if not text1 or not text2:
-        return 0.0
-    
-    tokens1 = set(tokenize(text1.lower()))
-    tokens2 = set(tokenize(text2.lower()))
-    
-    if not tokens1 and not tokens2:
-        return 1.0
-    if not tokens1 or not tokens2:
-        return 0.0
-        
-    intersection = tokens1.intersection(tokens2)
-    union = tokens1.union(tokens2)
-    
-    return len(intersection) / len(union)
-
-class MemoryManager:
-    def __init__(self, data_dir: str):
-        self.memory_file = os.path.join(data_dir, "memory.json")
-        self.ensure_file_exists()
-        
-    def extract_memory_from_chat(self, chat_history: List[Dict], session_id: str = None) -> List[Dict]:
-        """
-        Extract memory entries from chat history as a fallback when LLM fails.
-        
-        Args:
-            chat_history: List of chat messages with 'role' and 'content' keys
-            session_id: Optional session ID to associate with extracted memories
-            
-        Returns:
-            List of memory entries with text, timestamp, and optional session_id
-        """
-        memories = []
-        
-        for msg in chat_history:
-            if msg.get("role") == "assistant":
-                content = str(msg.get("content", ""))
-                lines = content.split('\n')
-                
-                for line in lines:
-                    line = line.strip()
-                    # Look for bullet points or numbered lists that might contain memories
-                    if re.match(r'^[-*•]|\d+\.', line):
-                        # Extract the text after the bullet/number
-                        text_match = re.match(r'^[-*•]|\d+\.\s*(.*)', line)
-                        if text_match:
-                            text = text_match.group(1).strip()
-                            if text:
-                                memories.append({
-                                    "text": text,
-                                    "timestamp": int(datetime.now().timestamp()),
-                                    "session_id": session_id
-                                })
-                    # If we see a heading that suggests memories
-                    elif re.search(r'memory|fact|note|remember', line, re.I):
-                        pass
-                    # If we see a clear separator or end
-                    elif re.match(r'^={3,}|-{3,}|_{3,}', line):
-                        pass
-                        
-        return memories
-        
-    def process_inline_memory_command(self, message: str) -> Tuple[bool, str]:
-        """
-        Check if a message is an inline memory command (e.g. "remember: X").
-        
-        Args:
-            message: The user message to check
-            
-        Returns:
-            Tuple of (is_command, extracted_text) where is_command is True if 
-            the message matches the memory command pattern
-        """
-        # Pattern for memory commands: "remember: X", "memorize: X", "save: X", etc.
-        pattern = r'^(?:remember|memorize|save|note|store)[:\-]?\s+(.+)$'
-        match = re.match(pattern, message.strip(), re.IGNORECASE)
-        
-        if match:
-            return True, match.group(1).strip()
-        else:
-            return False, ""
-    
-    def ensure_file_exists(self):
-        """Create memory file if it doesn't exist."""
-        if not os.path.exists(self.memory_file):
-            with open(self.memory_file, 'w', encoding='utf-8') as f:
-                json.dump([], f, ensure_ascii=False, indent=2)
-    
-    def load_all(self) -> List[Dict]:
-        """Load all memory entries from JSON file (unfiltered)."""
-        if not os.path.exists(self.memory_file):
-            return []
-
-        try:
-            with open(self.memory_file, "r", encoding="utf-8") as f:
-                data = json.load(f)
-                if isinstance(data, list):
-                    return self._validate_entries(data)
-        except (json.JSONDecodeError, PermissionError) as e:
-            logger.error("Error loading memory.json: %s", e)
-            return self._migrate_from_legacy()
-
-        return []
-
-    def load(self, owner: str = None) -> List[Dict]:
-        """Load memory entries, filtered by owner."""
-        entries = self.load_all()
-        if owner is None:
-            return entries
-        return [e for e in entries if e.get("owner") == owner]
-
-    def claim_ownerless(self, owner: str):
-        """Assign all ownerless memory entries to the given owner. Run once to migrate."""
-        entries = self.load_all()
-        changed = False
-        for e in entries:
-            if not e.get("owner"):
-                e["owner"] = owner
-                changed = True
-        if changed:
-            self.save(entries)
-            logger.info("Claimed %d ownerless memories for %s", sum(1 for e in entries if e.get("owner") == owner), owner)
-    
-    def _validate_entries(self, entries: List[Dict]) -> List[Dict]:
-        """Ensure all entries have required fields."""
-        validated = []
-        for entry in entries:
-            if "id" not in entry:
-                entry["id"] = str(uuid.uuid4())
-            if "timestamp" not in entry:
-                entry["timestamp"] = int(time.time())
-            if "source" not in entry:
-                entry["source"] = "unknown"
-            if "category" not in entry:
-                entry["category"] = "fact"
-            validated.append(entry)
-        return validated
-    
-    def _migrate_from_legacy(self) -> List[Dict]:
-        """Migrate from old text format to JSON if needed."""
-        legacy_path = os.path.join(os.path.dirname(self.memory_file), "memory.txt")
-        if not os.path.exists(legacy_path):
-            return []
-            
-        logger.info("Converting legacy memory.txt to new JSON format")
-        try:
-            with open(legacy_path, "r", encoding="utf-8") as f:
-                lines = [ln.strip() for ln in f.readlines() if ln.strip()]
-            
-            entries = []
-            for line in lines:
-                entries.append({
-                    "id": str(uuid.uuid4()),
-                    "text": line,
-                    "timestamp": int(time.time()),
-                    "source": "user",
-                    "category": "fact"
-                })
-            
-            self.save(entries)
-            return entries
-        except Exception as e:
-            logger.error("Failed to convert legacy memory: %s", e)
-            return []
-    
-    def save(self, entries: List[Dict]):
-        """Save memory entries to JSON file."""
-        # Validate entries before saving
-        for entry in entries:
-            if "id" not in entry:
-                entry["id"] = str(uuid.uuid4())
-            if "timestamp" not in entry:
-                entry["timestamp"] = int(time.time())
-            if "source" not in entry:
-                entry["source"] = "user"
-            if "category" not in entry:
-                entry["category"] = "fact"
-        
-        # Use atomic write
-        tmp_file = self.memory_file + ".tmp"
-        with open(tmp_file, "w", encoding="utf-8") as f:
-            json.dump(entries, f, ensure_ascii=False, indent=2)
-        os.replace(tmp_file, self.memory_file)
-    
-    def add_entry(self, text: str, source: str = "user", category: str = "fact", owner: str = None) -> Dict:
-        """Add a new memory entry."""
-        if not text.strip():
-            raise ValueError("Memory text cannot be empty")
-
-        entry = {
-            "id": str(uuid.uuid4()),
-            "text": text.strip(),
-            "timestamp": int(time.time()),
-            "source": source,
-            "category": category
-        }
-        if owner:
-            entry["owner"] = owner
-        return entry
-    
-    def find_duplicates(self, text: str, entries: List[Dict] = None) -> List[Dict]:
-        """Find duplicate memory entries based on text content."""
-        if entries is None:
-            entries = self.load()
-            
-        text_lower = text.strip().lower()
-        return [entry for entry in entries if entry["text"].lower() == text_lower]
-            
-    def categorize_memory_by_relevance(self, message: str, memories: list):
-        """Categorize memories by type and relevance"""
-        categories = {
-            "contacts": [],
-            "preferences": [],
-            "facts": [],
-            "tasks": []
-        }
-        
-        msg_lower = message.lower()
-        
-        for mem in memories:
-            text_lower = mem["text"].lower()
-            
-            # Contact info
-            if any(word in text_lower for word in ["phone", "email", "address", "lives", "works"]):
-                if any(word in msg_lower for word in ["contact", "phone", "address", "email"]):
-                    categories["contacts"].append(mem)
-            
-            # Personal preferences
-            elif any(word in text_lower for word in ["likes", "dislikes", "prefers", "favorite"]):
-                if any(word in msg_lower for word in ["like", "prefer", "favorite", "want"]):
-                    categories["preferences"].append(mem)
-            
-            # Tasks and todos
-            elif any(word in text_lower for word in ["todo", "task", "remind", "meeting"]):
-                if any(word in msg_lower for word in ["todo", "task", "schedule", "remind"]):
-                    categories["tasks"].append(mem)
-            
-            # General facts - only if very relevant
-            else:
-                if get_text_similarity(message, mem["text"]) > 0.4:
-                    categories["facts"].append(mem)
-        
-        return categories
-
-    def get_relevant_memories(self, query: str, memories: list, threshold: float = 0.05, max_items: int = 8):
-        """Get memories that are relevant to the query based on text similarity and semantic keyword matching."""
-        if not memories or not query.strip():
-            return []
-            
-        # Define keyword categories for semantic matching
-        identity_words = ["name", "who", "i", "am", "called", "identity", "myself", "me", "my"]
-        contact_words = ["phone", "email", "address", "contact", "number", "where", "located", "reach"]
-        preference_words = ["like", "prefer", "favorite", "want", "love", "hate", "dislike", "enjoy", "interested"]
-        task_words = ["todo", "task", "remind", "meeting", "appointment", "schedule", "deadline"]
-        fact_words = ["what", "when", "where", "how", "why", "explain", "describe", "information", "know"]
-        
-        query_lower = query.lower()
-        
-        # Determine query type based on keywords
-        query_type = None
-        if any(word in query_lower for word in identity_words):
-            query_type = "identity"
-        elif any(word in query_lower for word in contact_words):
-            query_type = "contact"
-        elif any(word in query_lower for word in preference_words):
-            query_type = "preference"
-        elif any(word in query_lower for word in task_words):
-            query_type = "task"
-        elif any(word in query_lower for word in fact_words):
-            query_type = "fact"
-        
-        relevant = []
-        identity_memories = []
-        other_memories = []
-        
-        # Separate identity memories from others
-        for memory in memories:
-            memory_text = memory["text"].lower()
-            # Check if this is an identity memory (contains name patterns or identity indicators)
-            is_identity = any([
-                re.search(r'\b[A-Z][a-z]+ [A-Z][a-z]+\b', memory["text"]),
-                any(word in memory_text for word in ["name is", "i'm", "i am", "called", "my name", "named", "call me"])
-            ])
-            if is_identity:
-                identity_memories.append(memory)
-            else:
-                other_memories.append(memory)
-        
-        # For identity queries, include all identity memories regardless of similarity
-        if query_type == "identity" and identity_memories:
-            # Give them high scores to ensure they're included first
-            for memory in identity_memories:
-                relevant.append((0.9, memory))  # High score for identity memories in identity queries
-        
-        # Process other memories with similarity scoring
-        for memory in other_memories:
-            memory_text = memory["text"].lower()
-            memory_tokens = set(tokenize(memory_text))
-            query_tokens = set(tokenize(query_lower))
-            
-            # Calculate base Jaccard similarity
-            if not query_tokens or not memory_tokens:
-                continue
-                
-            base_similarity = len(query_tokens & memory_tokens) / len(query_tokens | memory_tokens)
-            final_score = base_similarity
-            
-            # Apply boosts based on semantic matching
-            if query_type == "contact":
-                # Boost memories with contact information
-                has_contact_info = any(word in memory_text for word in ["@gmail.com", "@", ".com", 
-                                                                     "phone", "number", "address", 
-                                                                     "http", "www", "tel:"])
-                if has_contact_info:
-                    final_score *= 1.4  # 40% boost for contact-related memories
-            
-            elif query_type == "preference":
-                # Boost memories with preference indicators
-                has_preference = any(word in memory_text for word in ["like", "love", "hate", "dislike", 
-                                                                   "prefer", "favorite", "enjoy", "interested"])
-                if has_preference:
-                    final_score *= 1.3  # 30% boost for preference-related memories
-            
-            elif query_type == "task":
-                # Boost memories with task indicators
-                has_task = any(word in memory_text for word in ["todo", "task", "remind", "meeting", 
-                                                              "appointment", "schedule", "deadline", "need to"])
-                if has_task:
-                    final_score *= 1.3  # 30% boost for task-related memories
-            
-            # Always consider exact phrase matches as highly relevant
-            if query.lower() in memory["text"].lower():
-                final_score = max(final_score, 0.8)  # Ensure high relevance for exact matches
-            
-            # Include memory if it meets threshold after boosts
-            if final_score >= threshold:
-                relevant.append((final_score, memory))
-        
-        # Sort by final score (descending) and return top matches
-        relevant.sort(key=lambda x: x[0], reverse=True)
-        return [mem for _, mem in relevant[:max_items]]
+__all__ = [
+    "MemoryManager",
+    "MemoryStoreUnreadable",
+    "get_text_similarity",
+    "tokenize",
+]

services/memory/memory_extractor.py

@@ -17,6 +17,8 @@ import os
 import re
 from typing import Optional
 
+from src.memory import MemoryStoreUnreadable
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,7 +36,7 @@ def _fingerprint_entries(entries) -> str:
     only on id+text+category. Any add/edit/delete invalidates it."""
     items = sorted(
         (str(e.get("id", "")), e.get("text", ""), e.get("category", ""))
-        for e in entries
+        for e in _memory_dicts(entries)
     )
     h = hashlib.sha256()
     for triple in items:
@@ -42,10 +44,16 @@ def _fingerprint_entries(entries) -> str:
     return h.hexdigest()
 
 
+def _memory_dicts(entries):
+    for entry in entries or []:
+        if isinstance(entry, dict):
+            yield entry
+
+
 def _load_tidy_state(memory_manager) -> dict:
     path = _tidy_state_path(memory_manager)
     try:
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             data = json.load(f)
         return data if isinstance(data, dict) else {}
     except (FileNotFoundError, json.JSONDecodeError):
@@ -57,7 +65,7 @@ def _save_tidy_state(memory_manager, owner: Optional[str], fingerprint: str) ->
     state = _load_tidy_state(memory_manager)
     state[owner or ""] = {"fingerprint": fingerprint}
     try:
-        with open(path, "w") as f:
+        with open(path, "w", encoding="utf-8") as f:
             json.dump(state, f, indent=2)
     except OSError as e:
         logger.warning(f"Could not persist tidy fingerprint: {e}")
@@ -82,6 +90,29 @@ EXTRACT_SYSTEM_PROMPT = (
 # How many recent messages to include for extraction
 CONTEXT_WINDOW = 6
 
+PERSONA_MEMORY_SYSTEM_PROMPT = (
+    "You maintain concise continuity notes for one active chat persona. "
+    "Update the existing notes using only durable details established in the transcript. "
+    "Keep details that help the same persona stay consistent in future conversations: "
+    "relationship context, names, preferences, recurring story details, boundaries, and unresolved threads. "
+    "Do not store generic chat events, temporary wording, assistant reasoning, or one-off requests. "
+    "Never invent details. Return only the updated notes as short bullet points, max 12 bullets. "
+    "If there is nothing worth keeping, return the existing notes unchanged or an empty string."
+)
+
+HEALTH_PERSONA_MEMORY_SYSTEM_PROMPT = (
+    "You maintain a cautious health-record brief for a medical reasoning persona. "
+    "Update the existing brief using only medically durable information from the transcript. "
+    "Keep facts that may matter in future health conversations: confirmed diagnoses, chronic conditions, "
+    "surgeries/procedures, allergies, regular medications/supplements, important test results, clinicians/hospitals, "
+    "ongoing symptoms or care plans, and the user's preferences for medical explanations. "
+    "Use uncertainty labels when needed: 'reported', 'possible', 'asked about', 'unclear'. "
+    "Do not turn guesses into diagnoses. Do not store casual one-off symptoms unless they are recurring, severe, "
+    "or tied to an ongoing episode. Never invent facts. Return only the updated brief with these headings when useful: "
+    "Medical profile, Medications/allergies, Episodes/open questions, Preferences. Max 16 concise bullets total. "
+    "If nothing medically durable changed, return the existing brief unchanged or an empty string."
+)
+
 AUDIT_SYSTEM_PROMPT = (
     "You are a memory database curator. Be CONSERVATIVE: remove only TRUE "
     "duplicates and clearly useless entries. Every distinct fact must survive. "
@@ -104,6 +135,20 @@ AUDIT_SYSTEM_PROMPT = (
 )
 
 AUDIT_INTERVAL = 5  # audit every N new memories added
+AUTO_PINNED_IDENTITY_LIMIT = 5
+
+
+def _is_owner_memory(entry, owner):
+    if owner:
+        return entry.get("owner") == owner or entry.get("owner") is None
+    return True
+
+
+def _is_auto_pinned_identity(entry):
+    return (
+        bool(entry.get("pinned"))
+        and (entry.get("category") or "").lower() in {"identity", "contact"}
+    )
 _extractions_since_audit = 0
 
 
@@ -186,11 +231,19 @@ def _fallback_memory_candidates(messages) -> list[dict]:
             if place:
                 add(f"User lives in {place}.", "identity")
 
-        m = re.search(r"\bi (?:prefer|like|love|hate|do not like|don't like)\s+([^.!?\n]{4,100})", text, re.I)
+        m = re.search(r"\bi (prefer|like|love|hate|do not like|don't like)\s+([^.!?\n]{4,100})", text, re.I)
         if m:
-            preference = _clean_memory_value(m.group(1), 100)
+            preference = _clean_memory_value(m.group(2), 100)
             if preference:
-                add(f"User prefers {preference}.", "preference")
+                # The same pattern catches likes and dislikes; keep the stored
+                # sentiment faithful instead of recording every match as a
+                # preference ("I hate cilantro" must not become "User prefers
+                # cilantro").
+                verb = m.group(1).lower()
+                if verb in ("hate", "do not like", "don't like"):
+                    add(f"User dislikes {preference}.", "preference")
+                else:
+                    add(f"User prefers {preference}.", "preference")
 
         m = re.search(
             r"\bi (?:(?:want|would like|plan|hope) to|wanna) "
@@ -211,7 +264,7 @@ def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) ->
     new_tokens = set(new_text.lower().split())
     if not new_tokens:
         return False
-    for entry in existing:
+    for entry in _memory_dicts(existing):
         old_tokens = set(entry.get("text", "").lower().split())
         if not old_tokens:
             continue
@@ -222,6 +275,43 @@ def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) ->
     return False
 
 
+def _parse_extraction_json(raw: str) -> list:
+    """Parse the extraction LLM's reply into a list of facts, tolerating
+    reasoning-model noise.
+
+    The model emits <think>…</think> (and sometimes a prose preamble or a
+    ```json fence) AROUND the JSON array; without stripping it, json.loads
+    bombs and the run silently yields "0 candidates". Pure str -> list (no
+    LLM/network); returns [] on any parse failure instead of raising.
+    """
+    text = (raw or "").strip()
+    try:
+        from src.text_helpers import strip_think as _strip_think
+        text = _strip_think(text, prose=True, prompt_echo=True).strip()
+    except Exception:
+        pass
+    if text.startswith("```"):
+        text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+    # JSON may still be embedded in surrounding commentary (leading prose or
+    # trailing remarks like "[...] Done!") — slice from the first '[' to the
+    # last ']' whenever both exist. Slice unconditionally: a reply that starts
+    # with '[' can still carry trailing commentary that breaks json.loads.
+    _start = text.find("[")
+    _end = text.rfind("]")
+    if 0 <= _start < _end:
+        text = text[_start : _end + 1]
+
+    try:
+        facts = json.loads(text)
+    except json.JSONDecodeError:
+        logger.debug("Memory extraction returned non-JSON: %r", (raw or "")[:120])
+        return []
+    except Exception:
+        logger.debug("Memory extraction returned non-JSON: %r", (raw or "")[:120])
+        return []
+    return facts if isinstance(facts, list) else []
+
+
 async def extract_and_store(
     session,
     memory_manager,
@@ -235,6 +325,10 @@ async def extract_and_store(
     Designed to run as a background task (asyncio.create_task).
     Errors are logged, never raised.
     """
+    if not endpoint_url or not model:
+        logger.debug("[memory-extract] No model or URL provided, skipping")
+        return
+
     try:
         from src.llm_core import llm_call_async
 
@@ -245,11 +339,55 @@ async def extract_and_store(
         if len(recent) < 2:
             return  # Need at least a user message and assistant response
 
-        fallback_facts = _fallback_memory_candidates(recent)
+        # Strip media (images/audio) from messages — background memory extraction
+        # only needs the text. The VL-generated descriptions are already in the
+        # text content of the messages. This avoids sending image tokens to
+        # non-vision models and prevents accidental "vision grounding" triggers.
+        stripped_recent = []
+        for msg in recent:
+            role = msg.get("role")
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                # Filter out multimodal blocks that aren't text
+                text_only = [b for b in content if isinstance(b, dict) and b.get("type") == "text"]
+                if not text_only and content:
+                    continue
+                content = text_only
+            stripped_recent.append({"role": role, "content": content})
 
+        if not stripped_recent:
+            return
+
+        fallback_facts = _fallback_memory_candidates(stripped_recent)
+
+        # Flatten the window into a SINGLE user message instead of appending the
+        # raw alternating role messages. Passed as raw chat messages, the model
+        # treats the window as a conversation to CONTINUE rather than a transcript
+        # to ANALYZE, so it reliably extracts nothing — typically returning `[]`
+        # (and, depending on the input, sometimes an empty or <think>-only
+        # completion when the window ends on an assistant turn). This was the real
+        # cause of auto-memory logging "0 candidates" on every run. Reframing it as
+        # one "analyze this transcript, return the JSON array" user message makes
+        # the model actually extract. Controlled repro on this model: 0/6 trials
+        # with the old structure vs 6/6 with this one. The skill extractor flattens
+        # for the same reason.
+        def _flatten_msg(m):
+            c = m.get("content", "")
+            if isinstance(c, list):
+                c = " ".join(
+                    b.get("text", "") for b in c
+                    if isinstance(b, dict) and b.get("type") == "text"
+                )
+            return f"{m.get('role', '?')}: {c}"
+
+        transcript = "\n\n".join(_flatten_msg(m) for m in stripped_recent)
         extraction_messages = [
             {"role": "system", "content": EXTRACT_SYSTEM_PROMPT},
-        ] + recent
+            {"role": "user", "content": (
+                "Conversation to analyze:\n\n" + transcript
+                + "\n\nReturn the JSON array of durable facts now (or [] if none)."
+            )},
+        ]
 
         facts = []
         try:
@@ -258,19 +396,20 @@ async def extract_and_store(
                 model,
                 extraction_messages,
                 temperature=0.1,
-                max_tokens=500,
+                # A reasoning model spends most of its budget on <think> tokens
+                # BEFORE emitting the JSON, so the old 500 truncated the response
+                # before any JSON appeared → every run logged "0 candidates". The
+                # audit path hit the same wall and raised to 16384; extraction's
+                # output (a short facts list) is small, so an ample ceiling is
+                # enough once thinking has room.
+                max_tokens=4096,
                 headers=headers,
             )
 
-            # Parse JSON from response (handle markdown fences if model wraps them)
-            text = raw.strip()
-            if text.startswith("```"):
-                text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-
-            try:
-                facts = json.loads(text)
-            except json.JSONDecodeError:
-                logger.debug("Memory extraction returned non-JSON")
+            # Parse JSON, tolerating reasoning-model noise (<think> blocks, a
+            # ```json fence, and leading/trailing commentary). See
+            # _parse_extraction_json — returns [] rather than raising.
+            facts = _parse_extraction_json(raw)
         except Exception as e:
             logger.warning(f"LLM memory extraction failed; using fallback candidates if available: {e}")
 
@@ -287,8 +426,18 @@ async def extract_and_store(
         # Get owner from session
         _owner = getattr(session, 'owner', None)
 
-        existing = memory_manager.load_all()
+        # Strict load: this is a read-modify-write. Degrading to [] here would
+        # save only the newly extracted facts and drop the entire store.
+        try:
+            existing = memory_manager.load_all_for_update()
+        except MemoryStoreUnreadable as e:
+            logger.error("Skipping auto memory extraction, store unreadable: %s", e)
+            return
         added = 0
+        auto_pinned_identity_count = sum(
+            1 for entry in existing
+            if _is_owner_memory(entry, _owner) and _is_auto_pinned_identity(entry)
+        )
 
         for fact in facts:
             if isinstance(fact, str):
@@ -296,19 +445,37 @@ async def extract_and_store(
                 category = "fact"
             elif isinstance(fact, dict):
                 fact_text = fact.get("text", "").strip()
-                category = fact.get("category", "fact")
+                category = str(fact.get("category", "fact") or "fact")
             else:
                 continue
 
             if not fact_text or len(fact_text) < 5:
                 continue
 
-            # Dedup: check vector similarity first (fast), then exact text match
+            # Dedup: check vector similarity first (fast), then exact text match.
+            # A runtime embedding/ChromaDB failure (backend OOM, model evicted,
+            # remote endpoint down) must not abort the whole batch — fall through
+            # to the text/fuzzy dedup below instead of losing every validated
+            # fact extracted this session. (`.healthy` is only set at init, so
+            # it does not catch failures that develop later.)
             if memory_vector and memory_vector.healthy:
-                existing_id = memory_vector.find_similar(fact_text, threshold=0.72)
+                try:
+                    existing_id = memory_vector.find_similar(fact_text, threshold=0.72)
+                except Exception as e:
+                    logger.warning(f"Memory dedup (vector) unavailable, using text fallback: {e}")
+                    existing_id = None
                 if existing_id:
-                    logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}")
-                    continue
+                    # The vector store is a single shared collection with no
+                    # owner metadata, so find_similar can return ANOTHER
+                    # tenant's memory. Only treat it as a duplicate when the
+                    # match is this user's own (or a legacy unowned) memory —
+                    # otherwise the user's freshly-extracted fact would be
+                    # silently dropped. Mirror the owner predicate used by the
+                    # text dedup below; cross-tenant/stale matches fall through.
+                    _match = next((e for e in existing if e.get("id") == existing_id), None)
+                    if _match is not None and (_match.get("owner") == _owner or _match.get("owner") is None):
+                        logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}")
+                        continue
 
             # Text dedup fallback: exact match + fuzzy similarity
             user_existing = [e for e in existing if e.get("owner") == _owner or e.get("owner") is None] if _owner else existing
@@ -320,9 +487,15 @@ async def extract_and_store(
                 continue
 
             entry = memory_manager.add_entry(fact_text, source="auto", category=category, owner=_owner)
-            # Auto-pin identity facts (name, job, location) — core context
-            if category == "identity":
+            # Auto-pin only the first few identity/contact facts. Extra identity
+            # memories are still saved, but they must be recalled by relevance
+            # instead of riding along in every prompt forever.
+            if (
+                category.lower() in {"identity", "contact"}
+                and auto_pinned_identity_count < AUTO_PINNED_IDENTITY_LIMIT
+            ):
                 entry["pinned"] = True
+                auto_pinned_identity_count += 1
             if hasattr(session, "session_id"):
                 entry["session_id"] = session.session_id
             elif hasattr(session, "name"):
@@ -330,9 +503,14 @@ async def extract_and_store(
 
             existing.append(entry)
 
-            # Add to vector index
+            # Add to vector index. The JSON store (saved below) is the source of
+            # truth and the keyword path can still retrieve this entry, so a vector
+            # write failure must not drop the fact or abort the remaining batch.
             if memory_vector and memory_vector.healthy:
-                memory_vector.add(entry["id"], fact_text)
+                try:
+                    memory_vector.add(entry["id"], fact_text)
+                except Exception as e:
+                    logger.warning(f"Memory vector add failed for {entry['id']}: {e}")
 
             added += 1
 
@@ -361,6 +539,88 @@ async def extract_and_store(
         logger.error(f"Memory extraction failed: {e}")
 
 
+async def update_persona_memory(
+    session,
+    preset_manager,
+    character_name: str,
+    endpoint_url: str,
+    model: str,
+    headers: Optional[dict] = None,
+    schema: str = "general",
+):
+    """Update the active persona's continuity notes from recent conversation.
+
+    Persona memory is stored with the persona/template data, not in the global
+    memory DB, so deleting a saved persona also deletes its notes.
+    """
+    character_name = (character_name or "").strip()
+    if not character_name or not endpoint_url or not model or preset_manager is None:
+        return
+
+    try:
+        from src.llm_core import llm_call_async
+        from src.text_helpers import strip_think
+
+        custom = {}
+        try:
+            custom = preset_manager.presets.get("custom", {}) if isinstance(preset_manager.presets, dict) else {}
+        except Exception:
+            custom = {}
+        existing_memory = ""
+        if isinstance(custom, dict) and custom.get("character_name") == character_name:
+            existing_memory = custom.get("persona_memory", "") or ""
+
+        messages = session.get_context_messages()
+        recent = messages[-CONTEXT_WINDOW:] if len(messages) > CONTEXT_WINDOW else messages
+        if len(recent) < 2:
+            return
+
+        lines = []
+        for msg in recent:
+            role = msg.get("role")
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                content = " ".join(
+                    b.get("text", "") for b in content
+                    if isinstance(b, dict) and b.get("type") == "text"
+                )
+            content = str(content or "").strip()
+            if content:
+                lines.append(f"{role}: {content}")
+        if not lines:
+            return
+
+        system_prompt = HEALTH_PERSONA_MEMORY_SYSTEM_PROMPT if schema == "health" else PERSONA_MEMORY_SYSTEM_PROMPT
+        raw = await llm_call_async(
+            endpoint_url,
+            model,
+            [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": (
+                    f"Persona name: {character_name}\n\n"
+                    f"Existing continuity notes:\n{existing_memory or '(none)'}\n\n"
+                    "Recent transcript:\n"
+                    + "\n\n".join(lines)
+                    + "\n\nReturn only the updated continuity notes."
+                )},
+            ],
+            temperature=0.1,
+            max_tokens=1200,
+            headers=headers,
+        )
+
+        updated = strip_think(str(raw or ""), prose=True, prompt_echo=True).strip()
+        updated = re.sub(r"^```(?:text|markdown)?\s*|\s*```$", "", updated, flags=re.I | re.S).strip()
+        if len(updated) > 6000:
+            updated = updated[:6000].rstrip()
+        if updated == existing_memory:
+            return
+        if preset_manager.update_persona_memory(character_name, updated):
+            logger.info("Updated persona memory for %s", character_name)
+    except Exception as e:
+        logger.warning("Persona memory update failed: %s", e)
+
+
 async def audit_memories(
     memory_manager,
     memory_vector,
@@ -503,24 +763,38 @@ async def audit_memories(
 
         # Merge audited entries back with other users' entries
         if owner:
-            all_entries = memory_manager.load_all()
+            # Strict load: the merge below reconstructs the whole file. If this
+            # degraded to [] we would save only this owner's audited slice and
+            # destroy every other tenant's memories.
+            try:
+                all_entries = memory_manager.load_all_for_update()
+            except MemoryStoreUnreadable as e:
+                logger.error("Aborting memory audit save, store unreadable: %s", e)
+                return {
+                    "before": before_count,
+                    "after": before_count,
+                    "error": "store_unreadable",
+                }
             audited_ids = {e["id"] for e in final_entries}
             other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
             # Also keep legacy entries that weren't part of this audit
             for e in all_entries:
                 if e.get("owner") is None and e["id"] not in audited_ids and e["id"] not in {o["id"] for o in other_entries}:
                     other_entries.append(e)
-            memory_manager.save(final_entries + other_entries)
+            saved_entries = final_entries + other_entries
         else:
-            memory_manager.save(final_entries)
+            saved_entries = final_entries
+        memory_manager.save(saved_entries)
         logger.info(
             f"Memory audit complete: {before_count} -> {after_count} entries "
             f"({before_count - after_count} removed/merged)"
         )
 
-        # Rebuild vector index
+        # Rebuild vector index from the full saved set, not just this owner's
+        # slice — otherwise the shared collection is wiped of every other
+        # owner's entries until they happen to run their own audit.
         if memory_vector and memory_vector.healthy:
-            memory_vector.rebuild(final_entries)
+            memory_vector.rebuild(saved_entries)
 
         # Persist the post-tidy fingerprint so the next call short-circuits
         # if nothing has changed in the meantime.

services/memory/memory_vector.py

@@ -1,175 +1,5 @@
-"""
-memory_vector.py
+"""Compatibility import for the canonical memory vector store."""
 
-ChromaDB-backed vector store for memory entries.
-Shares the EmbeddingClient with RAG to save memory.
-Stores pre-computed embeddings (ChromaDB does not manage embedding).
-"""
+from src.memory_vector import MemoryVectorStore
 
-import logging
-from typing import List, Dict, Optional
-
-logger = logging.getLogger(__name__)
-
-
-class MemoryVectorStore:
-    """Vector index over memory entries for semantic retrieval."""
-
-    COLLECTION_NAME = "odysseus_memories"
-
-    def __init__(self, data_dir: str, embedding_model=None):
-        self._model = embedding_model
-        self._collection = None
-        self._healthy = False
-
-        self._initialize()
-
-    def _initialize(self):
-        try:
-            from src.chroma_client import get_chroma_client
-
-            if self._model is None:
-                from src.embeddings import get_embedding_client
-                self._model = get_embedding_client()
-                if self._model is None:
-                    raise RuntimeError("No embedding backend available")
-                logger.info(f"MemoryVectorStore using embeddings: {self._model.url}")
-
-            client = get_chroma_client()
-            self._collection = client.get_or_create_collection(
-                name=self.COLLECTION_NAME,
-                metadata={"hnsw:space": "cosine"},
-            )
-
-            self._healthy = True
-            count = self._collection.count()
-            logger.info(f"MemoryVectorStore ready (entries={count})")
-
-        except Exception as e:
-            logger.error(f"MemoryVectorStore init failed: {e}")
-
-    @property
-    def healthy(self) -> bool:
-        return self._healthy
-
-    def _embed(self, texts: List[str]) -> List[List[float]]:
-        vecs = self._model.encode(texts, normalize_embeddings=True)
-        return vecs.tolist()
-
-    def count(self) -> int:
-        """Return the number of stored vectors."""
-        if not self._healthy:
-            return 0
-        return self._collection.count()
-
-    def add(self, memory_id: str, text: str):
-        """Add a single memory entry to the vector index."""
-        if not self._healthy:
-            return
-        # Skip if already exists
-        existing = self._collection.get(ids=[memory_id])
-        if existing["ids"]:
-            return
-        embeddings = self._embed([text])
-        self._collection.add(
-            ids=[memory_id],
-            embeddings=embeddings,
-            documents=[text],
-            metadatas=[{"source": "memory"}],
-        )
-
-    def remove(self, memory_id: str):
-        """Remove a memory entry. O(1) — no rebuild needed."""
-        if not self._healthy:
-            return
-        try:
-            self._collection.delete(ids=[memory_id])
-        except Exception as e:
-            logger.warning(f"memory remove {memory_id}: {e}")
-
-    def search(self, query: str, k: int = 8) -> List[Dict]:
-        """Search for the most relevant memory IDs by semantic similarity.
-        Returns list of {"memory_id": str, "score": float}.
-
-        ChromaDB cosine distance = 1 - cosine_similarity.
-        We convert back: similarity = 1.0 - distance.
-        """
-        if not self._healthy or self._collection.count() == 0:
-            return []
-
-        embeddings = self._embed([query])
-        actual_k = min(k, self._collection.count())
-        results = self._collection.query(
-            query_embeddings=embeddings,
-            n_results=actual_k,
-        )
-
-        out = []
-        for idx, mid in enumerate(results["ids"][0]):
-            distance = results["distances"][0][idx]
-            out.append({
-                "memory_id": mid,
-                "score": round(1.0 - distance, 4),
-            })
-        return out
-
-    def find_similar(self, text: str, threshold: float = 0.92) -> Optional[str]:
-        """Check if a near-duplicate exists. Returns memory_id if found, else None."""
-        if not self._healthy or self._collection.count() == 0:
-            return None
-
-        embeddings = self._embed([text])
-        results = self._collection.query(
-            query_embeddings=embeddings,
-            n_results=1,
-        )
-
-        if results["ids"][0]:
-            distance = results["distances"][0][0]
-            similarity = 1.0 - distance
-            if similarity >= threshold:
-                return results["ids"][0][0]
-        return None
-
-    def rebuild(self, memories: List[Dict]):
-        """Rebuild the entire index from a list of memory entries.
-        Each entry must have 'id' and 'text' keys."""
-        if not self._healthy:
-            return
-
-        from src.chroma_client import get_chroma_client
-
-        # Delete and recreate collection for a clean rebuild
-        client = get_chroma_client()
-        try:
-            client.delete_collection(self.COLLECTION_NAME)
-        except Exception:
-            pass
-        self._collection = client.get_or_create_collection(
-            name=self.COLLECTION_NAME,
-            metadata={"hnsw:space": "cosine"},
-        )
-
-        texts = []
-        ids = []
-        for mem in memories:
-            text = mem.get("text", "").strip()
-            mid = mem.get("id", "")
-            if text and mid:
-                texts.append(text)
-                ids.append(mid)
-
-        if texts:
-            # Batch in chunks of 100 to avoid oversized requests
-            for i in range(0, len(texts), 100):
-                batch_texts = texts[i:i + 100]
-                batch_ids = ids[i:i + 100]
-                embeddings = self._embed(batch_texts)
-                self._collection.add(
-                    ids=batch_ids,
-                    embeddings=embeddings,
-                    documents=batch_texts,
-                    metadatas=[{"source": "memory"}] * len(batch_ids),
-                )
-
-        logger.info(f"MemoryVectorStore rebuilt with {len(ids)} entries")
+__all__ = ["MemoryVectorStore"]

services/memory/service.py

@@ -7,6 +7,8 @@ import os
 
 from .memory import MemoryManager
 from .memory_vector import MemoryVectorStore
+from src.memory_provider import MemoryRecord, NativeMemoryProvider
+from src.constants import DATA_DIR
 
 
 @dataclass
@@ -37,11 +39,38 @@ class MemoryService:
         results = await service.recall("preferences")
     """
 
-    def __init__(self, data_dir: str = "data"):
+    def __init__(self, data_dir: str = DATA_DIR):
         self.manager = MemoryManager(data_dir)
         self.vector_store = MemoryVectorStore(data_dir) if os.path.exists(
             os.path.join(data_dir, "memory_vectors")
         ) else None
+        self.provider = NativeMemoryProvider(self.manager, self.vector_store)
+
+    def _sync_provider(self) -> None:
+        self.provider.memory_vector = self.vector_store
+
+    @staticmethod
+    def _to_memory(entry: Dict[str, Any], metadata: Optional[Dict[str, Any]] = None) -> Memory:
+        return Memory(
+            id=entry.get("id", ""),
+            text=entry.get("text", ""),
+            timestamp=entry.get("timestamp", 0),
+            session_id=entry.get("session_id"),
+            metadata=metadata or {},
+        )
+
+    @staticmethod
+    def _record_to_memory(record: MemoryRecord, metadata: Optional[Dict[str, Any]] = None) -> Memory:
+        merged_metadata = dict(record.metadata)
+        if metadata:
+            merged_metadata.update(metadata)
+        return Memory(
+            id=record.id,
+            text=record.text,
+            timestamp=record.timestamp,
+            session_id=record.session_id,
+            metadata=merged_metadata,
+        )
 
     async def remember(self, text: str, session_id: Optional[str] = None) -> Memory:
         """
@@ -54,31 +83,9 @@ class MemoryService:
         Returns:
             Created Memory object
         """
-        import uuid
-        import time
-
-        memory_id = str(uuid.uuid4())[:8]
-        timestamp = int(time.time())
-
-        entry = {
-            "id": memory_id,
-            "text": text,
-            "timestamp": timestamp,
-            "session_id": session_id,
-        }
-
-        self.manager.add_memory(entry)
-
-        # Also add to vector store if available
-        if self.vector_store:
-            self.vector_store.add(text, {"id": memory_id, "session_id": session_id})
-
-        return Memory(
-            id=memory_id,
-            text=text,
-            timestamp=timestamp,
-            session_id=session_id,
-        )
+        self._sync_provider()
+        record = await self.provider.remember(text, session_id=session_id)
+        return self._record_to_memory(record)
 
     async def recall(self, query: str, top_k: int = 5) -> MemorySearchResult:
         """
@@ -91,47 +98,29 @@ class MemoryService:
         Returns:
             MemorySearchResult with matching memories
         """
-        # Try vector search first
-        if self.vector_store:
-            results = self.vector_store.search(query, k=top_k)
-            memories = [
-                Memory(
-                    id=r.get("id", ""),
-                    text=r.get("text", ""),
-                    timestamp=r.get("timestamp", 0),
-                    session_id=r.get("session_id"),
-                    metadata=r.get("metadata", {}),
-                )
-                for r in results
-            ]
-            return MemorySearchResult(memories=memories, query=query, total=len(memories))
-
-        # Fallback to keyword search
-        results = self.manager.search_memories(query, limit=top_k)
+        self._sync_provider()
+        results = await self.provider.recall(query, top_k=top_k)
         memories = [
-            Memory(
-                id=m.get("id", ""),
-                text=m.get("text", ""),
-                timestamp=m.get("timestamp", 0),
-                session_id=m.get("session_id"),
-            )
-            for m in results
+            self._record_to_memory(hit.memory, metadata={"score": hit.score})
+            if hit.score is not None
+            else self._record_to_memory(hit.memory)
+            for hit in results
         ]
         return MemorySearchResult(memories=memories, query=query, total=len(memories))
 
     def get_all(self, limit: int = 100) -> List[Memory]:
         """Get all memories."""
-        memories = self.manager.get_memories(limit=limit)
-        return [
-            Memory(
-                id=m.get("id", ""),
-                text=m.get("text", ""),
-                timestamp=m.get("timestamp", 0),
-                session_id=m.get("session_id"),
-            )
-            for m in memories
-        ]
+        records = self.manager.load_all()[:limit]
+        return [self._to_memory(m) for m in records]
 
     def delete(self, memory_id: str) -> bool:
         """Delete a memory by ID."""
-        return self.manager.delete_memory(memory_id)
+        memories = self.manager.load_all()
+        remaining = [m for m in memories if m.get("id") != memory_id]
+        if len(remaining) == len(memories):
+            return False
+
+        self.manager.save(remaining)
+        if self.vector_store and self.vector_store.healthy:
+            self.vector_store.remove(memory_id)
+        return True

services/memory/skill_extractor.py

@@ -28,6 +28,10 @@ SKILL_EXTRACT_PROMPT = (
     "(personal errands, a specific person/place/date, casual conversation).\n"
     "- A pure question/answer or explanation with no transferable method.\n"
     "- The agent failed, gave up, or the approach is not worth repeating.\n\n"
+    "- Routine use of an existing tool, or a generic checklist with no new discovery.\n"
+    "Prefer a specific successful workaround, an unexpected pitfall, or a verified "
+    "sequence that would save rediscovery. Preserve exact useful commands and "
+    "verification steps, but replace private identifiers and credentials with placeholders.\n\n"
     "When (and only when) a genuine reusable procedure exists, return a JSON "
     "object with:\n"
     '- "title": short name (under 10 words)\n'
@@ -48,6 +52,77 @@ MIN_CONFIDENCE = 0.6
 CONTEXT_WINDOW = 12
 
 
+def _skill_dicts(skills):
+    for skill in skills or []:
+        if isinstance(skill, dict):
+            yield skill
+
+
+def _has_duplicate_title(skills, title: str) -> bool:
+    wanted = title.lower()
+    for skill in _skill_dicts(skills):
+        existing = skill.get("title", "")
+        if isinstance(existing, str) and existing.lower() == wanted:
+            return True
+    return False
+
+
+def _extract_json_object(text: str) -> Optional[dict]:
+    """Best-effort extraction of a JSON object from an LLM response.
+
+    The response may be wrapped in code fences or surrounded by prose. Uses
+    json.JSONDecoder().raw_decode() to locate the boundaries of complete JSON
+    objects starting at each '{' position. Nested objects are filtered out to
+    keep only top-level candidates. If multiple non-overlapping valid JSON
+    objects are found, it is treated as ambiguous and returns None. Otherwise,
+    returns the single valid candidate dictionary.
+    """
+    if not text:
+        return None
+    s = text.strip()
+    if s.startswith("```"):
+        s = s.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+
+    decoder = json.JSONDecoder()
+    candidates = []
+
+    start = s.find("{")
+    while start != -1:
+        try:
+            obj, idx = decoder.raw_decode(s[start:])
+            end_pos = start + idx
+            if isinstance(obj, dict):
+                candidates.append((start, end_pos, obj))
+        except (json.JSONDecodeError, ValueError):
+            pass
+        start = s.find("{", start + 1)
+
+    # Filter out nested candidates to identify top-level dictionaries
+    top_level = []
+    for c in candidates:
+        is_nested = False
+        for other in candidates:
+            if other == c:
+                continue
+            if other[0] <= c[0] and c[1] <= other[1]:
+                is_nested = True
+                break
+        if not is_nested:
+            top_level.append(c)
+
+    if not top_level:
+        return None
+
+    if len(top_level) > 1:
+        logger.debug(
+            "[skill-extract] Found multiple non-overlapping JSON objects: %s",
+            [item[2].get("title") for item in top_level]
+        )
+        return None
+
+    return top_level[0][2]
+
+
 async def maybe_extract_skill(
     session,
     skills_manager,
@@ -59,6 +134,10 @@ async def maybe_extract_skill(
     owner: Optional[str] = None,
 ):
     """Extract a skill if the agent run was complex enough."""
+    if not model:
+        logger.debug("[skill-extract] No model provided, skipping")
+        return None
+
     # Quiet by default; flip to DEBUG when chasing extractor issues.
     logger.debug(
         "[skill-extract] start: rounds=%d tools=%d model=%s owner=%s",
@@ -78,9 +157,23 @@ async def maybe_extract_skill(
             logger.debug("[skill-extract] no recent messages, skipping")
             return None
 
+        # Strip media (images/audio) from messages
+        stripped_recent = []
+        for msg in recent:
+            content = msg.get("content", "")
+            if isinstance(content, list):
+                text_only = [b for b in content if isinstance(b, dict) and b.get("type") == "text"]
+                if not text_only and content:
+                    continue
+                content = text_only
+            stripped_recent.append({"role": msg.get("role"), "content": content})
+
+        if not stripped_recent:
+            return None
+
         # Build conversation summary for extraction
         conv_lines = []
-        for msg in recent:
+        for msg in stripped_recent:
             role = msg.get("role", "?")
             content = msg.get("content", "")
             if isinstance(content, list):
@@ -136,21 +229,14 @@ async def maybe_extract_skill(
         except Exception:
             pass
 
-        # Parse JSON
-        text = response.strip()
-        if text.startswith("```"):
-            text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-        # After strip_think, the JSON may still be embedded inside surrounding
-        # commentary — slice from the first '{' to the matching last '}'.
-        if text and text[0] != "{":
-            _start = text.find("{")
-            _end = text.rfind("}")
-            if 0 <= _start < _end:
-                text = text[_start : _end + 1]
-
-        data = json.loads(text)
-        if not data or not isinstance(data, dict):
-            logger.debug("[skill-extract] parsed JSON not a dict, dropping")
+        # Parse JSON. The object may be wrapped in code fences or surrounded by
+        # commentary (and may contain a stray/invalid brace fragment before
+        # the real object — including one that makes the response itself look
+        # like it starts with '{'), so use a tolerant extractor that tries the
+        # whole string first and then each '{' candidate left-to-right.
+        data = _extract_json_object(response)
+        if not data:
+            logger.debug("[skill-extract] no JSON object found in response, dropping")
             return None
 
         title = data.get("title", "").strip()
@@ -173,10 +259,13 @@ async def maybe_extract_skill(
 
         # Check for duplicate skills
         existing = skills_manager.load(owner=owner)
-        for sk in existing:
-            if sk.get("title", "").lower() == title.lower():
-                logger.debug("[skill-extract] '%s' already exists — dropped as duplicate", title)
-                return None
+        if _has_duplicate_title(existing, title):
+            logger.debug("[skill-extract] '%s' already exists — dropped as duplicate", title)
+            return None
+
+        # Automatic approval happens only after the audit has passed. A new
+        # extraction begins as a draft so it cannot enter chat context early.
+        _initial_status = "draft"
 
         entry = skills_manager.add_skill(
             title=title,
@@ -188,6 +277,7 @@ async def maybe_extract_skill(
             confidence=data.get("confidence", 0.7),
             session_id=getattr(session, "session_id", None),
             owner=owner,
+            status=_initial_status,
         )
         try:
             from src.event_bus import fire_event

services/memory/skill_format.py

@@ -50,7 +50,7 @@ import json
 import logging
 import re
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 
 logger = logging.getLogger(__name__)
@@ -100,6 +100,18 @@ def _parse_scalar(raw: str) -> Any:
     if raw.lower() in ("null", "none", "~"):
         return None
     if (raw[0] == raw[-1]) and raw[0] in ("'", '"'):
+        if raw[0] == '"':
+            # _emit_scalar writes double-quoted scalars with json.dumps, so
+            # decode the escapes instead of only stripping the quotes. Without
+            # this, `\"` / `\\` / `\uXXXX` stayed verbatim in the value and the
+            # next save escaped their backslashes again, doubling them on every
+            # load/save cycle (issue #5210).
+            try:
+                return json.loads(raw)
+            except ValueError:
+                # Hand-written file using escapes JSON rejects (e.g. a bare
+                # Windows path). Keep the previous literal reading.
+                pass
         return raw[1:-1]
     # Try number
     try:
@@ -171,6 +183,26 @@ def parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]:
     return fm, body
 
 
+# Characters that force a quoted scalar. The punctuation would otherwise change
+# how the value reads back; the second row is every character str.splitlines()
+# treats as a line break, and parse_frontmatter() reads one scalar per line, so
+# emitting one of those bare would split the value across lines.
+_FM_MUST_QUOTE = (
+    ":", "#", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@",
+    "\n", "\r", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029",
+)
+
+# json.dumps escapes every C0 control character, but with ensure_ascii=False it
+# passes NEL / LINE SEPARATOR / PARAGRAPH SEPARATOR through literally, and
+# str.splitlines() still breaks on all three. Re-escape exactly those, which
+# json.loads decodes again on the way in, so the pair stays symmetric.
+_FM_POST_DUMPS_ESCAPES = (
+    ("\x85", "\\u0085"),
+    ("\u2028", "\\u2028"),
+    ("\u2029", "\\u2029"),
+)
+
+
 def _emit_scalar(v: Any) -> str:
     if v is None:
         return "null"
@@ -181,8 +213,15 @@ def _emit_scalar(v: Any) -> str:
     if isinstance(v, list):
         return "[" + ", ".join(_emit_scalar(x) for x in v) + "]"
     s = str(v)
-    if any(c in s for c in (":", "#", "\n", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@")):
-        return json.dumps(s)
+    if any(c in s for c in _FM_MUST_QUOTE):
+        # ensure_ascii=False keeps non-ASCII text as itself. SKILL.md is UTF-8 at
+        # both ends (skills.py reads it, atomic_write_text writes it), so the
+        # \uXXXX form bought nothing and leaked into the parsed value (#5210).
+        out = json.dumps(s, ensure_ascii=False)
+        for ch, esc in _FM_POST_DUMPS_ESCAPES:
+            if ch in out:
+                out = out.replace(ch, esc)
+        return out
     return s
 
 
@@ -441,4 +480,4 @@ class Skill:
 
 
 def _now_iso() -> str:
-    return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

services/memory/skill_importer.py

@@ -0,0 +1,487 @@
+"""Import SKILL.md bundles from public GitHub (or skills.sh → GitHub) URLs."""
+from __future__ import annotations
+
+import ipaddress
+import logging
+import os
+import time
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional, Tuple, cast
+from urllib.parse import quote, urljoin, urlparse
+
+import httpcore
+import httpx
+
+from src.url_safety import _default_resolver, check_outbound_url
+
+logger = logging.getLogger(__name__)
+
+MAX_FILES = 64
+MAX_TOTAL_BYTES = 2_000_000
+MAX_FILE_BYTES = 400_000
+ALLOWED_SUFFIXES = (
+    ".md", ".txt", ".json", ".yaml", ".yml", ".py", ".sh", ".toml",
+    ".js", ".ts", ".css", ".html", ".xml", ".csv",
+)
+TEXT_NAMES = {"skill.md", "license", "license.md", "readme.md"}
+_GITHUB_HOSTS = frozenset({
+    "github.com", "www.github.com", "api.github.com", "raw.githubusercontent.com",
+})
+_SKILLS_SH_HOSTS = frozenset({"skills.sh", "www.skills.sh"})
+
+
+def _github_host(url: str) -> str:
+    return (urlparse(str(url)).hostname or "").lower()
+
+
+def _assert_github_url(url: str, *, context: str = "URL") -> None:
+    host = _github_host(url)
+    if host not in _GITHUB_HOSTS:
+        raise SkillImportError(
+            f"{context} must stay on GitHub (got {host or 'unknown host'})"
+        )
+
+
+@dataclass
+class ResolvedSource:
+    owner: str
+    repo: str
+    ref: str
+    path: str  # directory or file path inside repo (no leading slash)
+
+
+class SkillImportError(ValueError):
+    pass
+
+
+def _safe_relpath(rel: str) -> str:
+    rel = (rel or "").replace("\\", "/").strip().lstrip("/")
+    if not rel or rel.startswith("..") or "/../" in f"/{rel}/":
+        raise SkillImportError(f"unsafe path: {rel!r}")
+    parts = [p for p in rel.split("/") if p and p != "."]
+    if any(p == ".." for p in parts):
+        raise SkillImportError(f"unsafe path: {rel!r}")
+    return "/".join(parts)
+
+
+def _is_text_file(name: str) -> bool:
+    low = name.lower()
+    if low in TEXT_NAMES:
+        return True
+    return any(low.endswith(s) for s in ALLOWED_SUFFIXES)
+
+
+# Max redirect hops to follow manually while re-validating each one.
+_MAX_FETCH_REDIRECTS = 5
+
+
+def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
+    """Parse and de-duplicate one resolver snapshot in resolver order."""
+    ips: List[ipaddress._BaseAddress] = []
+    seen = set()
+    for raw in raw_ips:
+        if not isinstance(raw, str):
+            continue
+        try:
+            ip = ipaddress.ip_address(raw.split("%", 1)[0])
+        except ValueError:
+            continue
+        if ip in seen:
+            continue
+        seen.add(ip)
+        ips.append(ip)
+    return ips
+
+
+def _resolve_and_check_url(url: str) -> List[ipaddress._BaseAddress]:
+    """Return the exact address snapshot approved for one fetch hop."""
+    resolved_ips: List[str] = []
+
+    def _recording_resolver(host: str) -> List[str]:
+        answers = list(_default_resolver(host))
+        resolved_ips[:] = answers
+        return answers
+
+    ok, reason = check_outbound_url(
+        url,
+        block_private=True,
+        resolver=_recording_resolver,
+    )
+    if not ok:
+        raise SkillImportError(f"outbound URL blocked: {reason}")
+
+    pinned_ips = _validated_ips(resolved_ips)
+    if not pinned_ips:
+        raise SkillImportError("outbound URL blocked: host did not resolve to a usable address")
+    return pinned_ips
+
+
+# Backward compatibility alias for tests importing _check_fetch_url directly
+_check_fetch_url = _resolve_and_check_url
+
+
+class _PinnedBackend(httpcore.NetworkBackend):
+    """Connect only to addresses from one validated DNS snapshot."""
+
+    def __init__(self, ips: List[ipaddress._BaseAddress]):
+        self._ips = [str(ip) for ip in ips]
+        self._real = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options=None,
+    ):
+        deadline = None if timeout is None else time.monotonic() + timeout
+        last_exc: Optional[Exception] = None
+        for ip in self._ips:
+            remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
+            try:
+                return self._real.connect_tcp(
+                    ip,
+                    port,
+                    remaining,
+                    local_address,
+                    socket_options,
+                )
+            except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
+                last_exc = exc
+                if deadline is not None and time.monotonic() >= deadline:
+                    break
+        if last_exc is not None:
+            raise last_exc
+        raise httpcore.ConnectError("no validated address available")
+
+    def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return self._real.connect_unix_socket(path, timeout, socket_options)
+
+    def sleep(self, seconds: float) -> None:
+        return self._real.sleep(seconds)
+
+
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.LocalProtocolError: httpx.LocalProtocolError,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ProxyError: httpx.ProxyError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedTransport(httpx.BaseTransport):
+    """Pin socket connects while preserving URL authority, Host, and TLS SNI."""
+
+    def __init__(self, ips: List[ipaddress._BaseAddress]):
+        self._pinned_ips = list(ips)
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=httpx.create_ssl_context(),
+            http1=True,
+            http2=False,
+            network_backend=_PinnedBackend(ips),
+        )
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        core_request = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        core_response = None
+        try:
+            core_response = self._pool.handle_request(core_request)
+            content = b"".join(cast(Iterable[bytes], core_response.stream))
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+        finally:
+            if core_response is not None:
+                core_response.close()
+
+        return httpx.Response(
+            status_code=core_response.status,
+            headers=core_response.headers,
+            content=content,
+            extensions=core_response.extensions,
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
+
+def _get_checked(
+    url: str,
+    *,
+    headers: Optional[dict] = None,
+    timeout: float = 30.0,
+) -> httpx.Response:
+    """GET that follows redirects manually, re-running the SSRF guard per hop.
+
+    ``httpx``'s ``follow_redirects=True`` validates only the initial URL, so a
+    ``3xx`` to an internal address (``169.254.169.254``, ``127.0.0.1``, …) would
+    still be connected to before any post-hoc host check. Following redirects by
+    hand lets us re-validate every hop, closing that blind-SSRF gap.
+    """
+    current = url
+    for _ in range(_MAX_FETCH_REDIRECTS + 1):
+        pinned_ips = _resolve_and_check_url(current)
+        with httpx.Client(
+            transport=_PinnedTransport(pinned_ips),
+            follow_redirects=False,
+            timeout=timeout,
+        ) as client:
+            r = client.get(current, headers=headers)
+
+        if r.status_code in (301, 302, 303, 307, 308):
+            location = r.headers.get("location")
+            if not location:
+                return r
+            current = urljoin(str(r.url), location)
+            continue
+        return r
+    raise SkillImportError("too many redirects while fetching skill bundle")
+
+
+def parse_skill_source(url: str) -> ResolvedSource:
+    """Normalize skills.sh / GitHub web URLs into owner/repo/ref/path."""
+    url = (url or "").strip()
+    if not url:
+        raise SkillImportError("URL is required")
+
+    # ``urlparse`` only reports an unambiguous scheme when the URL carries the
+    # ``scheme://`` form. Opaque schemes (``mailto:``, ``javascript:``) and a
+    # schemeless ``host:port`` both parse a "scheme" that is not one, so they
+    # fall through to the host check below and are rejected on the host instead.
+    scheme = urlparse(url).scheme.lower()
+    if scheme not in ("http", "https"):
+        if scheme and url.lower().startswith(f"{scheme}://"):
+            raise SkillImportError(f"unsupported URL scheme: {scheme}")
+        # Schemeless "github.com/owner/repo" — accept only a supported host.
+        rough_host = (urlparse("//" + url).hostname or "").lower()
+        if rough_host not in _GITHUB_HOSTS and rough_host not in _SKILLS_SH_HOSTS:
+            raise SkillImportError("Only GitHub or skills.sh URLs are supported")
+        url = "https://" + url
+
+    parsed = urlparse(url)
+    hostname = (parsed.hostname or "").lower()
+    if hostname not in _GITHUB_HOSTS and hostname not in _SKILLS_SH_HOSTS:
+        raise SkillImportError("Only GitHub or skills.sh URLs are supported")
+
+    # A skills.sh link is only usable if it redirects to an exact supported
+    # GitHub host. Scraping the page body for a github.com link cannot work:
+    # skill pages only ever link the repository root, never the skill's
+    # subdirectory, so the scrape resolves every skill in a repo to the same
+    # (wrong) bundle. Fail with an actionable message instead.
+    if hostname in _SKILLS_SH_HOSTS:
+        r = _get_checked(url, timeout=20.0)
+        if r.status_code >= 400:
+            raise _github_response_error(r)
+        final = str(r.url)
+        if _github_host(final) not in _GITHUB_HOSTS:
+            raise SkillImportError(
+                "skills.sh did not redirect to GitHub — open the skill's "
+                "repository on GitHub, navigate to the exact skill folder or "
+                "SKILL.md file, and paste that URL; the repository-root link "
+                "alone is not sufficient"
+            )
+        url = final
+
+    # Update parsed and hostname to reflect the new GitHub URL
+    parsed = urlparse(url)
+    hostname = (parsed.hostname or "").lower()
+
+    _assert_github_url(url)
+
+    if hostname == "raw.githubusercontent.com":
+        # /owner/repo/ref/path/to/file
+        bits = [p for p in parsed.path.split("/") if p]
+        if len(bits) < 4:
+            raise SkillImportError("Invalid raw GitHub URL")
+        owner, repo, ref = bits[0], bits[1], bits[2]
+        path = "/".join(bits[3:])
+        return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path)
+
+    bits = [p for p in parsed.path.split("/") if p]
+    if len(bits) < 2:
+        raise SkillImportError("Invalid GitHub URL")
+    owner, repo = bits[0], bits[1]
+    ref = "main"
+    path = ""
+
+    if len(bits) >= 4 and bits[2] in ("tree", "blob"):
+        ref = bits[3]
+        path = "/".join(bits[4:])
+    elif len(bits) == 2:
+        path = ""
+    else:
+        raise SkillImportError("GitHub URL must include /tree/<branch>/... or /blob/<branch>/...")
+
+    return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path)
+
+
+def _raw_url(src: ResolvedSource, rel_path: str) -> str:
+    rel = _safe_relpath(rel_path)
+    return f"https://raw.githubusercontent.com/{src.owner}/{src.repo}/{quote(src.ref, safe='')}/{quote(rel, safe='/')}"
+
+
+def _api_contents_url(src: ResolvedSource, rel_path: str = "") -> str:
+    rel = _safe_relpath(rel_path) if rel_path else ""
+    base = f"https://api.github.com/repos/{src.owner}/{src.repo}/contents"
+    if rel:
+        base += f"/{quote(rel, safe='/')}"
+    return f"{base}?ref={quote(src.ref, safe='')}"
+
+
+def _github_response_error(response: httpx.Response) -> SkillImportError:
+    """Turn a failed GitHub HTTP response into a user-visible import error."""
+    status = response.status_code
+    detail = ""
+    try:
+        body = response.json()
+        if isinstance(body, dict):
+            detail = str(body.get("message") or "").strip()
+    except Exception:
+        detail = (response.text or "").strip()[:200]
+
+    low = detail.lower()
+    if status == 403 and "rate limit" in low:
+        return SkillImportError(
+            "GitHub API rate limit exceeded — try again in a bit"
+            + (f" ({detail})" if detail else "")
+        )
+    if status == 404:
+        return SkillImportError("path not found on GitHub")
+    if detail:
+        return SkillImportError(f"GitHub request failed ({status}): {detail}")
+    return SkillImportError(f"GitHub request failed ({status})")
+
+
+def _fetch_bytes(url: str) -> bytes:
+    r = _get_checked(url, headers={"Accept": "application/vnd.github+json"}, timeout=30.0)
+    if r.status_code >= 400:
+        raise _github_response_error(r)
+    _assert_github_url(str(r.url), context="redirect target")
+    if len(r.content) > MAX_FILE_BYTES:
+        raise SkillImportError(f"file too large: {url}")
+    return r.content
+
+
+def _fetch_text(url: str) -> str:
+    data = _fetch_bytes(url)
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise SkillImportError(f"non-text file: {url}") from e
+
+
+def _list_github_dir(src: ResolvedSource, rel_dir: str, out: Dict[str, str], *, depth: int = 0) -> None:
+    if depth > 4 or len(out) >= MAX_FILES:
+        return
+    url = _api_contents_url(src, rel_dir)
+    r = _get_checked(url, headers={"Accept": "application/vnd.github+json"}, timeout=30.0)
+    if r.status_code >= 400:
+        raise _github_response_error(r)
+    _assert_github_url(str(r.url), context="redirect target")
+    entries = r.json()
+    if not isinstance(entries, list):
+        raise SkillImportError("expected a directory on GitHub")
+    total = sum(len(v.encode("utf-8")) for v in out.values())
+    for ent in entries:
+        if len(out) >= MAX_FILES or total >= MAX_TOTAL_BYTES:
+            break
+        if not isinstance(ent, dict):
+            continue
+        name = ent.get("name") or ""
+        ent_type = ent.get("type")
+        rel = _safe_relpath(f"{rel_dir}/{name}" if rel_dir else name)
+        if ent_type == "dir":
+            _list_github_dir(src, rel, out, depth=depth + 1)
+            total = sum(len(v.encode("utf-8")) for v in out.values())
+            continue
+        if ent_type != "file" or not _is_text_file(name):
+            continue
+        dl = ent.get("download_url")
+        if not dl:
+            continue
+        _assert_github_url(dl, context="download URL")
+        text = _fetch_text(dl)
+        total += len(text.encode("utf-8"))
+        if total > MAX_TOTAL_BYTES:
+            raise SkillImportError("skill bundle exceeds size limit")
+        out[rel] = text
+
+
+def fetch_skill_bundle(url: str) -> Tuple[Dict[str, str], ResolvedSource]:
+    """Download SKILL.md and sibling text assets. Returns relative_path → content."""
+    src = parse_skill_source(url)
+    files: Dict[str, str] = {}
+
+    path = _safe_relpath(src.path) if src.path else ""
+    if path.lower().endswith("skill.md"):
+        files[path] = _fetch_text(_raw_url(src, path))
+        parent = "/".join(path.split("/")[:-1])
+        if parent:
+            try:
+                _list_github_dir(src, parent, files)
+            except SkillImportError:
+                pass
+        return files, src
+
+    if path:
+        try:
+            _fetch_text(_raw_url(src, f"{path}/SKILL.md"))
+            _list_github_dir(src, path, files)
+            return files, src
+        except Exception:
+            pass
+        try:
+            text = _fetch_text(_raw_url(src, path))
+            if path.lower().endswith(".md"):
+                files[path] = text
+                return files, src
+        except Exception:
+            pass
+        _list_github_dir(src, path, files)
+    else:
+        _list_github_dir(src, "", files)
+
+    if not any(p.lower().endswith("skill.md") for p in files):
+        # Flat repo root with SKILL.md only
+        try:
+            files["SKILL.md"] = _fetch_text(_raw_url(src, "SKILL.md"))
+        except Exception as e:
+            raise SkillImportError(
+                "No SKILL.md found — link to a skill folder or SKILL.md on GitHub"
+            ) from e
+    return files, src
+
+
+def pick_skill_md(files: Dict[str, str]) -> Tuple[str, str]:
+    for rel, content in files.items():
+        if rel.lower().endswith("skill.md"):
+            return rel, content
+    raise SkillImportError("bundle has no SKILL.md")
+
+
+def default_category_from_source(src: ResolvedSource) -> str:
+    return "imported"

services/memory/skill_lifecycle.py

@@ -0,0 +1,20 @@
+"""Bounded automatic review queue for user-owned procedural memory."""
+import time
+
+
+def automatic_audit_candidates(skills, limit=8, now=None):
+    """Retry transient checks daily and failed repairs weekly, oldest first."""
+    now = time.time() if now is None else now
+    pending = []
+    for skill in skills:
+        if not skill.get("name") or skill.get("source") == "builtin" or skill.get("status") == "binned":
+            continue
+        verdict = skill.get("audit_verdict")
+        if verdict in {"pass", "skipped"}:
+            continue
+        checked = float(skill.get("audited_at") or 0)
+        delay = 7 * 86400 if verdict in {"fail", "needs_work"} else 86400
+        if not verdict or now - checked >= delay:
+            pending.append(skill)
+    pending.sort(key=lambda skill: float(skill.get("audited_at") or 0))
+    return pending[:max(1, limit)]

services/memory/skills.py

@@ -6,8 +6,8 @@ YAML frontmatter and a structured markdown body (When to Use / Procedure /
 Pitfalls / Verification). See `skill_format.py` for the format.
 
 Usage counters (`uses`, `last_used`) live in a sidecar
-`data/skills/_usage.json` keyed by skill name so the SKILL.md content
-doesn't churn on every retrieval.
+`data/skills/_usage.json` keyed by owner plus skill name so the SKILL.md
+content doesn't churn on every retrieval.
 
 Ownership: skills declare `owner: <username>` in frontmatter. Single-user
 deployments can leave that blank.
@@ -54,6 +54,25 @@ def _to_float(x, default: float = 0.0) -> float:
         return default
 
 
+def _approval_policy(owner: Optional[str]) -> tuple[bool, float]:
+    """Read the user's automatic skill-approval gate without breaking retrieval."""
+    try:
+        from routes.prefs_routes import _load_for_user
+        prefs = _load_for_user(owner) or {}
+    except Exception:
+        prefs = {}
+    try:
+        from src.settings import get_setting
+        default_minimum = float(get_setting("skill_autosave_min_confidence", 0.85))
+    except Exception:
+        default_minimum = 0.85
+    try:
+        minimum = float(prefs.get("skill_min_confidence", default_minimum))
+    except (TypeError, ValueError):
+        minimum = default_minimum
+    return bool(prefs.get("auto_approve_skills", True)), max(0.0, min(1.0, minimum))
+
+
 # ---------------------------------------------------------------------------
 # SkillsManager
 # ---------------------------------------------------------------------------
@@ -89,7 +108,7 @@ class SkillsManager:
         if not os.path.exists(self.usage_file):
             return {}
         try:
-            with open(self.usage_file) as f:
+            with open(self.usage_file, encoding="utf-8") as f:
                 d = json.load(f)
             return d if isinstance(d, dict) else {}
         except Exception:
@@ -101,33 +120,77 @@ class SkillsManager:
             atomic_write_json(self.usage_file, usage, indent=2)
         except Exception:
             tmp = self.usage_file + ".tmp"
-            with open(tmp, "w") as f:
+            with open(tmp, "w", encoding="utf-8") as f:
                 json.dump(usage, f, indent=2)
             os.replace(tmp, self.usage_file)
 
+    @staticmethod
+    def _usage_key(name: str, owner: Optional[str] = None) -> str:
+        # Skill names are not globally unique once multiple owners are present.
+        # Keep the usage sidecar keyed the same way the skill file is scoped.
+        return f"{owner}::{name}" if owner else name
+
+    def _usage_entry(self, usage: Dict[str, Dict], name: str, owner: Optional[str] = None) -> Dict:
+        key = self._usage_key(name, owner)
+        entry = usage.get(key)
+        if isinstance(entry, dict):
+            return entry
+        return {}
+
     def set_audit(self, name: str, verdict: str, by_teacher: bool = False,
-                  worker_model: str = "", teacher_model: str = "") -> None:
+                  worker_model: str = "", teacher_model: str = "",
+                  owner: Optional[str] = None, saved_turns: Optional[int] = None,
+                  saved_tool_calls: Optional[int] = None,
+                  baseline_verdict: Optional[str] = None,
+                  usefulness: Optional[float] = None,
+                  audit_summary: Optional[str] = None) -> None:
         """Record the last test/audit result for a skill in the usage sidecar
         (so it surfaces in load() without touching SKILL.md). Drives the
         'verified' check + teacher mark on the card."""
         import time as _t
         usage = self._load_usage()
-        e = usage.setdefault(name, {"uses": 0, "last_used": None})
+        key = self._usage_key(name, owner)
+        e = usage.setdefault(key, {"uses": 0, "last_used": None})
         e["audit_verdict"] = verdict
+        # Replace, rather than retain, the explanation from a previous run.
+        e["audit_summary"] = str(audit_summary or "")[:2000]
+        # Version 2 fixes audit-arm isolation and separates functional success
+        # from baseline utility. Legacy inconclusive results are not evidence
+        # under that protocol and should be eligible for a clean re-audit.
+        e["audit_version"] = 2
         e["audit_by_teacher"] = bool(by_teacher)
         if worker_model:
             e["audit_worker_model"] = worker_model
         if teacher_model:
             e["audit_teacher_model"] = teacher_model
+        if saved_turns is not None:
+            try:
+                e["saved_turns"] = int(saved_turns)
+            except (TypeError, ValueError):
+                e.pop("saved_turns", None)
+        if saved_tool_calls is not None:
+            try:
+                e["saved_tool_calls"] = int(saved_tool_calls)
+            except (TypeError, ValueError):
+                e.pop("saved_tool_calls", None)
+        if baseline_verdict is not None:
+            e["baseline_verdict"] = str(baseline_verdict or "unknown")
+        if usefulness is not None:
+            try:
+                e["usefulness"] = float(usefulness)
+            except (TypeError, ValueError):
+                e.pop("usefulness", None)
         e["audited_at"] = _t.time()
         self._save_usage(usage)
 
     def set_necessity(self, name: str, necessary: bool,
-                      redundant_with=None, reason: str = "") -> None:
+                      redundant_with=None, reason: str = "",
+                      owner: Optional[str] = None) -> None:
         """Record the advisory 'is this skill necessary?' judgment in the usage
         sidecar. Surfaced on the card as a flag; never acts on the skill."""
         usage = self._load_usage()
-        e = usage.setdefault(name, {"uses": 0, "last_used": None})
+        key = self._usage_key(name, owner)
+        e = usage.setdefault(key, {"uses": 0, "last_used": None})
         e["necessity"] = {
             "necessary": bool(necessary),
             "redundant_with": list(redundant_with or []),
@@ -148,7 +211,7 @@ class SkillsManager:
 
     def _read_skill(self, path: str) -> Optional[Skill]:
         try:
-            with open(path) as f:
+            with open(path, encoding="utf-8") as f:
                 text = f.read()
             return Skill.from_markdown(text, path=path)
         except Exception as e:
@@ -180,6 +243,8 @@ class SkillsManager:
             sk = self._read_skill(path)
             if not sk:
                 continue
+            if sk.source == "builtin":
+                continue
             owner = (sk.owner or "").strip()
             if owner == primary_owner:
                 continue
@@ -207,21 +272,34 @@ class SkillsManager:
             if not sk:
                 continue
             d = sk.to_dict()
-            u = usage.get(sk.name) or {}
+            u = self._usage_entry(usage, sk.name, sk.owner)
             d["uses"] = int(u.get("uses", 0))
             d["last_used"] = u.get("last_used")
-            d["audit_verdict"] = u.get("audit_verdict")
+            audit_verdict = u.get("audit_verdict")
+            try:
+                audit_version = int(u.get("audit_version") or 0)
+            except (TypeError, ValueError):
+                audit_version = 0
+            if audit_verdict == "inconclusive" and audit_version < 2:
+                audit_verdict = None
+            d["audit_verdict"] = audit_verdict
+            d["audit_summary"] = u.get("audit_summary", "") if audit_verdict else ""
+            d["audit_version"] = audit_version
             d["audit_by_teacher"] = bool(u.get("audit_by_teacher"))
             d["audit_worker_model"] = u.get("audit_worker_model")
             d["audit_teacher_model"] = u.get("audit_teacher_model")
-            d["audited_at"] = u.get("audited_at")
+            d["audited_at"] = u.get("audited_at") if audit_verdict else None
+            d["saved_turns"] = u.get("saved_turns")
+            d["saved_tool_calls"] = u.get("saved_tool_calls")
+            d["baseline_verdict"] = u.get("baseline_verdict")
+            d["usefulness"] = u.get("usefulness")
             d["necessity"] = u.get("necessity")
             out.append(d)
             seen_names.add(sk.name)
         # Legacy JSON entries — surfaced as draft, not editable from new flow
         if os.path.exists(self.legacy_file):
             try:
-                with open(self.legacy_file) as f:
+                with open(self.legacy_file, encoding="utf-8") as f:
                     legacy = json.load(f)
                 if isinstance(legacy, list):
                     for row in legacy:
@@ -267,7 +345,11 @@ class SkillsManager:
         # leaked legacy / un-stamped skills to every authenticated user.
         # Hide them now; the owner needs to be backfilled on disk if those
         # skills should be visible to a specific user.
-        return [s for s in entries if s.get("owner") == owner]
+        return [
+            s for s in entries
+            if s.get("owner") == owner
+            or (s.get("source") == "builtin" and not s.get("owner"))
+        ]
 
     # ----------------------------------------------------------------------
     # CRUD — disk-backed
@@ -308,6 +390,7 @@ class SkillsManager:
         # never auto-skipped — a human asked for it. The every-X AI audit
         # handles the fuzzier near-duplicates this cheap check won't catch.
         _all = self.load_all()
+        _dedup_pool = _all if owner is None else [s for s in _all if s.get("owner") == owner]
         if source != "user":
             cand = _tokenize(" ".join([
                 nm, (description or title or ""),
@@ -315,7 +398,7 @@ class SkillsManager:
                 " ".join(procedure if procedure is not None else (steps or [])),
             ]))
             if cand:
-                for s in _all:
+                for s in _dedup_pool:
                     ex = _tokenize(" ".join([
                         s.get("name", ""), s.get("description", ""),
                         s.get("when_to_use", ""),
@@ -326,7 +409,7 @@ class SkillsManager:
                         # existing skill's usage and return it so the caller
                         # knows it already exists.
                         try:
-                            self.record_use(s["name"])
+                            self.record_use(s["name"], owner=s.get("owner"))
                         except Exception:
                             pass
                         return {**s, "_deduped": True, "_duplicate_of": s.get("name")}
@@ -363,19 +446,81 @@ class SkillsManager:
 
         return sk.to_dict()
 
-    def update_skill(self, skill_id: str, updates: Dict) -> bool:
+    def import_bundle_from_files(
+        self,
+        files: Dict[str, str],
+        *,
+        owner: Optional[str] = None,
+        source_url: str = "",
+        category: str = "imported",
+    ) -> Dict:
+        """Install a fetched skill bundle (relative path → text) under skills/."""
+        from .skill_importer import SkillImportError, pick_skill_md, _safe_relpath
+        from core.atomic_io import atomic_write_text
+
+        if not files:
+            raise SkillImportError("empty bundle")
+        _rel, skill_md = pick_skill_md(files)
+        sk = Skill.from_markdown(skill_md)
+        nm = slugify(sk.name or _rel.split("/")[-2] or "skill")
+        cat = slugify(category or sk.category or "imported", fallback="imported")
+
+        existing = {s["name"] for s in self.load_all()}
+        base = nm
+        i = 2
+        while nm in existing:
+            nm = f"{base}-{i}"
+            i += 1
+
+        skill_dir = self._skill_dir(cat, nm)
+        os.makedirs(skill_dir, exist_ok=True)
+
+        # Preserve bundle layout (templates/, references/, etc.) under the skill dir.
+        for rel, content in files.items():
+            safe = _safe_relpath(rel)
+            dest = os.path.join(skill_dir, safe)
+            os.makedirs(os.path.dirname(dest), exist_ok=True)
+            atomic_write_text(dest, content)
+
+        sk.name = nm
+        sk.category = cat
+        sk.owner = owner
+        sk.source = "imported"
+        if source_url:
+            extra = (sk.body_extra or "").strip()
+            note = f"Imported from {source_url}"
+            sk.body_extra = f"{extra}\n\n{note}".strip() if extra else note
+        atomic_write_text(self._skill_file(cat, nm), sk.to_markdown())
+        sk.path = self._skill_file(cat, nm)
+        return sk.to_dict()
+
+    def update_skill(self, skill_id: str, updates: Dict, owner: Optional[str] = None) -> bool:
         """`skill_id` is the slug name. Allows updating any field plus
-        renames if `name` changes (file is moved on disk)."""
+        renames if `name` changes (file is moved on disk).
+
+        The call is owner-scoped: it matches a skill on disk only if
+        `skill.owner == owner` (string compare; both empty-string and
+        None mean "ownerless"). When `owner is None` (the default), the
+        call only matches skills whose own `owner` field is empty —
+        callers that want to edit an owned skill must pass the matching
+        owner explicitly. This prevents a caller with one owner from
+        mutating a file owned by another user that happens to share
+        the same slug across category directories. The `owner` key in
+        `updates` is also ignored — ownership is not an editable field
+        via this path; rename or admin tooling is required for that.
+        """
         for path in self._iter_skill_files():
             sk = self._read_skill(path)
             if not sk or sk.name != skill_id:
                 continue
+            if (sk.owner or "") != (owner or ""):
+                continue
+
             old_dir = os.path.dirname(path)
 
-            # Apply updates in a Skill-shape friendly way
             scalar_keys = (
                 "description", "version", "category", "status", "confidence",
-                "source", "teacher_model", "owner", "when_to_use",
+                "source", "teacher_model", "when_to_use",
                 "body_extra",
             )
             for k in scalar_keys:
@@ -414,18 +559,21 @@ class SkillsManager:
                 os.rename(old_dir, new_dir)
                 # Also rename usage key
                 usage = self._load_usage()
-                if skill_id in usage:
-                    usage[sk.name] = usage.pop(skill_id)
+                old_usage_key = self._usage_key(skill_id, sk.owner)
+                if old_usage_key in usage:
+                    usage[self._usage_key(sk.name, sk.owner)] = usage.pop(old_usage_key)
                     self._save_usage(usage)
             self._write_skill(sk)
             return True
         return False
 
-    def delete_skill(self, skill_id: str) -> bool:
+    def delete_skill(self, skill_id: str, owner: Optional[str] = None) -> bool:
         for path in self._iter_skill_files():
             sk = self._read_skill(path)
             if not sk or sk.name != skill_id:
                 continue
+            if (sk.owner or "") != (owner or ""):
+                continue
             skill_dir = os.path.dirname(path)
             try:
                 # Remove the whole skill dir
@@ -439,15 +587,17 @@ class SkillsManager:
                 logger.warning(f"Failed to remove skill dir {skill_dir}: {e}")
                 return False
             usage = self._load_usage()
-            if skill_id in usage:
-                del usage[skill_id]
+            usage_key = self._usage_key(skill_id, sk.owner)
+            if usage_key in usage:
+                del usage[usage_key]
                 self._save_usage(usage)
             return True
         return False
 
-    def record_use(self, skill_id: str) -> None:
+    def record_use(self, skill_id: str, owner: Optional[str] = None) -> None:
         usage = self._load_usage()
-        entry = usage.setdefault(skill_id, {"uses": 0, "last_used": None})
+        key = self._usage_key(skill_id, owner)
+        entry = usage.setdefault(key, {"uses": 0, "last_used": None})
         entry["uses"] = int(entry.get("uses", 0)) + 1
         entry["last_used"] = int(time.time())
         self._save_usage(usage)
@@ -456,24 +606,40 @@ class SkillsManager:
     # Reading a single skill (used by the skill_view tool)
     # ----------------------------------------------------------------------
 
-    def read_skill_md(self, name: str) -> Optional[str]:
+    def read_skill_md(self, name: str, owner: Optional[str] = None) -> Optional[str]:
         for path in self._iter_skill_files():
             sk = self._read_skill(path)
-            if sk and sk.name == name:
-                try:
-                    with open(path) as f:
-                        return f.read()
-                except Exception:
-                    return None
+            if not sk or sk.name != name:
+                continue
+            # Built-in skills are shared, ownerless procedures. ``load``
+            # exposes them to every owner, so direct progressive-disclosure
+            # reads must apply the same visibility rule as the index/list
+            # path. Previously a built-in appeared in `list` but `view`
+            # returned not-found for authenticated users.
+            if not (
+                (sk.owner or "") == (owner or "")
+                or (sk.source == "builtin" and not (sk.owner or ""))
+            ):
+                continue
+            try:
+                with open(path, encoding="utf-8") as f:
+                    return f.read()
+            except Exception:
+                return None
         return None
 
-    def read_skill_reference(self, name: str, ref_path: str) -> Optional[str]:
+    def read_skill_reference(self, name: str, ref_path: str, owner: Optional[str] = None) -> Optional[str]:
         """Read a sub-file under the skill's directory (references/, etc).
         Refuses path traversal."""
         for path in self._iter_skill_files():
             sk = self._read_skill(path)
             if not sk or sk.name != name:
                 continue
+            if not (
+                (sk.owner or "") == (owner or "")
+                or (sk.source == "builtin" and not (sk.owner or ""))
+            ):
+                continue
             base = os.path.realpath(os.path.dirname(path))
             target = os.path.realpath(os.path.join(base, ref_path))
             if os.path.commonpath([base, target]) != base or target == os.path.dirname(path):
@@ -481,7 +647,7 @@ class SkillsManager:
             if not os.path.isfile(target):
                 return None
             try:
-                with open(target) as f:
+                with open(target, encoding="utf-8") as f:
                     return f.read()
             except Exception:
                 return None
@@ -501,19 +667,12 @@ class SkillsManager:
         """Return the `[{name, description, category, status}]` list the
         agent sees in its system prompt.
 
-        Includes:
-          - All published skills.
-          - Drafts written by the teacher-escalation loop
-            (`source == "teacher-escalation"`). The whole point of
-            the teacher loop is for the student to find the new
-            procedure on the very next turn — waiting for a manual
-            publish click defeats the loop.
-
-        Excludes user-created drafts (status=draft, source != teacher-
-        escalation) — those are work-in-progress and pollute the
-        prompt with half-finished procedures.
+        Includes built-ins plus user skills that have passed their audit and
+        meet the owner's current automatic-approval threshold. A persistent
+        ``published`` flag is not sufficient: a changed threshold or a legacy
+        record must not make an unaudited skill eligible for prompt injection.
         """
-        active_toolsets = active_toolsets or []
+        auto_approve, min_confidence = _approval_policy(owner)
         out = []
         for s in self.load(owner=owner):
             status = s.get("status")
@@ -524,16 +683,32 @@ class SkillsManager:
                     pass  # let it through
                 else:
                     continue
+            # A stale published record must not remain injectable after an
+            # audit has recorded a failure. Inconclusive is not a failure.
+            audit_verdict = str(s.get("audit_verdict") or "").lower()
+            if audit_verdict in {"needs_work", "fail"}:
+                continue
+            if s.get("source") != "builtin" and auto_approve:
+                if status != "published" or audit_verdict != "pass":
+                    continue
+                if _to_float(s.get("confidence"), 0.0) < min_confidence:
+                    continue
+            necessity = s.get("necessity") or {}
+            if isinstance(necessity, dict) and necessity.get("necessary") is False:
+                continue
             # Platform gating
             if platform and s.get("platforms") and platform not in s["platforms"]:
                 continue
-            # requires_toolsets: hide unless every required toolset is active
+            # requires_toolsets: hide unless every required toolset is active.
+            # active_toolsets=None means the caller doesn't know the active
+            # set (API listings, chat preface) — don't gate in that case;
+            # only an explicit list filters.
             req = s.get("requires_toolsets") or []
-            if req and not all(t in active_toolsets for t in req):
+            if req and active_toolsets is not None and not all(t in active_toolsets for t in req):
                 continue
             # fallback_for_toolsets: hide when any of those toolsets is active
             fb = s.get("fallback_for_toolsets") or []
-            if fb and any(t in active_toolsets for t in fb):
+            if fb and active_toolsets and any(t in active_toolsets for t in fb):
                 continue
             out.append({
                 "name": s["name"],
@@ -557,6 +732,8 @@ class SkillsManager:
         threshold: float = 0.3,
         max_items: int = 5,
         min_confidence: float = 0.0,
+        available_toolsets: Optional[Iterable[str]] = None,
+        platform: Optional[str] = None,
     ) -> List[Dict]:
         if skills is None:
             skills = self.load_all()
@@ -568,26 +745,62 @@ class SkillsManager:
         # without a manual publish click. The UI flags teacher-written
         # entries with a 🎓 badge so users can demote / delete bad
         # ones when they spot them.
-        skills = [s for s in skills if s.get("status") in ("published", "draft")]
-        # Confidence gate (used by prompt-injection, NOT by search): a DRAFT
-        # skill must clear the bar to be injected. Published skills are already
-        # vetted, so they always qualify. Missing confidence = treat as 1.0
-        # (legacy skills shouldn't silently vanish). 0 disables the gate.
+        skills = [
+            s for s in skills
+            if s.get("status") in ("published", "draft")
+            and str(s.get("audit_verdict") or "").lower()
+            not in {"needs_work", "fail", "skipped"}
+        ]
+        available = set(available_toolsets) if available_toolsets is not None else None
+        if available is not None:
+            skills = [
+                skill for skill in skills
+                if all(tool in available for tool in (skill.get("requires_toolsets") or []))
+                and not any(tool in available for tool in (skill.get("fallback_for_toolsets") or []))
+            ]
+        if platform:
+            skills = [
+                skill for skill in skills
+                if not skill.get("platforms") or platform in skill.get("platforms", [])
+            ]
+        # Prompt injection is fail-closed for user skills. Built-ins are
+        # shipped procedures; every other skill needs a passing audit and a
+        # confidence score at the user's current threshold.
         if min_confidence > 0:
             def _passes(s):
-                if s.get("status") == "published":
+                if s.get("source") == "builtin":
                     return True
-                c = s.get("confidence")
-                if c is None:
-                    return True  # unset → don't filter (legacy)
-                return _to_float(c, 1.0) >= min_confidence  # unparseable → pass
+                return (
+                    s.get("status") == "published"
+                    and str(s.get("audit_verdict") or "").lower() == "pass"
+                    and _to_float(s.get("confidence"), 0.0) >= min_confidence
+                )
             skills = [s for s in skills if _passes(s)]
         if not skills:
             return []
 
         query_tokens = _tokenize(query)
+        semantic_scores: Dict[int, float] = {}
+        semantic_enabled = str(
+            os.environ.get("ODYSSEUS_SKILL_SEMANTIC_RETRIEVAL", "1")
+        ).strip().lower() not in {"0", "false", "no", "off"}
+        if semantic_enabled:
+            try:
+                from src.skill_index import semantic_skill_scores
+
+                semantic_scores = semantic_skill_scores(query, skills)
+            except Exception as exc:
+                logger.debug("Semantic skill retrieval unavailable: %s", exc)
+        try:
+            semantic_threshold = float(
+                os.environ.get("ODYSSEUS_SKILL_SEMANTIC_THRESHOLD", "0.4")
+            )
+        except (TypeError, ValueError):
+            semantic_threshold = 0.4
+        semantic_threshold = max(-1.0, min(1.0, semantic_threshold))
+
         scored = []
-        for sk in skills:
+        for position, sk in enumerate(skills):
             text = " ".join([
                 sk.get("name", ""),
                 sk.get("description", ""),
@@ -595,16 +808,22 @@ class SkillsManager:
                 " ".join(sk.get("tags", []) or []),
                 " ".join(sk.get("procedure", []) or []),
             ])
-            score = _jaccard(query_tokens, _tokenize(text))
+            lexical_score = _jaccard(query_tokens, _tokenize(text))
             for tag in sk.get("tags", []) or []:
-                if tag and tag in query.lower():
-                    score = max(score, 0.3) * 1.3
+                # Match tags as whole tokens, not substrings: `tag in query`
+                # boosted e.g. a "ai" tag for any query containing "email".
+                tag_tokens = _tokenize(tag)
+                if tag_tokens and tag_tokens <= query_tokens:
+                    lexical_score = max(lexical_score, 0.3) * 1.3
             if query.lower() in (sk.get("description") or "").lower():
-                score = max(score, 0.6)
+                lexical_score = max(lexical_score, 0.6)
+            semantic_score = semantic_scores.get(position, -1.0)
+            if lexical_score < threshold and semantic_score < semantic_threshold:
+                continue
+            score = max(lexical_score, semantic_score)
             score *= 1.0 + _to_float(sk.get("confidence"), 0.5) * 0.1
             if sk.get("uses", 0) > 0:
                 score *= 1.05
-            if score >= threshold:
-                scored.append((score, sk))
+            scored.append((score, sk))
         scored.sort(key=lambda x: x[0], reverse=True)
         return [sk for _, sk in scored[:max_items]]

services/research/research_handler.py

@@ -14,9 +14,12 @@ import time
 from pathlib import Path
 from typing import Optional, Dict
 
+from src.research_utils import is_low_quality
+from src.constants import DEEP_RESEARCH_DIR
+
 logger = logging.getLogger(__name__)
 
-RESEARCH_DATA_DIR = Path("data/deep_research")
+RESEARCH_DATA_DIR = Path(DEEP_RESEARCH_DIR)
 
 
 class ResearchHandler:
@@ -114,7 +117,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return {
                     "status": data.get("status", "done"),
                     "progress": {},
@@ -151,7 +154,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return data.get("result")
             except Exception:
                 pass
@@ -171,7 +174,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return data.get("sources")
             except Exception:
                 pass
@@ -179,13 +182,16 @@ class ResearchHandler:
 
     @staticmethod
     def _extract_sources(findings: list) -> list:
-        """Extract deduplicated [{url, title}] from findings."""
+        """Extract deduplicated [{url, title}] from findings, filtering low-quality ones."""
         seen = set()
         sources = []
         for f in findings:
+            if not isinstance(f, dict):
+                continue
             url = f.get("url", "")
             title = f.get("title", "") or url
-            if url and url not in seen:
+            summary = f.get("summary", "") or f.get("evidence", "")
+            if url and url not in seen and not is_low_quality(summary):
                 seen.add(url)
                 sources.append({"url": url, "title": title})
         return sources
@@ -219,7 +225,7 @@ class ResearchHandler:
                 "started_at": entry["started_at"],
                 "completed_at": time.time(),
             }
-            path.write_text(json.dumps(data))
+            path.write_text(json.dumps(data), encoding="utf-8")
             logger.info(f"Research result saved to {path}")
         except Exception as e:
             logger.error(f"Failed to save research result: {e}")
@@ -281,6 +287,7 @@ class ResearchHandler:
                 query, report, stats, elapsed,
                 findings=researcher.findings,
                 evolving_report=researcher.evolving_report,
+                analyzed_urls=getattr(researcher, "analyzed_urls", None),
             )
 
         except Exception as e:
@@ -327,7 +334,8 @@ class ResearchHandler:
 
     def _format_research_report(
         self, query: str, full_report: str, stats: dict, elapsed: float,
-        findings: list = None, evolving_report: str = None,
+        findings: Optional[list] = None, evolving_report: Optional[str] = None,
+        analyzed_urls: Optional[list] = None,
     ) -> str:
         """Format research report with sources list and expandable raw findings."""
         summary_lines = [
@@ -338,19 +346,34 @@ class ResearchHandler:
         ]
         summary_text = " | ".join(summary_lines)
 
-        # Build sources list with clickable links
+        # Build sources list with clickable links. Keep the curated Sources
+        # section filtered for citation quality, but also list every unique URL
+        # the research run inspected so the "URLs Analyzed" count is auditable.
         sources_section = ""
-        if findings:
+        analyzed_urls_section = ""
+        url_items = analyzed_urls if analyzed_urls is not None else findings
+        if findings or url_items:
             seen_urls = set()
             source_lines = []
-            for f in findings:
+            analyzed_seen = set()
+            analyzed_lines = []
+            for f in findings or []:
                 url = f.get("url", "")
                 title = f.get("title", "") or url
-                if url and url not in seen_urls:
+                summary = f.get("summary", "") or f.get("evidence", "")
+                if url and url not in seen_urls and not is_low_quality(summary):
                     seen_urls.add(url)
                     source_lines.append(f"- [{title}]({url})")
+            for item in url_items or []:
+                url = item.get("url", "")
+                title = item.get("title", "") or url
+                if url and url not in analyzed_seen:
+                    analyzed_seen.add(url)
+                    analyzed_lines.append(f"{len(analyzed_lines) + 1}. [{title}]({url})")
             if source_lines:
                 sources_section = "\n### Sources\n\n" + "\n".join(source_lines) + "\n"
+            if analyzed_lines:
+                analyzed_urls_section = "\n### Analyzed URLs\n\n" + "\n".join(analyzed_lines) + "\n"
 
         # Build raw findings section (individual extractions per source)
         raw_findings_section = ""
@@ -386,6 +409,7 @@ class ResearchHandler:
 {full_report}
 
 {sources_section}
+{analyzed_urls_section}
 {collected_section}
 ---
 

services/research/service.py

@@ -1,11 +1,16 @@
 # services/research/service.py
 """Research service — deep research with LLM-in-the-loop."""
 
+import re
 from dataclasses import dataclass, field
 from typing import List, Optional, Callable
 
 from .research_handler import ResearchHandler
 
+# Markdown source links emitted by ResearchHandler._format_research_report,
+# e.g. "- [Some Title](https://example.com/page)".
+_SOURCE_LINK_RE = re.compile(r"^\s*-\s*\[(?P<title>[^\]]*)\]\((?P<url>[^)]+)\)\s*$")
+
 
 @dataclass
 class ResearchSource:
@@ -75,26 +80,71 @@ class ResearchService:
 
         duration = time.time() - start
 
-        # Parse result into structured format
-        sources = [
-            ResearchSource(
-                url=s.get("url", ""),
-                title=s.get("title", ""),
-                snippet=s.get("snippet", ""),
-                relevance=s.get("relevance", 0.0),
+        # call_research_service returns a formatted markdown report string
+        # (see ResearchHandler.call_research_service -> _format_research_report),
+        # not a dict. Treat it as such; tolerate an unexpected dict/None defensively.
+        if isinstance(result, dict):
+            sources = [
+                ResearchSource(
+                    url=s.get("url", ""),
+                    title=s.get("title", ""),
+                    snippet=s.get("snippet", ""),
+                    relevance=s.get("relevance", 0.0),
+                )
+                for s in result.get("sources", [])
+                if isinstance(s, dict)
+            ]
+            return ResearchResult(
+                query=topic,
+                summary=result.get("summary", result.get("answer", "")),
+                sources=sources,
+                sections=result.get("sections", []),
+                tokens_used=result.get("tokens_used", 0),
+                duration_seconds=duration,
             )
-            for s in result.get("sources", [])
-        ]
 
+        report = result if isinstance(result, str) else ""
         return ResearchResult(
             query=topic,
-            summary=result.get("summary", result.get("answer", "")),
-            sources=sources,
-            sections=result.get("sections", []),
-            tokens_used=result.get("tokens_used", 0),
+            summary=report,
+            sources=self._parse_sources(report),
             duration_seconds=duration,
         )
 
+    @staticmethod
+    def _parse_sources(report: str) -> List[ResearchSource]:
+        """Extract sources from the markdown ### Sources section of a report.
+
+        ResearchHandler emits one ``- [title](url)`` link per deduplicated
+        finding under a ``### Sources`` heading. Parse only that section so
+        inline links elsewhere in the body are not mistaken for sources.
+        """
+        if not report:
+            return []
+        sources: List[ResearchSource] = []
+        seen = set()
+        in_sources = False
+        for line in report.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("###") or stripped.startswith("##"):
+                in_sources = stripped.lower().lstrip("#").strip() == "sources"
+                continue
+            if not in_sources:
+                continue
+            match = _SOURCE_LINK_RE.match(line)
+            if not match:
+                continue
+            url = match.group("url").strip()
+            if not url or url in seen:
+                continue
+            seen.add(url)
+            sources.append(
+                # snippet is required on ResearchSource; markdown source links
+                # carry no snippet, so default to empty (matches the dict path).
+                ResearchSource(url=url, title=match.group("title").strip(), snippet="")
+            )
+        return sources
+
     def start_background(
         self,
         session_id: str,

services/search/analytics.py

@@ -6,21 +6,29 @@ from collections import Counter
 from pathlib import Path
 from typing import Dict, Any
 
+from core.constants import DATA_DIR
+
 from .cache import cache_metrics
 
 logger = logging.getLogger(__name__)
 
-# Dedicated error logger with file handler
-_error_log_path = Path(__file__).resolve().parent.parent / "search_engine_error.log"
-_error_handler = logging.FileHandler(_error_log_path, encoding="utf-8")
-_error_handler.setLevel(logging.WARNING)
-_error_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
+# Dedicated error logger — write to the data logs directory (writable on both
+# native runs and Docker, where DATA_DIR resolves to the bind-mounted volume).
+_log_dir = Path(DATA_DIR) / "logs"
+_error_log_path = _log_dir / "search_engine_error.log"
 error_logger = logging.getLogger("search_engine_error")
-error_logger.addHandler(_error_handler)
 error_logger.propagate = False
+try:
+    _log_dir.mkdir(parents=True, exist_ok=True)
+    _error_handler = logging.FileHandler(_error_log_path, encoding="utf-8")
+    _error_handler.setLevel(logging.WARNING)
+    _error_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
+    error_logger.addHandler(_error_handler)
+except Exception as _e:
+    logging.getLogger(__name__).warning("search_engine_error log handler unavailable: %s", _e)
 
-# Analytics file
-ANALYTICS_FILE = Path(__file__).resolve().parent.parent / "search_analytics.json"
+# Analytics file — also in the writable logs volume.
+ANALYTICS_FILE = _log_dir / "search_analytics.json"
 
 
 # ----------------------------------------------------------------------
@@ -45,32 +53,36 @@ class RateLimitError(SearchEngineError):
 # ----------------------------------------------------------------------
 # Analytics helpers
 # ----------------------------------------------------------------------
+def _default_analytics() -> Dict[str, Any]:
+    return {
+        "total_queries": 0,
+        "successful_queries": 0,
+        "failed_queries": 0,
+        "cache_hits": 0,
+        "cache_misses": 0,
+        "query_patterns": {},
+    }
+
+
 def _load_analytics() -> Dict[str, Any]:
     """Load analytics data from the JSON file, creating defaults if missing."""
     if not ANALYTICS_FILE.exists():
-        default = {
-            "total_queries": 0,
-            "successful_queries": 0,
-            "failed_queries": 0,
-            "cache_hits": 0,
-            "cache_misses": 0,
-            "query_patterns": {},
-        }
+        default = _default_analytics()
         _save_analytics(default)
         return default
     try:
         with open(ANALYTICS_FILE, "r", encoding="utf-8") as f:
-            return json.load(f)
+            data = json.load(f)
+        # Merge over defaults so a file written by an older schema (or a
+        # partial write) still has every counter — _record_query indexes
+        # these keys directly and would otherwise raise KeyError.
+        merged = _default_analytics()
+        if isinstance(data, dict):
+            merged.update(data)
+        return merged
     except Exception as e:
         logger.warning(f"Failed to load analytics file: {e}")
-        return {
-            "total_queries": 0,
-            "successful_queries": 0,
-            "failed_queries": 0,
-            "cache_hits": 0,
-            "cache_misses": 0,
-            "query_patterns": {},
-        }
+        return _default_analytics()
 
 
 def _save_analytics(data: Dict[str, Any]) -> None:

services/search/cache.py

@@ -6,17 +6,23 @@ from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Dict
 
+from core.constants import DATA_DIR
+
 logger = logging.getLogger(__name__)
 
 # Cache directories
-CACHE_DIR = Path(__file__).resolve().parent.parent / "cache"
+CACHE_DIR = Path(DATA_DIR) / "cache"
 SEARCH_CACHE_DIR = CACHE_DIR / "search"
 CONTENT_CACHE_DIR = CACHE_DIR / "content"
 CACHE_MAX_ENTRIES = 1000
 
-# Create cache directories
-SEARCH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
-CONTENT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+# Create cache directories. Guarded so an unwritable path (e.g. a read-only
+# mount) degrades to no-disk-cache instead of crashing module import.
+try:
+    SEARCH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    CONTENT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+except OSError as _e:
+    logger.warning("Search cache directory unavailable (%s); disk cache disabled", _e)
 
 # Track cache size for LRU eviction
 search_cache_index: Dict[str, datetime] = {}

services/search/content.py

@@ -1,19 +1,20 @@
 """Webpage content fetching with caching, PDF extraction, and summarization helpers."""
 
+import copy
 import io
-import ipaddress
 import json
 import os
 import re
 import logging
-import socket
 from datetime import datetime, timedelta
 from typing import List
-from urllib.parse import urljoin, urlparse
 
 import httpx
 from bs4 import BeautifulSoup
 
+from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
+from src import outbound_fetch as _outbound_fetch
+
 from .analytics import RateLimitError, error_logger
 from .cache import (
     CONTENT_CACHE_DIR,
@@ -24,73 +25,39 @@ from .cache import (
 
 logger = logging.getLogger(__name__)
 
-_PRIVATE_NETWORKS = (
-    ipaddress.ip_network("0.0.0.0/8"),
-    ipaddress.ip_network("10.0.0.0/8"),
-    ipaddress.ip_network("127.0.0.0/8"),
-    ipaddress.ip_network("169.254.0.0/16"),
-    ipaddress.ip_network("172.16.0.0/12"),
-    ipaddress.ip_network("192.168.0.0/16"),
-    ipaddress.ip_network("::1/128"),
-    ipaddress.ip_network("fc00::/7"),
-    ipaddress.ip_network("fe80::/10"),
-)
+def _is_private_address(addr):
+    return _outbound_fetch._is_private_address(addr)
 
 
-def _is_private_address(addr: ipaddress._BaseAddress) -> bool:
-    return addr.is_private or addr.is_loopback or addr.is_link_local or any(addr in net for net in _PRIVATE_NETWORKS)
+def _resolve_hostname_ips(hostname):
+    return _outbound_fetch._resolve_hostname_ips(hostname)
 
 
-def _resolve_hostname_ips(hostname: str) -> list[ipaddress._BaseAddress]:
-    try:
-        infos = socket.getaddrinfo(hostname, None)
-    except Exception:
-        return []
-    out = []
-    for info in infos:
-        try:
-            out.append(ipaddress.ip_address(info[4][0]))
-        except Exception:
-            continue
-    return out
+def _public_http_url(url):
+    return _outbound_fetch._public_http_url(url, resolver=_resolve_hostname_ips)
 
 
-def _public_http_url(url: str) -> bool:
-    try:
-        parsed = urlparse(url)
-        if parsed.scheme not in ("http", "https"):
-            return False
-        host = (parsed.hostname or "").strip()
-        if not host:
-            return False
-        lower = host.lower()
-        if lower in ("localhost", "metadata", "metadata.google.internal"):
-            return False
-        if lower.endswith((".local", ".localhost", ".internal", ".lan", ".intranet")):
-            return False
-        try:
-            return not _is_private_address(ipaddress.ip_address(host))
-        except ValueError:
-            pass
-        addrs = _resolve_hostname_ips(host)
-        return bool(addrs) and not any(_is_private_address(a) for a in addrs)
-    except Exception:
-        return False
+def _resolve_public_ips(url):
+    return _outbound_fetch._resolve_public_ips(url, resolver=_resolve_hostname_ips)
 
 
-def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response:
-    current = url
-    for _ in range(max_redirects + 1):
-        if not _public_http_url(current):
-            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
-        response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False)
-        if response.status_code not in (301, 302, 303, 307, 308):
-            return response
-        location = response.headers.get("location")
-        if not location:
-            return response
-        current = urljoin(str(response.url), location)
-    raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
+_PinnedBackend = _outbound_fetch._PinnedBackend
+_PinnedTransport = _outbound_fetch._PinnedTransport
+BodyTooLargeError = _outbound_fetch.BodyTooLargeError
+_CappedFetch = _outbound_fetch._CappedFetch
+
+
+def _get_public_url(url, headers, timeout, max_redirects=5, max_bytes=None):
+    return _outbound_fetch._get_public_url(
+        url,
+        headers=headers,
+        timeout=timeout,
+        max_redirects=max_redirects,
+        max_bytes=max_bytes,
+        resolve_public_ips=_resolve_public_ips,
+        transport_factory=_PinnedTransport,
+    )
+
 
 # PDF extraction (optional dependency)
 try:
@@ -98,6 +65,49 @@ try:
 except ImportError:
     pdf_extract_text = None  # type: ignore
 
+try:
+    from pypdf import PdfReader
+except ImportError:
+    PdfReader = None  # type: ignore
+
+
+def _extract_pdf_text(pdf_bytes: bytes, url: str = "") -> str:
+    """Extract PDF text with available permissive dependencies."""
+    # Prefer pypdf's layout mode. Plain text extraction and pdfminer often
+    # collapse table columns into an ambiguous number stream, which makes a
+    # correct source passage easy for the model to misread.
+    if PdfReader is not None:
+        try:
+            reader = PdfReader(io.BytesIO(pdf_bytes))
+            pages: List[str] = []
+            for idx, page in enumerate(reader.pages):
+                try:
+                    try:
+                        page_text = page.extract_text(extraction_mode="layout") or ""
+                    except TypeError:
+                        page_text = page.extract_text() or ""
+                except Exception as e:
+                    logger.warning(f"pypdf extraction failed for {url} page {idx + 1}: {e}")
+                    page_text = ""
+                if page_text.strip():
+                    pages.append(f"[Page {idx + 1}]\n{page_text.strip()}")
+            if pages:
+                return "\n\n".join(pages)
+        except Exception as e:
+            logger.warning(f"pypdf extraction failed for {url}: {e}")
+
+    if pdf_extract_text is not None:
+        try:
+            text = pdf_extract_text(io.BytesIO(pdf_bytes)) or ""
+            if text.strip():
+                return text
+        except Exception as e:
+            logger.warning(f"pdfminer extraction failed for {url}: {e}")
+
+    if PdfReader is None and pdf_extract_text is None:
+        logger.error("No PDF text extractor installed; install pdfminer.six or pypdf.")
+    return ""
+
 
 # ----------------------------------------------------------------------
 # HTML extraction helpers
@@ -115,6 +125,28 @@ def _extract_meta(soup: BeautifulSoup) -> dict:
     return {"description": description, "keywords": keywords}
 
 
+def _extract_og_image(soup: BeautifulSoup) -> str:
+    """Extract the best representative image URL from meta tags.
+
+    Only returns absolute http(s) URLs -- skips relative paths and data URIs.
+    """
+    candidates = []
+    for prop in ("og:image", "og:image:url", "og:image:secure_url"):
+        tag = soup.find("meta", attrs={"property": prop})
+        if tag and tag.get("content", "").strip():
+            candidates.append(tag["content"].strip())
+    tag = soup.find("meta", attrs={"name": "twitter:image"})
+    if tag and tag.get("content", "").strip():
+        candidates.append(tag["content"].strip())
+    tag = soup.find("meta", attrs={"name": "thumbnail"})
+    if tag and tag.get("content", "").strip():
+        candidates.append(tag["content"].strip())
+    for url in candidates:
+        if url.startswith(("https://", "http://")) and not url.endswith((".svg", ".ico")):
+            return url
+    return ""
+
+
 def _extract_lists(soup: BeautifulSoup) -> List[List[str]]:
     """Return a list of lists, each inner list representing a <ul>/<ol>."""
     all_lists = []
@@ -189,9 +221,19 @@ def _empty_result(url: str, error: str = "") -> dict:
 # ----------------------------------------------------------------------
 # Main content fetcher
 # ----------------------------------------------------------------------
-def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict:
-    """Fetch and extract meaningful content from a webpage with caching."""
-    cache_key = generate_cache_key(url)
+def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
+                          max_bytes: int = None) -> dict:
+    """Fetch and extract meaningful content from a webpage with caching.
+
+    ``max_bytes`` raises the download budget per call (clamped to the hard
+    cap); the default is the soft cap. When the body is cut short the result
+    carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can
+    tell the model the content is partial (#3812).
+    """
+    effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
+    # The cap is part of the cache identity: a truncated soft-cap fetch must
+    # not be served to a later full-budget request for the same URL.
+    cache_key = generate_cache_key(f"{url}#cap={effective_cap}")
     cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache"
 
     # Check cache
@@ -214,18 +256,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
     # Fetch
     try:
         headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "User-Agent": WEB_FETCH_USER_AGENT,
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate",
             "Connection": "keep-alive",
         }
-        response = _get_public_url(url, headers=headers, timeout=timeout)
+        response = _get_public_url(url, headers=headers, timeout=timeout,
+                                   max_bytes=effective_cap)
 
         if response.status_code == 429:
             raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})")
 
         response.raise_for_status()
+    except BodyTooLargeError as e:
+        error_logger.warning(f"Refused oversized body for {url}: {e}")
+        return _empty_result(url, f"TooLarge: {e}")
+    except httpx.HTTPStatusError as e:
+        error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}")
+        return _empty_result(url, f"HTTP {e.response.status_code}: {e}")
     except httpx.RequestError as e:
         error_logger.error(f"NetworkError fetching {url} (attempt {retry_attempt}): {e}")
         return _empty_result(url, f"NetworkError: {e}")
@@ -233,19 +281,54 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
         error_logger.error(str(e))
         return _empty_result(url, str(e))
 
+    # Size bookkeeping shared by every content branch below. getattr keeps
+    # plain httpx.Response stand-ins (tests) working without the cap fields.
+    _size_fields = {
+        "truncated": getattr(response, "truncated", False),
+        "fetched_bytes": len(response.content),
+        "total_bytes": getattr(response, "declared_bytes", None),
+    }
+
     # PDF handling
     content_type = response.headers.get("Content-Type", "").lower()
     if "application/pdf" in content_type or url.lower().endswith(".pdf"):
-        if pdf_extract_text is None:
-            logger.error("pdfminer.six is not installed; cannot extract PDF text.")
-            pdf_text = ""
-        else:
+        if (
+            _size_fields["truncated"]
+            and effective_cap < WEB_FETCH_HARD_MAX_BYTES
+            and (
+                _size_fields["total_bytes"] is None
+                or _size_fields["total_bytes"] <= WEB_FETCH_HARD_MAX_BYTES
+            )
+        ):
             try:
-                pdf_bytes = io.BytesIO(response.content)
-                pdf_text = pdf_extract_text(pdf_bytes)
+                response = _get_public_url(
+                    url,
+                    headers=headers,
+                    timeout=timeout,
+                    max_bytes=WEB_FETCH_HARD_MAX_BYTES,
+                )
+                _size_fields = {
+                    "truncated": getattr(response, "truncated", False),
+                    "fetched_bytes": len(response.content),
+                    "total_bytes": getattr(response, "declared_bytes", None),
+                }
+                effective_cap = WEB_FETCH_HARD_MAX_BYTES
+            except BodyTooLargeError as e:
+                error_logger.warning(f"Refused oversized PDF body for {url}: {e}")
+                return _empty_result(url, f"TooLarge: {e}")
             except Exception as e:
-                logger.warning(f"PDF extraction failed for {url}: {e}")
-                pdf_text = ""
+                logger.warning(f"Full-budget PDF retry failed for {url}: {e}")
+        if _size_fields["truncated"]:
+            # A PDF cut mid-stream is not parseable; unlike text there is no
+            # useful partial result, so report the budget problem instead.
+            _declared = _size_fields["total_bytes"]
+            error = (
+                f"TooLarge: PDF decoded body exceeded the {effective_cap:,}-byte fetch budget"
+                + (f" (declared compressed size {_declared:,} bytes)" if _declared else "")
+                + "; retry with a larger budget if it fits under the hard cap"
+            )
+            return {**_empty_result(url, error), **_size_fields}
+        pdf_text = _extract_pdf_text(response.content, url)
         result = {
             "url": url,
             "title": os.path.basename(url),
@@ -259,6 +342,42 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
             "js_message": "",
             "success": bool(pdf_text),
             "error": "" if pdf_text else "Failed to extract PDF text",
+            **_size_fields,
+        }
+        _cache_result(cache_file, cache_key, result, url)
+        return result
+
+    # Plain-text / Markdown / JSON handling. Sources like
+    # raw.githubusercontent.com serve Markdown as `text/plain`, JSON APIs and
+    # raw config files serve `application/json`, and a lot of code and tool
+    # docs live in `.md` / `.txt`. These have no HTML structure, so the HTML
+    # branch below would extract nothing and report "no readable text content".
+    # Return the body verbatim instead. The `is_html` guard keeps real HTML
+    # (including `application/xhtml+xml`) on the parsing path; the `json` check
+    # covers `application/json` and `+json` suffixes; the URL-suffix fallback
+    # catches servers that mislabel text files as `application/octet-stream`.
+    is_html = "html" in content_type
+    is_json = "json" in content_type
+    url_path = url.lower().split("?", 1)[0].split("#", 1)[0]
+    looks_like_text_file = url_path.endswith(
+        (".md", ".markdown", ".txt", ".text", ".json", ".jsonl")
+    )
+    if not is_html and (content_type.startswith("text/") or is_json or looks_like_text_file):
+        text_body = (response.text or "").strip()
+        result = {
+            "url": url,
+            "title": os.path.basename(url_path) or url,
+            "content": text_body,
+            "lists": [],
+            "tables": [],
+            "code_blocks": [],
+            "meta_description": "",
+            "meta_keywords": "",
+            "js_rendered": False,
+            "js_message": "",
+            "success": bool(text_body),
+            "error": "" if text_body else "Empty response body",
+            **_size_fields,
         }
         _cache_result(cache_file, cache_key, result, url)
         return result
@@ -275,10 +394,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
     title_tag = soup.find("title")
     title_text = title_tag.get_text(strip=True) if title_tag else ""
     meta_info = _extract_meta(soup)
+    og_image = _extract_og_image(soup)
     js_rendered = _detect_js_frameworks(soup)
     js_message = "Page appears to be rendered by a JavaScript framework; content may be incomplete." if js_rendered else ""
 
-    # Main textual content (heuristic)
+    # Main textual content (heuristic): prefer semantic / "content"-classed
+    # containers to skip nav/footer/boilerplate; tuned for article pages.
     main_content = ""
     content_areas = soup.find_all(
         ["main", "article", "section", "div"],
@@ -287,12 +408,23 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
     if content_areas:
         for area in content_areas[:3]:
             main_content += area.get_text(separator=" ", strip=True) + " "
-    if not main_content:
+    main_content = re.sub(r"\s+", " ", main_content).strip()
+
+    # If the heuristic finds only a tiny wrapper, fall back to body text with
+    # obvious boilerplate stripped so UI/deep-research search results do not
+    # look empty for app/landing pages.
+    THIN_CONTENT_CHARS = 600
+    if len(main_content) < THIN_CONTENT_CHARS:
         body = soup.find("body")
         if body:
-            main_content = body.get_text(separator=" ", strip=True)
-
-    main_content = re.sub(r"\s+", " ", main_content).strip()[:8000]
+            body_copy = copy.copy(body)
+            for noise in body_copy.find_all(
+                ["script", "style", "noscript", "template", "nav", "header", "footer", "aside"]
+            ):
+                noise.extract()
+            body_text = re.sub(r"\s+", " ", body_copy.get_text(separator=" ", strip=True)).strip()
+            if len(body_text) > len(main_content):
+                main_content = body_text
 
     result = {
         "url": url,
@@ -303,10 +435,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
         "code_blocks": _extract_code_blocks(soup),
         "meta_description": meta_info.get("description", ""),
         "meta_keywords": meta_info.get("keywords", ""),
+        "og_image": og_image,
         "js_rendered": js_rendered,
         "js_message": js_message,
         "success": True,
         "error": "",
+        **_size_fields,
     }
     _cache_result(cache_file, cache_key, result, url)
     return result
@@ -348,13 +482,18 @@ def get_tldr(text: str, max_sentences: int = 3) -> str:
 
 def extract_quotes(text: str) -> List[str]:
     """Return quoted excerpts that are at least 15 characters long."""
-    return [m.group(1).strip() for m in re.finditer(r'["\']([^"\']{15,}?)["\']', text)]
+    # Backreference the opening quote so the closing quote must match it —
+    # otherwise `"text'` (open double, close single) is treated as a quote.
+    return [m.group(2).strip() for m in re.finditer(r'(["\'])([^"\']{15,}?)\1', text)]
 
 
 def extract_statistics(text: str) -> List[str]:
     """Find numbers, percentages, dates and simple measurements."""
+    # Match a comma-grouped number (1,000,000) OR a plain digit run (50000) —
+    # the old `\d{1,3}(?:,\d{3})*` matched only the first 3 digits of a
+    # comma-less number, and the trailing `\b` dropped a closing `%`.
     pattern = re.compile(
-        r"\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?\b",
+        r"\b(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?",
         re.IGNORECASE,
     )
     return [m.group(0).strip() for m in pattern.finditer(text)]

services/search/core.py

@@ -2,11 +2,15 @@
 
 import json
 import logging
+import re
+import xml.etree.ElementTree as ET
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Dict, Any, Optional, List, Set
 from urllib.parse import urlparse
 
+import httpx
+
 from .analytics import (
     NetworkError,
     ParseError,
@@ -30,6 +34,7 @@ from .providers import (
     tavily_search,
     serper_search,
     _get_search_settings,
+    _get_provider_key,
     _get_result_count,
 )
 from .content import (
@@ -48,30 +53,56 @@ SEARCH_CONFIG: Dict[str, Any] = {
 }
 
 
+def _is_secret_key(name: str) -> bool:
+    """True for config keys that hold a credential (e.g. ``brave_api_key``)."""
+    return name.endswith(("_api_key", "_key", "_token", "_secret"))
+
+
 def get_search_config() -> Dict[str, Any]:
-    """Get current search configuration including active provider info."""
+    """Get current search configuration including active provider info.
+
+    Never returns stored API keys: callers — including the unauthenticated
+    ``GET /api/search/config`` route — only need key *presence* via
+    ``has_api_key``, not the secret itself (#1661).
+    """
     config = SEARCH_CONFIG.copy()
     settings = _get_search_settings()
     provider = settings.get("search_provider", "searxng")
     config["active_provider"] = provider
-    config["has_api_key"] = bool((settings.get("search_api_key") or "").strip())
+    config["has_api_key"] = bool(_get_provider_key(provider))
     config["result_count"] = _get_result_count()
     if provider == "searxng":
         from .providers import _get_search_instance
         config["search_url"] = _get_search_instance()
-    return config
+    # Strip any string-valued credential so secrets never reach the response;
+    # the boolean has_api_key flag (presence only) is preserved.
+    return {
+        k: v for k, v in config.items()
+        if not (isinstance(v, str) and _is_secret_key(k))
+    }
 
 
 def update_search_config(api_key: str = None, **kwargs):
-    """Update search configuration (e.g. Brave API key)."""
-    if api_key:
-        SEARCH_CONFIG["brave_api_key"] = api_key
+    """Merge non-secret search config into SEARCH_CONFIG.
+
+    Provider API keys are intentionally NOT cached here. They are read on demand
+    from settings/env via ``_get_provider_key`` (e.g. ``brave_search``), so the
+    previous ``SEARCH_CONFIG["brave_api_key"] = api_key`` cache was never used
+    for search and only leaked the decrypted key through ``get_search_config`` /
+    ``GET /api/search/config`` (#1661). ``api_key`` is accepted for backward
+    compatibility but no longer stored.
+    """
+    for k, v in kwargs.items():
+        if not _is_secret_key(k):
+            SEARCH_CONFIG[k] = v
 
 
 def _call_provider(provider_name: str, query: str, count: int, time_filter: str = None) -> List[dict]:
     """Call a search provider by name. Returns list of results or empty list."""
     if provider_name == "searxng":
         return searxng_search_api(query, count, time_filter=time_filter)
+    elif provider_name == "searxng_yep":
+        return searxng_search_api(query, count, time_filter=time_filter, engines="yep")
     elif provider_name == "brave":
         return brave_search(query, count, time_filter)
     elif provider_name == "duckduckgo":
@@ -102,7 +133,484 @@ def _build_provider_chain(primary: str) -> List[str]:
     for fb in fallbacks:
         if fb and fb != primary and fb not in chain and fb != "disabled":
             chain.append(fb)
-    return chain
+    from .providers import provider_configured
+    configured = [provider for provider in chain if provider_configured(provider)]
+    for provider in set(chain) - set(configured):
+        logger.warning("Skipping unconfigured search provider: %s", provider)
+    if primary == "searxng" and configured == ["searxng"]:
+        # No usable configured fallback: try a separate engine on the same
+        # private metasearch instance before reporting retrieval failure.
+        configured.append("searxng_yep")
+    return configured
+
+
+_SEARCH_QUERY_FILLER = {
+    "what", "whats", "what's", "which", "when", "where", "year", "from",
+    "any", "info", "information", "details", "update", "updates",
+    "with", "this", "that", "search", "lookup", "look", "find", "tell",
+    "about", "quick", "please", "pls", "official", "links", "source",
+    "sources", "news", "headlines", "breaking", "latest", "current",
+    "newest", "recent", "today", "now",
+    "release", "releases", "version", "versions", "changelog", "github",
+    "gitlab", "weather", "forecast", "forecasts", "tomorrow", "hourly",
+    "daily", "temperature", "temperatures", "conditions", "rain", "raining",
+    "chance", "precipitation",
+    "january", "february", "march", "april", "may", "june", "july",
+    "august", "september", "october", "november", "december",
+    "the", "and", "or", "but", "are", "was", "were", "does", "did",
+    "can", "could", "should", "would", "will", "has", "have", "had",
+    "for", "into", "onto", "near", "over", "under",
+}
+
+_SHORT_QUERY_SUBJECTS = {"ai", "ar", "eu", "uk", "us", "vr"}
+
+_WEATHER_QUERY_HINTS = {
+    "weather", "forecast", "forecasts", "temperature", "temperatures",
+    "rain", "raining", "precipitation", "humid", "humidity", "wind",
+}
+_WEATHER_RESULT_HINTS = {
+    "weather", "forecast", "temperature", "temperatures", "rain",
+    "precipitation", "humidity", "wind", "accuweather", "meteoblue",
+    "weather-atlas", "weather25", "weather365", "easeweather",
+}
+
+
+def _meaningful_query_terms(query: str) -> list[str]:
+    return [
+        term
+        for term in re.findall(r"[a-z0-9]+", str(query or "").lower())
+        if (len(term) > 2 or term in _SHORT_QUERY_SUBJECTS)
+        and not term.isdigit()
+        and term not in _SEARCH_QUERY_FILLER
+    ]
+
+
+def _result_has_query_overlap(query: str, result: dict) -> bool:
+    terms = _meaningful_query_terms(query)
+    if not terms:
+        return True
+    text = " ".join(
+        str(result.get(key) or "").lower()
+        for key in ("title", "snippet", "url")
+    )
+    query_tokens = set(re.findall(r"[a-z0-9]+", str(query or "").lower()))
+    if query_tokens & _WEATHER_QUERY_HINTS:
+        return (
+            any(re.search(rf"\b{re.escape(term)}\b", text) for term in terms)
+            and any(marker in text for marker in _WEATHER_RESULT_HINTS)
+        )
+    result_tokens = set(re.findall(r"[a-z0-9]+", text))
+
+    def lexical_root(word: str) -> str:
+        for suffix in ("ation", "ition", "ence", "ance", "ment", "ents", "ent", "ant", "ing", "ed", "es", "s"):
+            if word.endswith(suffix) and len(word) - len(suffix) >= 6:
+                return word[:-len(suffix)]
+        return word
+
+    result_roots = {lexical_root(token) for token in result_tokens}
+    matched_terms = {
+        term for term in terms
+        if term in result_tokens or lexical_root(term) in result_roots
+    }
+    # A single broad token is not enough evidence for a detailed entity/event
+    # query.  For example, SearXNG may answer "Sweden 78 year old British woman
+    # deportation Brexit ..." with generic Sweden tourism pages.  Treat that as
+    # an empty provider result so the configured fallback gets a chance.
+    minimum_matches = 2 if len(set(terms)) >= 4 else 1
+    return len(matched_terms) >= minimum_matches
+
+
+def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]:
+    if not results:
+        return []
+    relevant = [result for result in results if _result_has_query_overlap(query, result)]
+    # Only reject a provider when it returned a fully off-topic page set. Mixed
+    # result pages are common; ranking can handle those.
+    return relevant if relevant else []
+
+
+_SCHOLARLY_QUERY_CUE_RE = re.compile(
+    r"\b(?:paper|preprint|arxiv|proceedings|table\s+\d+|figure\s+\d+|"
+    r"appendix\s+[a-z0-9]+|benchmark(?:s)?)\b",
+    re.IGNORECASE,
+)
+_SCHOLARLY_TITLE_FILLER = _SEARCH_QUERY_FILLER | {
+    "paper", "preprint", "arxiv", "proceedings", "table", "figure",
+    "appendix", "authors", "author", "extract", "locate", "read",
+}
+_ARXIV_IDENTIFIER_RE = re.compile(
+    r"(?i)(?:\barxiv\s*:\s*|\barxiv\.org/(?:abs|pdf|html)/)?"
+    r"(?P<identifier>\d{4}\.\d{4,5}(?:v\d+)?)\b"
+)
+_FORMAL_PUBLICATION_CUE_RE = re.compile(
+    r"\b(?:publish(?:ed|ing|cation)?|venue|conference|journal|proceedings|doi)\b",
+    re.IGNORECASE,
+)
+
+
+def _exact_arxiv_identifier_results(query: str) -> list[dict]:
+    """Return deterministic official landing pages for explicit arXiv IDs."""
+    seen: set[str] = set()
+    results: list[dict] = []
+    for match in _ARXIV_IDENTIFIER_RE.finditer(str(query or "")):
+        identifier = match.group("identifier")
+        canonical = re.sub(r"v\d+$", "", identifier, flags=re.IGNORECASE)
+        if canonical in seen:
+            continue
+        seen.add(canonical)
+        results.append({
+            "title": f"arXiv:{canonical} — exact identifier match",
+            "url": f"https://arxiv.org/abs/{canonical}",
+            "snippet": (
+                "Official arXiv landing page resolved directly from the exact "
+                "identifier in the query."
+            ),
+            "source": "arxiv",
+        })
+    return results
+
+
+def _title_before_explicit_arxiv_identifier(query: str) -> str:
+    """Extract a probable title that precedes an explicit arXiv identifier."""
+
+    text = re.sub(r"\s+", " ", str(query or "")).strip()
+    match = _ARXIV_IDENTIFIER_RE.search(text)
+    if not match or not _FORMAL_PUBLICATION_CUE_RE.search(text):
+        return ""
+    candidate = text[:match.start()].strip(" \t,;:-'\"")
+    candidate = re.sub(
+        r"\barxiv(?:\.org)?(?:\s*:\s*|\s+(?:abs|pdf|html)\s*[/ :]*)?$",
+        "",
+        candidate,
+        flags=re.IGNORECASE,
+    ).strip(" \t,;:-'\"")
+    candidate = re.sub(
+        r"^(?:(?:please\s+)?(?:find|locate|search\s+for|look\s+up|verify|check)\s+)"
+        r"(?:(?:the|this)\s+)?(?:paper\s+)?",
+        "",
+        candidate,
+        flags=re.IGNORECASE,
+    ).strip(" \t,;:-'\"")
+    return candidate if len(_normalized_title_terms(candidate)) >= 2 else ""
+
+
+def _normalized_title_terms(value: str) -> list[str]:
+    return [
+        token
+        for token in re.findall(r"[a-z0-9]+", str(value or "").lower())
+        if len(token) > 1 and token not in _SCHOLARLY_TITLE_FILLER
+    ]
+
+
+def _is_distinctive_short_scholarly_title(value: str) -> bool:
+    """Recognize compact model/report names without accepting generic phrases."""
+
+    terms = _normalized_title_terms(value)
+    if not 1 <= len(terms) <= 2:
+        return False
+    text = str(value or "").strip()
+    return bool(
+        re.search(r"\d", text)
+        or re.search(r"\b[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*\b", text)
+    )
+
+
+def _scholarly_title_from_query(query: str) -> str:
+    """Extract a probable paper title only from clearly scholarly searches."""
+
+    text = re.sub(r"\s+", " ", str(query or "")).strip()
+    if not text or not _SCHOLARLY_QUERY_CUE_RE.search(text):
+        return ""
+
+    quoted = [
+        candidate.strip()
+        for candidate in re.findall(r'["“”]([^"“”]{4,180})["“”]', text)
+        if len(_normalized_title_terms(candidate)) >= 3
+        or _is_distinctive_short_scholarly_title(candidate)
+    ]
+    if quoted:
+        return max(quoted, key=lambda candidate: len(_normalized_title_terms(candidate)))
+
+    before_paper = re.search(
+        r"(?:^|\b(?:find|locate|read|from|about)\s+)(.{4,160}?)\s+"
+        r"(?:paper|preprint)\b",
+        text,
+        re.IGNORECASE,
+    )
+    if before_paper:
+        candidate = before_paper.group(1).strip(" ,:;-'")
+        if (
+            len(_normalized_title_terms(candidate)) >= 3
+            or _is_distinctive_short_scholarly_title(candidate)
+        ):
+            return candidate
+
+    before_locator = re.match(
+        r"(.{2,80}?)\s+(?:table|figure)\s+\d+\b",
+        text,
+        re.IGNORECASE,
+    )
+    if before_locator:
+        candidate = before_locator.group(1).strip(" ,:;-'\"")
+        if _is_distinctive_short_scholarly_title(candidate):
+            return candidate
+    return ""
+
+
+def _result_strongly_matches_title(title: str, result: dict) -> bool:
+    wanted = set(_normalized_title_terms(title))
+    found = set(_normalized_title_terms(str(result.get("title") or "")))
+    if len(wanted) < 2 or not found:
+        return False
+    overlap = len(wanted & found) / len(wanted)
+    return overlap >= (1.0 if len(wanted) == 2 else 0.8)
+
+
+def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve a paper title through arXiv's public Atom API."""
+
+    try:
+        response = httpx.get(
+            "https://export.arxiv.org/api/query",
+            params={
+                "search_query": f'ti:"{title}"',
+                "start": 0,
+                "max_results": max(1, min(int(count), 5)),
+            },
+            headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
+            timeout=12.0,
+            follow_redirects=True,
+        )
+        response.raise_for_status()
+        root = ET.fromstring(response.text)
+    except Exception as exc:
+        logger.info("arXiv title lookup failed for %r: %s", title, exc)
+        return []
+
+    namespace = {"atom": "http://www.w3.org/2005/Atom"}
+    matches: list[dict] = []
+    for entry in root.findall("atom:entry", namespace):
+        result_title = " ".join(
+            (entry.findtext("atom:title", default="", namespaces=namespace) or "").split()
+        )
+        if not _result_strongly_matches_title(title, {"title": result_title}):
+            continue
+        entry_id = (entry.findtext("atom:id", default="", namespaces=namespace) or "").strip()
+        arxiv_id = entry_id.rstrip("/").rsplit("/", 1)[-1]
+        if not arxiv_id:
+            continue
+        summary = " ".join(
+            (entry.findtext("atom:summary", default="", namespaces=namespace) or "").split()
+        )
+        matches.append({
+            "title": result_title,
+            "url": f"https://arxiv.org/abs/{arxiv_id}",
+            "snippet": summary,
+            "source": "arxiv",
+        })
+    return matches
+
+
+def _openalex_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve an exact scholarly title through OpenAlex metadata."""
+
+    try:
+        # OpenAlex treats a literal question mark as query syntax and returns
+        # HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".
+        search_title = re.sub(r"[?]+", " ", str(title or "")).strip()
+        response = httpx.get(
+            "https://api.openalex.org/works",
+            params={
+                "search": search_title,
+                "per-page": max(1, min(int(count), 5)),
+                "select": (
+                    "display_name,doi,primary_location,publication_year,type"
+                ),
+            },
+            headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
+            timeout=12.0,
+            follow_redirects=True,
+        )
+        response.raise_for_status()
+        payload = response.json()
+    except Exception as exc:
+        logger.info("OpenAlex title lookup failed for %r: %s", title, exc)
+        return []
+
+    matches: list[dict] = []
+    for item in payload.get("results", []):
+        result_title = str(item.get("display_name") or "").strip()
+        if not _result_strongly_matches_title(title, {"title": result_title}):
+            continue
+        location = item.get("primary_location") or {}
+        url = str(location.get("landing_page_url") or item.get("doi") or "").strip()
+        if url.startswith("http://arxiv.org/"):
+            url = "https://" + url[len("http://"):]
+        if not url:
+            continue
+        snippet = "Exact scholarly-title match from OpenAlex metadata."
+        venue = str(location.get("raw_source_name") or "").strip()
+        year = item.get("publication_year")
+        publication_type = str(item.get("type") or "").strip()
+        version = str(location.get("version") or "").strip()
+        formal_parts: list[str] = []
+        if venue:
+            formal_parts.append(f"{venue}, {year}" if year else venue)
+        elif year:
+            formal_parts.append(str(year))
+        if publication_type:
+            formal_parts.append(f"type: {publication_type}")
+        if version:
+            formal_parts.append(f"version: {version}")
+        if formal_parts:
+            snippet += f" Formal publication: {'; '.join(formal_parts)}."
+        matches.append({
+            "title": result_title,
+            "url": url,
+            "snippet": snippet,
+            "source": "openalex",
+        })
+    return matches
+
+
+def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:
+    """Retry a noisy scholarly query as a bare title, then use arXiv API."""
+
+    try:
+        simplified = searxng_search_api(title, count=max(3, count))
+    except Exception as exc:
+        logger.info("Simplified scholarly search failed for %r: %s", title, exc)
+        simplified = []
+    exact = [
+        result for result in simplified
+        if _result_strongly_matches_title(title, result)
+    ]
+    if exact:
+        return exact[:count]
+    openalex = _openalex_title_results(title, count)
+    if openalex:
+        return openalex
+    return _arxiv_title_results(title, count)
+
+
+def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve a clear paper title without waiting on generic search providers."""
+
+    # OpenAlex typically resolves titles in under a second and often returns
+    # the official arXiv landing page. The arXiv API remains the fallback.
+    openalex = _openalex_title_results(title, count)
+    if openalex:
+        return openalex
+    return _arxiv_title_results(title, count)
+
+
+def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:
+    """Prepend an exact arXiv match when a scholarly SERP missed its title."""
+
+    current = list(results or [])
+    identifier_results = _exact_arxiv_identifier_results(query)
+    if identifier_results:
+        title = _title_before_explicit_arxiv_identifier(query)
+        formal_results: list[dict] = []
+        if title:
+            formal_results = [
+                item
+                for item in _openalex_title_results(title, min(count, 3))
+                if "arxiv.org/" not in str(item.get("url") or "").lower()
+            ]
+        exact_urls = {str(item["url"]) for item in identifier_results}
+        formal_urls = {str(item.get("url") or "") for item in formal_results}
+        return (
+            formal_results
+            + identifier_results
+            + [
+                item for item in current
+                if str(item.get("url") or "") not in exact_urls | formal_urls
+            ]
+        )[:count]
+    title = _scholarly_title_from_query(query)
+    if not title:
+        return current
+    exact_current = [
+        item for item in current
+        if _result_strongly_matches_title(title, item)
+    ]
+    if exact_current:
+        exact_ids = {id(item) for item in exact_current}
+        return (exact_current + [item for item in current if id(item) not in exact_ids])[:count]
+    arxiv_results = _scholarly_title_results(title, min(count, 3))
+    if not arxiv_results:
+        return current
+    seen = {str(item.get("url") or "") for item in arxiv_results}
+    return (arxiv_results + [item for item in current if str(item.get("url") or "") not in seen])[:count]
+
+
+def _subject_first_weather_query(query: str) -> str:
+    """Rewrite natural weather questions into the shape SearXNG handles best."""
+    text = re.sub(r"\s+", " ", str(query or "")).strip(" ?")
+    if not text:
+        return text
+    if not (set(re.findall(r"[a-z0-9]+", text.lower())) & _WEATHER_QUERY_HINTS):
+        return text
+    loc_match = re.search(
+        r"\b(?:weather|forecast)\s+(?:in|for|at)\s+(.+)$",
+        text,
+        re.IGNORECASE,
+    )
+    if not loc_match:
+        loc_match = re.search(
+            r"\b(?:weather|forecast)\b.*?\b(?:in|for|at)\s+(.+)$",
+            text,
+            re.IGNORECASE,
+        )
+    if not loc_match:
+        return text
+    location = loc_match.group(1).strip(" ?.,")
+    timing = ""
+    timing_match = re.search(
+        r"\b(today|tomorrow|tonight|this\s+week|next\s+week|now|current)\b",
+        location,
+        re.IGNORECASE,
+    )
+    if timing_match:
+        timing = timing_match.group(1).lower()
+        location = (
+            location[: timing_match.start()] + location[timing_match.end():]
+        ).strip(" ?.,")
+    if not location:
+        return text
+    return re.sub(r"\s+", " ", f"{location} weather forecast {timing}").strip()
+
+
+def _provider_friendly_query(query: str) -> str:
+    """Convert generic question grammar to keyword order without changing its topic."""
+    text = _subject_first_weather_query(query)
+    match = re.fullmatch(
+        r"(?:what|which)\s+(year|date|time)\s+(?:did|does|do|was|were|is|are)\s+(.+)",
+        text,
+        re.IGNORECASE,
+    )
+    if match:
+        return f"{match.group(2).strip()} {match.group(1).lower()}"
+    # Search providers already receive recency separately. Remove a leading
+    # conversational request shell so ranking is driven by the subject rather
+    # than words such as "any", "latest", and "information".
+    cleaned = re.sub(
+        r"^(?:can|could|would)\s+you\s+(?:find|search|look\s+up)\s+",
+        "",
+        text,
+        flags=re.IGNORECASE,
+    )
+    cleaned = re.sub(
+        r"^(?:any\s+)?(?:latest|current|recent)?\s*"
+        r"(?:news|info(?:rmation)?|updates?|details?)\s+(?:on|about)\s+",
+        "",
+        cleaned,
+        flags=re.IGNORECASE,
+    )
+    if cleaned.strip():
+        return cleaned.strip()
+    return text
 
 
 # ----------------------------------------------------------------------
@@ -110,6 +618,7 @@ def _build_provider_chain(primary: str) -> List[str]:
 # ----------------------------------------------------------------------
 def searxng_search_results(query: str, count: int = 10, time_filter: str = None) -> list[dict]:
     """Perform a web search using configured provider with caching and retry."""
+    provider_query = _provider_friendly_query(query)
     settings = _get_search_settings()
     search_provider = settings.get("search_provider", "searxng")
     result_count = _get_result_count()
@@ -117,7 +626,17 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
     if count == 10:
         count = result_count
 
-    cache_key = generate_cache_key(f"{query}|{count}|{time_filter}")
+    # A named scholarly work has a deterministic metadata path. Resolve that
+    # first instead of spending the full tool deadline retrying generic search
+    # providers; the returned official URL lets the agent proceed to PDF tools.
+    scholarly_title = _scholarly_title_from_query(provider_query)
+    if scholarly_title:
+        direct_results = _direct_scholarly_title_results(scholarly_title, count)
+        if direct_results:
+            _record_query(provider_query, True, cache_hit=False)
+            return direct_results[:count]
+
+    cache_key = generate_cache_key(f"{provider_query}|{count}|{time_filter}")
     cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
 
     # Check cache
@@ -130,8 +649,22 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
             if expiry and datetime.now() < expiry:
                 logger.debug(f"Search cache hit for query: {query}")
                 results = cached_data["data"]
-                _record_query(query, bool(results), cache_hit=True)
-                return results
+                # Ranking/relevance logic evolves independently from provider
+                # results. Re-apply it on cache hits so stale cached ordering
+                # does not preserve bad SERP choices after a harness fix.
+                results = _filter_low_relevance_results(provider_query, results)
+                if results:
+                    results = rank_search_results(provider_query, results)
+                results = _augment_scholarly_results(provider_query, results, count)
+                if results:
+                    _record_query(query, True, cache_hit=True)
+                    return results
+                logger.info(
+                    "Search cache hit for %r became empty after relevance filtering; refetching",
+                    provider_query,
+                )
+                cache_file.unlink(missing_ok=True)
+                search_cache_index.pop(cache_key, None)
             else:
                 cache_file.unlink(missing_ok=True)
                 search_cache_index.pop(cache_key, None)
@@ -153,7 +686,8 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
         for attempt in range(2):
             try:
                 logger.info(f"Attempting {provider_name} search (attempt {attempt + 1})")
-                results = _call_provider(provider_name, query, count, time_filter)
+                results = _call_provider(provider_name, provider_query, count, time_filter)
+                results = _filter_low_relevance_results(provider_query, results)
                 if results:
                     logger.info(f"{provider_name} search succeeded with {len(results)} results")
                     break
@@ -164,11 +698,14 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
         if results:
             break
 
+    results = _augment_scholarly_results(provider_query, results, count)
+
     success = bool(results)
-    _record_query(query, success, cache_hit=False)
+    _record_query(provider_query, success, cache_hit=False)
 
     if success:
-        results = rank_search_results(query, results)
+        results = rank_search_results(provider_query, results)
+        results = _augment_scholarly_results(provider_query, results, count)
         try:
             expiry = datetime.now() + _cache_duration_for_query(query)
             cache_data = {
@@ -181,10 +718,10 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
             search_cache_index[cache_key] = datetime.now()
             cleanup_cache(SEARCH_CACHE_DIR, search_cache_index, timedelta(hours=1))
         except Exception as e:
-            logger.warning(f"Failed to write search cache for {query}: {e}")
+            logger.warning(f"Failed to write search cache for {provider_query}: {e}")
 
     if not success:
-        logger.error(f"All search providers failed for query: {query}")
+        logger.error(f"All search providers failed for query: {provider_query}")
 
     return results
 
@@ -203,7 +740,10 @@ def invalidate_search_cache(query: Optional[str] = None) -> None:
         search_cache_index.clear()
         logger.info("All search cache entries have been cleared.")
     else:
-        cache_key = generate_cache_key(f"{query}|10|None")
+        # Match the key the write path stores: searxng_search_results replaces
+        # the caller's default count with the configured _get_result_count()
+        # (default 5), so a hardcoded "|10|None" never matched a real entry.
+        cache_key = generate_cache_key(f"{query}|{_get_result_count()}|None")
         cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
         if cache_file.exists():
             try:
@@ -232,7 +772,8 @@ def comprehensive_web_search(
     return_sources: bool = False,
 ):
     """Perform comprehensive web search with content fetching and advanced filtering."""
-    logger.info(f"Starting comprehensive search for: {query}")
+    provider_query = _provider_friendly_query(query)
+    logger.info(f"Starting comprehensive search for: {provider_query}")
     if time_filter:
         logger.info(f"Applying time filter: {time_filter}")
 
@@ -257,7 +798,8 @@ def comprehensive_web_search(
         empty = False
         for attempt in range(2):
             try:
-                search_results = _call_provider(provider_name, query, fetch_count, time_filter)
+                search_results = _call_provider(provider_name, provider_query, fetch_count, time_filter)
+                search_results = _filter_low_relevance_results(provider_query, search_results)
                 if search_results:
                     provider_attempts[provider_name] = f"ok ({len(search_results)})"
                     logger.info(f"Comprehensive search: {provider_name} returned {len(search_results)} results")
@@ -273,6 +815,12 @@ def comprehensive_web_search(
         elif empty:
             provider_attempts[provider_name] = "empty"
 
+    search_results = _augment_scholarly_results(
+        provider_query,
+        search_results,
+        fetch_count,
+    )
+
     if not search_results:
         tally = ", ".join(f"{p}:{r}" for p, r in provider_attempts.items()) or "no providers configured"
         any_errors = any(r.startswith("error") for r in provider_attempts.values())
@@ -287,7 +835,12 @@ def comprehensive_web_search(
         logger.warning(msg)
         return (msg, []) if return_sources else msg
 
-    search_results = rank_search_results(query, search_results)
+    search_results = rank_search_results(provider_query, search_results)
+    search_results = _augment_scholarly_results(
+        provider_query,
+        search_results,
+        fetch_count,
+    )
 
     # URL filter helper
     def url_passes_filters(url: str) -> bool:
@@ -328,6 +881,12 @@ def comprehensive_web_search(
         for r in search_results if r.get("url")
     ]
 
+    # Map each URL to its [i] number in the sources list so fetched content
+    # blocks can be labeled with the SAME index the model cites.
+    _url_index = {
+        r["url"]: i for i, r in enumerate(search_results, 1) if r.get("url")
+    }
+
     # Fetch content in parallel
     fetched_content = []
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -340,6 +899,10 @@ def comprehensive_web_search(
             try:
                 result = future.result()
                 if result["success"] and result["content"] and len(result["content"]) >= min_content_length:
+                    # Remember which source this fetch belongs to: redirects
+                    # can change result["url"] and completion order is
+                    # arbitrary, so the block label cannot be recomputed later.
+                    result["source_index"] = _url_index.get(url)
                     fetched_content.append(result)
             except Exception as e:
                 logger.error(f"Exception while fetching {url}: {str(e)}")
@@ -361,7 +924,7 @@ def comprehensive_web_search(
 
     output_parts.append("=" * 70)
     output_parts.append("WEB SEARCH RESULTS AND FETCHED CONTENT")
-    output_parts.append(f"Query: {query}")
+    output_parts.append(f"Query: {provider_query}")
     output_parts.append(f"Searched {len(search_results)} results, fetched {len(fetched_content)} pages")
     output_parts.append("=" * 70)
     output_parts.append("")
@@ -380,8 +943,15 @@ def comprehensive_web_search(
         output_parts.append("FETCHED PAGE CONTENT:")
         output_parts.append("-" * 50)
 
-        for i, content in enumerate(fetched_content, 1):
-            output_parts.append(f"\n[CONTENT {i}] From: {content['url']}")
+        # Emit blocks in source order, numbered with the same [i] as the
+        # sources list, so [CONTENT 2] really is content from source [2].
+        # Before this, blocks were numbered 1..N in fetch COMPLETION order,
+        # which matched neither the sources list nor each other run to run.
+        fetched_content.sort(key=lambda c: c.get("source_index") or len(search_results) + 1)
+        for content in fetched_content:
+            _idx = content.get("source_index")
+            _label = f"[CONTENT {_idx}]" if _idx else "[CONTENT]"
+            output_parts.append(f"\n{_label} From: {content['url']}")
             output_parts.append(f"Title: {content['title']}")
             output_parts.append("-" * 30)
 

services/search/providers.py

@@ -3,19 +3,19 @@
 import json
 import logging
 import os
+import re
 from typing import List, Optional
+from urllib.parse import urljoin, urlparse, parse_qs
 
 import httpx
 from bs4 import BeautifulSoup
 
-from src.constants import SEARXNG_INSTANCE
+from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT
 from .analytics import RateLimitError, error_logger
 from .query import build_enhanced_query
 
 logger = logging.getLogger(__name__)
 
-REQUEST_TIMEOUT = 20
-
 # Provider registry — maps setting value to (label, needs_key, needs_url)
 PROVIDER_INFO = {
     "searxng":  ("SearXNG",           False, True),
@@ -34,9 +34,16 @@ def _get_search_settings() -> dict:
     """Return search settings from admin config, falling back to env defaults."""
     try:
         from src.settings import load_settings
-        return load_settings()
+        settings = dict(load_settings())
     except Exception:
-        return {}
+        settings = {}
+    # Headless/native deployments do not necessarily have an admin settings
+    # database.  Require an explicit Odysseus-prefixed override so ordinary UI
+    # configuration remains authoritative by default.
+    env_provider = os.environ.get("ODYSSEUS_SEARCH_PROVIDER", "").strip().lower()
+    if env_provider:
+        settings["search_provider"] = env_provider
+    return settings
 
 
 def _get_search_instance() -> str:
@@ -63,7 +70,22 @@ def _get_provider_key(provider: str) -> str:
         if val:
             return val
     # Legacy fallback: old shared search_api_key field
-    return (settings.get("search_api_key") or "").strip()
+    legacy = (settings.get("search_api_key") or "").strip()
+    if legacy:
+        return legacy
+    env_map = {
+        # DATA_BRAVE_API_KEY is the historical Odysseus name; BRAVE_API_KEY is
+        # the standard name used by headless runners and the Brave SDK.
+        "brave": ("DATA_BRAVE_API_KEY", "BRAVE_API_KEY"),
+        "google_pse": ("GOOGLE_API_KEY",),
+        "tavily": ("TAVILY_API_KEY",),
+        "serper": ("SERPER_API_KEY",),
+    }
+    for env_name in env_map.get(provider, ()):
+        value = (os.environ.get(env_name) or "").strip()
+        if value:
+            return value
+    return ""
 
 
 def _get_result_count() -> int:
@@ -75,9 +97,77 @@ def _get_result_count() -> int:
         return 5
 
 
+def provider_configured(provider: str) -> bool:
+    """Configuration readiness only; a configured engine can still fail upstream."""
+    if provider in {"searxng", "searxng_yep", "duckduckgo"}:
+        return True
+    if provider not in {"brave", "google_pse", "tavily", "serper"}:
+        return False
+    if not _get_provider_key(provider):
+        return False
+    if provider == "google_pse":
+        return bool(_get_search_settings().get("google_pse_cx") or os.environ.get("GOOGLE_PSE_CX"))
+    return True
+
+
+# Canonical SafeSearch levels: "strict" (default), "moderate", "off".
+# Each provider has its own knob name and value space -- see _safesearch_for(...).
+_SAFESEARCH_LEVELS = ("strict", "moderate", "off")
+
+
+def _get_safesearch_level() -> str:
+    """Return configured SafeSearch level normalized to a canonical value."""
+    settings = _get_search_settings()
+    raw = (settings.get("search_safesearch") or "strict").strip().lower()
+    if raw in _SAFESEARCH_LEVELS:
+        return raw
+    aliases = {
+        "on": "strict", "high": "strict", "2": "strict",
+        "medium": "moderate", "1": "moderate", "default": "moderate",
+        "none": "off", "disabled": "off", "0": "off",
+    }
+    return aliases.get(raw, "strict")
+
+
+def _safesearch_for(provider: str) -> Optional[str]:
+    """Translate the canonical SafeSearch level into provider-specific values."""
+    level = _get_safesearch_level()
+    if provider == "searxng":
+        return {"strict": "2", "moderate": "1", "off": "0"}[level]
+    if provider == "brave":
+        return level
+    if provider == "duckduckgo_lib":
+        return {"strict": "on", "moderate": "moderate", "off": "off"}[level]
+    if provider == "duckduckgo_html":
+        return {"strict": "1", "moderate": "-1", "off": "-2"}[level]
+    if provider == "google_pse":
+        return None if level == "off" else "active"
+    if provider == "serper":
+        return None if level == "off" else "active"
+    return None
+
+
 # ── SearXNG ──
 
 _NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "idag")
+_NEWS_EVENT_HINT_RE = re.compile(
+    r"\b(?:deport(?:ation|ed|ing)?|arrest(?:ed|s)?|election(?:s)?|"
+    r"evacuat(?:e|ed|ion)|flood(?:ing|s|ed)?|sanction(?:s|ed)?)\b",
+    re.IGNORECASE,
+)
+_SOFTWARE_RELEASE_HINTS = (
+    "github",
+    "gitlab",
+    "release",
+    "releases",
+    "version",
+    "versions",
+    "changelog",
+    "change log",
+    "pypi",
+    "npm",
+    "package",
+)
 
 # Default general engines (google/duckduckgo/brave/startpage/wikipedia) are
 # routinely rate-limited / CAPTCHA-blocked on this instance and return nothing.
@@ -86,12 +176,13 @@ _NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "i
 _GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "bing,mojeek,presearch")
 
 
-def searxng_search_api(query: str, count: int = 10, categories: str = "general",
-                       time_filter: Optional[str] = None) -> List[dict]:
+def searxng_search_api(query: str, count: Optional[int] = None, categories: str = "general",
+                       time_filter: Optional[str] = None, *, engines: Optional[str] = None) -> List[dict]:
     """Search using SearXNG JSON API. Returns list of {title, url, snippet}."""
+    count = count if count is not None else _get_result_count()
     instance = _get_search_instance()
     api_key = ""
-    headers = {"User-Agent": "Mozilla/5.0"}
+    headers = {"User-Agent": WEB_FETCH_USER_AGENT}
     if api_key:
         headers["Authorization"] = f"Bearer {api_key}"
     # News/fresh queries do badly in the 'general' category — it favours
@@ -104,9 +195,26 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
     # languages and brand-ambiguous terms bleed in foreign SEO pages (e.g.
     # "Odyssey" → Honda Japan, "Trojan" → Japanese malware blogs, "Polyphemus"
     # → Chinese math forums). The news path already did this; general didn't.
-    params = {"q": query, "format": "json", "language": "en"}
+    params = {
+        "q": query,
+        "format": "json",
+        "language": "en",
+        "safesearch": _safesearch_for("searxng"),
+    }
     q_lc = query.lower()
-    is_news = time_filter is not None or any(h in q_lc for h in _NEWS_HINTS)
+    # Fresh software-version queries are usually better served by general
+    # search or canonical project pages than by the news vertical. For example
+    # "latest ollama release version github" can return a sparse news result
+    # that gets filtered as irrelevant, while general engines find GitHub.
+    is_software_release_query = any(h in q_lc for h in _SOFTWARE_RELEASE_HINTS)
+    is_news = (
+        not is_software_release_query
+        and (
+            time_filter is not None
+            or any(h in q_lc for h in _NEWS_HINTS)
+            or bool(_NEWS_EVENT_HINT_RE.search(query))
+        )
+    )
     if is_news and categories == "general":
         params["categories"] = "news"
         if time_filter in ("day", "week", "month", "year"):
@@ -119,6 +227,9 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
         # set returns 0 on this instance — see _GENERAL_ENGINES).
         if categories == "general" and _GENERAL_ENGINES:
             params["engines"] = _GENERAL_ENGINES
+    if engines:
+        params["categories"] = "general"
+        params["engines"] = engines
     try:
         def _parse_results(results):
             return [
@@ -126,6 +237,10 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
                     "title": r.get("title", ""),
                     "url": r.get("url", ""),
                     "snippet": r.get("content", ""),
+                    "provider": "searxng",
+                    "engines": r.get("engines", []),
+                    "published_date": r.get("publishedDate"),
+                    "query": query,
                 }
                 for r in results[:count]
                 if r.get("url")
@@ -153,6 +268,7 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
                 "format": "json",
                 "language": "en",
                 "categories": "general",
+                "safesearch": _safesearch_for("searxng"),
             }
             if _GENERAL_ENGINES:
                 fallback["engines"] = _GENERAL_ENGINES
@@ -197,13 +313,13 @@ def searxng_search(query, max_results=10):
     """Search using SearXNG instance - parsing HTML."""
     instance = _get_search_instance()
     api_key = ""
-    req_headers = {"User-Agent": "Mozilla/5.0"}
+    req_headers = {"User-Agent": WEB_FETCH_USER_AGENT}
     if api_key:
         req_headers["Authorization"] = f"Bearer {api_key}"
     try:
         response = httpx.get(
             f"{instance}/search",
-            params={"q": query},
+            params={"q": query, "safesearch": _safesearch_for("searxng")},
             headers=req_headers,
             timeout=10,
         )
@@ -228,8 +344,9 @@ def searxng_search(query, max_results=10):
 
 # ── Brave ──
 
-def brave_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
+def brave_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
     """Search using Brave API with key from admin settings or env var."""
+    count = count if count is not None else _get_result_count()
     api_key = _get_provider_key("brave") or os.environ.get("DATA_BRAVE_API_KEY") or ""
     return _brave_search_impl(query, count, time_filter, search_config={"brave_api_key": api_key})
 
@@ -248,7 +365,11 @@ def _brave_search_impl(query: str, count: int, time_filter: Optional[str] = None
         return []
 
     headers = {"X-Subscription-Token": brave_api_key, "Accept": "application/json"}
-    params = {"q": enhanced_query, "count": count}
+    params = {
+        "q": enhanced_query,
+        "count": count,
+        "safesearch": _safesearch_for("brave"),
+    }
     if time_filter:
         time_map = {"day": "day", "week": "week", "month": "month", "year": "year"}
         if time_filter in time_map:
@@ -297,14 +418,41 @@ def _brave_search_impl(query: str, count: int, time_filter: Optional[str] = None
 
 # ── DuckDuckGo (free, no key) ──
 
-def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
+def _is_duckduckgo_host(host: str) -> bool:
+    """True only for duckduckgo.com and its subdomains."""
+    host = (host or "").lower()
+    return host == "duckduckgo.com" or host.endswith(".duckduckgo.com")
+
+
+def _resolve_ddg_redirect(raw: str) -> str:
+    """Resolve a DuckDuckGo /l/?uddg= redirect URL to its destination."""
+    if not raw:
+        return raw
+    resolved = raw
+    if resolved.startswith("//"):
+        resolved = "https:" + resolved
+    elif resolved.startswith("/"):
+        resolved = urljoin("https://html.duckduckgo.com", resolved)
+    try:
+        parsed = urlparse(resolved)
+        if _is_duckduckgo_host(parsed.hostname) and parsed.path.rstrip("/") == "/l":
+            qs = parse_qs(parsed.query)
+            if "uddg" in qs:
+                return qs["uddg"][0]
+    except Exception:
+        pass
+    return resolved
+
+
+def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
     """Search using DuckDuckGo via the duckduckgo-search library. No API key needed."""
+    count = count if count is not None else _get_result_count()
     def _html_fallback() -> List[dict]:
         try:
             response = httpx.get(
                 "https://html.duckduckgo.com/html/",
-                params={"q": query},
-                headers={"User-Agent": "Mozilla/5.0"},
+                params={"q": query, "kp": _safesearch_for("duckduckgo_html")},
+                headers={"User-Agent": WEB_FETCH_USER_AGENT},
                 timeout=REQUEST_TIMEOUT,
             )
             response.raise_for_status()
@@ -314,7 +462,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
                 link = result.select_one(".result__a")
                 if not link:
                     continue
-                url = link.get("href", "")
+                url = _resolve_ddg_redirect(link.get("href", ""))
                 if not url:
                     continue
                 snippet_el = result.select_one(".result__snippet")
@@ -330,7 +478,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
             return []
 
     try:
-        from duckduckgo_search import DDGS
+        from ddgs import DDGS
     except ImportError:
         logger.warning("duckduckgo-search package not installed; using HTML fallback")
         return _html_fallback()
@@ -342,7 +490,12 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
 
     try:
         ddgs = DDGS()
-        raw = ddgs.text(query, max_results=count, timelimit=timelimit)
+        raw = ddgs.text(
+            query,
+            max_results=count,
+            timelimit=timelimit,
+            safesearch=_safesearch_for("duckduckgo_lib"),
+        )
         results = []
         for item in raw:
             url = item.get("href", "")
@@ -362,7 +515,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
 
 # ── Google Programmable Search Engine ──
 
-def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
+def google_pse_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
     """Search using Google PSE (Custom Search JSON API).
 
     Requires two keys in settings:
@@ -370,6 +523,7 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
       - google_pse_cx: Programmable Search Engine ID (cx)
     Or env vars GOOGLE_API_KEY and GOOGLE_PSE_CX.
     """
+    count = count if count is not None else _get_result_count()
     settings = _get_search_settings()
     api_key = _get_provider_key("google_pse") or os.environ.get("GOOGLE_API_KEY", "")
     cx = (settings.get("google_pse_cx") or "").strip() or os.environ.get("GOOGLE_PSE_CX", "")
@@ -384,6 +538,9 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
         "q": query,
         "num": min(count, 10),  # Google PSE max is 10 per request
     }
+    safe = _safesearch_for("google_pse")
+    if safe:
+        params["safe"] = safe
     if time_filter:
         # dateRestrict: d[number], w[number], m[number], y[number]
         time_map = {"day": "d1", "week": "w1", "month": "m1", "year": "y1"}
@@ -399,7 +556,6 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
         if response.status_code == 429:
             raise RateLimitError("Google PSE rate limit hit")
         response.raise_for_status()
-        data = response.json()
     except httpx.RequestError as e:
         error_logger.error(f"Google PSE search failed: {e}")
         return []
@@ -407,6 +563,12 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
         error_logger.error(str(e))
         return []
 
+    try:
+        data = response.json()
+    except json.JSONDecodeError as e:
+        error_logger.error(f"Google PSE returned invalid JSON: {e}")
+        return []
+
     results = []
     for item in data.get("items", [])[:count]:
         url = item.get("link", "")
@@ -424,8 +586,9 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
 
 # ── Tavily ──
 
-def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
+def tavily_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
     """Search using Tavily API. Requires search_api_key or TAVILY_API_KEY env var."""
+    count = count if count is not None else _get_result_count()
     api_key = _get_provider_key("tavily") or os.environ.get("TAVILY_API_KEY", "")
     if not api_key:
         logger.warning("Tavily: no API key configured")
@@ -451,7 +614,6 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
         if response.status_code == 429:
             raise RateLimitError("Tavily rate limit hit")
         response.raise_for_status()
-        data = response.json()
     except httpx.RequestError as e:
         error_logger.error(f"Tavily search failed: {e}")
         return []
@@ -459,6 +621,12 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
         error_logger.error(str(e))
         return []
 
+    try:
+        data = response.json()
+    except json.JSONDecodeError as e:
+        error_logger.error(f"Tavily returned invalid JSON: {e}")
+        return []
+
     results = []
     for item in data.get("results", [])[:count]:
         url = item.get("url", "")
@@ -477,8 +645,9 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
 
 # ── Serper.dev ──
 
-def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
+def serper_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
     """Search using Serper.dev API. Requires search_api_key or SERPER_API_KEY env var."""
+    count = count if count is not None else _get_result_count()
     api_key = _get_provider_key("serper") or os.environ.get("SERPER_API_KEY", "")
     if not api_key:
         logger.warning("Serper: no API key configured")
@@ -488,6 +657,9 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
         "q": query,
         "num": count,
     }
+    safe = _safesearch_for("serper")
+    if safe:
+        payload["safe"] = safe
     if time_filter:
         time_map = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m", "year": "qdr:y"}
         if time_filter in time_map:
@@ -503,7 +675,6 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
         if response.status_code == 429:
             raise RateLimitError("Serper rate limit hit")
         response.raise_for_status()
-        data = response.json()
     except httpx.RequestError as e:
         error_logger.error(f"Serper search failed: {e}")
         return []
@@ -511,6 +682,12 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
         error_logger.error(str(e))
         return []
 
+    try:
+        data = response.json()
+    except json.JSONDecodeError as e:
+        error_logger.error(f"Serper returned invalid JSON: {e}")
+        return []
+
     results = []
     for item in data.get("organic", [])[:count]:
         url = item.get("link", "")

services/search/query.py

@@ -13,23 +13,36 @@ logger = logging.getLogger(__name__)
 # ----------------------------------------------------------------------
 def _detect_question_type(query: str) -> Optional[str]:
     """Return the leading question word if present (who, what, when, where, why, how)."""
+    if not isinstance(query, str):
+        return None
     q = query.strip().lower()
     for word in ("who", "what", "when", "where", "why", "how"):
-        if q.startswith(word):
+        # Require a whole-word match: a bare prefix mis-flags ordinary queries
+        # like "whatsapp pricing" (-> what) or "however ..." (-> how), which
+        # then get spurious boost terms OR-appended in enhance_query.
+        if q == word or q.startswith(word + " "):
             return word
     return None
 
 
 def _extract_entities(query: str) -> Dict[str, List[str]]:
     """Lightweight entity extraction: capitalized words and date patterns."""
+    if not isinstance(query, str):
+        return {"names": [], "dates": []}
     entities: Dict[str, List[str]] = {"names": [], "dates": []}
     qtype = _detect_question_type(query)
     cleaned = query
     if qtype:
         cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
-    for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
-        entities["names"].append(token)
-    for year in re.findall(r"\b(19|20)\d{2}\b", cleaned):
+    # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
+    # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
+    # "São" (shredded). Keep the ASCII behaviour — the word boundary already
+    # excludes camelCase mid-word capitals — by requiring an all-alphabetic
+    # token of length > 1 whose first character is uppercase.
+    for token in re.findall(r"\b\w+\b", cleaned):
+        if len(token) > 1 and token[0].isupper() and token.isalpha():
+            entities["names"].append(token)
+    for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
         entities["dates"].append(year)
     month_day_year = re.findall(
         r"\b(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\s+\d{1,2},?\s*\d{4}\b",
@@ -42,12 +55,16 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
 
 def _split_multi_part(query: str) -> List[str]:
     """Split a query into sub-queries on common conjunctions."""
+    if not isinstance(query, str):
+        return []
     parts = re.split(r"\s+and\s+|\s+or\s+|;", query, flags=re.I)
     return [p.strip() for p in parts if p.strip()]
 
 
 def _extract_site_filter(query: str) -> Tuple[str, Optional[str]]:
     """Detect a 'site:example.com' token. Returns (query_without_token, site_or_None)."""
+    if not isinstance(query, str):
+        return "", None
     match = re.search(r"\bsite:([^\s]+)", query, flags=re.I)
     if match:
         site = match.group(1)
@@ -68,6 +85,8 @@ def _boost_entities_in_query(base_query: str, entities: Dict[str, List[str]]) ->
 
 def enhance_query(original_query: str) -> Tuple[str, Optional[str]]:
     """Process the original query: site filter, question type boosts, entity extraction."""
+    if not isinstance(original_query, str):
+        original_query = ""
     query_without_site, site = _extract_site_filter(original_query)
     sub_queries = _split_multi_part(query_without_site)
 
@@ -117,6 +136,8 @@ def build_enhanced_query(query: str, time_filter: str = None) -> str:
 def _is_news_query(query: str) -> bool:
     """Lightweight heuristic to decide if a query is news-oriented."""
     news_terms = {"news", "latest", "breaking", "today", "today's", "current", "updates", "happening"}
+    if not isinstance(query, str):
+        return False
     tokens = set(re.findall(r"\b\w+\b", query.lower()))
     return bool(tokens & news_terms)
 

services/search/ranking.py

@@ -2,17 +2,59 @@
 
 import re
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
+_AGE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S")
+
+
+def _utcnow_naive() -> datetime:
+    """Naive UTC 'now'. Matches the naive, UTC-style published dates parsed below,
+    and is safe on Python 3.14 where ``datetime.utcnow()`` is removed (#1116)."""
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+def recency_score(age_str: Optional[str], now: Optional[datetime] = None) -> float:
+    """Score how recent a result is: 1.0 for <=7 days old, 0.0 for >=30 days.
+
+    The age is measured against UTC, not local time. The previous code used
+    ``datetime.now()`` (local) against UTC-style published dates, so the age was
+    skewed by the host's UTC offset; it was also a latent crash once neighbouring
+    code moves to timezone-aware datetimes (#1116). ``now`` is injectable for tests.
+    """
+    if not age_str:
+        return 0.0
+    dt = None
+    for fmt in _AGE_FORMATS:
+        try:
+            dt = datetime.strptime(age_str, fmt)
+            break
+        except Exception:
+            dt = None
+    if not dt:
+        return 0.0
+    now = now or _utcnow_naive()
+    days_old = (now - dt).days
+    if days_old <= 7:
+        return 1.0
+    if days_old >= 30:
+        return 0.0
+    return (30 - days_old) / 23
+
+
 _NEWS_HINTS = {"news", "nyheter", "headlines", "breaking", "latest", "today", "idag"}
 _SPORTS_HINTS = {
     "sport", "sports", "soccer", "football", "hockey", "nba", "nfl", "mlb",
     "fifa", "world cup", "championship", "quarterfinal", "eliminates",
 }
+# Word-boundary match so "sport" does not fire inside "transport"/"passport"
+# and a domain like "transport.gov" is not mistaken for a sports site.
+_SPORTS_HINT_RE = re.compile(
+    r"\b(?:" + "|".join(re.escape(h) for h in _SPORTS_HINTS) + r")\b"
+)
 _LOW_VALUE_NEWS_DOMAINS = {
     "facebook.com", "www.facebook.com", "sports.yahoo.com", "yahoo.com",
     "www.yahoo.com", "msn.com", "www.msn.com",
@@ -25,6 +67,22 @@ _TRUSTED_NEWS_DOMAINS = {
     "www.theguardian.com", "euronews.com", "www.euronews.com",
     "dw.com", "www.dw.com", "government.se", "www.government.se",
 }
+_SOFTWARE_RELEASE_HINTS = {
+    "github", "gitlab", "release", "releases", "version", "versions",
+    "changelog", "package", "pypi", "npm",
+}
+_PRODUCT_SPEC_HINTS = {
+    "product", "hardware", "device", "phone", "laptop", "desktop", "computer",
+    "chip", "cpu", "gpu", "mac", "iphone", "ipad", "android", "camera",
+    "console", "kindle", "tesla", "car", "model", "price", "pricing", "cost",
+    "buy", "shop", "order", "preorder", "pre-order", "spec", "specs",
+    "specifications", "available", "availability", "ship", "shipping",
+    "released", "launch", "launched", "vram", "memory", "ram", "storage",
+}
+_COMMERCE_OR_SPEC_PATH_HINTS = (
+    "/shop", "/buy", "/store", "/product", "/products", "/spec", "/specs",
+    "/support", "/tech-specs", "/technical-specifications",
+)
 
 
 def _domain(url: str) -> str:
@@ -34,25 +92,40 @@ def _domain(url: str) -> str:
         return ""
 
 
+def _has_word(text: str, term: str) -> bool:
+    """True if ``term`` appears in ``text`` as a whole word.
+
+    Query terms are matched on word boundaries so a short term doesn't match
+    inside an unrelated word: "us" must not match "business"/"music", "port"
+    must not match "transport"/"support". This mirrors the tokenization used to
+    build ``query_terms`` (``\\b\\w+\\b``). #1473 converted the title and sports
+    checks to word boundaries; the snippet and subject-term checks below use
+    the same helper so the whole file stays consistent.
+    """
+    return re.search(rf"\b{re.escape(term)}\b", text) is not None
+
+
 def rank_search_results(query: str, results: List[dict]) -> List[dict]:
     """Rank search results by title relevance, snippet quality, domain authority, and recency."""
     query_terms = [t.lower() for t in re.findall(r"\b\w+\b", query)]
     query_lc = query.lower()
     is_news_query = any(term in _NEWS_HINTS for term in query_terms)
-    is_sports_query = any(hint in query_lc for hint in _SPORTS_HINTS)
+    is_sports_query = bool(_SPORTS_HINT_RE.search(query_lc))
+    is_software_release_query = any(term in _SOFTWARE_RELEASE_HINTS for term in query_terms)
+    is_product_spec_query = any(term in _PRODUCT_SPEC_HINTS for term in query_terms)
 
     def title_score(title: str) -> float:
         if not title:
             return 0.0
         title_lc = title.lower()
-        matches = sum(1 for term in query_terms if re.search(rf"\b{re.escape(term)}\b", title_lc))
+        matches = sum(1 for term in query_terms if _has_word(title_lc, term))
         return matches / len(query_terms) if query_terms else 0.0
 
     def snippet_score(snippet: str) -> float:
         if not snippet:
             return 0.0
         length_factor = min(len(snippet), 200) / 200
-        term_hits = sum(1 for term in query_terms if term in snippet.lower())
+        term_hits = sum(1 for term in query_terms if _has_word(snippet.lower(), term))
         term_factor = term_hits / len(query_terms) if query_terms else 0.0
         return (length_factor + term_factor) / 2
 
@@ -68,24 +141,6 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             return 0.7
         return 0.4
 
-    def recency_score(age_str: Optional[str]) -> float:
-        if not age_str:
-            return 0.0
-        for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"):
-            try:
-                dt = datetime.strptime(age_str, fmt)
-                break
-            except Exception:
-                dt = None
-        if not dt:
-            return 0.0
-        days_old = (datetime.now() - dt).days
-        if days_old <= 7:
-            return 1.0
-        if days_old >= 30:
-            return 0.0
-        return (30 - days_old) / 23
-
     def news_quality_adjustment(title: str, snippet: str, url: str) -> float:
         if not is_news_query:
             return 0.0
@@ -98,15 +153,50 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             adjustment += 0.4
         if netloc in _LOW_VALUE_NEWS_DOMAINS:
             adjustment -= 0.8
-        if not is_sports_query and any(hint in text or hint in netloc for hint in _SPORTS_HINTS):
+        if not is_sports_query and (_SPORTS_HINT_RE.search(text) or _SPORTS_HINT_RE.search(netloc)):
             adjustment -= 1.5
         # A country/news query should not rank a page whose title/snippet barely
         # mentions the country above actual news pages for that country.
         subject_terms = [t for t in query_terms if t not in _NEWS_HINTS]
-        if subject_terms and not any(t in text or t in netloc for t in subject_terms):
+        if subject_terms and not any(_has_word(text, t) or _has_word(netloc, t) for t in subject_terms):
             adjustment -= 1.0
         return adjustment
 
+    def software_release_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_software_release_query:
+            return 0.0
+        netloc = _domain(url)
+        path = urlparse(url).path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if netloc in {"github.com", "www.github.com", "gitlab.com", "www.gitlab.com"}:
+            adjustment += 1.6
+        if "/releases" in path or "/tags" in path:
+            adjustment += 1.2
+        if any(_has_word(text, term) for term in ("release", "releases", "changelog", "version")):
+            adjustment += 0.4
+        if netloc in {"releasealert.dev", "releases.sh", "releasebot.io"}:
+            adjustment -= 0.8
+        return adjustment
+
+    def product_spec_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_product_spec_query:
+            return 0.0
+        parsed = urlparse(url)
+        netloc = parsed.netloc.lower()
+        path = parsed.path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if any(hint in path for hint in _COMMERCE_OR_SPEC_PATH_HINTS):
+            adjustment += 1.1
+        if re.search(r"\b(?:official|specs?|specifications|tech specs|buy|shop|store|price|pricing|available|ships?)\b", text):
+            adjustment += 0.5
+        if netloc.endswith(".com") and any(_has_word(netloc, term) for term in query_terms if len(term) >= 4):
+            adjustment += 0.4
+        if re.search(r"\b(?:rumor|rumour|leak|may|could|expected|reportedly|unannounced)\b", text):
+            adjustment -= 0.8
+        return adjustment
+
     ranked = []
     for result in results:
         title = result.get("title", "")
@@ -120,6 +210,8 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             + 1.5 * domain_score(url)
             + 1.0 * recency_score(age)
             + news_quality_adjustment(title, snippet, url)
+            + software_release_adjustment(title, snippet, url)
+            + product_spec_adjustment(title, snippet, url)
         )
         ranked.append((score, result))
 

services/search/service.py

@@ -62,17 +62,24 @@ class SearchService:
             SearchResponse with results
         """
         depth = depth or self.default_depth
-        fetch_content = fetch_content if fetch_content is not None else self.fetch_content
 
-        # Use existing search implementation
-        raw_results = await comprehensive_web_search(
+        # comprehensive_web_search is synchronous and, with return_sources=True,
+        # returns (context_str, [{"url", "title"}, ...]). Run it off the event
+        # loop so we don't block it, and use the source list as the result rows.
+        # `fetch_content` is accepted for API compatibility; the comprehensive
+        # search always fetches page content.
+        import asyncio
+        _context, raw_results = await asyncio.to_thread(
+            comprehensive_web_search,
             query,
-            max_results=10 * depth,
-            fetch_content=fetch_content,
+            max_pages=10 * depth,
+            return_sources=True,
         )
 
         results = []
         for r in raw_results:
+            if not isinstance(r, dict):
+                continue
             results.append(SearchResult(
                 url=r.get("url", ""),
                 title=r.get("title", ""),

services/shell/service.py

@@ -125,10 +125,11 @@ class ShellService:
                 asyncio.create_task(_reader(proc.stderr, "stderr")),
             ]
 
+            loop = asyncio.get_running_loop()
             finished = 0
-            deadline = asyncio.get_event_loop().time() + timeout
+            deadline = loop.time() + timeout
             while finished < 2:
-                remaining = deadline - asyncio.get_event_loop().time()
+                remaining = deadline - loop.time()
                 if remaining <= 0:
                     raise asyncio.TimeoutError()
 

services/stt/stt_service.py

@@ -40,6 +40,8 @@ class STTService:
     @property
     def available(self) -> bool:
         settings = self._load_settings()
+        if settings.get("stt_enabled") is False:
+            return False
         provider = settings["stt_provider"]
         if provider == "disabled":
             return False
@@ -57,17 +59,29 @@ class STTService:
         if self._whisper_model is None:
             try:
                 from faster_whisper import WhisperModel
-                settings = self._load_settings()
-                model_size = settings.get("stt_model", "base")
-                # Use CPU by default; will use CUDA if available
-                import torch
-                device = "cuda" if torch.cuda.is_available() else "cpu"
-                compute_type = "float16" if device == "cuda" else "int8"
-                self._whisper_model = WhisperModel(model_size, device=device, compute_type=compute_type)
-                logger.info(f"faster-whisper model '{model_size}' loaded on {device}")
             except ImportError:
                 logger.warning("faster-whisper not installed. Install with: pip install faster-whisper")
                 return None
+            try:
+                settings = self._load_settings()
+                model_size = settings.get("stt_model", "base")
+                # faster-whisper runs on CTranslate2, not torch. torch is only
+                # used (optionally) to detect a CUDA device for acceleration —
+                # if it's missing or unusable we just run on CPU. Keeping this
+                # probe separate (and tolerant of any failure, e.g. a broken
+                # CUDA/torch install that raises OSError on import) means a
+                # torch-less or torch-broken machine still does CPU
+                # transcription instead of failing with a misleading
+                # "faster-whisper not installed" error.
+                try:
+                    import torch
+                    use_cuda = torch.cuda.is_available()
+                except Exception:
+                    use_cuda = False
+                device = "cuda" if use_cuda else "cpu"
+                compute_type = "float16" if device == "cuda" else "int8"
+                self._whisper_model = WhisperModel(model_size, device=device, compute_type=compute_type)
+                logger.info(f"faster-whisper model '{model_size}' loaded on {device}")
             except Exception as e:
                 logger.error(f"Failed to load whisper model: {e}")
                 return None
@@ -77,6 +91,7 @@ class STTService:
         model = self._get_whisper()
         if not model:
             return None
+        tmp_path = None
         try:
             # Write to temp file (faster-whisper needs a file path or file-like)
             with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp:
@@ -90,14 +105,14 @@ class STTService:
             segments, info = model.transcribe(tmp_path, **kwargs)
             text = " ".join(seg.text.strip() for seg in segments)
 
-            # Cleanup
-            Path(tmp_path).unlink(missing_ok=True)
-
             logger.info(f"Local STT: {len(text)} chars, lang={info.language}, prob={info.language_probability:.2f}")
             return text
         except Exception as e:
             logger.error(f"Local STT transcription failed: {e}", exc_info=True)
             return None
+        finally:
+            if tmp_path:
+                Path(tmp_path).unlink(missing_ok=True)
 
     # ── API endpoint ──
 
@@ -140,6 +155,8 @@ class STTService:
 
     def transcribe(self, audio_bytes: bytes) -> Optional[str]:
         settings = self._load_settings()
+        if settings.get("stt_enabled") is False:
+            return None
         provider = settings["stt_provider"]
         model = settings["stt_model"]
         language = settings.get("stt_language", "")

services/tts/tts_service.py

@@ -2,6 +2,7 @@
 """Multi-provider TTS service — dispatches to local Kokoro, OpenAI-compatible API, or browser."""
 
 import io
+import os
 import wave
 import logging
 import hashlib
@@ -9,9 +10,23 @@ import httpx
 from pathlib import Path
 from typing import Optional, Dict, Any
 
+from src.constants import TTS_CACHE_DIR
+
 logger = logging.getLogger(__name__)
 
 
+def _safe_speed(value, default: float = 1.0) -> float:
+    """Parse the stored tts_speed defensively. The settings layer tolerates
+    corrupt/agent-written config, so a non-numeric or empty value (e.g. an agent
+    setting "speech speed" = "fast", or a hand-edited settings.json) must not
+    crash synthesis or the stats endpoint with a ValueError."""
+    try:
+        speed = float(value)
+    except (TypeError, ValueError):
+        return default
+    return speed if speed > 0 else default
+
+
 class TTSService:
     """Multi-provider TTS service.
 
@@ -23,10 +38,15 @@ class TTSService:
       "endpoint:<id>"   — OpenAI-compatible /audio/speech via ModelEndpoint
     """
 
-    def __init__(self, cache_dir: str = "data/tts_cache"):
+    def __init__(self, cache_dir: str = TTS_CACHE_DIR):
         self.cache_dir = Path(cache_dir)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         self._kokoro = None  # lazy-init
+        
+        try:
+            self.max_cache_bytes = int(os.getenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", 500 * 1024 * 1024))
+        except ValueError:
+            self.max_cache_bytes = 500 * 1024 * 1024
 
     # ── Settings ──
 
@@ -34,6 +54,7 @@ class TTSService:
         from src.settings import load_settings
         saved = load_settings()
         return {
+            "tts_enabled": saved.get("tts_enabled", True),
             "tts_provider": saved.get("tts_provider", "disabled"),
             "tts_model": saved.get("tts_model", "tts-1"),
             "tts_voice": saved.get("tts_voice", "alloy"),
@@ -43,6 +64,8 @@ class TTSService:
     @property
     def available(self) -> bool:
         settings = self._load_settings()
+        if settings.get("tts_enabled") is False:
+            return False
         provider = settings["tts_provider"]
         if provider == "disabled":
             return False
@@ -51,7 +74,7 @@ class TTSService:
         if provider == "local":
             kokoro = self._get_kokoro()
             return kokoro is not None and kokoro.available
-        if provider.startswith("endpoint:"):
+        if isinstance(provider, str) and provider.startswith("endpoint:"):
             return True  # assume reachable; errors surface at synthesis time
         return False
 
@@ -72,6 +95,53 @@ class TTSService:
         ext = ".mp3" if (len(data) >= 3 and (data[:3] == b'ID3' or (data[0] == 0xff and (data[1] & 0xe0) == 0xe0))) else ".wav"
         (self.cache_dir / f"{key}{ext}").write_bytes(data)
 
+        self._enforce_cache_limit()
+
+    def _enforce_cache_limit(self):
+            """Evicts oldest files if the cache exceeds the configured byte limit."""
+            if self.max_cache_bytes <= 0:
+                return
+
+            try:
+                files = []
+                total_size = 0
+
+                # Safely scan files and sum sizes, ignoring files deleted mid-scan
+                for f in self.cache_dir.iterdir():
+                    try:
+                        if f.is_file() and f.suffix.lower() in (".mp3", ".wav"):
+                            files.append(f)
+                            total_size += f.stat().st_size
+                    except OSError:
+                        continue
+
+                if total_size > self.max_cache_bytes:
+                    logger.info(
+                        f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files."
+                    )
+
+                    # Sort files by modification time (oldest first)
+                    try:
+                        files.sort(key=lambda f: f.stat().st_mtime)
+                    except OSError as e:
+                        logger.warning(f"Failed to sort cache files by mtime: {e}")
+
+                    # Trim down to 80% of max capacity
+                    target_size = self.max_cache_bytes * 0.8
+
+                    while files and total_size > target_size:
+                        f = files.pop(0)
+                        try:
+                            size = f.stat().st_size
+                            f.unlink()
+                            total_size -= size
+                        except OSError as e:
+                            logger.warning(f"Failed to evict cache file {f}: {e}")
+                            continue
+
+            except Exception as e:
+                logger.warning(f"Error enforcing TTS cache limit: {e}", exc_info=True)
+
     def clear_cache(self):
         count = 0
         for f in self.cache_dir.glob("*.*"):
@@ -128,10 +198,12 @@ class TTSService:
 
     def synthesize(self, text: str, use_cache: bool = True) -> Optional[bytes]:
         settings = self._load_settings()
+        if settings.get("tts_enabled") is False:
+            return None
         provider = settings["tts_provider"]
         model = settings["tts_model"]
         voice = settings["tts_voice"]
-        speed = float(settings.get("tts_speed", "1"))
+        speed = _safe_speed(settings.get("tts_speed", "1"))
 
         if provider in ("disabled", "browser"):
             return None
@@ -183,7 +255,7 @@ class TTSService:
         provider = settings["tts_provider"]
         tts_enabled = settings.get("tts_enabled", True)
 
-        cache_files = list(self.cache_dir.glob("*.wav"))
+        cache_files = list(self.cache_dir.glob("*.wav")) + list(self.cache_dir.glob("*.mp3"))
         cache_size = sum(f.stat().st_size for f in cache_files)
 
         is_available = self.available and tts_enabled
@@ -193,7 +265,7 @@ class TTSService:
             "provider": provider,
             "model": settings["tts_model"],
             "voice": settings["tts_voice"],
-            "speed": float(settings.get("tts_speed", "1")),
+            "speed": _safe_speed(settings.get("tts_speed", "1")),
             "cache_entries": len(cache_files),
             "cache_size_mb": round(cache_size / (1024 * 1024), 2),
         }

services/youtube/youtube_handler.py

@@ -59,21 +59,45 @@ def init_youtube():
 
 
 def is_youtube_url(url: str) -> bool:
+    if not isinstance(url, str):
+        return False
     return "youtube.com" in url or "youtu.be" in url
 
 
+# youtube.com-shaped hosts. music.youtube.com serves the same /watch and
+# /shorts paths, so links shared from YouTube Music must resolve too.
+_YT_HOSTS = ("www.youtube.com", "youtube.com", "m.youtube.com", "music.youtube.com")
+# Path prefixes whose first following segment is the video id. Covers the
+# /embed/ player, Shorts (/shorts/), live streams (/live/), and the legacy
+# /v/ embed — all of which `is_youtube_url` already treats as YouTube, so
+# they must be extractable or the link is silently dropped (neither web-fetched
+# nor transcript-fetched) by the chat pipeline.
+_YT_PATH_PREFIXES = ("/embed/", "/shorts/", "/live/", "/v/")
+
+
 def extract_youtube_id(url: str) -> Optional[str]:
-    """Extract YouTube video ID from various URL formats."""
+    """Extract a YouTube video ID from the common URL shapes:
+    watch?v=, youtu.be/<id>, /embed/<id>, /shorts/<id>, /live/<id>, /v/<id>,
+    across youtube.com / m.youtube.com / music.youtube.com / youtu.be."""
+    if not isinstance(url, str):
+        return None
     parsed = urllib.parse.urlparse(url)
-    if parsed.hostname in ("www.youtube.com", "youtube.com", "m.youtube.com"):
+    host = (parsed.hostname or "").lower()
+    if host in _YT_HOSTS:
         if parsed.path == "/watch":
             params = urllib.parse.parse_qs(parsed.query)
-            if "v" in params:
+            if params.get("v"):
                 return params["v"][0]
-        elif parsed.path.startswith("/embed/"):
-            return parsed.path.split("/")[-1]
-    elif parsed.hostname == "youtu.be":
-        return parsed.path[1:]
+        else:
+            for prefix in _YT_PATH_PREFIXES:
+                if parsed.path.startswith(prefix):
+                    vid = parsed.path[len(prefix):].split("/")[0]
+                    if vid:
+                        return vid
+    elif host == "youtu.be":
+        vid = parsed.path.lstrip("/").split("/")[0]
+        if vid:
+            return vid
     return None
 
 
@@ -166,6 +190,8 @@ def format_transcript_for_context(
     if segments:
         ctx += "Timestamped Transcript:\n"
         for seg in segments:
+            if not isinstance(seg, dict):
+                continue
             ctx += f"[{seg['timestamp']}] {seg['text']}\n"
         # Check length — fall back to plain text if too long
         if len(ctx) > 12000:
@@ -198,15 +224,24 @@ async def fetch_youtube_comments(
             f"https://www.youtube.com/watch?v={video_id}",
         ]
 
-        proc = await asyncio.wait_for(
-            asyncio.create_subprocess_exec(
-                *cmd,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            ),
-            timeout=timeout,
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
         )
-        stdout, stderr = await proc.communicate()
+        # Bound the wait on the process actually finishing, not on spawning it.
+        # create_subprocess_exec returns as soon as the child starts, so wrapping
+        # it in wait_for never enforces the timeout — proc.communicate() is the
+        # blocking step. Kill and reap the child if it overruns so it does not
+        # linger after we return.
+        try:
+            stdout, stderr = await asyncio.wait_for(
+                proc.communicate(), timeout=timeout
+            )
+        except asyncio.TimeoutError:
+            proc.kill()
+            await proc.wait()
+            raise
 
         if proc.returncode != 0:
             return {"success": False, "error": f"yt-dlp failed: {stderr.decode()[:200]}", "comments": []}
@@ -254,6 +289,8 @@ def format_comments_for_context(comments_data: Dict[str, Any], url: str) -> str:
     ctx += f"URL: {url}\n\n"
 
     for i, c in enumerate(comments, 1):
+        if not isinstance(c, dict):
+            continue
         likes = c.get("likes", 0)
         likes_str = f" [{likes} likes]" if likes else ""
         ctx += f"{i}. @{c['author']}{likes_str}: {c['text']}\n\n"