refactor(model-routing): centralize explicit foreground fallback policy (#6020)

* refactor(model-routing): centralize explicit foreground fallback policy

Make foreground fallback an explicit per-user, availability-only policy shared by streaming Chat, non-stream Chat, and Agent runs.

Preserve strict defaults, owner/model and credential boundaries, pinned Agent routes, and truthful per-round provenance/accounting. Carry provider-reported model identifiers through native streaming adapters, non-stream responses, and caches, and keep legacy default_model_fallbacks as tombstoned raw storage that generic settings APIs and agent tools cannot expose or mutate.

* fix(agent-loop): restore rebase-dropped qwen routing, workspace prompt, and temperature clamp

* fix(model-routing): thread selected endpoint identity, fix cost classification and fallback eligibility

* fix(chat): restore stream helpers and harden run stop lifecycle

* fix(model-routing): let numeric provider codes win over symbolic rate-limit statuses

* fix(agent-loop): apply qwen temperature and notes-tool clamps per fallback candidate

* fix(chat): honor queued stop across resend and reload canonical terminal on EOF

* fix(chat): track stop queue and cleanup ownership by per-send generation

* fix(agent-loop): preserve requested temperature for non-qwen fallback candidates

* fix(chat): reserve send ownership before any await and scope stop to the current send

* fix(chat): clear the previous run identity at send reservation

---------

Co-authored-by: RaresKeY <158580472+RaresKeY@users.noreply.github.com>
Co-authored-by: StressTestor <212606152+StressTestor@users.noreply.github.com>

src/endpoint_resolver.py

@@ -5,6 +5,7 @@ Consolidates the 4+ copies of normalize_base / resolve_endpoint logic into one p
 """
 
 import json
+import ipaddress
 import logging
 import socket
 import subprocess
@@ -27,6 +28,43 @@ _NON_CHAT_MODEL = (
 )
 
 
+def endpoint_cost_tracked(url: str, endpoint_kind: Optional[str] = None) -> bool:
+    """Return whether token cost should be tracked for a concrete route.
+
+    This is intentionally a non-secret route classification.  It mirrors the
+    frontend's local/subscription exclusions without exposing endpoint URLs to
+    message metadata.
+    """
+
+    try:
+        parsed = urlparse(url or "")
+        host = (parsed.hostname or "").lower().rstrip(".")
+        path = (parsed.path or "").rstrip("/")
+    except Exception:
+        return False
+    if not host:
+        return False
+    if host == "chatgpt.com" and (
+        path == "/backend-api/codex" or path.startswith("/backend-api/codex/")
+    ):
+        return False
+    kind = str(endpoint_kind or "auto").strip().lower()
+    if kind == "local":
+        return False
+    if kind in {"api", "proxy"}:
+        return True
+    if host in {"localhost", "0.0.0.0", "host.docker.internal"} or host.endswith(".local"):
+        return False
+    try:
+        ip = ipaddress.ip_address(host)
+        return ip.is_global
+    except ValueError:
+        pass
+    if "." not in host:
+        return False
+    return True
+
+
 def _first_chat_model(models) -> Optional[str]:
     """First model that isn't an embedding/tts/etc.; falls back to models[0]."""
     for m in (models or []):
@@ -396,10 +434,14 @@ def resolve_endpoint(
         db.close()
 
 
-def resolve_endpoint_by_id(
-    ep_id: str, model: Optional[str] = None, owner: Optional[str] = None
-) -> Optional[Tuple[str, str, Dict]]:
-    """Resolve a specific endpoint id (+ optional model) to (chat_url, model, headers).
+def _resolve_endpoint_by_id_with_descriptor(
+    ep_id: str,
+    model: Optional[str] = None,
+    owner: Optional[str] = None,
+    *,
+    require_exact_model: bool = False,
+) -> Optional[Tuple[Tuple[str, str, Dict], dict]]:
+    """Resolve a concrete endpoint/model plus its non-secret descriptor.
 
     Returns None if the endpoint doesn't exist or is disabled. Used to turn
     a configured fallback entry ({endpoint_id, model}) into a dispatch target.
@@ -426,15 +468,34 @@ def resolve_endpoint_by_id(
         chat_url = build_chat_url(base)
         headers = build_headers(api_key, base)
         m = (model or "").strip()
-        # Drop a model the user disabled on the endpoint, then pick the first
-        # enabled chat model rather than a hidden one.
-        if m and m in _endpoint_hidden_models(ep):
-            m = ""
-        if not m:
-            m = _first_chat_model(_endpoint_enabled_models(ep)) or ""
+        enabled_models = _endpoint_enabled_models(ep)
+        if require_exact_model:
+            # Explicit foreground fallback entries are concrete choices. A
+            # hidden or known-missing model must disable the entry instead of
+            # silently substituting another model from the endpoint.
+            if not m or m in _endpoint_hidden_models(ep):
+                return None
+            if enabled_models and m not in enabled_models:
+                return None
+        else:
+            # Legacy Utility/Vision chains retain their model-repair behavior.
+            if m and m in _endpoint_hidden_models(ep):
+                m = ""
+            if not m:
+                m = _first_chat_model(enabled_models) or ""
         if not m:
             return None
-        return chat_url, m, headers
+        return (
+            (chat_url, m, headers),
+            {
+                "endpoint_id": ep.id,
+                "endpoint_label": getattr(ep, "name", None) or ep.id,
+                "endpoint_cost_tracked": endpoint_cost_tracked(
+                    chat_url,
+                    getattr(ep, "endpoint_kind", None),
+                ),
+            },
+        )
     except Exception as e:
         logger.debug(f"Could not resolve endpoint {ep_id}: {e}")
         return None
@@ -442,11 +503,101 @@ def resolve_endpoint_by_id(
         db.close()
 
 
-def resolve_chat_fallback_candidates(owner: Optional[str] = None) -> list:
-    """Compatibility shim for the retired default-chat fallback chain."""
+def resolve_endpoint_by_id(
+    ep_id: str,
+    model: Optional[str] = None,
+    owner: Optional[str] = None,
+    *,
+    require_exact_model: bool = False,
+) -> Optional[Tuple[str, str, Dict]]:
+    """Resolve a specific endpoint id (+ optional model) to its runtime route."""
 
-    del owner
-    return []
+    resolved = _resolve_endpoint_by_id_with_descriptor(
+        ep_id,
+        model,
+        owner=owner,
+        require_exact_model=require_exact_model,
+    )
+    return resolved[0] if resolved else None
+
+
+def resolve_route_descriptor(
+    endpoint_url: str,
+    model: str,
+    headers: Optional[Dict] = None,
+    owner: Optional[str] = None,
+) -> dict:
+    """Return the visible endpoint identity for an already-resolved route.
+
+    Headers are compared only inside the process so two endpoints using the
+    same provider URL/model but different credentials remain distinguishable.
+    No credential material is returned or logged.
+    """
+
+    if not endpoint_url or not model:
+        return {
+            "endpoint_id": None,
+            "endpoint_label": "Selected route",
+            "endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
+        }
+    db = SessionLocal()
+    try:
+        q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
+        if owner:
+            from src.auth_helpers import owner_filter
+            q = owner_filter(q, ModelEndpoint, owner)
+        expected = (endpoint_url.rstrip("/"), model, headers or {})
+        for ep in q.all():
+            resolved = _resolve_endpoint_by_id_with_descriptor(
+                ep.id,
+                model,
+                owner=owner,
+                require_exact_model=True,
+            )
+            if not resolved:
+                continue
+            candidate, descriptor = resolved
+            actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
+            if actual == expected:
+                return descriptor
+    except Exception as e:
+        logger.debug("Could not identify selected endpoint route: %s", e)
+    finally:
+        db.close()
+    return {
+        "endpoint_id": None,
+        "endpoint_label": "Selected route",
+        "endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
+    }
+
+
+def resolve_route_descriptor_by_id(
+    endpoint_id: str,
+    endpoint_url: str,
+    model: str,
+    headers: Optional[Dict] = None,
+    owner: Optional[str] = None,
+) -> Optional[dict]:
+    """Resolve a selected route's identity without relying on row order.
+
+    The explicit endpoint id is still verified against the resolved runtime
+    route. This prevents stale or mismatched request metadata from being used
+    for attribution while disambiguating endpoints whose routes are otherwise
+    identical.
+    """
+
+    resolved = _resolve_endpoint_by_id_with_descriptor(
+        endpoint_id,
+        model,
+        owner=owner,
+        require_exact_model=True,
+    )
+    if not resolved:
+        return None
+    candidate, descriptor = resolved
+    expected = ((endpoint_url or "").rstrip("/"), model, headers or {})
+    actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
+    return descriptor if actual == expected else None
 
 
 def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list:
@@ -460,17 +611,62 @@ def resolve_vision_fallback_candidates(owner: Optional[str] = None) -> list:
 
 
 def _resolve_fallback_candidates(setting_key: str, owner: Optional[str] = None) -> list:
-    out = []
     try:
         from src.settings import get_user_setting, load_settings
         settings = load_settings()
         chain = get_user_setting(setting_key, owner or "", settings.get(setting_key) or []) or []
     except Exception:
-        return out
-    for entry in chain:
+        return []
+    return resolve_fallback_entries(chain, owner=owner)
+
+
+def resolve_fallback_entries(
+    entries,
+    owner: Optional[str] = None,
+    *,
+    require_exact_model: bool = False,
+) -> list:
+    """Resolve ordered endpoint/model entries within the caller's owner scope."""
+
+    out = []
+    for entry in entries or []:
         if not isinstance(entry, dict):
             continue
-        resolved = resolve_endpoint_by_id(entry.get("endpoint_id", ""), entry.get("model", ""), owner=owner)
-        if resolved:
+        resolved = resolve_endpoint_by_id(
+            entry.get("endpoint_id", ""),
+            entry.get("model", ""),
+            owner=owner,
+            require_exact_model=require_exact_model,
+        )
+        if resolved and resolved not in out:
             out.append(resolved)
     return out
+
+
+def resolve_fallback_entries_with_descriptors(
+    entries,
+    owner: Optional[str] = None,
+    *,
+    require_exact_model: bool = False,
+) -> list:
+    """Resolve ordered entries while retaining safe endpoint provenance."""
+
+    out = []
+    seen = []
+    for entry in entries or []:
+        if not isinstance(entry, dict):
+            continue
+        resolved = _resolve_endpoint_by_id_with_descriptor(
+            entry.get("endpoint_id", ""),
+            entry.get("model", ""),
+            owner=owner,
+            require_exact_model=require_exact_model,
+        )
+        if not resolved:
+            continue
+        candidate, descriptor = resolved
+        if any(candidate == prior for prior in seen):
+            continue
+        seen.append(candidate)
+        out.append((candidate, descriptor))
+    return out