refactor(model-routing): centralize explicit foreground fallback policy (#6020)

* refactor(model-routing): centralize explicit foreground fallback policy

Make foreground fallback an explicit per-user, availability-only policy shared by streaming Chat, non-stream Chat, and Agent runs.

Preserve strict defaults, owner/model and credential boundaries, pinned Agent routes, and truthful per-round provenance/accounting. Carry provider-reported model identifiers through native streaming adapters, non-stream responses, and caches, and keep legacy default_model_fallbacks as tombstoned raw storage that generic settings APIs and agent tools cannot expose or mutate.

* fix(agent-loop): restore rebase-dropped qwen routing, workspace prompt, and temperature clamp

* fix(model-routing): thread selected endpoint identity, fix cost classification and fallback eligibility

* fix(chat): restore stream helpers and harden run stop lifecycle

* fix(model-routing): let numeric provider codes win over symbolic rate-limit statuses

* fix(agent-loop): apply qwen temperature and notes-tool clamps per fallback candidate

* fix(chat): honor queued stop across resend and reload canonical terminal on EOF

* fix(chat): track stop queue and cleanup ownership by per-send generation

* fix(agent-loop): preserve requested temperature for non-qwen fallback candidates

* fix(chat): reserve send ownership before any await and scope stop to the current send

* fix(chat): clear the previous run identity at send reservation

---------

Co-authored-by: RaresKeY <158580472+RaresKeY@users.noreply.github.com>
Co-authored-by: StressTestor <212606152+StressTestor@users.noreply.github.com>
This commit is contained in:
Alexandre Teixeira
2026-08-14 08:10:30 +01:00
committed by GitHub
co-authored by RaresKeY StressTestor
parent b52296471b
commit c4369305f0
46 changed files with 11418 additions and 1006 deletions
+1090 -291
View File
File diff suppressed because it is too large Load Diff
+84 -26
View File
@@ -17,13 +17,14 @@ close / navigation / refresh). It does NOT survive a server restart.
import asyncio
import json
import logging
import uuid
from typing import AsyncGenerator, Dict, Optional
logger = logging.getLogger(__name__)
class _Run:
__slots__ = ("buffer", "subscribers", "status", "task", "evict_task")
__slots__ = ("buffer", "subscribers", "status", "task", "evict_task", "run_id")
def __init__(self) -> None:
self.buffer: list = [] # ordered SSE event strings (replay log)
@@ -31,6 +32,9 @@ class _Run:
self.status: str = "running" # running | done | error | stopped
self.task: Optional[asyncio.Task] = None
self.evict_task: Optional[asyncio.Task] = None
# Stable across every subscription/replay of this exact detached run.
# The browser uses it to make local cost accounting replay-idempotent.
self.run_id: str = uuid.uuid4().hex
_RUNS: Dict[str, _Run] = {}
@@ -53,13 +57,24 @@ def _publish(run: _Run, ev: str) -> None:
pass
def _schedule_evict(session_id: str) -> None:
def _wake_run_subscribers(run: _Run) -> None:
"""Close subscribers even when the drain task never reached its body."""
for q in list(run.subscribers):
try:
q.put_nowait((None, None))
except Exception:
pass
def _schedule_evict(session_id: str, expected_run: Optional[_Run] = None) -> None:
"""(Re)arm a grace-period eviction for a terminal run with no subscribers.
Identity-checked so a run that gets replaced/reused is never evicted by a
stale timer."""
run = _RUNS.get(session_id)
if run is None:
return
if expected_run is not None and run is not expected_run:
return
if run.evict_task and not run.evict_task.done():
run.evict_task.cancel()
@@ -85,25 +100,38 @@ def get_status(session_id: str) -> Optional[str]:
return r.status if r else None
async def _drain(session_id: str, agen: AsyncGenerator[str, None],
def get_run_id(session_id: str) -> Optional[str]:
"""Return the opaque identity of the current detached run, if present."""
r = _RUNS.get(session_id)
return r.run_id if r else None
def get_active_run(session_id: str) -> Optional[_Run]:
"""Return the exact active run currently registered for a session."""
r = _RUNS.get(session_id)
return r if r and r.status == "running" else None
async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None],
prev_task: Optional[asyncio.Task] = None) -> None:
"""Pull every event from the wrapped generator into the run buffer, fanning
each out to live subscribers. Runs to completion regardless of subscribers."""
run = _RUNS.get(session_id)
if run is None:
return
subscribers_woken = False
def _wake_subscribers() -> None:
nonlocal subscribers_woken
if subscribers_woken:
return
subscribers_woken = True
_wake_run_subscribers(run)
# If this run replaced an in-flight one (rapid double-send), wait for that
# one to fully finish first. Its CancelledError handler calls aclose(), which
# persists its partial response — letting it complete before we start writing
# keeps the two runs' session saves sequential instead of interleaved.
if prev_task is not None and not prev_task.done():
try:
await asyncio.wait({prev_task})
except asyncio.CancelledError:
raise # our own cancellation — propagate
except Exception:
pass
try:
if prev_task is not None and not prev_task.done():
await asyncio.wait({prev_task})
async for ev in agen:
_publish(run, ev)
if run.status == "running":
@@ -116,6 +144,16 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None],
await agen.aclose()
except Exception:
pass
# A rapid third replacement can cancel this task while it is still
# waiting for its predecessor. Close this run's subscribers promptly,
# but keep the task alive until the predecessor finishes so the next
# run still observes the transitive session-save ordering barrier.
_wake_subscribers()
if prev_task is not None and not prev_task.done():
try:
await asyncio.shield(prev_task)
except (asyncio.CancelledError, Exception):
pass
except Exception as e:
logger.error("[agent-run] %s failed: %s", session_id, e, exc_info=True)
run.status = "error"
@@ -127,15 +165,11 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None],
_publish(run, "data: [DONE]\n\n")
finally:
# Wake every subscriber with the end sentinel so their SSE closes.
for q in list(run.subscribers):
try:
q.put_nowait((None, None))
except Exception:
pass
_wake_subscribers()
# Run is terminal — arm the grace timer so it (and its buffer) is
# eventually freed even if nobody ever reconnects. subscribe() cancels
# this on connect and re-arms on disconnect.
_schedule_evict(session_id)
_schedule_evict(session_id, run)
def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
@@ -145,20 +179,37 @@ def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
prev_task: Optional[asyncio.Task] = None
if prev:
if prev.task and not prev.task.done():
# A task cancelled before its first instruction never enters
# _drain(), so its except/finally blocks cannot update status or
# wake a response already bound to this exact run. Terminalize it
# synchronously before cancelling; _drain's cleanup is idempotent
# when the task had already started.
if prev.status == "running":
prev.status = "stopped"
_wake_run_subscribers(prev)
prev.task.cancel()
prev_task = prev.task # new run awaits this before it starts writing
if prev.evict_task and not prev.evict_task.done():
prev.evict_task.cancel()
run = _Run()
_RUNS[session_id] = run
run.task = asyncio.create_task(_drain(session_id, agen, prev_task))
run.task = asyncio.create_task(_drain(session_id, run, agen, prev_task))
return run
async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
async def subscribe(
session_id: str,
expected_run: Optional[_Run] = None,
) -> AsyncGenerator[str, None]:
"""Replay the run's buffer from the start, then stream live until it ends.
Safe to call repeatedly (reconnect) and from multiple clients at once."""
run = _RUNS.get(session_id)
Safe to call repeatedly (reconnect) and from multiple clients at once.
``expected_run`` binds a lazy StreamingResponse body to the same run whose
identity was put in its response headers. Without that binding, a rapid
replacement between response construction and body iteration could replay
the replacement run under the prior run's identity.
"""
run = expected_run or _RUNS.get(session_id)
if run is None:
return
q: asyncio.Queue = asyncio.Queue()
@@ -201,12 +252,19 @@ async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
# Last subscriber gone on a finished run — (re)arm eviction so the
# buffer doesn't linger indefinitely.
if not run.subscribers and run.status != "running":
_schedule_evict(session_id)
_schedule_evict(session_id, run)
def stop(session_id: str) -> bool:
"""Cancel an in-flight run (the wrapped generator saves its partial)."""
def stop(session_id: str, expected_run_id: Optional[str] = None) -> bool:
"""Cancel the matching in-flight run (which saves its partial output).
A stale browser may issue Stop after another tab has replaced the session's
run. Once the caller knows its opaque run identity, fail closed rather than
cancelling that newer run.
"""
run = _RUNS.get(session_id)
if not expected_run_id or run is None or run.run_id != expected_run_id:
return False
if run and run.task and not run.task.done():
run.task.cancel()
return True
+18 -6
View File
@@ -510,7 +510,12 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
# set/get/list/delete operate on the REAL app settings (the same store
# the Settings panel writes), so changing a model / voice / search
# engine / reminder channel from chat actually takes effect.
from src.settings import load_settings, save_settings, DEFAULT_SETTINGS
from src.settings import (
DEFAULT_SETTINGS,
RETIRED_SETTING_KEYS,
load_settings,
save_settings,
)
# Secrets/credentials the agent must NOT write: kept read-only (masked)
# so API keys never flow through chat. User sets these in the panel.
@@ -562,6 +567,9 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
return k2
return _ALIASES_SET.get(k2, (k or "").strip())
def _is_managed_key(key):
return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS
_ENUMS = {
"image_quality": ["low", "medium", "high"],
"reminder_channel": ["browser", "email", "ntfy", "webhook"],
@@ -624,14 +632,18 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
if action == "list":
s = load_settings()
shown = {k: _mask(k, v) for k, v in s.items() if k in DEFAULT_SETTINGS and not isinstance(v, dict)}
shown = {
k: _mask(k, v)
for k, v in s.items()
if _is_managed_key(k) and not isinstance(v, dict)
}
return {"response": f"{len(shown)} settings (use get/set with a key)", "settings": shown, "exit_code": 0}
elif action == "get":
key = _resolve(args.get("key", ""))
if not key:
return {"error": "key is required", "exit_code": 1}
if key not in DEFAULT_SETTINGS:
if not _is_managed_key(key):
return {"error": f"Unknown setting '{args.get('key')}'. Use action='list' to see them.", "exit_code": 1}
val = load_settings().get(key, DEFAULT_SETTINGS.get(key))
return {"response": f"{key} = {_mask(key, val)}", "value": _mask(key, val), "exit_code": 0}
@@ -642,11 +654,11 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
if not raw:
return {"error": "key is required", "exit_code": 1}
key = _resolve(raw)
if key not in DEFAULT_SETTINGS:
if not _is_managed_key(key):
return {"error": f"Unknown setting '{raw}'. Use action='list' to see available settings.", "exit_code": 1}
if _is_secret(key):
return {"response": f"'{key}' is a credential/secret. For security I can't set it from chat. Open Settings and set it there.", "exit_code": 0}
# Structured settings (dicts/lists like keybinds, default_model_fallbacks)
# Structured settings (dicts/lists like keybinds or vision fallbacks)
# have no safe scalar coercion; _coerce would pass a bare string
# straight through and clobber the structure. Refuse them here; they're
# edited in their dedicated panels. (reset/delete still restore the
@@ -675,7 +687,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
elif action == "delete" or action == "reset":
key = _resolve(args.get("key", ""))
if key not in DEFAULT_SETTINGS:
if not _is_managed_key(key):
return {"error": f"Unknown setting '{args.get('key')}'.", "exit_code": 1}
if _is_secret(key):
return {"response": f"'{key}' is a credential. Reset it in the panel.", "exit_code": 0}
+62 -2
View File
@@ -282,7 +282,9 @@ def trim_for_context(messages: List[Dict], context_length: int, reserve_tokens:
if essential_system:
sys_text = essential_system[0].get("content", "")
if len(sys_text) > 2000:
essential_system[0] = {"role": "system", "content": sys_text[:2000] + "\n[System prompt truncated for context limits]"}
truncated_system = dict(essential_system[0])
truncated_system["content"] = sys_text[:2000] + "\n[System prompt truncated for context limits]"
essential_system[0] = truncated_system
trimmed = essential_system + convo_msgs
if estimate_tokens(trimmed) <= budget:
return _sanitize_tool_messages(essential_system + protected_msgs + convo_msgs)
@@ -325,6 +327,9 @@ async def maybe_compact(
messages: List[Dict],
headers: Optional[Dict] = None,
owner: Optional[str] = None,
*,
persist: bool = True,
compaction_state: Optional[Dict[str, Any]] = None,
) -> tuple:
"""Check context usage and compact if above threshold.
@@ -416,7 +421,17 @@ async def maybe_compact(
# offset — session.history INCLUDES the system messages, but
# split_point is indexed against convo_msgs which does NOT. Without
# this, the slice drops the leading system message(s).
_update_session_history(session, split_point, summary, system_msg_count=len(system_msgs))
if compaction_state is not None:
compaction_state.update({
"split_point": split_point,
"summary": summary,
"system_msg_count": len(system_msgs),
"applied": False,
})
if persist:
_update_session_history(session, split_point, summary, system_msg_count=len(system_msgs))
if compaction_state is not None:
compaction_state["applied"] = True
new_used = estimate_tokens(compacted)
logger.info(
@@ -427,6 +442,51 @@ async def maybe_compact(
return compacted, context_length, True
def apply_compaction_state(session, compaction_state: Optional[Dict[str, Any]]) -> bool:
"""Persist a route-specific compaction after that route commits output.
Candidate prompts may be compacted speculatively while an explicit
foreground fallback chain is being tried. Persisting at construction time
would let an unavailable route rewrite history before another route answers,
so callers hold this small plan and apply only the winning route's plan.
"""
state = compaction_state if isinstance(compaction_state, dict) else None
if not state or state.get("applied"):
return False
summary = state.get("summary")
split_point = state.get("split_point")
system_msg_count = state.get("system_msg_count", 0)
if not isinstance(summary, str) or not isinstance(split_point, int):
return False
_update_session_history(
session,
split_point,
summary,
system_msg_count=system_msg_count if isinstance(system_msg_count, int) else 0,
)
state["applied"] = True
return True
def apply_compaction_state_for_session(
session_id: Optional[str],
compaction_state: Optional[Dict[str, Any]],
) -> bool:
"""Resolve an in-memory session and apply a deferred compaction plan."""
if not session_id:
return False
try:
from core.models import get_session_manager_instance
manager = get_session_manager_instance()
session = manager.get_session(session_id) if manager else None
except Exception:
session = None
return apply_compaction_state(session, compaction_state) if session else False
def _update_session_history(session, split_point: int, summary: str,
system_msg_count: int = 0):
"""Update the in-memory session history after compaction.
+216 -20
View File
@@ -5,6 +5,7 @@ Consolidates the 4+ copies of normalize_base / resolve_endpoint logic into one p
"""
import json
import ipaddress
import logging
import socket
import subprocess
@@ -27,6 +28,43 @@ _NON_CHAT_MODEL = (
)
def endpoint_cost_tracked(url: str, endpoint_kind: Optional[str] = None) -> bool:
"""Return whether token cost should be tracked for a concrete route.
This is intentionally a non-secret route classification. It mirrors the
frontend's local/subscription exclusions without exposing endpoint URLs to
message metadata.
"""
try:
parsed = urlparse(url or "")
host = (parsed.hostname or "").lower().rstrip(".")
path = (parsed.path or "").rstrip("/")
except Exception:
return False
if not host:
return False
if host == "chatgpt.com" and (
path == "/backend-api/codex" or path.startswith("/backend-api/codex/")
):
return False
kind = str(endpoint_kind or "auto").strip().lower()
if kind == "local":
return False
if kind in {"api", "proxy"}:
return True
if host in {"localhost", "0.0.0.0", "host.docker.internal"} or host.endswith(".local"):
return False
try:
ip = ipaddress.ip_address(host)
return ip.is_global
except ValueError:
pass
if "." not in host:
return False
return True
def _first_chat_model(models) -> Optional[str]:
"""First model that isn't an embedding/tts/etc.; falls back to models[0]."""
for m in (models or []):
@@ -396,10 +434,14 @@ def resolve_endpoint(
db.close()
def resolve_endpoint_by_id(
ep_id: str, model: Optional[str] = None, owner: Optional[str] = None
) -> Optional[Tuple[str, str, Dict]]:
"""Resolve a specific endpoint id (+ optional model) to (chat_url, model, headers).
def _resolve_endpoint_by_id_with_descriptor(
ep_id: str,
model: Optional[str] = None,
owner: Optional[str] = None,
*,
require_exact_model: bool = False,
) -> Optional[Tuple[Tuple[str, str, Dict], dict]]:
"""Resolve a concrete endpoint/model plus its non-secret descriptor.
Returns None if the endpoint doesn't exist or is disabled. Used to turn
a configured fallback entry ({endpoint_id, model}) into a dispatch target.
@@ -426,15 +468,34 @@ def resolve_endpoint_by_id(
chat_url = build_chat_url(base)
headers = build_headers(api_key, base)
m = (model or "").strip()
# Drop a model the user disabled on the endpoint, then pick the first
# enabled chat model rather than a hidden one.
if m and m in _endpoint_hidden_models(ep):
m = ""
if not m:
m = _first_chat_model(_endpoint_enabled_models(ep)) or ""
enabled_models = _endpoint_enabled_models(ep)
if require_exact_model:
# Explicit foreground fallback entries are concrete choices. A
# hidden or known-missing model must disable the entry instead of
# silently substituting another model from the endpoint.
if not m or m in _endpoint_hidden_models(ep):
return None
if enabled_models and m not in enabled_models:
return None
else:
# Legacy Utility/Vision chains retain their model-repair behavior.
if m and m in _endpoint_hidden_models(ep):
m = ""
if not m:
m = _first_chat_model(enabled_models) or ""
if not m:
return None
return chat_url, m, headers
return (
(chat_url, m, headers),
{
"endpoint_id": ep.id,
"endpoint_label": getattr(ep, "name", None) or ep.id,
"endpoint_cost_tracked": endpoint_cost_tracked(
chat_url,
getattr(ep, "endpoint_kind", None),
),
},
)
except Exception as e:
logger.debug(f"Could not resolve endpoint {ep_id}: {e}")
return None
@@ -442,11 +503,101 @@ def resolve_endpoint_by_id(
db.close()
def resolve_chat_fallback_candidates(owner: Optional[str] = None) -> list:
"""Compatibility shim for the retired default-chat fallback chain."""
def resolve_endpoint_by_id(
ep_id: str,
model: Optional[str] = None,
owner: Optional[str] = None,
*,
require_exact_model: bool = False,
) -> Optional[Tuple[str, str, Dict]]:
"""Resolve a specific endpoint id (+ optional model) to its runtime route."""
del owner
return []
resolved = _resolve_endpoint_by_id_with_descriptor(
ep_id,
model,
owner=owner,
require_exact_model=require_exact_model,
)
return resolved[0] if resolved else None
def resolve_route_descriptor(
endpoint_url: str,
model: str,
headers: Optional[Dict] = None,
owner: Optional[str] = None,
) -> dict:
"""Return the visible endpoint identity for an already-resolved route.
Headers are compared only inside the process so two endpoints using the
same provider URL/model but different credentials remain distinguishable.
No credential material is returned or logged.
"""
if not endpoint_url or not model:
return {
"endpoint_id": None,
"endpoint_label": "Selected route",
"endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
}
db = SessionLocal()
try:
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
if owner:
from src.auth_helpers import owner_filter
q = owner_filter(q, ModelEndpoint, owner)
expected = (endpoint_url.rstrip("/"), model, headers or {})
for ep in q.all():
resolved = _resolve_endpoint_by_id_with_descriptor(
ep.id,
model,
owner=owner,
require_exact_model=True,
)
if not resolved:
continue
candidate, descriptor = resolved
actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
if actual == expected:
return descriptor
except Exception as e:
logger.debug("Could not identify selected endpoint route: %s", e)
finally:
db.close()
return {
"endpoint_id": None,
"endpoint_label": "Selected route",
"endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
}
def resolve_route_descriptor_by_id(
endpoint_id: str,
endpoint_url: str,
model: str,
headers: Optional[Dict] = None,
owner: Optional[str] = None,
) -> Optional[dict]:
"""Resolve a selected route's identity without relying on row order.
The explicit endpoint id is still verified against the resolved runtime
route. This prevents stale or mismatched request metadata from being used
for attribution while disambiguating endpoints whose routes are otherwise
identical.
"""
resolved = _resolve_endpoint_by_id_with_descriptor(
endpoint_id,
model,
owner=owner,
require_exact_model=True,
)
if not resolved:
return None
candidate, descriptor = resolved
expected = ((endpoint_url or "").rstrip("/"), model, headers or {})
actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
return descriptor if actual == expected else None
def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list:
@@ -460,17 +611,62 @@ def resolve_vision_fallback_candidates(owner: Optional[str] = None) -> list:
def _resolve_fallback_candidates(setting_key: str, owner: Optional[str] = None) -> list:
out = []
try:
from src.settings import get_user_setting, load_settings
settings = load_settings()
chain = get_user_setting(setting_key, owner or "", settings.get(setting_key) or []) or []
except Exception:
return out
for entry in chain:
return []
return resolve_fallback_entries(chain, owner=owner)
def resolve_fallback_entries(
entries,
owner: Optional[str] = None,
*,
require_exact_model: bool = False,
) -> list:
"""Resolve ordered endpoint/model entries within the caller's owner scope."""
out = []
for entry in entries or []:
if not isinstance(entry, dict):
continue
resolved = resolve_endpoint_by_id(entry.get("endpoint_id", ""), entry.get("model", ""), owner=owner)
if resolved:
resolved = resolve_endpoint_by_id(
entry.get("endpoint_id", ""),
entry.get("model", ""),
owner=owner,
require_exact_model=require_exact_model,
)
if resolved and resolved not in out:
out.append(resolved)
return out
def resolve_fallback_entries_with_descriptors(
entries,
owner: Optional[str] = None,
*,
require_exact_model: bool = False,
) -> list:
"""Resolve ordered entries while retaining safe endpoint provenance."""
out = []
seen = []
for entry in entries or []:
if not isinstance(entry, dict):
continue
resolved = _resolve_endpoint_by_id_with_descriptor(
entry.get("endpoint_id", ""),
entry.get("model", ""),
owner=owner,
require_exact_model=require_exact_model,
)
if not resolved:
continue
candidate, descriptor = resolved
if any(candidate == prior for prior in seen):
continue
seen.append(candidate)
out.append((candidate, descriptor))
return out
+189 -14
View File
@@ -1,22 +1,155 @@
"""Foreground Chat and Agent model-routing policy.
"""Explicit foreground Chat and Agent model-routing policy."""
The selected session model is strict by default. Historical
``default_model_fallbacks`` values remain stored for compatibility, but this
policy intentionally does not read or migrate them.
"""
from dataclasses import dataclass
from typing import Any, Collection, Dict, FrozenSet, Optional, Tuple
from typing import Any, Dict, Optional
from src.endpoint_resolver import (
endpoint_cost_tracked,
resolve_fallback_entries,
resolve_fallback_entries_with_descriptors,
resolve_route_descriptor,
resolve_route_descriptor_by_id,
)
_DEFAULT_FALLBACK_ENTRY_RESOLVER = resolve_fallback_entries
FOREGROUND_FALLBACK_ENABLED_KEY = "foreground_fallback_enabled"
FOREGROUND_FALLBACK_LIST_KEY = "foreground_model_fallbacks"
FOREGROUND_AVAILABILITY_STATUSES: FrozenSet[int] = frozenset({
408, 425, 429, 500, 502, 503, 504, 507, 508, 529,
})
MAX_FOREGROUND_FALLBACKS = 10
@dataclass(frozen=True)
class ForegroundModelPolicy:
"""Resolved per-user foreground fallback policy."""
enabled: bool = False
fallback_candidates: Tuple[tuple, ...] = ()
fallback_descriptors: Tuple[dict, ...] = ()
eligible_statuses: FrozenSet[int] = FOREGROUND_AVAILABILITY_STATUSES
fallback_on_empty: bool = False
def _load_policy_preferences(owner: Optional[str]) -> dict:
"""Load only preferences that explicitly belong to ``owner``.
The generic preferences loader intentionally treats a legacy flat store as
the single-user preferences object. That compatibility must not cross an
authentication transition: once a named owner is present, foreground
fallback consent exists only in an actual ``_users[owner]`` dictionary.
"""
from routes import prefs_routes
if owner is None:
prefs = prefs_routes._load_for_user(None)
return dict(prefs) if isinstance(prefs, dict) else {}
raw = prefs_routes._load()
users = raw.get("_users") if isinstance(raw, dict) else None
if not isinstance(users, dict):
return {}
prefs = users.get(owner)
return dict(prefs) if isinstance(prefs, dict) else {}
def resolve_foreground_model_policy(
owner: Optional[str] = None,
allowed_models: Optional[Collection[str]] = None,
) -> ForegroundModelPolicy:
"""Resolve an explicit owner-scoped policy, failing closed to strict mode.
The policy is stored in user preferences even when authentication is
disabled. Historical ``default_model_fallbacks`` values are deliberately
unrelated and are never read or migrated.
"""
try:
prefs = _load_policy_preferences(owner)
except Exception:
return ForegroundModelPolicy()
if prefs.get(FOREGROUND_FALLBACK_ENABLED_KEY) is not True:
return ForegroundModelPolicy()
entries = prefs.get(FOREGROUND_FALLBACK_LIST_KEY)
if not isinstance(entries, list) or not entries:
return ForegroundModelPolicy()
if allowed_models is not None:
allowed = frozenset(allowed_models)
entries = [
entry for entry in entries
if (
isinstance(entry, dict)
and isinstance(entry.get("model"), str)
and entry.get("model") in allowed
)
]
if not entries:
return ForegroundModelPolicy()
entries = entries[:MAX_FOREGROUND_FALLBACKS]
if resolve_fallback_entries is not _DEFAULT_FALLBACK_ENTRY_RESOLVER:
# Preserve the long-standing resolver seam used by downstream tests and
# integrations. Production uses the descriptor-aware resolver below.
compatibility_candidates = resolve_fallback_entries(
entries,
owner=owner,
require_exact_model=True,
)
# Known limitation of this test-only seam: alignment matches on model
# alone, so when two entries share a model and the resolver skips the
# first, the surviving candidate inherits the skipped entry's
# endpoint_id. Production uses the descriptor-aware branch below,
# which is unaffected.
resolved_routes = []
remaining_entries = list(entries)
for candidate in compatibility_candidates:
matching_index = next(
(
index for index, entry in enumerate(remaining_entries)
if isinstance(entry, dict)
and entry.get("model") == candidate[1]
),
None,
)
matching_entry = (
remaining_entries.pop(matching_index)
if matching_index is not None
else {}
)
descriptor = {
"endpoint_id": matching_entry.get("endpoint_id"),
"endpoint_label": matching_entry.get("endpoint_id") or "Fallback route",
"endpoint_cost_tracked": endpoint_cost_tracked(candidate[0]),
}
resolved_routes.append((candidate, descriptor))
else:
resolved_routes = resolve_fallback_entries_with_descriptors(
entries,
owner=owner,
require_exact_model=True,
)
candidates = [candidate for candidate, _descriptor in resolved_routes]
if not candidates:
return ForegroundModelPolicy()
return ForegroundModelPolicy(
enabled=True,
fallback_candidates=tuple(candidates),
fallback_descriptors=tuple(
dict(descriptor) for _candidate, descriptor in resolved_routes
),
)
def resolve_foreground_fallback_candidates(owner: Optional[str] = None) -> list:
"""Return fallback candidates for a foreground Chat or Agent request.
"""Return only candidates explicitly enabled by the current user."""
Foreground routing is strict, so no alternate endpoint/model is eligible.
``owner`` is accepted to keep this policy boundary owner-aware.
"""
del owner
return []
return list(resolve_foreground_model_policy(owner).fallback_candidates)
def build_foreground_model_candidates(
@@ -24,8 +157,50 @@ def build_foreground_model_candidates(
model: str,
headers: Optional[Dict[str, Any]] = None,
owner: Optional[str] = None,
policy: Optional[ForegroundModelPolicy] = None,
) -> list:
"""Build the ordered candidate list for a foreground request."""
policy = policy or resolve_foreground_model_policy(owner)
primary = (endpoint_url, model, headers or {})
return [primary] + resolve_foreground_fallback_candidates(owner=owner)
candidates = [primary]
for candidate in policy.fallback_candidates:
if candidate not in candidates:
candidates.append(candidate)
return candidates
def build_foreground_route_descriptors(
endpoint_url: str,
model: str,
headers: Optional[Dict[str, Any]] = None,
owner: Optional[str] = None,
policy: Optional[ForegroundModelPolicy] = None,
selected_endpoint_id: Optional[str] = None,
) -> list:
"""Build safe route metadata parallel to foreground candidates."""
policy = policy or resolve_foreground_model_policy(owner)
selected = None
if selected_endpoint_id:
selected = resolve_route_descriptor_by_id(
selected_endpoint_id,
endpoint_url,
model,
headers or {},
owner=owner,
)
if selected is None:
selected = resolve_route_descriptor(endpoint_url, model, headers or {}, owner=owner)
primary = (endpoint_url, model, headers or {})
candidates = [primary]
descriptors = [selected]
for candidate, descriptor in zip(
policy.fallback_candidates,
policy.fallback_descriptors,
):
if candidate in candidates:
continue
candidates.append(candidate)
descriptors.append(dict(descriptor))
return descriptors
+885 -126
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -12,6 +12,7 @@ class ChatRequest(BaseModel):
use_research: Optional[bool] = Field(default=False, description="Enable deep research")
time_filter: Optional[str] = Field(default=None, description="Time filter for search")
preset_id: Optional[str] = Field(default=None, description="Preset identifier")
selected_endpoint_id: Optional[str] = Field(default=None, description="Selected model endpoint ID")
@field_validator('message')
@classmethod
+19 -1
View File
@@ -14,6 +14,13 @@ from src.constants import SETTINGS_FILE, FEATURES_FILE
logger = logging.getLogger(__name__)
# Keys retained in the raw settings store for compatibility and rollback, but
# deliberately unavailable through generic settings APIs or agent tools. They
# must stay in ``DEFAULT_SETTINGS`` so old files continue to load without data
# loss; callers that present or mutate settings should use this set as a
# tombstone boundary.
RETIRED_SETTING_KEYS = frozenset({"default_model_fallbacks"})
# Tiny TTL cache for settings/features. get_setting() is called on hot paths
# (every chat, every preprocess); without this it re-parses the JSON each call.
# Picks up edits within _CACHE_TTL seconds, which is fine for human-edited config.
@@ -197,6 +204,17 @@ DEFAULT_SETTINGS = {
},
}
def without_retired_settings(settings: dict) -> dict:
"""Return a shallow copy suitable for generic settings interfaces."""
if not isinstance(settings, dict):
return {}
return {
key: value
for key, value in settings.items()
if key not in RETIRED_SETTING_KEYS
}
DEFAULT_FEATURES = {
"web_search": True,
"web_fetch": True,
@@ -269,7 +287,7 @@ _PER_USER_KEYS = {
# Default chat endpoint / model — without per-user resolution every new
# account inherited whatever the most-recent admin picked, which then
# got injected into the chat composer on first open.
"default_endpoint_id", "default_model", "default_model_fallbacks",
"default_endpoint_id", "default_model",
"utility_endpoint_id", "utility_model", "utility_model_fallbacks",
"research_endpoint_id", "research_model",
}
-5
View File
@@ -1,7 +1,6 @@
"""Shared resolver for background-task AI endpoints."""
from src.endpoint_resolver import (
resolve_chat_fallback_candidates,
resolve_endpoint,
resolve_utility_fallback_candidates,
)
@@ -32,7 +31,6 @@ def resolve_task_candidates(
2. Utility endpoint/model
3. Default endpoint/model
4. Utility fallback chain
5. Retired default-fallback compatibility hook (currently empty)
"""
candidates = []
@@ -49,9 +47,6 @@ def resolve_task_candidates(
_append(*resolve_endpoint("default", owner=owner))
for url, model, headers in resolve_utility_fallback_candidates(owner=owner):
_append(url, model, headers)
for url, model, headers in resolve_chat_fallback_candidates(owner=owner):
_append(url, model, headers)
return candidates