mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-28 11:02:21 +02:00
refactor(model-routing): centralize explicit foreground fallback policy (#6020)
* refactor(model-routing): centralize explicit foreground fallback policy Make foreground fallback an explicit per-user, availability-only policy shared by streaming Chat, non-stream Chat, and Agent runs. Preserve strict defaults, owner/model and credential boundaries, pinned Agent routes, and truthful per-round provenance/accounting. Carry provider-reported model identifiers through native streaming adapters, non-stream responses, and caches, and keep legacy default_model_fallbacks as tombstoned raw storage that generic settings APIs and agent tools cannot expose or mutate. * fix(agent-loop): restore rebase-dropped qwen routing, workspace prompt, and temperature clamp * fix(model-routing): thread selected endpoint identity, fix cost classification and fallback eligibility * fix(chat): restore stream helpers and harden run stop lifecycle * fix(model-routing): let numeric provider codes win over symbolic rate-limit statuses * fix(agent-loop): apply qwen temperature and notes-tool clamps per fallback candidate * fix(chat): honor queued stop across resend and reload canonical terminal on EOF * fix(chat): track stop queue and cleanup ownership by per-send generation * fix(agent-loop): preserve requested temperature for non-qwen fallback candidates * fix(chat): reserve send ownership before any await and scope stop to the current send * fix(chat): clear the previous run identity at send reservation --------- Co-authored-by: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Co-authored-by: StressTestor <212606152+StressTestor@users.noreply.github.com>
This commit is contained in:
co-authored by
RaresKeY
StressTestor
parent
b52296471b
commit
c4369305f0
+1090
-291
File diff suppressed because it is too large
Load Diff
+84
-26
@@ -17,13 +17,14 @@ close / navigation / refresh). It does NOT survive a server restart.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import AsyncGenerator, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _Run:
|
||||
__slots__ = ("buffer", "subscribers", "status", "task", "evict_task")
|
||||
__slots__ = ("buffer", "subscribers", "status", "task", "evict_task", "run_id")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.buffer: list = [] # ordered SSE event strings (replay log)
|
||||
@@ -31,6 +32,9 @@ class _Run:
|
||||
self.status: str = "running" # running | done | error | stopped
|
||||
self.task: Optional[asyncio.Task] = None
|
||||
self.evict_task: Optional[asyncio.Task] = None
|
||||
# Stable across every subscription/replay of this exact detached run.
|
||||
# The browser uses it to make local cost accounting replay-idempotent.
|
||||
self.run_id: str = uuid.uuid4().hex
|
||||
|
||||
|
||||
_RUNS: Dict[str, _Run] = {}
|
||||
@@ -53,13 +57,24 @@ def _publish(run: _Run, ev: str) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _schedule_evict(session_id: str) -> None:
|
||||
def _wake_run_subscribers(run: _Run) -> None:
|
||||
"""Close subscribers even when the drain task never reached its body."""
|
||||
for q in list(run.subscribers):
|
||||
try:
|
||||
q.put_nowait((None, None))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _schedule_evict(session_id: str, expected_run: Optional[_Run] = None) -> None:
|
||||
"""(Re)arm a grace-period eviction for a terminal run with no subscribers.
|
||||
Identity-checked so a run that gets replaced/reused is never evicted by a
|
||||
stale timer."""
|
||||
run = _RUNS.get(session_id)
|
||||
if run is None:
|
||||
return
|
||||
if expected_run is not None and run is not expected_run:
|
||||
return
|
||||
if run.evict_task and not run.evict_task.done():
|
||||
run.evict_task.cancel()
|
||||
|
||||
@@ -85,25 +100,38 @@ def get_status(session_id: str) -> Optional[str]:
|
||||
return r.status if r else None
|
||||
|
||||
|
||||
async def _drain(session_id: str, agen: AsyncGenerator[str, None],
|
||||
def get_run_id(session_id: str) -> Optional[str]:
|
||||
"""Return the opaque identity of the current detached run, if present."""
|
||||
r = _RUNS.get(session_id)
|
||||
return r.run_id if r else None
|
||||
|
||||
|
||||
def get_active_run(session_id: str) -> Optional[_Run]:
|
||||
"""Return the exact active run currently registered for a session."""
|
||||
r = _RUNS.get(session_id)
|
||||
return r if r and r.status == "running" else None
|
||||
|
||||
|
||||
async def _drain(session_id: str, run: _Run, agen: AsyncGenerator[str, None],
|
||||
prev_task: Optional[asyncio.Task] = None) -> None:
|
||||
"""Pull every event from the wrapped generator into the run buffer, fanning
|
||||
each out to live subscribers. Runs to completion regardless of subscribers."""
|
||||
run = _RUNS.get(session_id)
|
||||
if run is None:
|
||||
return
|
||||
subscribers_woken = False
|
||||
|
||||
def _wake_subscribers() -> None:
|
||||
nonlocal subscribers_woken
|
||||
if subscribers_woken:
|
||||
return
|
||||
subscribers_woken = True
|
||||
_wake_run_subscribers(run)
|
||||
|
||||
# If this run replaced an in-flight one (rapid double-send), wait for that
|
||||
# one to fully finish first. Its CancelledError handler calls aclose(), which
|
||||
# persists its partial response — letting it complete before we start writing
|
||||
# keeps the two runs' session saves sequential instead of interleaved.
|
||||
if prev_task is not None and not prev_task.done():
|
||||
try:
|
||||
await asyncio.wait({prev_task})
|
||||
except asyncio.CancelledError:
|
||||
raise # our own cancellation — propagate
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if prev_task is not None and not prev_task.done():
|
||||
await asyncio.wait({prev_task})
|
||||
async for ev in agen:
|
||||
_publish(run, ev)
|
||||
if run.status == "running":
|
||||
@@ -116,6 +144,16 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None],
|
||||
await agen.aclose()
|
||||
except Exception:
|
||||
pass
|
||||
# A rapid third replacement can cancel this task while it is still
|
||||
# waiting for its predecessor. Close this run's subscribers promptly,
|
||||
# but keep the task alive until the predecessor finishes so the next
|
||||
# run still observes the transitive session-save ordering barrier.
|
||||
_wake_subscribers()
|
||||
if prev_task is not None and not prev_task.done():
|
||||
try:
|
||||
await asyncio.shield(prev_task)
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("[agent-run] %s failed: %s", session_id, e, exc_info=True)
|
||||
run.status = "error"
|
||||
@@ -127,15 +165,11 @@ async def _drain(session_id: str, agen: AsyncGenerator[str, None],
|
||||
_publish(run, "data: [DONE]\n\n")
|
||||
finally:
|
||||
# Wake every subscriber with the end sentinel so their SSE closes.
|
||||
for q in list(run.subscribers):
|
||||
try:
|
||||
q.put_nowait((None, None))
|
||||
except Exception:
|
||||
pass
|
||||
_wake_subscribers()
|
||||
# Run is terminal — arm the grace timer so it (and its buffer) is
|
||||
# eventually freed even if nobody ever reconnects. subscribe() cancels
|
||||
# this on connect and re-arms on disconnect.
|
||||
_schedule_evict(session_id)
|
||||
_schedule_evict(session_id, run)
|
||||
|
||||
|
||||
def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
|
||||
@@ -145,20 +179,37 @@ def start(session_id: str, agen: AsyncGenerator[str, None]) -> _Run:
|
||||
prev_task: Optional[asyncio.Task] = None
|
||||
if prev:
|
||||
if prev.task and not prev.task.done():
|
||||
# A task cancelled before its first instruction never enters
|
||||
# _drain(), so its except/finally blocks cannot update status or
|
||||
# wake a response already bound to this exact run. Terminalize it
|
||||
# synchronously before cancelling; _drain's cleanup is idempotent
|
||||
# when the task had already started.
|
||||
if prev.status == "running":
|
||||
prev.status = "stopped"
|
||||
_wake_run_subscribers(prev)
|
||||
prev.task.cancel()
|
||||
prev_task = prev.task # new run awaits this before it starts writing
|
||||
if prev.evict_task and not prev.evict_task.done():
|
||||
prev.evict_task.cancel()
|
||||
run = _Run()
|
||||
_RUNS[session_id] = run
|
||||
run.task = asyncio.create_task(_drain(session_id, agen, prev_task))
|
||||
run.task = asyncio.create_task(_drain(session_id, run, agen, prev_task))
|
||||
return run
|
||||
|
||||
|
||||
async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
|
||||
async def subscribe(
|
||||
session_id: str,
|
||||
expected_run: Optional[_Run] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Replay the run's buffer from the start, then stream live until it ends.
|
||||
Safe to call repeatedly (reconnect) and from multiple clients at once."""
|
||||
run = _RUNS.get(session_id)
|
||||
Safe to call repeatedly (reconnect) and from multiple clients at once.
|
||||
|
||||
``expected_run`` binds a lazy StreamingResponse body to the same run whose
|
||||
identity was put in its response headers. Without that binding, a rapid
|
||||
replacement between response construction and body iteration could replay
|
||||
the replacement run under the prior run's identity.
|
||||
"""
|
||||
run = expected_run or _RUNS.get(session_id)
|
||||
if run is None:
|
||||
return
|
||||
q: asyncio.Queue = asyncio.Queue()
|
||||
@@ -201,12 +252,19 @@ async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
|
||||
# Last subscriber gone on a finished run — (re)arm eviction so the
|
||||
# buffer doesn't linger indefinitely.
|
||||
if not run.subscribers and run.status != "running":
|
||||
_schedule_evict(session_id)
|
||||
_schedule_evict(session_id, run)
|
||||
|
||||
|
||||
def stop(session_id: str) -> bool:
|
||||
"""Cancel an in-flight run (the wrapped generator saves its partial)."""
|
||||
def stop(session_id: str, expected_run_id: Optional[str] = None) -> bool:
|
||||
"""Cancel the matching in-flight run (which saves its partial output).
|
||||
|
||||
A stale browser may issue Stop after another tab has replaced the session's
|
||||
run. Once the caller knows its opaque run identity, fail closed rather than
|
||||
cancelling that newer run.
|
||||
"""
|
||||
run = _RUNS.get(session_id)
|
||||
if not expected_run_id or run is None or run.run_id != expected_run_id:
|
||||
return False
|
||||
if run and run.task and not run.task.done():
|
||||
run.task.cancel()
|
||||
return True
|
||||
|
||||
@@ -510,7 +510,12 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
# set/get/list/delete operate on the REAL app settings (the same store
|
||||
# the Settings panel writes), so changing a model / voice / search
|
||||
# engine / reminder channel from chat actually takes effect.
|
||||
from src.settings import load_settings, save_settings, DEFAULT_SETTINGS
|
||||
from src.settings import (
|
||||
DEFAULT_SETTINGS,
|
||||
RETIRED_SETTING_KEYS,
|
||||
load_settings,
|
||||
save_settings,
|
||||
)
|
||||
|
||||
# Secrets/credentials the agent must NOT write: kept read-only (masked)
|
||||
# so API keys never flow through chat. User sets these in the panel.
|
||||
@@ -562,6 +567,9 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
return k2
|
||||
return _ALIASES_SET.get(k2, (k or "").strip())
|
||||
|
||||
def _is_managed_key(key):
|
||||
return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS
|
||||
|
||||
_ENUMS = {
|
||||
"image_quality": ["low", "medium", "high"],
|
||||
"reminder_channel": ["browser", "email", "ntfy", "webhook"],
|
||||
@@ -624,14 +632,18 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
|
||||
if action == "list":
|
||||
s = load_settings()
|
||||
shown = {k: _mask(k, v) for k, v in s.items() if k in DEFAULT_SETTINGS and not isinstance(v, dict)}
|
||||
shown = {
|
||||
k: _mask(k, v)
|
||||
for k, v in s.items()
|
||||
if _is_managed_key(k) and not isinstance(v, dict)
|
||||
}
|
||||
return {"response": f"{len(shown)} settings (use get/set with a key)", "settings": shown, "exit_code": 0}
|
||||
|
||||
elif action == "get":
|
||||
key = _resolve(args.get("key", ""))
|
||||
if not key:
|
||||
return {"error": "key is required", "exit_code": 1}
|
||||
if key not in DEFAULT_SETTINGS:
|
||||
if not _is_managed_key(key):
|
||||
return {"error": f"Unknown setting '{args.get('key')}'. Use action='list' to see them.", "exit_code": 1}
|
||||
val = load_settings().get(key, DEFAULT_SETTINGS.get(key))
|
||||
return {"response": f"{key} = {_mask(key, val)}", "value": _mask(key, val), "exit_code": 0}
|
||||
@@ -642,11 +654,11 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
if not raw:
|
||||
return {"error": "key is required", "exit_code": 1}
|
||||
key = _resolve(raw)
|
||||
if key not in DEFAULT_SETTINGS:
|
||||
if not _is_managed_key(key):
|
||||
return {"error": f"Unknown setting '{raw}'. Use action='list' to see available settings.", "exit_code": 1}
|
||||
if _is_secret(key):
|
||||
return {"response": f"'{key}' is a credential/secret. For security I can't set it from chat. Open Settings and set it there.", "exit_code": 0}
|
||||
# Structured settings (dicts/lists like keybinds, default_model_fallbacks)
|
||||
# Structured settings (dicts/lists like keybinds or vision fallbacks)
|
||||
# have no safe scalar coercion; _coerce would pass a bare string
|
||||
# straight through and clobber the structure. Refuse them here; they're
|
||||
# edited in their dedicated panels. (reset/delete still restore the
|
||||
@@ -675,7 +687,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
|
||||
elif action == "delete" or action == "reset":
|
||||
key = _resolve(args.get("key", ""))
|
||||
if key not in DEFAULT_SETTINGS:
|
||||
if not _is_managed_key(key):
|
||||
return {"error": f"Unknown setting '{args.get('key')}'.", "exit_code": 1}
|
||||
if _is_secret(key):
|
||||
return {"response": f"'{key}' is a credential. Reset it in the panel.", "exit_code": 0}
|
||||
|
||||
@@ -282,7 +282,9 @@ def trim_for_context(messages: List[Dict], context_length: int, reserve_tokens:
|
||||
if essential_system:
|
||||
sys_text = essential_system[0].get("content", "")
|
||||
if len(sys_text) > 2000:
|
||||
essential_system[0] = {"role": "system", "content": sys_text[:2000] + "\n[System prompt truncated for context limits]"}
|
||||
truncated_system = dict(essential_system[0])
|
||||
truncated_system["content"] = sys_text[:2000] + "\n[System prompt truncated for context limits]"
|
||||
essential_system[0] = truncated_system
|
||||
trimmed = essential_system + convo_msgs
|
||||
if estimate_tokens(trimmed) <= budget:
|
||||
return _sanitize_tool_messages(essential_system + protected_msgs + convo_msgs)
|
||||
@@ -325,6 +327,9 @@ async def maybe_compact(
|
||||
messages: List[Dict],
|
||||
headers: Optional[Dict] = None,
|
||||
owner: Optional[str] = None,
|
||||
*,
|
||||
persist: bool = True,
|
||||
compaction_state: Optional[Dict[str, Any]] = None,
|
||||
) -> tuple:
|
||||
"""Check context usage and compact if above threshold.
|
||||
|
||||
@@ -416,7 +421,17 @@ async def maybe_compact(
|
||||
# offset — session.history INCLUDES the system messages, but
|
||||
# split_point is indexed against convo_msgs which does NOT. Without
|
||||
# this, the slice drops the leading system message(s).
|
||||
_update_session_history(session, split_point, summary, system_msg_count=len(system_msgs))
|
||||
if compaction_state is not None:
|
||||
compaction_state.update({
|
||||
"split_point": split_point,
|
||||
"summary": summary,
|
||||
"system_msg_count": len(system_msgs),
|
||||
"applied": False,
|
||||
})
|
||||
if persist:
|
||||
_update_session_history(session, split_point, summary, system_msg_count=len(system_msgs))
|
||||
if compaction_state is not None:
|
||||
compaction_state["applied"] = True
|
||||
|
||||
new_used = estimate_tokens(compacted)
|
||||
logger.info(
|
||||
@@ -427,6 +442,51 @@ async def maybe_compact(
|
||||
return compacted, context_length, True
|
||||
|
||||
|
||||
def apply_compaction_state(session, compaction_state: Optional[Dict[str, Any]]) -> bool:
|
||||
"""Persist a route-specific compaction after that route commits output.
|
||||
|
||||
Candidate prompts may be compacted speculatively while an explicit
|
||||
foreground fallback chain is being tried. Persisting at construction time
|
||||
would let an unavailable route rewrite history before another route answers,
|
||||
so callers hold this small plan and apply only the winning route's plan.
|
||||
"""
|
||||
|
||||
state = compaction_state if isinstance(compaction_state, dict) else None
|
||||
if not state or state.get("applied"):
|
||||
return False
|
||||
summary = state.get("summary")
|
||||
split_point = state.get("split_point")
|
||||
system_msg_count = state.get("system_msg_count", 0)
|
||||
if not isinstance(summary, str) or not isinstance(split_point, int):
|
||||
return False
|
||||
_update_session_history(
|
||||
session,
|
||||
split_point,
|
||||
summary,
|
||||
system_msg_count=system_msg_count if isinstance(system_msg_count, int) else 0,
|
||||
)
|
||||
state["applied"] = True
|
||||
return True
|
||||
|
||||
|
||||
def apply_compaction_state_for_session(
|
||||
session_id: Optional[str],
|
||||
compaction_state: Optional[Dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Resolve an in-memory session and apply a deferred compaction plan."""
|
||||
|
||||
if not session_id:
|
||||
return False
|
||||
try:
|
||||
from core.models import get_session_manager_instance
|
||||
|
||||
manager = get_session_manager_instance()
|
||||
session = manager.get_session(session_id) if manager else None
|
||||
except Exception:
|
||||
session = None
|
||||
return apply_compaction_state(session, compaction_state) if session else False
|
||||
|
||||
|
||||
def _update_session_history(session, split_point: int, summary: str,
|
||||
system_msg_count: int = 0):
|
||||
"""Update the in-memory session history after compaction.
|
||||
|
||||
+216
-20
@@ -5,6 +5,7 @@ Consolidates the 4+ copies of normalize_base / resolve_endpoint logic into one p
|
||||
"""
|
||||
|
||||
import json
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
import subprocess
|
||||
@@ -27,6 +28,43 @@ _NON_CHAT_MODEL = (
|
||||
)
|
||||
|
||||
|
||||
def endpoint_cost_tracked(url: str, endpoint_kind: Optional[str] = None) -> bool:
|
||||
"""Return whether token cost should be tracked for a concrete route.
|
||||
|
||||
This is intentionally a non-secret route classification. It mirrors the
|
||||
frontend's local/subscription exclusions without exposing endpoint URLs to
|
||||
message metadata.
|
||||
"""
|
||||
|
||||
try:
|
||||
parsed = urlparse(url or "")
|
||||
host = (parsed.hostname or "").lower().rstrip(".")
|
||||
path = (parsed.path or "").rstrip("/")
|
||||
except Exception:
|
||||
return False
|
||||
if not host:
|
||||
return False
|
||||
if host == "chatgpt.com" and (
|
||||
path == "/backend-api/codex" or path.startswith("/backend-api/codex/")
|
||||
):
|
||||
return False
|
||||
kind = str(endpoint_kind or "auto").strip().lower()
|
||||
if kind == "local":
|
||||
return False
|
||||
if kind in {"api", "proxy"}:
|
||||
return True
|
||||
if host in {"localhost", "0.0.0.0", "host.docker.internal"} or host.endswith(".local"):
|
||||
return False
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
return ip.is_global
|
||||
except ValueError:
|
||||
pass
|
||||
if "." not in host:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _first_chat_model(models) -> Optional[str]:
|
||||
"""First model that isn't an embedding/tts/etc.; falls back to models[0]."""
|
||||
for m in (models or []):
|
||||
@@ -396,10 +434,14 @@ def resolve_endpoint(
|
||||
db.close()
|
||||
|
||||
|
||||
def resolve_endpoint_by_id(
|
||||
ep_id: str, model: Optional[str] = None, owner: Optional[str] = None
|
||||
) -> Optional[Tuple[str, str, Dict]]:
|
||||
"""Resolve a specific endpoint id (+ optional model) to (chat_url, model, headers).
|
||||
def _resolve_endpoint_by_id_with_descriptor(
|
||||
ep_id: str,
|
||||
model: Optional[str] = None,
|
||||
owner: Optional[str] = None,
|
||||
*,
|
||||
require_exact_model: bool = False,
|
||||
) -> Optional[Tuple[Tuple[str, str, Dict], dict]]:
|
||||
"""Resolve a concrete endpoint/model plus its non-secret descriptor.
|
||||
|
||||
Returns None if the endpoint doesn't exist or is disabled. Used to turn
|
||||
a configured fallback entry ({endpoint_id, model}) into a dispatch target.
|
||||
@@ -426,15 +468,34 @@ def resolve_endpoint_by_id(
|
||||
chat_url = build_chat_url(base)
|
||||
headers = build_headers(api_key, base)
|
||||
m = (model or "").strip()
|
||||
# Drop a model the user disabled on the endpoint, then pick the first
|
||||
# enabled chat model rather than a hidden one.
|
||||
if m and m in _endpoint_hidden_models(ep):
|
||||
m = ""
|
||||
if not m:
|
||||
m = _first_chat_model(_endpoint_enabled_models(ep)) or ""
|
||||
enabled_models = _endpoint_enabled_models(ep)
|
||||
if require_exact_model:
|
||||
# Explicit foreground fallback entries are concrete choices. A
|
||||
# hidden or known-missing model must disable the entry instead of
|
||||
# silently substituting another model from the endpoint.
|
||||
if not m or m in _endpoint_hidden_models(ep):
|
||||
return None
|
||||
if enabled_models and m not in enabled_models:
|
||||
return None
|
||||
else:
|
||||
# Legacy Utility/Vision chains retain their model-repair behavior.
|
||||
if m and m in _endpoint_hidden_models(ep):
|
||||
m = ""
|
||||
if not m:
|
||||
m = _first_chat_model(enabled_models) or ""
|
||||
if not m:
|
||||
return None
|
||||
return chat_url, m, headers
|
||||
return (
|
||||
(chat_url, m, headers),
|
||||
{
|
||||
"endpoint_id": ep.id,
|
||||
"endpoint_label": getattr(ep, "name", None) or ep.id,
|
||||
"endpoint_cost_tracked": endpoint_cost_tracked(
|
||||
chat_url,
|
||||
getattr(ep, "endpoint_kind", None),
|
||||
),
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not resolve endpoint {ep_id}: {e}")
|
||||
return None
|
||||
@@ -442,11 +503,101 @@ def resolve_endpoint_by_id(
|
||||
db.close()
|
||||
|
||||
|
||||
def resolve_chat_fallback_candidates(owner: Optional[str] = None) -> list:
|
||||
"""Compatibility shim for the retired default-chat fallback chain."""
|
||||
def resolve_endpoint_by_id(
|
||||
ep_id: str,
|
||||
model: Optional[str] = None,
|
||||
owner: Optional[str] = None,
|
||||
*,
|
||||
require_exact_model: bool = False,
|
||||
) -> Optional[Tuple[str, str, Dict]]:
|
||||
"""Resolve a specific endpoint id (+ optional model) to its runtime route."""
|
||||
|
||||
del owner
|
||||
return []
|
||||
resolved = _resolve_endpoint_by_id_with_descriptor(
|
||||
ep_id,
|
||||
model,
|
||||
owner=owner,
|
||||
require_exact_model=require_exact_model,
|
||||
)
|
||||
return resolved[0] if resolved else None
|
||||
|
||||
|
||||
def resolve_route_descriptor(
|
||||
endpoint_url: str,
|
||||
model: str,
|
||||
headers: Optional[Dict] = None,
|
||||
owner: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Return the visible endpoint identity for an already-resolved route.
|
||||
|
||||
Headers are compared only inside the process so two endpoints using the
|
||||
same provider URL/model but different credentials remain distinguishable.
|
||||
No credential material is returned or logged.
|
||||
"""
|
||||
|
||||
if not endpoint_url or not model:
|
||||
return {
|
||||
"endpoint_id": None,
|
||||
"endpoint_label": "Selected route",
|
||||
"endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
|
||||
}
|
||||
db = SessionLocal()
|
||||
try:
|
||||
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
|
||||
if owner:
|
||||
from src.auth_helpers import owner_filter
|
||||
q = owner_filter(q, ModelEndpoint, owner)
|
||||
expected = (endpoint_url.rstrip("/"), model, headers or {})
|
||||
for ep in q.all():
|
||||
resolved = _resolve_endpoint_by_id_with_descriptor(
|
||||
ep.id,
|
||||
model,
|
||||
owner=owner,
|
||||
require_exact_model=True,
|
||||
)
|
||||
if not resolved:
|
||||
continue
|
||||
candidate, descriptor = resolved
|
||||
actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
|
||||
if actual == expected:
|
||||
return descriptor
|
||||
except Exception as e:
|
||||
logger.debug("Could not identify selected endpoint route: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
return {
|
||||
"endpoint_id": None,
|
||||
"endpoint_label": "Selected route",
|
||||
"endpoint_cost_tracked": endpoint_cost_tracked(endpoint_url),
|
||||
}
|
||||
|
||||
|
||||
def resolve_route_descriptor_by_id(
|
||||
endpoint_id: str,
|
||||
endpoint_url: str,
|
||||
model: str,
|
||||
headers: Optional[Dict] = None,
|
||||
owner: Optional[str] = None,
|
||||
) -> Optional[dict]:
|
||||
"""Resolve a selected route's identity without relying on row order.
|
||||
|
||||
The explicit endpoint id is still verified against the resolved runtime
|
||||
route. This prevents stale or mismatched request metadata from being used
|
||||
for attribution while disambiguating endpoints whose routes are otherwise
|
||||
identical.
|
||||
"""
|
||||
|
||||
resolved = _resolve_endpoint_by_id_with_descriptor(
|
||||
endpoint_id,
|
||||
model,
|
||||
owner=owner,
|
||||
require_exact_model=True,
|
||||
)
|
||||
if not resolved:
|
||||
return None
|
||||
candidate, descriptor = resolved
|
||||
expected = ((endpoint_url or "").rstrip("/"), model, headers or {})
|
||||
actual = (candidate[0].rstrip("/"), candidate[1], candidate[2] or {})
|
||||
return descriptor if actual == expected else None
|
||||
|
||||
|
||||
def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list:
|
||||
@@ -460,17 +611,62 @@ def resolve_vision_fallback_candidates(owner: Optional[str] = None) -> list:
|
||||
|
||||
|
||||
def _resolve_fallback_candidates(setting_key: str, owner: Optional[str] = None) -> list:
|
||||
out = []
|
||||
try:
|
||||
from src.settings import get_user_setting, load_settings
|
||||
settings = load_settings()
|
||||
chain = get_user_setting(setting_key, owner or "", settings.get(setting_key) or []) or []
|
||||
except Exception:
|
||||
return out
|
||||
for entry in chain:
|
||||
return []
|
||||
return resolve_fallback_entries(chain, owner=owner)
|
||||
|
||||
|
||||
def resolve_fallback_entries(
|
||||
entries,
|
||||
owner: Optional[str] = None,
|
||||
*,
|
||||
require_exact_model: bool = False,
|
||||
) -> list:
|
||||
"""Resolve ordered endpoint/model entries within the caller's owner scope."""
|
||||
|
||||
out = []
|
||||
for entry in entries or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
resolved = resolve_endpoint_by_id(entry.get("endpoint_id", ""), entry.get("model", ""), owner=owner)
|
||||
if resolved:
|
||||
resolved = resolve_endpoint_by_id(
|
||||
entry.get("endpoint_id", ""),
|
||||
entry.get("model", ""),
|
||||
owner=owner,
|
||||
require_exact_model=require_exact_model,
|
||||
)
|
||||
if resolved and resolved not in out:
|
||||
out.append(resolved)
|
||||
return out
|
||||
|
||||
|
||||
def resolve_fallback_entries_with_descriptors(
|
||||
entries,
|
||||
owner: Optional[str] = None,
|
||||
*,
|
||||
require_exact_model: bool = False,
|
||||
) -> list:
|
||||
"""Resolve ordered entries while retaining safe endpoint provenance."""
|
||||
|
||||
out = []
|
||||
seen = []
|
||||
for entry in entries or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
resolved = _resolve_endpoint_by_id_with_descriptor(
|
||||
entry.get("endpoint_id", ""),
|
||||
entry.get("model", ""),
|
||||
owner=owner,
|
||||
require_exact_model=require_exact_model,
|
||||
)
|
||||
if not resolved:
|
||||
continue
|
||||
candidate, descriptor = resolved
|
||||
if any(candidate == prior for prior in seen):
|
||||
continue
|
||||
seen.append(candidate)
|
||||
out.append((candidate, descriptor))
|
||||
return out
|
||||
|
||||
+189
-14
@@ -1,22 +1,155 @@
|
||||
"""Foreground Chat and Agent model-routing policy.
|
||||
"""Explicit foreground Chat and Agent model-routing policy."""
|
||||
|
||||
The selected session model is strict by default. Historical
|
||||
``default_model_fallbacks`` values remain stored for compatibility, but this
|
||||
policy intentionally does not read or migrate them.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Collection, Dict, FrozenSet, Optional, Tuple
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
from src.endpoint_resolver import (
|
||||
endpoint_cost_tracked,
|
||||
resolve_fallback_entries,
|
||||
resolve_fallback_entries_with_descriptors,
|
||||
resolve_route_descriptor,
|
||||
resolve_route_descriptor_by_id,
|
||||
)
|
||||
|
||||
_DEFAULT_FALLBACK_ENTRY_RESOLVER = resolve_fallback_entries
|
||||
|
||||
|
||||
FOREGROUND_FALLBACK_ENABLED_KEY = "foreground_fallback_enabled"
|
||||
FOREGROUND_FALLBACK_LIST_KEY = "foreground_model_fallbacks"
|
||||
FOREGROUND_AVAILABILITY_STATUSES: FrozenSet[int] = frozenset({
|
||||
408, 425, 429, 500, 502, 503, 504, 507, 508, 529,
|
||||
})
|
||||
MAX_FOREGROUND_FALLBACKS = 10
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ForegroundModelPolicy:
|
||||
"""Resolved per-user foreground fallback policy."""
|
||||
|
||||
enabled: bool = False
|
||||
fallback_candidates: Tuple[tuple, ...] = ()
|
||||
fallback_descriptors: Tuple[dict, ...] = ()
|
||||
eligible_statuses: FrozenSet[int] = FOREGROUND_AVAILABILITY_STATUSES
|
||||
fallback_on_empty: bool = False
|
||||
|
||||
|
||||
def _load_policy_preferences(owner: Optional[str]) -> dict:
|
||||
"""Load only preferences that explicitly belong to ``owner``.
|
||||
|
||||
The generic preferences loader intentionally treats a legacy flat store as
|
||||
the single-user preferences object. That compatibility must not cross an
|
||||
authentication transition: once a named owner is present, foreground
|
||||
fallback consent exists only in an actual ``_users[owner]`` dictionary.
|
||||
"""
|
||||
|
||||
from routes import prefs_routes
|
||||
|
||||
if owner is None:
|
||||
prefs = prefs_routes._load_for_user(None)
|
||||
return dict(prefs) if isinstance(prefs, dict) else {}
|
||||
|
||||
raw = prefs_routes._load()
|
||||
users = raw.get("_users") if isinstance(raw, dict) else None
|
||||
if not isinstance(users, dict):
|
||||
return {}
|
||||
prefs = users.get(owner)
|
||||
return dict(prefs) if isinstance(prefs, dict) else {}
|
||||
|
||||
|
||||
def resolve_foreground_model_policy(
|
||||
owner: Optional[str] = None,
|
||||
allowed_models: Optional[Collection[str]] = None,
|
||||
) -> ForegroundModelPolicy:
|
||||
"""Resolve an explicit owner-scoped policy, failing closed to strict mode.
|
||||
|
||||
The policy is stored in user preferences even when authentication is
|
||||
disabled. Historical ``default_model_fallbacks`` values are deliberately
|
||||
unrelated and are never read or migrated.
|
||||
"""
|
||||
|
||||
try:
|
||||
prefs = _load_policy_preferences(owner)
|
||||
except Exception:
|
||||
return ForegroundModelPolicy()
|
||||
|
||||
if prefs.get(FOREGROUND_FALLBACK_ENABLED_KEY) is not True:
|
||||
return ForegroundModelPolicy()
|
||||
|
||||
entries = prefs.get(FOREGROUND_FALLBACK_LIST_KEY)
|
||||
if not isinstance(entries, list) or not entries:
|
||||
return ForegroundModelPolicy()
|
||||
if allowed_models is not None:
|
||||
allowed = frozenset(allowed_models)
|
||||
entries = [
|
||||
entry for entry in entries
|
||||
if (
|
||||
isinstance(entry, dict)
|
||||
and isinstance(entry.get("model"), str)
|
||||
and entry.get("model") in allowed
|
||||
)
|
||||
]
|
||||
if not entries:
|
||||
return ForegroundModelPolicy()
|
||||
entries = entries[:MAX_FOREGROUND_FALLBACKS]
|
||||
|
||||
if resolve_fallback_entries is not _DEFAULT_FALLBACK_ENTRY_RESOLVER:
|
||||
# Preserve the long-standing resolver seam used by downstream tests and
|
||||
# integrations. Production uses the descriptor-aware resolver below.
|
||||
compatibility_candidates = resolve_fallback_entries(
|
||||
entries,
|
||||
owner=owner,
|
||||
require_exact_model=True,
|
||||
)
|
||||
# Known limitation of this test-only seam: alignment matches on model
|
||||
# alone, so when two entries share a model and the resolver skips the
|
||||
# first, the surviving candidate inherits the skipped entry's
|
||||
# endpoint_id. Production uses the descriptor-aware branch below,
|
||||
# which is unaffected.
|
||||
resolved_routes = []
|
||||
remaining_entries = list(entries)
|
||||
for candidate in compatibility_candidates:
|
||||
matching_index = next(
|
||||
(
|
||||
index for index, entry in enumerate(remaining_entries)
|
||||
if isinstance(entry, dict)
|
||||
and entry.get("model") == candidate[1]
|
||||
),
|
||||
None,
|
||||
)
|
||||
matching_entry = (
|
||||
remaining_entries.pop(matching_index)
|
||||
if matching_index is not None
|
||||
else {}
|
||||
)
|
||||
descriptor = {
|
||||
"endpoint_id": matching_entry.get("endpoint_id"),
|
||||
"endpoint_label": matching_entry.get("endpoint_id") or "Fallback route",
|
||||
"endpoint_cost_tracked": endpoint_cost_tracked(candidate[0]),
|
||||
}
|
||||
resolved_routes.append((candidate, descriptor))
|
||||
else:
|
||||
resolved_routes = resolve_fallback_entries_with_descriptors(
|
||||
entries,
|
||||
owner=owner,
|
||||
require_exact_model=True,
|
||||
)
|
||||
candidates = [candidate for candidate, _descriptor in resolved_routes]
|
||||
if not candidates:
|
||||
return ForegroundModelPolicy()
|
||||
|
||||
return ForegroundModelPolicy(
|
||||
enabled=True,
|
||||
fallback_candidates=tuple(candidates),
|
||||
fallback_descriptors=tuple(
|
||||
dict(descriptor) for _candidate, descriptor in resolved_routes
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resolve_foreground_fallback_candidates(owner: Optional[str] = None) -> list:
|
||||
"""Return fallback candidates for a foreground Chat or Agent request.
|
||||
"""Return only candidates explicitly enabled by the current user."""
|
||||
|
||||
Foreground routing is strict, so no alternate endpoint/model is eligible.
|
||||
``owner`` is accepted to keep this policy boundary owner-aware.
|
||||
"""
|
||||
|
||||
del owner
|
||||
return []
|
||||
return list(resolve_foreground_model_policy(owner).fallback_candidates)
|
||||
|
||||
|
||||
def build_foreground_model_candidates(
|
||||
@@ -24,8 +157,50 @@ def build_foreground_model_candidates(
|
||||
model: str,
|
||||
headers: Optional[Dict[str, Any]] = None,
|
||||
owner: Optional[str] = None,
|
||||
policy: Optional[ForegroundModelPolicy] = None,
|
||||
) -> list:
|
||||
"""Build the ordered candidate list for a foreground request."""
|
||||
|
||||
policy = policy or resolve_foreground_model_policy(owner)
|
||||
primary = (endpoint_url, model, headers or {})
|
||||
return [primary] + resolve_foreground_fallback_candidates(owner=owner)
|
||||
candidates = [primary]
|
||||
for candidate in policy.fallback_candidates:
|
||||
if candidate not in candidates:
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
|
||||
def build_foreground_route_descriptors(
|
||||
endpoint_url: str,
|
||||
model: str,
|
||||
headers: Optional[Dict[str, Any]] = None,
|
||||
owner: Optional[str] = None,
|
||||
policy: Optional[ForegroundModelPolicy] = None,
|
||||
selected_endpoint_id: Optional[str] = None,
|
||||
) -> list:
|
||||
"""Build safe route metadata parallel to foreground candidates."""
|
||||
|
||||
policy = policy or resolve_foreground_model_policy(owner)
|
||||
selected = None
|
||||
if selected_endpoint_id:
|
||||
selected = resolve_route_descriptor_by_id(
|
||||
selected_endpoint_id,
|
||||
endpoint_url,
|
||||
model,
|
||||
headers or {},
|
||||
owner=owner,
|
||||
)
|
||||
if selected is None:
|
||||
selected = resolve_route_descriptor(endpoint_url, model, headers or {}, owner=owner)
|
||||
primary = (endpoint_url, model, headers or {})
|
||||
candidates = [primary]
|
||||
descriptors = [selected]
|
||||
for candidate, descriptor in zip(
|
||||
policy.fallback_candidates,
|
||||
policy.fallback_descriptors,
|
||||
):
|
||||
if candidate in candidates:
|
||||
continue
|
||||
candidates.append(candidate)
|
||||
descriptors.append(dict(descriptor))
|
||||
return descriptors
|
||||
|
||||
+885
-126
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@ class ChatRequest(BaseModel):
|
||||
use_research: Optional[bool] = Field(default=False, description="Enable deep research")
|
||||
time_filter: Optional[str] = Field(default=None, description="Time filter for search")
|
||||
preset_id: Optional[str] = Field(default=None, description="Preset identifier")
|
||||
selected_endpoint_id: Optional[str] = Field(default=None, description="Selected model endpoint ID")
|
||||
|
||||
@field_validator('message')
|
||||
@classmethod
|
||||
|
||||
+19
-1
@@ -14,6 +14,13 @@ from src.constants import SETTINGS_FILE, FEATURES_FILE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys retained in the raw settings store for compatibility and rollback, but
|
||||
# deliberately unavailable through generic settings APIs or agent tools. They
|
||||
# must stay in ``DEFAULT_SETTINGS`` so old files continue to load without data
|
||||
# loss; callers that present or mutate settings should use this set as a
|
||||
# tombstone boundary.
|
||||
RETIRED_SETTING_KEYS = frozenset({"default_model_fallbacks"})
|
||||
|
||||
# Tiny TTL cache for settings/features. get_setting() is called on hot paths
|
||||
# (every chat, every preprocess); without this it re-parses the JSON each call.
|
||||
# Picks up edits within _CACHE_TTL seconds, which is fine for human-edited config.
|
||||
@@ -197,6 +204,17 @@ DEFAULT_SETTINGS = {
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def without_retired_settings(settings: dict) -> dict:
|
||||
"""Return a shallow copy suitable for generic settings interfaces."""
|
||||
if not isinstance(settings, dict):
|
||||
return {}
|
||||
return {
|
||||
key: value
|
||||
for key, value in settings.items()
|
||||
if key not in RETIRED_SETTING_KEYS
|
||||
}
|
||||
|
||||
DEFAULT_FEATURES = {
|
||||
"web_search": True,
|
||||
"web_fetch": True,
|
||||
@@ -269,7 +287,7 @@ _PER_USER_KEYS = {
|
||||
# Default chat endpoint / model — without per-user resolution every new
|
||||
# account inherited whatever the most-recent admin picked, which then
|
||||
# got injected into the chat composer on first open.
|
||||
"default_endpoint_id", "default_model", "default_model_fallbacks",
|
||||
"default_endpoint_id", "default_model",
|
||||
"utility_endpoint_id", "utility_model", "utility_model_fallbacks",
|
||||
"research_endpoint_id", "research_model",
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Shared resolver for background-task AI endpoints."""
|
||||
|
||||
from src.endpoint_resolver import (
|
||||
resolve_chat_fallback_candidates,
|
||||
resolve_endpoint,
|
||||
resolve_utility_fallback_candidates,
|
||||
)
|
||||
@@ -32,7 +31,6 @@ def resolve_task_candidates(
|
||||
2. Utility endpoint/model
|
||||
3. Default endpoint/model
|
||||
4. Utility fallback chain
|
||||
5. Retired default-fallback compatibility hook (currently empty)
|
||||
"""
|
||||
candidates = []
|
||||
|
||||
@@ -49,9 +47,6 @@ def resolve_task_candidates(
|
||||
_append(*resolve_endpoint("default", owner=owner))
|
||||
for url, model, headers in resolve_utility_fallback_candidates(owner=owner):
|
||||
_append(url, model, headers)
|
||||
for url, model, headers in resolve_chat_fallback_candidates(owner=owner):
|
||||
_append(url, model, headers)
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user