jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-11-07 13:42:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor(model-routing): centralize explicit foreground fallback policy (#6020)

Browse Source 
* refactor(model-routing): centralize explicit foreground fallback policy

Make foreground fallback an explicit per-user, availability-only policy shared by streaming Chat, non-stream Chat, and Agent runs.

Preserve strict defaults, owner/model and credential boundaries, pinned Agent routes, and truthful per-round provenance/accounting. Carry provider-reported model identifiers through native streaming adapters, non-stream responses, and caches, and keep legacy default_model_fallbacks as tombstoned raw storage that generic settings APIs and agent tools cannot expose or mutate.

* fix(agent-loop): restore rebase-dropped qwen routing, workspace prompt, and temperature clamp

* fix(model-routing): thread selected endpoint identity, fix cost classification and fallback eligibility

* fix(chat): restore stream helpers and harden run stop lifecycle

* fix(model-routing): let numeric provider codes win over symbolic rate-limit statuses

* fix(agent-loop): apply qwen temperature and notes-tool clamps per fallback candidate

* fix(chat): honor queued stop across resend and reload canonical terminal on EOF

* fix(chat): track stop queue and cleanup ownership by per-send generation

* fix(agent-loop): preserve requested temperature for non-qwen fallback candidates

* fix(chat): reserve send ownership before any await and scope stop to the current send

* fix(chat): clear the previous run identity at send reservation

---------

Co-authored-by: RaresKeY <158580472+RaresKeY@users.noreply.github.com>
Co-authored-by: StressTestor <212606152+StressTestor@users.noreply.github.com>
This commit is contained in:
Alexandre Teixeira
2026-08-14 08:10:30 +01:00
committed by GitHub
co-authored by RaresKeY StressTestor
parent b52296471b
commit c4369305f0
46 changed files with 11418 additions and 1006 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routes/chat_routes.py
+561 -37
View File
@@ -15,13 +15,28 @@ from pydantic import ValidationError
from core.models import ChatMessage
from src.request_models import ChatRequest
from src.llm_core import llm_call_async, stream_llm, stream_llm_with_fallback
from src.llm_core import (
_normalize_http_status,
llm_call_async,
llm_call_async_with_route_fallback,
stream_llm,
stream_llm_with_fallback,
)
from src.agent_loop import stream_agent_loop
from src import agent_runs
from src.model_context import estimate_tokens
from src.context_compactor import (
apply_compaction_state,
maybe_compact,
trim_for_context,
)
from src.chat_helpers import coerce_message_and_session
from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_url
from src.foreground_model_routing import build_foreground_model_candidates
from src.foreground_model_routing import (
build_foreground_model_candidates,
build_foreground_route_descriptors,
resolve_foreground_model_policy,
)
from src.session_search import search_session_messages
from src.prompt_security import untrusted_context_message
from core.exceptions import SessionNotFoundError
@@ -39,7 +54,9 @@ from routes.chat_helpers import (
build_chat_context,
save_assistant_response,
run_post_response_tasks,
accumulate_token_usage,
clean_thinking_for_save,
_allowed_models_for_request,
_enforce_chat_privileges,
)
from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent
@@ -57,6 +74,74 @@ logger = logging.getLogger(__name__)
_active_streams: Dict[str, dict] = {}
def _stream_failure_status(chunk: str) -> Optional[int]:
"""Extract a provider status without retaining provider-supplied detail."""
try:
for line in str(chunk or "").splitlines():
if not line.startswith("data: "):
continue
status = json.loads(line[6:]).get("status")
return _normalize_http_status(status)
except json.JSONDecodeError:
return None
return None
def _chat_candidate_request_factory(
messages,
fallback_context_length: int = 0,
*,
session=None,
owner: Optional[str] = None,
):
"""Shape one route-neutral Chat prompt for each candidate window."""
state = {
"requests": {},
"context_lengths": {},
"trim_stats": {},
"compactions": {},
"was_compacted": {},
}
async def factory(index, candidate_url, candidate_model, candidate_headers):
compaction_state = {}
candidate_messages, context_length, was_compacted = await maybe_compact(
session,
candidate_url,
candidate_model,
list(messages),
candidate_headers,
owner=owner,
persist=False,
compaction_state=compaction_state,
)
if not context_length:
context_length = fallback_context_length
request_messages = trim_for_context(candidate_messages, context_length)
state["requests"][index] = request_messages
state["context_lengths"][index] = context_length
state["compactions"][index] = compaction_state
state["was_compacted"][index] = was_compacted
state["trim_stats"][index] = {
"messages_before": len(messages),
"messages_after": len(request_messages),
"tokens_before": estimate_tokens(messages),
"tokens_after": estimate_tokens(request_messages),
}
return {"messages": request_messages}
return factory, state
def _candidate_index(candidates, actual_candidate) -> int:
for index, candidate in enumerate(candidates):
if candidate == actual_candidate:
return index
return 0
def _stream_set(session_id: str, **fields) -> None:
"""Update fields on the active-stream entry for `session_id`, or
no-op if the entry has already been popped. Using .get() avoids a
@@ -590,8 +675,8 @@ def setup_chat_routes(
# ------------------------------------------------------------------ #
# POST /api/chat (non-streaming)
# ------------------------------------------------------------------ #
@router.post("/api/chat", response_model=Dict[str, str])
async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, str]:
@router.post("/api/chat", response_model=Dict[str, Any])
async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, Any]:
_set_user_time_from_request(request)
message = chat_request.message
@@ -623,6 +708,8 @@ def setup_chat_routes(
400,
"No model selected for this chat. Open the model picker and choose one before sending.",
)
if not (getattr(sess, "endpoint_url", "") or "").strip():
raise HTTPException(400, "Selected model endpoint is not configured")
# Same allowed_models + daily-cap gate as chat_stream (mirror so the
# non-streaming path can't be used to bypass).
@@ -638,6 +725,11 @@ def setup_chat_routes(
if memory_response:
return {"response": memory_response}
foreground_policy = resolve_foreground_model_policy(
owner=owner,
allowed_models=_allowed_models_for_request(request),
)
# Build shared context (preset, preprocess, preface, compact)
ctx = await build_chat_context(
sess, request, chat_handler, chat_processor,
@@ -649,6 +741,7 @@ def setup_chat_routes(
time_filter=time_filter,
webhook_manager=webhook_manager,
allow_tool_preprocessing=allow_tool_preprocessing,
defer_context_shaping=foreground_policy.enabled,
)
# Research injection
@@ -662,24 +755,88 @@ def setup_chat_routes(
research_ctx = await research_handler.call_research_service(
message, _r_ep, _r_model, llm_headers=_r_headers
)
ctx.messages.insert(
len(ctx.preface),
untrusted_context_message("research context", research_ctx),
)
research_message = untrusted_context_message("research context", research_ctx)
ctx.messages.insert(len(ctx.preface), research_message)
if foreground_policy.enabled:
getattr(ctx, "route_messages", ctx.messages).insert(
len(ctx.preface),
research_message,
)
except Exception as e:
logger.error(f"Research failed: {e}")
reply = await llm_call_async(
foreground_candidates = build_foreground_model_candidates(
sess.endpoint_url,
sess.model,
ctx.messages,
headers=sess.headers,
sess.headers,
owner=owner,
policy=foreground_policy,
)
route_descriptors = build_foreground_route_descriptors(
sess.endpoint_url,
sess.model,
sess.headers,
owner=owner,
policy=foreground_policy,
selected_endpoint_id=chat_request.selected_endpoint_id,
)
candidate_request_factory = None
selected_context_length = getattr(ctx, "context_length", 0)
candidate_request_state = {
"context_lengths": {0: selected_context_length},
"requests": {0: ctx.messages},
"trim_stats": {},
}
request_messages = ctx.messages
if foreground_policy.enabled:
request_messages = getattr(ctx, "route_messages", ctx.messages)
candidate_request_factory, candidate_request_state = _chat_candidate_request_factory(
request_messages,
selected_context_length,
session=sess,
owner=owner,
)
requested_model = sess.model
reply, actual_candidate, actual_model = await llm_call_async_with_route_fallback(
foreground_candidates,
request_messages,
fallback_statuses=foreground_policy.eligible_statuses,
candidate_request_factory=candidate_request_factory,
temperature=ctx.preset.temperature,
max_tokens=ctx.preset.max_tokens,
prompt_type=preset_id,
session_id=session,
)
_clean_reply, _clean_md = clean_thinking_for_save(reply, {"model": sess.model})
actual_index = _candidate_index(foreground_candidates, actual_candidate)
apply_compaction_state(
sess,
candidate_request_state.get("compactions", {}).get(actual_index),
)
requested_route = route_descriptors[0]
actual_route = route_descriptors[actual_index]
actual_trim = candidate_request_state.get("trim_stats", {}).get(actual_index, {})
_clean_reply, _clean_md = clean_thinking_for_save(
reply,
{
"model": actual_model,
"requested_model": requested_model,
"endpoint_id": actual_route.get("endpoint_id"),
"endpoint_label": actual_route.get("endpoint_label"),
"requested_endpoint_id": requested_route.get("endpoint_id"),
"requested_endpoint_label": requested_route.get("endpoint_label"),
"context_length": candidate_request_state["context_lengths"].get(
actual_index,
selected_context_length,
),
"context_trimmed": bool(
actual_trim
and (
actual_trim.get("messages_after") < actual_trim.get("messages_before")
or actual_trim.get("tokens_after") < actual_trim.get("tokens_before")
)
),
},
)
sess.add_message(ChatMessage("assistant", _clean_reply, metadata=_clean_md))
from core.database import update_session_last_accessed
@@ -695,7 +852,15 @@ def setup_chat_routes(
allow_background_extraction=not tool_policy.block_all_tool_calls,
)
return {"response": reply}
return {
"response": reply,
"requested_model": requested_model,
"model": actual_model,
"requested_endpoint_id": requested_route.get("endpoint_id"),
"requested_endpoint_label": requested_route.get("endpoint_label"),
"endpoint_id": actual_route.get("endpoint_id"),
"endpoint_label": actual_route.get("endpoint_label"),
}
# ------------------------------------------------------------------ #
# POST /api/chat_stream
@@ -724,6 +889,11 @@ def setup_chat_routes(
use_research = form_data.get("use_research")
time_filter = form_data.get("time_filter")
preset_id = form_data.get("preset_id")
selected_endpoint_id = str(
form_data.get("selected_endpoint_id")
or (body or {}).get("selected_endpoint_id")
or ""
).strip()
# Issue #3229: API callers send JSON, not FormData. Read from the
# JSON body as fallback so callers who send {"allow_bash": true}
# actually get bash enabled.
@@ -896,6 +1066,8 @@ def setup_chat_routes(
400,
"No model selected for this chat. Open the model picker and choose one before sending.",
)
if not (getattr(sess, "endpoint_url", "") or "").strip():
raise HTTPException(400, "Selected model endpoint is not configured")
if (
chat_mode == "chat"
and isinstance(message, str)
@@ -971,6 +1143,10 @@ def setup_chat_routes(
last_user_message=message,
)
allow_tool_preprocessing = not pre_context_tool_policy.block_all_tool_calls
foreground_policy = resolve_foreground_model_policy(
owner=owner,
allowed_models=_allowed_models_for_request(request),
)
# Build shared context (stream path uses enhanced_message for context preface)
ctx = await build_chat_context(
@@ -993,6 +1169,7 @@ def setup_chat_routes(
# index would be useless / unwanted noise.
agent_mode=(chat_mode == "agent"),
allow_tool_preprocessing=allow_tool_preprocessing,
defer_context_shaping=foreground_policy.enabled,
)
_research_flags = {"do": do_research} # Mutable container for generator scope
@@ -1292,6 +1469,8 @@ def setup_chat_routes(
"what aspects matter most, are they comparing to something, what's their context "
"(moving, traveling, curiosity). Be conversational. Keep it short."
})
if foreground_policy.enabled:
getattr(ctx, "route_messages", ctx.messages).insert(0, dict(ctx.messages[0]))
_skip_research = True
else:
_skip_research = False
@@ -1388,7 +1567,12 @@ def setup_chat_routes(
_active_streams.pop(session, None)
return
messages = _ensure_current_request_is_latest_user(ctx.messages, message)
context_source = (
getattr(ctx, "route_messages", ctx.messages)
if foreground_policy.enabled
else ctx.messages
)
messages = _ensure_current_request_is_latest_user(context_source, message)
# Auto-compact notification
if ctx.was_compacted:
@@ -1400,25 +1584,56 @@ def setup_chat_routes(
thinking_response = ""
last_metrics = None
# Foreground Chat and Agent requests use one owner-aware policy
# boundary. Legacy `default_model_fallbacks` data is not eligible.
# Foreground Chat and Agent requests share one explicit owner-aware
# policy. Strict mode is the default; legacy values are unrelated.
_foreground_policy = foreground_policy
_foreground_candidates = build_foreground_model_candidates(
sess.endpoint_url,
sess.model,
sess.headers,
owner=_user,
policy=_foreground_policy,
)
_foreground_route_descriptors = build_foreground_route_descriptors(
sess.endpoint_url,
sess.model,
sess.headers,
owner=_user,
policy=_foreground_policy,
selected_endpoint_id=selected_endpoint_id,
)
_chat_request_factory = None
_selected_context_length = getattr(ctx, "context_length", 0)
_chat_request_state = {
"context_lengths": {0: _selected_context_length},
"requests": {0: messages},
"trim_stats": {},
}
if _foreground_policy.enabled:
_chat_request_factory, _chat_request_state = _chat_candidate_request_factory(
messages,
_selected_context_length,
session=sess,
owner=_user,
)
# Send model name early so the frontend can show it during streaming
_model_suffix = "Research" if effective_do_research else None
_model_info = {"type": "model_info", "model": sess.model}
_selected_route = _foreground_route_descriptors[0]
_model_info = {
"type": "model_info",
"model": sess.model,
"endpoint_id": _selected_route.get("endpoint_id"),
"endpoint_label": _selected_route.get("endpoint_label"),
}
if _model_suffix:
_model_info["suffix"] = _model_suffix
if ctx.preset.character_name:
_model_info["character_name"] = ctx.preset.character_name
yield f'data: {json.dumps(_model_info)}\n\n'
if image_generation_session:
_terminal_saved = False
if _is_image_generation_session(sess, owner=_user):
from src.settings import get_setting
if tool_policy.blocks("generate_image"):
_blocked_msg = tool_policy.reason_for("generate_image")
@@ -1521,6 +1736,16 @@ def setup_chat_routes(
_answered_by = None # set if the selected model failed and a fallback answered
_requested_model = sess.model
_actual_model = None
_requested_route = _foreground_route_descriptors[0]
_actual_route = _requested_route
_actual_candidate_index = 0
_chat_terminal_saved = False
def _commit_chat_compaction(candidate_index: int) -> bool:
return apply_compaction_state(
sess,
_chat_request_state.get("compactions", {}).get(candidate_index),
)
# ── Chat mode: call stream_llm directly, NO tools, NO document access ──
try:
async for chunk in stream_llm_with_fallback(
@@ -1536,11 +1761,21 @@ def setup_chat_routes(
prompt_type=preset_id,
tools=None,
session_id=session,
fallback_statuses=_foreground_policy.eligible_statuses,
fallback_on_empty=_foreground_policy.fallback_on_empty,
candidate_request_factory=_chat_request_factory,
candidate_route_descriptors=_foreground_route_descriptors,
):
if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
try:
data = json.loads(chunk[6:])
if "delta" in data:
if _commit_chat_compaction(_actual_candidate_index):
_compacted_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n'
# Reasoning tokens arrive flagged thinking:true.
# Forward them so the client can show a thinking
# indicator, but don't fold them into the saved
@@ -1556,29 +1791,82 @@ def setup_chat_routes(
# Forward the notice and remember the real model.
_answered_by = data.get("answered_by") or _answered_by
_actual_model = _actual_model or _answered_by
_actual_candidate_index = data.get("candidate_index", 0)
if not isinstance(_actual_candidate_index, int):
_actual_candidate_index = 0
if 0 <= _actual_candidate_index < len(_foreground_route_descriptors):
_actual_route = _foreground_route_descriptors[_actual_candidate_index]
if _commit_chat_compaction(_actual_candidate_index):
_compacted_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n'
data["selected_model"] = data.get("selected_model") or _requested_model
yield chunk
yield f'data: {json.dumps(data)}\n\n'
elif data.get("type") == "model_actual":
if _commit_chat_compaction(_actual_candidate_index):
_compacted_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n'
_actual_model = data.get("model") or _actual_model
data["requested_model"] = _requested_model
data["requested_endpoint_id"] = _requested_route.get("endpoint_id")
data["requested_endpoint_label"] = _requested_route.get("endpoint_label")
data["endpoint_id"] = _actual_route.get("endpoint_id")
data["endpoint_label"] = _actual_route.get("endpoint_label")
yield f'data: {json.dumps(data)}\n\n'
elif data.get("type") == "usage":
if _commit_chat_compaction(_actual_candidate_index):
_compacted_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
yield f'data: {json.dumps({"type": "compacted", "context_length": _compacted_length})}\n\n'
last_metrics = data.get("data", {})
_reported_model = last_metrics.get("model")
last_metrics["requested_model"] = _requested_model
last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
if ctx.context_trimmed:
last_metrics["requested_endpoint_id"] = _requested_route.get("endpoint_id")
last_metrics["requested_endpoint_label"] = _requested_route.get("endpoint_label")
last_metrics["endpoint_id"] = _actual_route.get("endpoint_id")
last_metrics["endpoint_label"] = _actual_route.get("endpoint_label")
if isinstance(
_actual_route.get("endpoint_cost_tracked"),
bool,
):
last_metrics["endpoint_cost_tracked"] = _actual_route.get(
"endpoint_cost_tracked"
)
_actual_context_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
_route_trim = _chat_request_state.get("trim_stats", {}).get(
_actual_candidate_index,
{},
)
if _route_trim and (
_route_trim.get("messages_after") < _route_trim.get("messages_before")
or _route_trim.get("tokens_after") < _route_trim.get("tokens_before")
):
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = _route_trim.get("messages_before")
last_metrics["context_messages_after_trim"] = _route_trim.get("messages_after")
last_metrics["context_tokens_before_trim"] = _route_trim.get("tokens_before")
last_metrics["context_tokens_after_trim"] = _route_trim.get("tokens_after")
elif ctx.context_trimmed:
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
request_context_tokens = ctx.context_tokens_after_trim or estimate_tokens(messages)
last_metrics["request_context_tokens"] = request_context_tokens
if ctx.context_length and request_context_tokens:
pct = min(round((request_context_tokens / ctx.context_length) * 100, 1), 100.0)
if _actual_context_length and last_metrics.get("input_tokens"):
pct = min(round((last_metrics["input_tokens"] / _actual_context_length) * 100, 1), 100.0)
last_metrics["context_percent"] = pct
last_metrics["context_length"] = ctx.context_length
last_metrics["context_length"] = _actual_context_length
# The frontend reads `tokens_per_second`; the raw usage event
# carries the backend's true gen speed as `gen_tps` (llama.cpp
# timings). Map it through so this direct-chat path shows real
@@ -1593,17 +1881,121 @@ def setup_chat_routes(
yield chunk
elif chunk.startswith("event: error"):
logger.warning(f"Stream error for {sess.model} on {sess.endpoint_url}: {chunk!r}")
if (
not _chat_terminal_saved
and (full_response.strip() or thinking_response.strip())
):
_failure_status = _stream_failure_status(chunk)
_failure_message = (
f"Model request failed (HTTP {_failure_status})"
if _failure_status is not None
else "Model request failed"
)
_terminal_content = full_response.strip()
_failure_note = f"[Response stopped: {_failure_message}]"
_terminal_content = (
f"{_terminal_content}\n\n{_failure_note}"
if _terminal_content
else _failure_note
)
_had_terminal_usage = bool(last_metrics)
_terminal_metrics = dict(last_metrics or {})
if not _had_terminal_usage:
_actual_request_messages = _chat_request_state["requests"].get(
_actual_candidate_index,
messages,
)
_actual_context_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
_estimated_input = estimate_tokens(_actual_request_messages)
_estimated_output = max(
len(full_response + thinking_response) // 4,
0,
)
_terminal_metrics.update({
"input_tokens": _estimated_input,
"output_tokens": _estimated_output,
"total_tokens": _estimated_input + _estimated_output,
"usage_source": "estimated",
"response_time": round(time.time() - _chat_start, 2),
"context_length": _actual_context_length,
"context_percent": (
min(
round(
(_estimated_input / _actual_context_length) * 100,
1,
),
100.0,
)
if _actual_context_length
else 0
),
})
_terminal_metrics.update({
"failed": True,
"failure": {
"status": _failure_status,
"message": _failure_message,
},
"model": _actual_model or _answered_by or _requested_model,
"requested_model": _requested_model,
"endpoint_id": _actual_route.get("endpoint_id"),
"endpoint_label": _actual_route.get("endpoint_label"),
"requested_endpoint_id": _requested_route.get("endpoint_id"),
"requested_endpoint_label": _requested_route.get("endpoint_label"),
})
if isinstance(
_actual_route.get("endpoint_cost_tracked"),
bool,
):
_terminal_metrics["endpoint_cost_tracked"] = _actual_route.get(
"endpoint_cost_tracked"
)
if thinking_response.strip():
_terminal_metrics["thinking"] = thinking_response.strip()
_commit_chat_compaction(_actual_candidate_index)
_saved_id = save_assistant_response(
sess,
session_manager,
session,
_terminal_content,
_terminal_metrics,
character_name=ctx.preset.character_name,
incognito=incognito,
)
accumulate_token_usage(session, _terminal_metrics)
_chat_terminal_saved = True
_stream_set(session, status="error")
if _saved_id:
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
yield f'data: {json.dumps({"type": "chat_terminal", "data": _terminal_metrics})}\n\n'
yield chunk
elif chunk.startswith("event: "):
yield chunk
elif chunk == "data: [DONE]\n\n":
if _chat_terminal_saved:
# Some providers append DONE after a terminal
# error. The failed partial is already saved;
# never re-save/post-process it as a success or
# advertise successful completion to the client.
continue
# Generate fallback metrics if LLM didn't send usage
if not last_metrics and full_response:
_elapsed = time.time() - _chat_start
_est_in = estimate_tokens(messages)
_est_out = len(full_response) // 4
_tps = round(_est_out / _elapsed, 2) if _elapsed > 0 else 0
_ctx_pct = min(round((_est_in / ctx.context_length) * 100, 1), 100.0) if ctx.context_length else 0
_actual_context_length = _chat_request_state["context_lengths"].get(
_actual_candidate_index,
_selected_context_length,
)
_actual_request_messages = _chat_request_state["requests"].get(
_actual_candidate_index,
messages,
)
_est_in = estimate_tokens(_actual_request_messages)
_ctx_pct = min(round((_est_in / _actual_context_length) * 100, 1), 100.0) if _actual_context_length else 0
last_metrics = {
"response_time": round(_elapsed, 2),
"input_tokens": _est_in,
@@ -1611,13 +2003,25 @@ def setup_chat_routes(
"tokens_per_second": _tps,
"request_context_tokens": _est_in,
"context_percent": _ctx_pct,
"context_length": ctx.context_length,
"context_length": _actual_context_length,
"model": _actual_model or _answered_by or _requested_model,
"requested_model": _requested_model,
"requested_endpoint_id": _requested_route.get("endpoint_id"),
"requested_endpoint_label": _requested_route.get("endpoint_label"),
"endpoint_id": _actual_route.get("endpoint_id"),
"endpoint_label": _actual_route.get("endpoint_label"),
"usage_source": "estimated",
}
if isinstance(
_actual_route.get("endpoint_cost_tracked"),
bool,
):
last_metrics["endpoint_cost_tracked"] = _actual_route.get(
"endpoint_cost_tracked"
)
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
if full_response:
_commit_chat_compaction(_actual_candidate_index)
_metrics_to_save = dict(last_metrics or {})
if thinking_response.strip() and not _metrics_to_save.get("thinking"):
_metrics_to_save["thinking"] = thinking_response.strip()
@@ -1652,6 +2056,10 @@ def setup_chat_routes(
"stopped": True,
"model": _actual_model or _answered_by or _requested_model,
"requested_model": _requested_model,
"endpoint_id": _actual_route.get("endpoint_id"),
"endpoint_label": _actual_route.get("endpoint_label"),
"requested_endpoint_id": _requested_route.get("endpoint_id"),
"requested_endpoint_label": _requested_route.get("endpoint_label"),
},
)
sess.add_message(ChatMessage("assistant", _stopped_content, metadata=_stopped_md))
@@ -1666,6 +2074,12 @@ def setup_chat_routes(
_answered_by = None # set if the selected model failed and a fallback answered
_requested_model = sess.model
_actual_model = None
_agent_requested_route = _foreground_route_descriptors[0]
_agent_actual_endpoint_id = _agent_requested_route.get("endpoint_id")
_agent_actual_endpoint_label = _agent_requested_route.get("endpoint_label")
_agent_round_models = {1: _requested_model}
_agent_round_endpoint_ids = {1: _agent_actual_endpoint_id}
_agent_round_endpoint_labels = {1: _agent_actual_endpoint_label}
try:
from src.settings import get_setting
from src.agent_tools import MAX_AGENT_ROUNDS as _DEFAULT_ROUNDS
@@ -1703,19 +2117,24 @@ def setup_chat_routes(
prompt_type=preset_id,
max_tool_calls=_tool_budget,
max_rounds=_max_rounds,
context_length=ctx.context_length,
context_length=_selected_context_length,
active_document=active_doc,
active_email=active_email_ctx,
session_id=session,
history_session=sess,
disabled_tools=disabled_tools if disabled_tools else None,
tool_policy=tool_policy,
owner=_user,
fallbacks=_foreground_candidates[1:],
route_descriptors=_foreground_route_descriptors,
fallback_statuses=_foreground_policy.eligible_statuses,
fallback_on_empty=_foreground_policy.fallback_on_empty,
plan_mode=plan_mode,
approved_plan=approved_plan or None,
workspace=workspace or None,
forced_tools=_forced_tools,
uploaded_files=ctx.uploaded_files,
defer_context_shaping=_foreground_policy.enabled,
):
if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
try:
@@ -1744,7 +2163,20 @@ def setup_chat_routes(
"plan_update",
):
if data.get("type") == "agent_step":
_agent_rounds = max(_agent_rounds, data.get("round", 1))
_event_round = data.get("round", 1)
_agent_rounds = max(_agent_rounds, _event_round)
_agent_round_models.setdefault(
_event_round,
_actual_model or _answered_by or _requested_model,
)
_agent_round_endpoint_ids.setdefault(
_event_round,
_agent_actual_endpoint_id,
)
_agent_round_endpoint_labels.setdefault(
_event_round,
_agent_actual_endpoint_label,
)
elif data.get("type") == "tool_start":
_agent_tool_calls += 1
yield chunk
@@ -1754,13 +2186,70 @@ def setup_chat_routes(
# model so metrics reflect it, not the masked
# selected model.
_answered_by = data.get("answered_by") or _answered_by
_actual_model = _actual_model or _answered_by
_actual_model = _answered_by or _actual_model
if "answered_by_endpoint_id" in data:
_agent_actual_endpoint_id = data.get("answered_by_endpoint_id")
if data.get("answered_by_endpoint_label"):
_agent_actual_endpoint_label = data.get("answered_by_endpoint_label")
_event_round = data.get("round") or max(_agent_rounds, 1)
_agent_round_models[_event_round] = _answered_by or _requested_model
_agent_round_endpoint_ids[_event_round] = _agent_actual_endpoint_id
_agent_round_endpoint_labels[_event_round] = _agent_actual_endpoint_label
data["selected_model"] = data.get("selected_model") or _requested_model
yield chunk
elif data.get("type") == "model_actual":
_actual_model = data.get("model") or _actual_model
if "endpoint_id" in data:
_agent_actual_endpoint_id = data.get("endpoint_id")
if data.get("endpoint_label"):
_agent_actual_endpoint_label = data.get("endpoint_label")
_event_round = data.get("round") or max(_agent_rounds, 1)
_agent_round_models[_event_round] = _actual_model or _requested_model
_agent_round_endpoint_ids[_event_round] = _agent_actual_endpoint_id
_agent_round_endpoint_labels[_event_round] = _agent_actual_endpoint_label
data["requested_model"] = _requested_model
yield f'data: {json.dumps(data)}\n\n'
elif data.get("type") == "agent_terminal":
terminal_metadata = dict(data.get("data") or {})
last_metrics = terminal_metadata
failure = terminal_metadata.get("failure") or {}
failure_status = _normalize_http_status(
failure.get("status")
)
failure_message = (
f"Model request failed (HTTP {failure_status})"
if failure_status is not None
else "Model request failed"
)
terminal_metadata["failure"] = {
"status": failure_status,
"message": failure_message,
}
terminal_content = full_response.strip()
failure_note = f"[Agent stopped: {failure_message}]"
if terminal_content:
terminal_content = f"{terminal_content}\n\n{failure_note}"
else:
terminal_content = failure_note
if not _terminal_saved:
_saved_id = save_assistant_response(
sess,
session_manager,
session,
terminal_content,
terminal_metadata,
character_name=ctx.preset.character_name,
web_sources=web_sources,
rag_sources=ctx.rag_sources,
used_memories=ctx.used_memories,
incognito=incognito,
)
_terminal_saved = True
accumulate_token_usage(session, terminal_metadata)
_stream_set(session, status="error")
if _saved_id:
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
yield chunk
elif data.get("type") == "metrics":
last_metrics = data.get("data", {})
_reported_model = last_metrics.get("model")
@@ -1772,7 +2261,16 @@ def setup_chat_routes(
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
_metrics_event = {"type": "metrics", "data": last_metrics}
# Inline teacher escalation marks its
# recursively emitted events at the SSE
# envelope. Preserve that non-secret marker
# when normalizing metrics so the browser's
# replay-stable ledger keeps primary and
# teacher segments distinct.
if data.get("teacher") is True:
_metrics_event["teacher"] = True
yield f'data: {json.dumps(_metrics_event)}\n\n'
except json.JSONDecodeError:
yield chunk
elif chunk.startswith("event: "):
@@ -1824,6 +2322,22 @@ def setup_chat_routes(
"stopped": True,
"model": _actual_model or _answered_by or _requested_model,
"requested_model": _requested_model,
"endpoint_id": _agent_actual_endpoint_id,
"endpoint_label": _agent_actual_endpoint_label,
"requested_endpoint_id": _agent_requested_route.get("endpoint_id"),
"requested_endpoint_label": _agent_requested_route.get("endpoint_label"),
"round_models": [
_agent_round_models.get(i, _actual_model or _requested_model)
for i in range(1, max(_agent_round_models, default=1) + 1)
],
"round_endpoint_ids": [
_agent_round_endpoint_ids.get(i)
for i in range(1, max(_agent_round_models, default=1) + 1)
],
"round_endpoint_labels": [
_agent_round_endpoint_labels.get(i)
for i in range(1, max(_agent_round_models, default=1) + 1)
],
},
)
sess.add_message(ChatMessage("assistant", _stopped_content2, metadata=_stopped_md2))
@@ -1866,8 +2380,12 @@ def setup_chat_routes(
if compare_mode:
return StreamingResponse(_safe_stream(), media_type="text/event-stream")
agent_runs.start(session, _safe_stream())
return StreamingResponse(agent_runs.subscribe(session), media_type="text/event-stream")
_detached_run = agent_runs.start(session, _safe_stream())
return StreamingResponse(
agent_runs.subscribe(session, _detached_run),
media_type="text/event-stream",
headers={"X-Odysseus-Run-Id": _detached_run.run_id},
)
# ------------------------------------------------------------------ #
# GET /api/chat/resume — reconnect to a detached run that's still going
@@ -1876,9 +2394,14 @@ def setup_chat_routes(
@router.get("/api/chat/resume/{session_id}")
async def chat_resume(request: Request, session_id: str) -> StreamingResponse:
_verify_session_owner(request, session_id)
if not agent_runs.is_active(session_id):
_active_run = agent_runs.get_active_run(session_id)
if _active_run is None:
raise HTTPException(404, "No active run for this session")
return StreamingResponse(agent_runs.subscribe(session_id), media_type="text/event-stream")
return StreamingResponse(
agent_runs.subscribe(session_id, _active_run),
media_type="text/event-stream",
headers={"X-Odysseus-Run-Id": _active_run.run_id},
)
# ------------------------------------------------------------------ #
# POST /api/chat/stop — cancel a detached run (Stop button). Closing the SSE
@@ -1887,7 +2410,8 @@ def setup_chat_routes(
@router.post("/api/chat/stop/{session_id}")
async def chat_stop(request: Request, session_id: str) -> Dict[str, Any]:
_verify_session_owner(request, session_id)
stopped = agent_runs.stop(session_id)
_expected_run_id = request.headers.get("X-Odysseus-Run-Id")
stopped = agent_runs.stop(session_id, _expected_run_id)
return {"stopped": stopped}
# ------------------------------------------------------------------ #