fix(model-routing): keep selected models strict (#5801)

Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com>

tests/test_foreground_model_routing.py

@@ -0,0 +1,277 @@
+"""Regression coverage for strict foreground model selection."""
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+import pytest
+import src.agent_loop as agent_loop
+import src.endpoint_resolver as endpoint_resolver
+import src.foreground_model_routing as foreground_model_routing
+import routes.chat_routes as chat_routes
+from src.foreground_model_routing import (
+    build_foreground_model_candidates,
+    resolve_foreground_fallback_candidates,
+)
+
+
+def _collect(gen):
+    async def _run():
+        return [chunk async for chunk in gen]
+
+    return asyncio.run(_run())
+
+
+class _EmptyQuery:
+    def filter(self, *args, **kwargs):
+        return self
+
+    def order_by(self, *args, **kwargs):
+        return self
+
+    def first(self):
+        return None
+
+
+class _EmptyDb:
+    def query(self, *args, **kwargs):
+        return _EmptyQuery()
+
+    def close(self):
+        return None
+
+
+class _RouteRequest:
+    def __init__(self, mode):
+        self.headers = {}
+        self.app = SimpleNamespace(state=SimpleNamespace(auth_manager=None))
+        self._form = {
+            "message": "hello",
+            "session": "session-1",
+            "mode": mode,
+            "compare_mode": "true",
+        }
+
+    async def form(self):
+        return self._form
+
+
+def _chat_stream_endpoint(monkeypatch, mode, captured):
+    session = SimpleNamespace(
+        endpoint_url="https://selected.example/v1",
+        model="selected-model",
+        headers={"Authorization": "Bearer selected"},
+        name="test",
+        history=[],
+        add_message=lambda message: None,
+    )
+    session_manager = SimpleNamespace(
+        get_session=lambda session_id: session,
+        save_sessions=lambda: None,
+    )
+    context = SimpleNamespace(
+        user="alice",
+        messages=[{"role": "user", "content": "hello"}],
+        preprocessed=SimpleNamespace(attachment_meta=[]),
+        auto_opened_docs=[],
+        rag_sources=[],
+        web_sources=[],
+        used_memories=[],
+        uploaded_files=[],
+        uprefs={},
+        was_compacted=False,
+        context_trimmed=False,
+        context_length=4096,
+        context_messages_before_trim=1,
+        context_messages_after_trim=1,
+        context_tokens_before_trim=10,
+        context_tokens_after_trim=10,
+        preset=SimpleNamespace(temperature=0.2, max_tokens=128, character_name=None),
+    )
+
+    async def fake_build_context(*args, **kwargs):
+        return context
+
+    async def fake_chat_stream(candidates, messages, **kwargs):
+        captured["chat"] = candidates
+        yield f'data: {json.dumps({"delta": "done"})}\n\n'
+        yield "data: [DONE]\n\n"
+
+    async def fake_agent_stream(endpoint_url, model, messages, **kwargs):
+        captured["agent"] = {
+            "primary": (endpoint_url, model, kwargs.get("headers")),
+            "fallbacks": kwargs.get("fallbacks"),
+        }
+        yield f'data: {json.dumps({"delta": "done"})}\n\n'
+        yield "data: [DONE]\n\n"
+
+    monkeypatch.setattr(chat_routes, "coerce_message_and_session", lambda *args, **kwargs: ("hello", "session-1"))
+    monkeypatch.setattr(chat_routes, "_verify_session_owner", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "effective_user", lambda request: "alice")
+    monkeypatch.setattr(chat_routes, "_clear_orphaned_session_endpoint", lambda *args, **kwargs: False)
+    monkeypatch.setattr(chat_routes, "_recover_empty_session_model", lambda *args, **kwargs: False)
+    monkeypatch.setattr(chat_routes, "_enforce_chat_privileges", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "resolve_session_auth", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "get_session_mode", lambda session_id: "chat")
+    monkeypatch.setattr(chat_routes, "set_session_mode", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "build_chat_context", fake_build_context)
+    monkeypatch.setattr(chat_routes, "SessionLocal", _EmptyDb)
+    monkeypatch.setattr(chat_routes, "_is_image_generation_session", lambda *args, **kwargs: False)
+    monkeypatch.setattr(chat_routes, "stream_llm_with_fallback", fake_chat_stream)
+    monkeypatch.setattr(chat_routes, "stream_agent_loop", fake_agent_stream)
+    monkeypatch.setattr(chat_routes, "save_assistant_response", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "run_post_response_tasks", lambda *args, **kwargs: None)
+    monkeypatch.setattr(chat_routes, "estimate_tokens", lambda messages: 10)
+    monkeypatch.setattr(
+        endpoint_resolver,
+        "resolve_chat_fallback_candidates",
+        lambda owner=None: [("https://legacy.example/v1", "legacy-model", {})],
+    )
+
+    import src.settings as settings
+
+    monkeypatch.setattr(
+        settings,
+        "get_setting",
+        lambda key, default=None: default,
+    )
+    monkeypatch.setattr(
+        settings,
+        "get_user_setting",
+        lambda key, owner="", default=None: (
+            [{"endpoint_id": "legacy", "model": "legacy-model"}]
+            if key == "default_model_fallbacks"
+            else default
+        ),
+    )
+
+    router = chat_routes.setup_chat_routes(
+        session_manager,
+        SimpleNamespace(),
+        SimpleNamespace(),
+        SimpleNamespace(),
+        SimpleNamespace(),
+        SimpleNamespace(),
+    )
+    return next(route.endpoint for route in router.routes if route.path == "/api/chat_stream")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("mode", ["chat", "agent"])
+async def test_chat_stream_route_keeps_selected_model_strict_with_legacy_data(monkeypatch, mode):
+    captured = {}
+    endpoint = _chat_stream_endpoint(monkeypatch, mode, captured)
+
+    response = await endpoint(_RouteRequest(mode))
+    async for _ in response.body_iterator:
+        pass
+
+    selected = (
+        "https://selected.example/v1",
+        "selected-model",
+        {"Authorization": "Bearer selected"},
+    )
+    if mode == "chat":
+        assert captured == {"chat": [selected]}
+    else:
+        assert captured == {"agent": {"primary": selected, "fallbacks": []}}
+
+
+def test_candidate_builder_appends_only_policy_authorized_fallbacks(monkeypatch):
+    """Chat and Agent share the same candidate-building policy boundary."""
+
+    authorized = [("https://opt-in.example/v1", "opt-in-model", {})]
+    monkeypatch.setattr(
+        foreground_model_routing,
+        "resolve_foreground_fallback_candidates",
+        lambda owner=None: authorized,
+    )
+
+    assert build_foreground_model_candidates(
+        "https://selected.example/v1",
+        "selected-model",
+        {"Authorization": "Bearer selected"},
+        owner="alice",
+    ) == [
+        ("https://selected.example/v1", "selected-model", {"Authorization": "Bearer selected"}),
+        *authorized,
+    ]
+
+
+def test_strict_policy_builds_only_the_selected_chat_candidate():
+    candidates = build_foreground_model_candidates(
+        "https://selected.example/v1",
+        "selected-model",
+        {"Authorization": "Bearer selected"},
+        owner="alice",
+    )
+
+    assert candidates == [
+        ("https://selected.example/v1", "selected-model", {"Authorization": "Bearer selected"})
+    ]
+
+
+def test_legacy_chat_resolver_is_disconnected():
+    assert endpoint_resolver.resolve_chat_fallback_candidates(owner="alice") == []
+
+
+def test_utility_resolver_does_not_inherit_legacy_chat_fallbacks(monkeypatch):
+    seen_keys = []
+
+    def fake_resolve(setting_key, owner=None):
+        seen_keys.append((setting_key, owner))
+        return [("https://utility.example/v1", "utility-model", {})]
+
+    monkeypatch.setattr(endpoint_resolver, "_resolve_fallback_candidates", fake_resolve)
+
+    assert endpoint_resolver.resolve_utility_fallback_candidates(owner="alice") == [
+        ("https://utility.example/v1", "utility-model", {})
+    ]
+    assert seen_keys == [("utility_model_fallbacks", "alice")]
+
+
+def test_multi_round_agent_uses_only_selected_model(monkeypatch):
+    """Every Agent round receives only the selected foreground candidate."""
+
+    seen_candidates = []
+    round_number = 0
+
+    monkeypatch.setattr(agent_loop, "get_setting", lambda key, default=None: default)
+    monkeypatch.setattr(agent_loop, "get_mcp_manager", lambda: None)
+    monkeypatch.setattr(agent_loop, "estimate_tokens", lambda *args, **kwargs: 10)
+
+    async def fake_stream(candidates, messages, **kwargs):
+        nonlocal round_number
+        round_number += 1
+        seen_candidates.append([(url, model) for url, model, _headers in candidates])
+        if round_number == 1:
+            call = {"name": "bash", "arguments": json.dumps({"command": "printf ok"})}
+            yield f'data: {json.dumps({"type": "tool_calls", "calls": [call]})}\n\n'
+        else:
+            yield f'data: {json.dumps({"delta": "done"})}\n\n'
+        yield "data: [DONE]\n\n"
+
+    async def fake_execute(block, *args, **kwargs):
+        return "bash", {"output": "ok", "exit_code": 0}
+
+    monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", fake_stream)
+    monkeypatch.setattr(agent_loop, "execute_tool_block", fake_execute)
+
+    fallbacks = resolve_foreground_fallback_candidates(owner="alice")
+    chunks = _collect(
+        agent_loop.stream_agent_loop(
+            "https://selected.example/v1",
+            "selected-model",
+            [{"role": "user", "content": "Run one tool and report back."}],
+            max_rounds=3,
+            relevant_tools={"bash"},
+            fallbacks=fallbacks,
+            _is_teacher_run=True,
+        )
+    )
+
+    assert seen_candidates == [
+        [("https://selected.example/v1", "selected-model")],
+        [("https://selected.example/v1", "selected-model")],
+    ]
+    assert any('"delta": "done"' in chunk for chunk in chunks)

tests/test_legacy_default_fallback_ui.py

@@ -0,0 +1,29 @@
+"""The retired default fallback editor must not imply active routing."""
+
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+
+
+_REPO = Path(__file__).resolve().parents[1]
+
+
+def test_legacy_default_fallback_editor_is_hidden():
+    soup = BeautifulSoup(
+        (_REPO / "static" / "index.html").read_text(encoding="utf-8"),
+        "html.parser",
+    )
+    editor = soup.find(id="set-defaultFallbacks")
+
+    assert editor is not None
+    assert editor.find_parent(class_="settings-row").has_attr("hidden")
+
+
+def test_default_model_save_does_not_rewrite_legacy_fallbacks():
+    source = (_REPO / "static" / "js" / "settings.js").read_text(encoding="utf-8")
+    start = source.index("async function initDefaultChat()")
+    end = source.index("/* ── Utility Model ── */", start)
+    default_chat_source = source[start:end]
+
+    assert "settings.default_model_fallbacks" not in default_chat_source
+    assert "default_model_fallbacks:" not in default_chat_source

tests/test_model_defaults.py

@@ -144,10 +144,10 @@ def test_get_default_chat_user_no_prefs_share_disabled_resolves_nothing(monkeypa
     assert test_data["model"] == "", "Should get empty model"
 
 
-def test_get_default_chat_user_no_prefs_share_enabled_resolves_global_defaults_fallbacks(monkeypatch):
+def test_get_default_chat_user_no_prefs_share_enabled_resolves_global_defaults(monkeypatch):
     """
     Non-admin user without personal preferences should resolve to global
-    defaults for ep_id, model, and fallbacks when share_defaults_with_users is enabled.
+    defaults for ep_id and model when share_defaults_with_users is enabled.
     """
 
     test_data = _run_get_default_chat_test(monkeypatch, share_defaults_enabled=True)
@@ -158,16 +158,45 @@ def test_get_default_chat_user_no_prefs_share_enabled_resolves_global_defaults_f
     assert test_data["endpoint_id"] == "global-ep-123", \
         "Should get global endpoint_id"
 
-def test_get_default_chat_user_no_prefs_share_enabled_resolves_global_defaults(monkeypatch):
+def test_get_default_chat_does_not_read_legacy_fallbacks(monkeypatch):
     """
-    Non-admin user without personal preferences should resolve to global
-    defaults for ep_id, model, and fallbacks when share_defaults_with_users is enabled.
+    The preserved legacy list must not influence default model resolution.
     """
 
-    test_data = _run_get_default_chat_test(monkeypatch, share_defaults_enabled=True, second_endpoint_only=True)
+    class LegacyReadGuard(dict):
+        def get(self, key, default=None):
+            if key == "default_model_fallbacks":
+                raise AssertionError("legacy fallback list was read")
+            return super().get(key, default)
 
-    assert test_data["model"] == "qwen-3.6", \
-        "model should be resolved from global default_model"
+    guarded_settings = LegacyReadGuard({
+        "default_endpoint_id": "global-ep-123",
+        "default_model": "qwen-3.6",
+        "default_model_fallbacks": [
+            {"endpoint_id": "fallback-ep", "model": "fallback-model"}
+        ],
+        "share_defaults_with_users": True,
+    })
+    monkeypatch.setattr(model_routes, "_load_settings", lambda: guarded_settings)
+    monkeypatch.setattr(prefs_routes, "_load_for_user", lambda user: LegacyReadGuard({}))
 
-    assert test_data["endpoint_id"] == "fallback-ep", \
-        "Should get global endpoint_id"
+    fake_auth_manager = MagicMock()
+    fake_auth_manager.is_admin = lambda user: False
+    endpoint = _FakeEndpoint(
+        id="global-ep-123",
+        base_url="http://global-endpoint:8000/v1",
+        is_enabled=True,
+    )
+    fake_db = _make_db_session([endpoint], user="regular_user")
+    monkeypatch.setattr(model_routes, "SessionLocal", lambda: fake_db)
+    monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url)
+    monkeypatch.setattr(model_routes, "build_chat_url", lambda base: f"{base}/chat")
+
+    router = model_routes.setup_model_routes(model_discovery=None)
+    get_default_chat = _get_default_chat_route(router)
+    fake_request = _make_request(user="regular_user", auth_manager=fake_auth_manager)
+
+    test_data = get_default_chat(fake_request)
+
+    assert test_data["endpoint_id"] == "global-ep-123"
+    assert test_data["model"] == "qwen-3.6"

tests/test_model_routes.py

@@ -90,7 +90,7 @@ def test_clear_speech_endpoint_settings_resets_tts_and_stt():
     }
 
 
-def test_endpoint_cleanup_removes_primary_and_fallback_references():
+def test_endpoint_cleanup_preserves_legacy_default_fallback_data():
     settings = {
         "default_endpoint_id": "dead",
         "default_model": "primary",
@@ -106,14 +106,12 @@ def test_endpoint_cleanup_removes_primary_and_fallback_references():
 
     assert _endpoint_settings_using_endpoint(settings, "dead", include_speech=True) == [
         "Default Model",
-        "Default Model Fallbacks",
         "Utility Model Fallbacks",
         "Vision Model Fallbacks",
         "Speech to Text",
     ]
     assert _clear_endpoint_settings_for_endpoint(settings, "dead", include_speech=True) == [
         "Default Model",
-        "Default Model Fallbacks",
         "Utility Model Fallbacks",
         "Vision Model Fallbacks",
         "Speech to Text",
@@ -121,6 +119,7 @@ def test_endpoint_cleanup_removes_primary_and_fallback_references():
     assert settings["default_endpoint_id"] == ""
     assert settings["default_model"] == ""
     assert settings["default_model_fallbacks"] == [
+        {"endpoint_id": "dead", "model": "fallback-a"},
         {"endpoint_id": "keep", "model": "fallback-b"},
     ]
     assert settings["utility_model_fallbacks"] == []
@@ -129,7 +128,7 @@ def test_endpoint_cleanup_removes_primary_and_fallback_references():
     assert settings["stt_model"] == "base"
 
 
-def test_endpoint_cleanup_updates_scoped_and_legacy_user_prefs():
+def test_endpoint_cleanup_updates_active_scoped_prefs_but_preserves_legacy_data():
     scoped = {
         "_users": {
             "alice": {
@@ -154,8 +153,10 @@ def test_endpoint_cleanup_updates_scoped_and_legacy_user_prefs():
     legacy = {
         "default_model_fallbacks": [{"endpoint_id": "dead", "model": "chat"}],
     }
-    assert _clear_user_pref_endpoint_refs(legacy, "dead") == 1
-    assert legacy["default_model_fallbacks"] == []
+    assert _clear_user_pref_endpoint_refs(legacy, "dead") == 0
+    assert legacy["default_model_fallbacks"] == [
+        {"endpoint_id": "dead", "model": "chat"}
+    ]
 
 
 # ── _default_endpoint_needs_assignment (add-endpoint auto-default) ──