odysseus/tests/test_utility_reasoning_effort.py

"""Opt-in ``utility_reasoning_effort`` for utility/background LLM calls.

The setting is resolved in ``task_llm_call_async`` and sent only to candidates
on the utility chain whose model advertises the level.  Explicit caller values
win; with the setting empty nothing changes.
"""
import asyncio

import pytest

from src import llm_core, task_endpoint
from src.chatgpt_subscription import validate_reasoning_effort

UTIL = ("https://api.openai.example/v1", "gpt-5.5", {})
UTIL_FALLBACK = ("https://fallback.example/v1", "gpt-5.5-mini", {})
OVERRIDE = ("https://override.example/v1", "gpt-5.5", {})
NO_EVIDENCE = ("https://local.example/v1", "tiny-local-model", {})


def _stub_resolvers(monkeypatch, utility):
    """Stub the resolvers in both modules that look the utility chain up."""
    from src import endpoint_resolver

    resolve = lambda prefix, *a, **k: {  # noqa: E731
        "utility": utility, "task": OVERRIDE, "default": NO_EVIDENCE,
    }[prefix]
    fallbacks = lambda owner=None: [UTIL_FALLBACK]  # noqa: E731
    for mod in (task_endpoint, endpoint_resolver):
        monkeypatch.setattr(mod, "resolve_endpoint", resolve)
        monkeypatch.setattr(mod, "resolve_utility_fallback_candidates", fallbacks)


@pytest.fixture
def chain(monkeypatch):
    """Stub the resolvers and capture what reaches llm_call_async per model."""
    _stub_resolvers(monkeypatch, UTIL)

    async def _quiet(*_a, **_k):
        return None
    monkeypatch.setattr(task_endpoint, "wait_for_interactive_quiet", _quiet)

    calls = {}

    async def fake_call(url, model, messages, **kw):
        calls[url] = kw.get("reasoning_effort")
        if url != UTIL_FALLBACK[0]:
            raise RuntimeError("force walk down the chain")
        return "ok"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
    return calls


def _set_effort(monkeypatch, value):
    monkeypatch.setattr(
        "src.settings.get_user_setting",
        lambda key, owner="", default=None: value if key == "utility_reasoning_effort" else default,
    )


def _run(**kw):
    return asyncio.run(task_endpoint.task_llm_call_async(
        [{"role": "user", "content": "hi"}], **kw,
    ))


def test_setting_applies_only_to_utility_chain(chain, monkeypatch):
    _set_effort(monkeypatch, "high")
    _run()
    # The Background Tasks route shares the model id with the utility model but
    # is a different (url, model) route, so it gets no effort.
    assert chain[OVERRIDE[0]] is None
    assert chain[UTIL[0]] == "high"
    assert chain[UTIL_FALLBACK[0]] == "high"


def test_utility_candidate_gets_validated_effort(chain, monkeypatch):
    _set_effort(monkeypatch, "HIGH")
    _run()
    assert chain[UTIL[0]] == "high"


def test_effort_without_evidence_is_dropped(chain, monkeypatch):
    _set_effort(monkeypatch, "high")
    _stub_resolvers(monkeypatch, ("https://local.example/v1", "tiny-local-model", {}))
    _run()
    assert chain["https://local.example/v1"] is None


def test_unsupported_level_is_dropped(chain, monkeypatch):
    _set_effort(monkeypatch, "bogus")
    _run()
    assert chain[UTIL[0]] is None


def test_empty_setting_changes_nothing(chain, monkeypatch):
    _set_effort(monkeypatch, "")
    _run()
    assert chain and all(effort is None for effort in chain.values())


def test_explicit_caller_effort_wins(chain, monkeypatch):
    _set_effort(monkeypatch, "high")
    _run(reasoning_effort="low")
    assert chain and all(effort == "low" for effort in chain.values())


def test_fallback_factory_can_override_messages_and_kwargs(monkeypatch):
    seen = []

    async def fake_call(url, model, messages, **kw):
        seen.append((model, messages, kw.get("reasoning_effort"), kw.get("temperature")))
        if len(seen) == 1:
            raise RuntimeError("first fails")
        return "ok"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)

    def factory(index, url, model, headers):
        return {"kwargs": {"reasoning_effort": f"e{index}"}}

    out = asyncio.run(llm_core.llm_call_async_with_fallback(
        [("http://a/v1", "m1", {}), ("http://b/v1", "m2", {})],
        [{"role": "user", "content": "x"}],
        temperature=0.1,
        candidate_request_factory=factory,
    ))
    assert out == "ok"
    assert [(s[0], s[2], s[3]) for s in seen] == [("m1", "e0", 0.1), ("m2", "e1", 0.1)]


def test_validate_reasoning_effort_needs_evidence():
    assert validate_reasoning_effort("gpt-5.5", "high") == "high"
    assert validate_reasoning_effort("tiny-local-model", "high") is None
    assert validate_reasoning_effort("gpt-5.5", "") is None


def test_chatgpt_wire_carries_effort_and_default_omits_it():
    sent = llm_core._build_chatgpt_responses_payload(
        "gpt-5.5", [{"role": "user", "content": "hi"}], 0.3, 64, reasoning_effort="high",
    )
    assert sent["reasoning"] == {"effort": "high"}
    plain = llm_core._build_chatgpt_responses_payload(
        "gpt-5.5", [{"role": "user", "content": "hi"}], 0.3, 64,
    )
    assert "reasoning" not in plain


def test_sync_llm_call_cache_key_includes_effort(monkeypatch):
    keys = []

    def fake_cached(key):
        keys.append(key)
        return "cached"

    monkeypatch.setattr(llm_core, "_get_cached_response", fake_cached)
    msgs = [{"role": "user", "content": "hi"}]
    llm_core.llm_call("http://x/v1", "m", msgs)
    llm_core.llm_call("http://x/v1", "m", msgs, reasoning_effort="high")
    llm_core.llm_call("http://x/v1", "m", msgs, reasoning_effort="HIGH")
    assert keys[0] != keys[1]
    assert keys[1] == keys[2]


def test_setting_is_registered_default_and_per_user():
    from src.settings import DEFAULT_SETTINGS, _PER_USER_KEYS

    assert DEFAULT_SETTINGS["utility_reasoning_effort"] == ""
    assert "utility_reasoning_effort" in _PER_USER_KEYS


# --- effort_for_call: setting, then the chat session's own effort ---

CHAT = ("https://chat.example/v1", "gpt-5.5", {})


class _Sess:
    def __init__(self, thinking_mode="effort:low", url=CHAT[0], model=CHAT[1], owner="alice"):
        self.thinking_mode = thinking_mode
        self.endpoint_url = url
        self.model = model
        self.owner = owner
        self.headers = {}


@pytest.fixture
def util(monkeypatch):
    """Utility chain = {UTIL}; the setting is controlled via ``settings[owner]``."""
    from src import utility_effort

    settings = {}
    monkeypatch.setattr(utility_effort, "utility_routes", lambda owner=None: {(UTIL[0], UTIL[1])})
    monkeypatch.setattr(
        "src.settings.get_user_setting",
        lambda key, owner="", default=None: (
            settings.get(owner, "") if key == "utility_reasoning_effort" else default
        ),
    )
    util_mod = utility_effort
    util_mod.settings = settings
    return util_mod


def test_setting_applies_to_utility_route_and_wins_over_chat_effort(util):
    util.settings["alice"] = "medium"
    assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess("effort:low")) == "medium"
    assert util.effort_for_call(UTIL[0], UTIL[1], "alice") == "medium"


def test_setting_is_per_user(util):
    util.settings["alice"] = "high"
    assert util.effort_for_call(UTIL[0], UTIL[1], "alice") == "high"
    assert util.effort_for_call(UTIL[0], UTIL[1], "bob") is None


def test_session_effort_inherited_on_the_session_route(util):
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) == "low"
    # Endpoint spelling differences (trailing slash) are still the same route.
    assert util.effort_for_call(CHAT[0] + "/", CHAT[1], "alice", _Sess("effort:low")) == "low"


def test_session_effort_not_inherited_on_another_route(util):
    assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess("effort:low")) is None
    assert util.effort_for_call(CHAT[0], "gpt-5.5-mini", "alice", _Sess("effort:low")) is None


def test_session_effort_off_or_missing_inherits_nothing(util):
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("off")) is None
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("")) is None


def test_no_session_means_no_inheritance(util):
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice") is None


def test_explicit_off_setting_blocks_session_effort_inheritance(util):
    util.settings["alice"] = "off"
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) is None
    assert util.candidate_effort_factory("alice") is None

    util.settings["alice"] = "default"
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) is None
    assert util.candidate_effort_factory("alice") is None


def test_inherited_effort_is_validated_against_the_model(util):
    assert util.effort_for_call(
        NO_EVIDENCE[0], NO_EVIDENCE[1], "alice",
        _Sess("effort:low", url=NO_EVIDENCE[0], model=NO_EVIDENCE[1]),
    ) is None
    assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:bogus")) is None


def test_effort_for_call_never_raises(util, monkeypatch):
    def boom(*_a, **_k):
        raise RuntimeError("db down")

    monkeypatch.setattr(util, "effort_for_route", boom)
    assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess()) is None


# --- no global fill: callers that are not utility call sites never get an effort ---

def test_llm_call_async_adds_no_default_effort(util, monkeypatch):
    # The owner-less helpers resolve at global scope; set it there and per user.
    util.settings[""] = util.settings["alice"] = "high"
    seen = []

    def fake_key(url, model, messages, temperature, max_tokens, headers=None,
                 thinking_mode=None, reasoning_effort=None):
        seen.append(reasoning_effort)
        return "k"

    monkeypatch.setattr(llm_core, "_get_cache_key", fake_key)
    monkeypatch.setattr(llm_core, "_get_cached_response", lambda key: "cached")
    msgs = [{"role": "user", "content": "hi"}]
    # Same (url, model) as the utility route, but not a utility call site.
    asyncio.run(llm_core.llm_call_async(UTIL[0], UTIL[1], msgs))
    llm_core.llm_call(UTIL[0], UTIL[1], msgs)
    asyncio.run(llm_core.llm_call_async(UTIL[0], UTIL[1], msgs, reasoning_effort="low"))
    assert seen == [None, None, "low"]


# --- call sites pass the effort explicitly ---

def _capture_llm(monkeypatch):
    sent = []

    async def fake_call(url, model, messages, **kw):
        sent.append((url, model, kw.get("reasoning_effort")))
        return "A title"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
    return sent


class _History:
    role = "user"
    content = "hello there"


def _title_session(**kw):
    sess = _Sess(**kw)
    sess.id = "s1"
    sess.history = [_History()]
    return sess


class _Manager:
    def update_session_name(self, *_a):
        pass


@pytest.fixture
def title_registered_endpoint(tmp_path, monkeypatch):
    import core.database as database
    import src.database as compat_database
    import src.endpoint_resolver as resolver
    from tests.helpers.database import disposable_database

    monkeypatch.setenv("AUTH_ENABLED", "true")
    monkeypatch.setattr("src.settings.load_settings", lambda: {})
    monkeypatch.setattr(resolver, "resolve_url", lambda url: url)
    with disposable_database(tmp_path) as factory:
        monkeypatch.setattr(compat_database, "SessionLocal", factory)
        monkeypatch.setattr(resolver, "SessionLocal", factory)
        with factory() as db:
            db.add(database.ModelEndpoint(id="chat", name="chat", owner="alice",
                                          base_url=CHAT[0], is_enabled=True))
            db.add(database.ModelEndpoint(id="shared", name="shared", owner=None,
                                          base_url="https://shared.example/v1", is_enabled=True))
            db.commit()
        yield


def test_auto_title_inherits_chat_effort_when_utility_is_blank(util, monkeypatch, title_registered_endpoint):
    from routes import chat_helpers
    from src.endpoint_resolver import build_chat_url

    sent = _capture_llm(monkeypatch)
    chat_url = build_chat_url(CHAT[0])
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session(url=chat_url)))
    assert sent == [(chat_url, CHAT[1], "low")]


@pytest.mark.parametrize("owner,url", [
    ("alice", "https://unregistered.example/v1"),
    ("bob", CHAT[0]),
    (None, "https://shared.example/v1"),
    ("", "https://shared.example/v1"),
    ("   ", "https://shared.example/v1"),
    (" alice ", "https://shared.example/v1"),
    ("system", "https://shared.example/v1"),
    (42, "https://shared.example/v1"),
])
def test_auto_title_rejects_unauthorized_chat_endpoint_when_utility_is_blank(
    util, monkeypatch, title_registered_endpoint, owner, url,
):
    from routes import chat_helpers

    sent = _capture_llm(monkeypatch)
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session(owner=owner, url=url)))
    assert sent == []


def test_auto_title_missing_owner_fails_closed_in_single_user_mode(util, monkeypatch, title_registered_endpoint):
    from routes import chat_helpers

    monkeypatch.setenv("AUTH_ENABLED", "false")
    sent = _capture_llm(monkeypatch)
    asyncio.run(chat_helpers.auto_name_session(
        _Manager(), _title_session(owner=None, url="https://shared.example/v1"),
    ))
    assert sent == []


def test_auto_title_setting_beats_chat_effort(util, monkeypatch):
    from routes import chat_helpers

    util.settings["alice"] = "medium"
    sent = _capture_llm(monkeypatch)
    monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
    assert sent == [(UTIL[0], UTIL[1], "medium")]


def test_auto_title_on_a_different_utility_model_gets_no_chat_effort(util, monkeypatch):
    from routes import chat_helpers

    sent = _capture_llm(monkeypatch)
    monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
    assert sent == [(UTIL[0], UTIL[1], None)]


def _capture_title_thinking(monkeypatch):
    sent = []

    async def fake_call(url, model, messages, **kw):
        sent.append((kw.get("reasoning_effort"), kw.get("thinking_mode")))
        return "A title"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
    return sent


def test_auto_title_turns_thinking_off_without_a_configured_effort(util, monkeypatch):
    from routes import chat_helpers

    sent = _capture_title_thinking(monkeypatch)
    monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
    assert sent == [(None, "off")]


def test_auto_title_keeps_thinking_default_with_a_configured_effort(util, monkeypatch):
    from routes import chat_helpers

    util.settings["alice"] = "medium"
    sent = _capture_title_thinking(monkeypatch)
    monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
    asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
    assert sent == [("medium", None)]


def test_memory_extraction_passes_chat_effort(util, monkeypatch):
    from services.memory import memory_extractor

    sent = []

    async def fake_call(url, model, messages, **kw):
        sent.append(kw.get("reasoning_effort"))
        return "[]"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)

    class Sess(_Sess):
        def get_context_messages(self):
            return [{"role": "user", "content": "I live in Oslo"},
                    {"role": "assistant", "content": "Noted"}]

    asyncio.run(memory_extractor.extract_and_store(
        Sess(), object(), object(), CHAT[0], CHAT[1], {},
    ))
    assert sent and sent[0] == "low"


def test_email_summary_sends_setting_effort_for_owner_only(util, monkeypatch):
    from routes.email import email_helpers

    util.settings["alice"] = "medium"
    sent = []

    async def fake_call(**kw):
        sent.append(kw.get("reasoning_effort"))
        return "- point"

    monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
    for owner in ("alice", "bob", None):
        asyncio.run(email_helpers._generate_email_summary(
            UTIL[0], UTIL[1], "a@x", "s", "body", owner=owner,
        ))
    assert sent == ["medium", None, None]


def test_candidate_effort_factory_scopes_to_utility_route(util):
    from src.utility_effort import candidate_effort_factory

    assert candidate_effort_factory("alice") is None
    util.settings["alice"] = "medium"
    factory = candidate_effort_factory("alice")
    assert factory(0, UTIL[0], UTIL[1], {}) == {"kwargs": {"reasoning_effort": "medium"}}
    assert factory(1, NO_EVIDENCE[0], NO_EVIDENCE[1], {}) == {}
    assert candidate_effort_factory("bob") is None


def test_fallback_route_efforts_and_factory(monkeypatch):
    from src.utility_effort import candidate_effort_factory, effort_for_route

    def fake_resolve_id(ep_id, model, owner=None):
        if ep_id == "ep-fallback":
            return ("https://fallback.example/v1", model, {})
        return None

    monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint_by_id", fake_resolve_id)
    monkeypatch.setattr(
        "src.settings.load_settings",
        lambda: {"utility_model_fallbacks": [{"endpoint_id": "ep-fallback", "model": "gpt-5.5-mini", "reasoning_effort": "low"}]},
    )
    monkeypatch.setattr(
        "src.settings.get_user_setting",
        lambda key, owner="", default=None: default,
    )

    factory = candidate_effort_factory("alice")
    assert factory is not None
    assert effort_for_route("https://api.openai.example/v1", "gpt-5.5", "alice") is None
    assert effort_for_route("https://fallback.example/v1", "gpt-5.5-mini", "alice") == "low"
    assert factory(0, "https://fallback.example/v1", "gpt-5.5-mini", {}) == {"kwargs": {"reasoning_effort": "low"}}


def test_sync_openai_compatible_wire_transmits_explicit_effort(monkeypatch):
    import httpx

    sent = []

    def fake_post(url, headers, **kwargs):
        sent.append(dict(kwargs["json"]))
        return httpx.Response(
            200, json={"choices": [{"message": {"content": "ok"}}]}
        )

    monkeypatch.setattr(llm_core, "_get_cached_response", lambda *_: None)
    monkeypatch.setattr(llm_core, "_set_cached_response", lambda *a, **k: None)
    monkeypatch.setattr(llm_core, "note_model_activity", lambda *a: None)
    monkeypatch.setattr(llm_core, "httpx_post_kimi_aware", fake_post)

    url = "https://example.test/v1"
    messages = [{"role": "user", "content": "hi"}]

    llm_core.llm_call(
        url, "gpt-5.5", messages, max_tokens=64,
        reasoning_effort="HIGH",
    )
    llm_core.llm_call(url, "gpt-5.5", messages, max_tokens=64)

    assert sent[0]["reasoning_effort"] == "high"
    assert "reasoning_effort" not in sent[1]


def test_async_openai_compatible_wire_transmits_explicit_effort(monkeypatch):
    import asyncio
    import httpx
    from contextlib import asynccontextmanager

    sent = []

    @asynccontextmanager
    async def fake_slot(*args, **kwargs):
        yield

    async def fake_post(client, url, headers, **kwargs):
        sent.append(dict(kwargs["json"]))
        return httpx.Response(
            200, json={"choices": [{"message": {"content": "ok"}}]}
        )

    monkeypatch.setattr(llm_core, "_get_cached_response", lambda *_: None)
    monkeypatch.setattr(llm_core, "_set_cached_response", lambda *a, **k: None)
    monkeypatch.setattr(llm_core, "_get_http_client", lambda: object())
    monkeypatch.setattr(llm_core, "_is_host_dead", lambda *_: False)
    monkeypatch.setattr(llm_core, "_local_model_slot", fake_slot)
    monkeypatch.setattr(llm_core, "note_model_activity", lambda *a: None)
    monkeypatch.setattr(llm_core, "httpx_post_kimi_aware_async", fake_post)

    async def exercise():
        url = "https://example.test/v1"
        messages = [{"role": "user", "content": "hi"}]

        await llm_core.llm_call_async(
            url, "gpt-5.5", messages, max_tokens=0,
            max_retries=1, reasoning_effort="medium",
        )
        await llm_core.llm_call_async(
            url, "gpt-5.5", messages, max_tokens=0,
            max_retries=1,
        )

    asyncio.run(exercise())

    assert sent[0]["reasoning_effort"] == "medium"
    assert "reasoning_effort" not in sent[1]