mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-15 11:22:21 +02:00
feat(settings): opt-in reasoning effort for utility model calls (#6572)
* feat(settings): opt-in reasoning effort for utility model calls Add utility_reasoning_effort (empty = provider default) and let utility calls inherit a chat session's effort only on that session's matching endpoint+model route. Explicit caller effort wins; otherwise the utility setting wins over session inheritance on utility-chain routes. Resolve the chain and setting per user and validate levels against model evidence. No-session work uses the setting only, with no chat-effort crossover. - src/utility_effort.py owns effort_for_call and candidate_effort_factory - remove global effort fill from llm_call/llm_call_async; utility call sites pass effort explicitly, so unrelated calls on the same route do not pick it up - wire auto-title, memory/persona/skill extraction, memory suggest/import/ audit, compaction, auto-sort, and per-owner background task candidates - wire no-session email writing-style, summary, and translation calls, plus skills evaluation, necessity/retrieval audits, and improvement - deliberately exclude email replies because thinking_mode='off' - add the optional candidate_request_factory to llm_call_async_with_fallback - add sync llm_call reasoning_effort and include it in the cache key - show the Utility Model effort selector only for advertised levels Fixes #6569 * fix(utility): exclude email replies and support inherited effort in settings Pass reasoning_effort='' to task_llm_call_async for scheduled reply drafting to honor the email reply exclusion. In AI settings, resolve the effective Default Chat endpoint and model when Utility is set to 'Same as chat' so supported reasoning effort levels can be selected. * feat(utility): support same-as-chat effort and session model fallback in compaction In AI settings, show 'Same as chat' as the default effort option when Utility Model is inherited, and support an explicit 'Provider default' (off) override. Pass the active session fallback in context compaction callers so compaction shares the live chat session's model and reasoning effort for maximum prompt/KV cache affinity. * fix(settings): clarify chat effort inheritance and keep option visible Label inherited effort as 'Same as chat (when supported)' and mention cache affinity and latency speedups in the help text. Ensure the effort control remains visible when the utility model is set to 'Same as chat' even if the default chat model lacks advertised reasoning levels, and populate candidate levels across configured endpoints so the user can configure effort or keep provider default. * feat(settings): per-fallback reasoning effort for utility model Each utility fallback row gets an Effort select, shown only when the selected model advertises reasoning levels. The choice is stored as reasoning_effort on the utility_model_fallbacks entry and sent only to that candidate; an empty value means provider default. * fix(utility): transmit reasoning effort in non-streaming calls --------- Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com>
This commit is contained in:
Mark Cockram
co-authored by
Alexandre Teixeira
Alexandre Teixeira
parent
afecd44ddf
commit
0e2c08300e
@@ -551,6 +551,7 @@ async def auto_name_session(session_manager, sess):
|
||||
try:
|
||||
from src.llm_core import llm_call_async
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
# Find first user message
|
||||
first_msg = ""
|
||||
@@ -603,6 +604,7 @@ async def auto_name_session(session_manager, sess):
|
||||
max_tokens=64,
|
||||
headers=t_headers,
|
||||
timeout=15,
|
||||
reasoning_effort=effort_for_call(t_url, t_model, owner, sess),
|
||||
)
|
||||
|
||||
title = title.strip().strip('"\'').strip()
|
||||
|
||||
@@ -443,9 +443,11 @@ async def _generate_email_summary(
|
||||
headers: dict | None = None,
|
||||
max_tokens: int = 8192,
|
||||
timeout: int = 180,
|
||||
owner: str | None = None,
|
||||
) -> str:
|
||||
"""Generate an interactive email summary through the shared LLM adapter."""
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
raw = await llm_call_async(
|
||||
url=url,
|
||||
@@ -456,6 +458,7 @@ async def _generate_email_summary(
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
workload="foreground",
|
||||
reasoning_effort=effort_for_call(url, model, owner),
|
||||
)
|
||||
return _normalize_email_summary(raw)
|
||||
|
||||
|
||||
@@ -1031,6 +1031,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
|
||||
fallback_url=url, fallback_model=model, fallback_headers=headers,
|
||||
owner=account_owner or None,
|
||||
temperature=0.7, max_tokens=1024, timeout=90,
|
||||
reasoning_effort="",
|
||||
)
|
||||
reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
|
||||
if reply:
|
||||
|
||||
@@ -5920,7 +5920,12 @@ def setup_email_routes():
|
||||
},
|
||||
]
|
||||
|
||||
style = await llm_call_async(url, model, messages, headers=headers, max_tokens=2048)
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
style = await llm_call_async(
|
||||
url, model, messages, headers=headers, max_tokens=2048,
|
||||
reasoning_effort=effort_for_call(url, model, owner),
|
||||
)
|
||||
style = _strip_think(style or "")
|
||||
if not style:
|
||||
return {"success": False, "error": "LLM failed to generate style description"}
|
||||
@@ -6004,6 +6009,7 @@ def setup_email_routes():
|
||||
headers=req_headers,
|
||||
max_tokens=8192,
|
||||
timeout=180,
|
||||
owner=owner,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
@@ -6125,6 +6131,8 @@ def setup_email_routes():
|
||||
if not candidates:
|
||||
return {"success": False, "error": "No LLM endpoint configured"}
|
||||
|
||||
from src.utility_effort import candidate_effort_factory
|
||||
|
||||
content = await llm_call_async_with_fallback(
|
||||
candidates,
|
||||
messages=[
|
||||
@@ -6152,6 +6160,7 @@ def setup_email_routes():
|
||||
temperature=0.2,
|
||||
max_tokens=8192,
|
||||
timeout=180,
|
||||
candidate_request_factory=candidate_effort_factory(owner),
|
||||
)
|
||||
model = candidates[0][1] if candidates else ""
|
||||
content = (content or "").strip()
|
||||
|
||||
@@ -237,6 +237,18 @@ def _keep_count_before_message(db_messages, before_msg_id: str | None) -> int |
|
||||
return None
|
||||
|
||||
|
||||
def _effort_mode(model: str, raw_effort: Any) -> str:
|
||||
"""Session ``thinking_mode`` for a picked effort: ``effort:<level>`` or ``off``.
|
||||
|
||||
Validated like the chat routes do, so llama-swap advertised efforts persist
|
||||
as well as ChatGPT subscription levels.
|
||||
"""
|
||||
from src.chatgpt_subscription import validate_reasoning_effort
|
||||
|
||||
effort = validate_reasoning_effort(model, raw_effort)
|
||||
return f"effort:{effort}" if effort else "off"
|
||||
|
||||
|
||||
def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
router = APIRouter(
|
||||
tags=["history"],
|
||||
@@ -1031,26 +1043,11 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
mode = getattr(session, "thinking_mode", "off") or "off"
|
||||
raw_effort = body.get("reasoning_effort")
|
||||
if raw_effort is not None:
|
||||
clean_effort = str(raw_effort).strip().lower()
|
||||
if clean_effort in {"", "default"}:
|
||||
mode = "off"
|
||||
else:
|
||||
from src.chatgpt_subscription import get_chatgpt_model_metadata
|
||||
meta = get_chatgpt_model_metadata(session.model)
|
||||
if meta and clean_effort in [lvl.lower() for lvl in meta.get("supported_reasoning_levels", [])]:
|
||||
mode = f"effort:{clean_effort}"
|
||||
else:
|
||||
mode = "off"
|
||||
mode = _effort_mode(session.model, raw_effort)
|
||||
elif "thinking_mode" in body:
|
||||
raw_mode = str(body.get("thinking_mode") or "").strip().lower()
|
||||
if raw_mode.startswith("effort:"):
|
||||
clean_effort = raw_mode[7:].strip()
|
||||
from src.chatgpt_subscription import get_chatgpt_model_metadata
|
||||
meta = get_chatgpt_model_metadata(session.model)
|
||||
if meta and clean_effort in [lvl.lower() for lvl in meta.get("supported_reasoning_levels", [])]:
|
||||
mode = f"effort:{clean_effort}"
|
||||
else:
|
||||
mode = "off"
|
||||
mode = _effort_mode(session.model, raw_mode[7:])
|
||||
elif raw_mode in {"", "on", "off"}:
|
||||
mode = raw_mode
|
||||
from src.model_profiles import supports_user_thinking_toggle
|
||||
@@ -1129,7 +1126,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
)
|
||||
|
||||
# Use utility model if available
|
||||
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
|
||||
from src.utility_effort import effort_for_call
|
||||
util_url, util_model, util_headers = resolve_endpoint(
|
||||
"utility",
|
||||
fallback_url=getattr(session, "endpoint_url", None),
|
||||
fallback_model=getattr(session, "model", None),
|
||||
fallback_headers=getattr(session, "headers", None),
|
||||
owner=owner or None,
|
||||
)
|
||||
compact_url = util_url or session.endpoint_url
|
||||
compact_model = util_model or session.model
|
||||
compact_headers = util_headers if util_url else session.headers
|
||||
@@ -1145,6 +1149,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
],
|
||||
temperature=0.2, max_tokens=1024,
|
||||
headers=compact_headers, timeout=30,
|
||||
reasoning_effort=effort_for_call(compact_url, compact_model, owner, session),
|
||||
)
|
||||
summary = normalize_compaction_summary(summary)
|
||||
|
||||
|
||||
@@ -261,6 +261,8 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
|
||||
)
|
||||
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
try:
|
||||
suggestion_text = await llm_call_async(
|
||||
t_url,
|
||||
@@ -269,6 +271,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
temperature=0.2,
|
||||
max_tokens=500,
|
||||
headers=t_headers,
|
||||
reasoning_effort=effort_for_call(t_url, t_model, _owner(request), sess),
|
||||
)
|
||||
try:
|
||||
suggestions = json.loads(suggestion_text)
|
||||
@@ -349,6 +352,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
endpoint_url = None
|
||||
model = None
|
||||
headers = {}
|
||||
sess = None
|
||||
|
||||
user = _owner(request)
|
||||
|
||||
@@ -453,6 +457,8 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
"Return ONLY valid JSON, no markdown fences."
|
||||
)
|
||||
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
try:
|
||||
raw = await llm_call_async(
|
||||
endpoint_url,
|
||||
@@ -464,6 +470,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
|
||||
temperature=0.2,
|
||||
max_tokens=2000,
|
||||
headers=headers,
|
||||
reasoning_effort=effort_for_call(endpoint_url, model, user, sess),
|
||||
)
|
||||
|
||||
# Parse JSON
|
||||
|
||||
@@ -509,6 +509,7 @@ def setup_session_routes(
|
||||
"has_documents": s.id in doc_session_ids,
|
||||
"has_images": s.id in img_session_ids,
|
||||
"mode": s.mode,
|
||||
"thinking_mode": s.thinking_mode or "off",
|
||||
"message_count": s.message_count or 0,
|
||||
})
|
||||
finally:
|
||||
@@ -1042,6 +1043,7 @@ def setup_session_routes(
|
||||
"id": s.id,
|
||||
"name": s.name,
|
||||
"model": s.model,
|
||||
"thinking_mode": s.thinking_mode or "off",
|
||||
"message_count": s.message_count or 0,
|
||||
"created_at": s.created_at.isoformat() if s.created_at else None,
|
||||
"updated_at": s.updated_at.isoformat() if s.updated_at else None,
|
||||
@@ -1238,9 +1240,16 @@ def setup_session_routes(
|
||||
from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
owner = getattr(session, "owner", None) or effective_user(request)
|
||||
url, model, headers = resolve_endpoint("utility", owner=owner)
|
||||
url, model, headers = resolve_endpoint(
|
||||
"utility",
|
||||
fallback_url=getattr(session, "endpoint_url", None),
|
||||
fallback_model=getattr(session, "model", None),
|
||||
fallback_headers=getattr(session, "headers", None),
|
||||
owner=owner,
|
||||
)
|
||||
if not url or not model:
|
||||
url, model, headers = session.endpoint_url, session.model, session.headers
|
||||
if not url or not model:
|
||||
@@ -1268,6 +1277,7 @@ def setup_session_routes(
|
||||
max_tokens=1024,
|
||||
headers=headers,
|
||||
timeout=60,
|
||||
reasoning_effort=effort_for_call(url, model, owner, session),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Manual compaction failed: %s", e)
|
||||
@@ -1311,6 +1321,7 @@ def setup_session_routes(
|
||||
users can clean junk without spending tokens.
|
||||
"""
|
||||
from src.llm_core import llm_call
|
||||
from src.utility_effort import effort_for_call
|
||||
user = effective_user(request)
|
||||
single_user_mode = not user and _auth_disabled()
|
||||
user_sessions = session_manager.get_sessions_for_user(user)
|
||||
@@ -1483,7 +1494,8 @@ def setup_session_routes(
|
||||
# reasoning model spends tokens thinking first — 4096 truncated the
|
||||
# JSON mid-output, so it never parsed ("invalid JSON for auto-sort").
|
||||
raw = llm_call(url, model, [{"role": "user", "content": prompt}],
|
||||
temperature=0.3, max_tokens=16384, headers=headers, timeout=120)
|
||||
temperature=0.3, max_tokens=16384, headers=headers, timeout=120,
|
||||
reasoning_effort=effort_for_call(url, model, user))
|
||||
logger.info(f"Auto-sort raw response ({len(raw)} chars): {raw[:300]}")
|
||||
# Extract JSON from response — handle markdown fences, leading text,
|
||||
# reasoning-model <think> blocks, and trailing commas.
|
||||
|
||||
+19
-6
@@ -21,6 +21,7 @@ from services.memory.skills import SkillsManager
|
||||
from src.auth_helpers import get_current_user
|
||||
from src.prompt_security import untrusted_context_message
|
||||
from src.text_helpers import strip_closed_think_blocks
|
||||
from src.utility_effort import effort_for_call
|
||||
from core.middleware import require_admin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -170,7 +171,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
|
||||
baseline_transcript: str = "",
|
||||
skill_stats: Optional[dict] = None,
|
||||
baseline_stats: Optional[dict] = None,
|
||||
workload: str = "foreground") -> dict:
|
||||
workload: str = "foreground", owner: Optional[str] = None) -> dict:
|
||||
"""LLM-as-judge: grade a skill test run from its transcript. Advisory only.
|
||||
|
||||
Robust against local reasoning models (strips <think>, lenient JSON,
|
||||
@@ -349,6 +350,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
|
||||
url, model, msgs,
|
||||
temperature=0.1, max_tokens=32768, headers=headers, timeout=180,
|
||||
workload=workload,
|
||||
reasoning_effort=effort_for_call(url, model, owner),
|
||||
)
|
||||
except Exception as e:
|
||||
# Don't give up on a transient first-attempt error — let the second
|
||||
@@ -368,7 +370,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
|
||||
|
||||
async def _eval_skill_necessity(skill_md: str, others: list, url: str, model: str,
|
||||
headers: Optional[dict],
|
||||
workload: str = "foreground") -> Optional[dict]:
|
||||
workload: str = "foreground", owner: Optional[str] = None) -> Optional[dict]:
|
||||
"""Advisory judge: is this skill worth keeping, or is it redundant / trivially
|
||||
unnecessary? Sees the OTHER skills' names+descriptions so it can spot
|
||||
duplicates. Returns {necessary, redundant_with, reason} or None. Never acts —
|
||||
@@ -401,6 +403,7 @@ async def _eval_skill_necessity(skill_md: str, others: list, url: str, model: st
|
||||
[{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}],
|
||||
temperature=0.1, max_tokens=8192, headers=headers, timeout=120,
|
||||
workload=workload,
|
||||
reasoning_effort=effort_for_call(url, model, owner),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Necessity check failed: {e}")
|
||||
@@ -453,7 +456,7 @@ def _should_check_retrieval_precision(skill: dict) -> bool:
|
||||
async def _eval_skill_retrieval_precision(skill_md: str, others: list,
|
||||
url: str, model: str,
|
||||
headers: Optional[dict],
|
||||
workload: str = "foreground") -> Optional[dict]:
|
||||
workload: str = "foreground", owner: Optional[str] = None) -> Optional[dict]:
|
||||
"""Advisory judge: would this skill's metadata make retrieval over-select it?
|
||||
|
||||
This is distinct from "does the procedure work?". It asks whether tags,
|
||||
@@ -491,6 +494,7 @@ async def _eval_skill_retrieval_precision(skill_md: str, others: list,
|
||||
[{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}],
|
||||
temperature=0.1, max_tokens=4096, headers=headers, timeout=90,
|
||||
workload=workload,
|
||||
reasoning_effort=effort_for_call(url, model, owner),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Retrieval precision check failed: {e}")
|
||||
@@ -667,6 +671,7 @@ async def _run_skill_test_job(
|
||||
baseline_transcript=baseline_transcript,
|
||||
skill_stats=skill_stats,
|
||||
baseline_stats=baseline_stats,
|
||||
owner=owner,
|
||||
)
|
||||
except Exception as e:
|
||||
job["verdict"] = {"verdict": "unknown", "confidence": 0, "summary": f"Eval failed: {e}", "issues": []}
|
||||
@@ -1157,12 +1162,13 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner,
|
||||
headers,
|
||||
skill_stats=stats,
|
||||
workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
return text, verdict
|
||||
|
||||
|
||||
async def _improve_skill_md(skill_md: str, verdict: dict, transcript: str, url, model, headers,
|
||||
workload: str = "foreground"):
|
||||
workload: str = "foreground", owner: Optional[str] = None):
|
||||
"""Have a model rewrite SKILL.md to fix the reviewer's issues. Returns the
|
||||
corrected markdown, or None if it couldn't produce a usable change."""
|
||||
import re as _re
|
||||
@@ -1192,7 +1198,8 @@ async def _improve_skill_md(skill_md: str, verdict: dict, transcript: str, url,
|
||||
[{"role": "system", "content": sys_prompt},
|
||||
{"role": "user", "content": user_msg}],
|
||||
temperature=0.2, max_tokens=16384, headers=headers, timeout=180,
|
||||
workload=workload)
|
||||
workload=workload,
|
||||
reasoning_effort=effort_for_call(url, model, owner))
|
||||
except Exception as e:
|
||||
logger.warning(f"Audit: improve call failed: {e}")
|
||||
return None
|
||||
@@ -1262,6 +1269,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
]
|
||||
nec = await _eval_skill_necessity(
|
||||
md, others, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if nec is not None:
|
||||
skills_manager.set_necessity(name, nec.get("necessary", True),
|
||||
@@ -1278,6 +1286,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
if _should_check_retrieval_precision(skill):
|
||||
rp = await _eval_skill_retrieval_precision(
|
||||
md, others, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if rp and not rp.get("ok"):
|
||||
issues = rp.get("issues") or ["metadata: retrieval: narrow tags and when_to_use to the intended trigger"]
|
||||
@@ -1288,7 +1297,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
"summary": rp.get("summary") or "Retrieval metadata is too broad.",
|
||||
"issues": issues,
|
||||
}, "Retrieval audit only: the procedure may work, but matching metadata is too broad.",
|
||||
url, model, headers, workload=workload)
|
||||
url, model, headers, workload=workload, owner=owner)
|
||||
if fixed and fixed.strip() != md.strip() and _apply_skill_md(skills_manager, name, fixed, owner):
|
||||
md = fixed
|
||||
refreshed = next((s for s in skills_manager.load(owner=owner) if s.get("name") == name), None)
|
||||
@@ -1364,6 +1373,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
skill_stats=skill_stats,
|
||||
baseline_stats=baseline_stats,
|
||||
workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
v = verdict.get("verdict")
|
||||
log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})")
|
||||
@@ -1397,6 +1407,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
log(f"{name}: pass, but fixing {len(meta_issues)} metadata issue(s)…")
|
||||
fixed = await _improve_skill_md(
|
||||
md, verdict, transcript, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if fixed and fixed.strip() != md.strip():
|
||||
_apply_skill_md(skills_manager, name, fixed, owner)
|
||||
@@ -1430,6 +1441,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
log(f"{name}: self-editing to fix issues…")
|
||||
new_md = await _improve_skill_md(
|
||||
md, verdict, transcript, url, model, headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if new_md and new_md.strip() != md.strip() and _apply_skill_md(skills_manager, name, new_md, owner):
|
||||
md = new_md
|
||||
@@ -1464,6 +1476,7 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
|
||||
log(f"{name}: teacher {t_model} rewriting the skill…")
|
||||
t_md = await _improve_skill_md(
|
||||
md, verdict, transcript, t_url, t_model, t_headers, workload=workload,
|
||||
owner=owner,
|
||||
)
|
||||
if t_md and t_md.strip() != md.strip() and _apply_skill_md(skills_manager, name, t_md, owner):
|
||||
md = t_md
|
||||
|
||||
@@ -331,6 +331,7 @@ async def extract_and_store(
|
||||
|
||||
try:
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
# Get last N messages from session
|
||||
messages = session.get_context_messages()
|
||||
@@ -404,6 +405,9 @@ async def extract_and_store(
|
||||
# enough once thinking has room.
|
||||
max_tokens=4096,
|
||||
headers=headers,
|
||||
reasoning_effort=effort_for_call(
|
||||
endpoint_url, model, getattr(session, "owner", None), session
|
||||
),
|
||||
)
|
||||
|
||||
# Parse JSON, tolerating reasoning-model noise (<think> blocks, a
|
||||
@@ -559,6 +563,7 @@ async def update_persona_memory(
|
||||
|
||||
try:
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
from src.text_helpers import strip_think
|
||||
|
||||
custom = {}
|
||||
@@ -607,6 +612,9 @@ async def update_persona_memory(
|
||||
temperature=0.1,
|
||||
max_tokens=1200,
|
||||
headers=headers,
|
||||
reasoning_effort=effort_for_call(
|
||||
endpoint_url, model, getattr(session, "owner", None), session
|
||||
),
|
||||
)
|
||||
|
||||
updated = strip_think(str(raw or ""), prose=True, prompt_echo=True).strip()
|
||||
@@ -643,6 +651,7 @@ async def audit_memories(
|
||||
"""
|
||||
try:
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
existing = memory_manager.load(owner=owner)
|
||||
if not existing:
|
||||
@@ -691,6 +700,7 @@ async def audit_memories(
|
||||
# Bound the call so the Tidy whirlpool can't spin indefinitely on a
|
||||
# slow/large generation.
|
||||
timeout=120,
|
||||
reasoning_effort=effort_for_call(endpoint_url, model, owner),
|
||||
)
|
||||
|
||||
# Parse the JSON list, tolerating reasoning-model noise: <think> blocks,
|
||||
|
||||
@@ -149,6 +149,7 @@ async def maybe_extract_skill(
|
||||
|
||||
try:
|
||||
from src.llm_core import llm_call_async
|
||||
from src.utility_effort import effort_for_call
|
||||
|
||||
# Get recent messages
|
||||
history = session.get_context_messages()
|
||||
@@ -204,6 +205,7 @@ async def maybe_extract_skill(
|
||||
],
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
reasoning_effort=effort_for_call(endpoint_url, model, owner, session),
|
||||
)
|
||||
logger.debug(
|
||||
"[skill-extract] LLM returned in %.1fs (len=%d, head=%r)",
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Optional
|
||||
from src.model_context import estimate_text_tokens, get_context_length, estimate_tokens
|
||||
from src.llm_core import llm_call_async
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
from src.utility_effort import effort_for_call
|
||||
from src.settings import get_setting
|
||||
from core.models import ChatMessage
|
||||
|
||||
@@ -556,7 +557,13 @@ async def maybe_compact(
|
||||
)
|
||||
|
||||
# Use utility model if configured, otherwise fall back to session model
|
||||
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner)
|
||||
util_url, util_model, util_headers = resolve_endpoint(
|
||||
"utility",
|
||||
fallback_url=endpoint_url,
|
||||
fallback_model=model,
|
||||
fallback_headers=headers,
|
||||
owner=owner,
|
||||
)
|
||||
compact_url = util_url or endpoint_url
|
||||
compact_model = util_model or model
|
||||
compact_headers = util_headers if util_url else headers
|
||||
@@ -580,6 +587,7 @@ async def maybe_compact(
|
||||
max_tokens=SUMMARY_MAX_TOKENS,
|
||||
headers=compact_headers,
|
||||
timeout=30,
|
||||
reasoning_effort=effort_for_call(compact_url, compact_model, owner, session),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Compaction summary failed: {e}")
|
||||
|
||||
+37
-4
@@ -2268,10 +2268,25 @@ def normalize_model_id(
|
||||
return a
|
||||
return None
|
||||
|
||||
|
||||
def _apply_explicit_nonstream_reasoning_effort(
|
||||
payload: Dict, reasoning_effort: Optional[str]
|
||||
) -> None:
|
||||
"""Apply explicit effort to an OpenAI-compatible non-streaming request.
|
||||
|
||||
No explicit effort preserves existing provider-specific defaults.
|
||||
Native Ollama, Anthropic and ChatGPT Responses use separate builders.
|
||||
"""
|
||||
effort = str(reasoning_effort or "").strip().lower()
|
||||
if effort and effort != "default":
|
||||
payload["reasoning_effort"] = effort
|
||||
|
||||
|
||||
def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
|
||||
max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
|
||||
timeout: int = LLMConfig.DEFAULT_TIMEOUT, prompt_type: Optional[str] = None,
|
||||
thinking_mode: Optional[str] = None) -> str:
|
||||
thinking_mode: Optional[str] = None,
|
||||
reasoning_effort: Optional[str] = None) -> str:
|
||||
"""Synchronous LLM call with optional prompt type enhancement."""
|
||||
h = _provider_headers(_detect_provider(url))
|
||||
# Tolerate headers that arrive as a JSON string (some sessions stored them
|
||||
@@ -2303,7 +2318,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
|
||||
provider = _detect_provider(url)
|
||||
cache_key = _get_cache_key(
|
||||
url, model, messages_copy, temperature, max_tokens, headers=headers,
|
||||
thinking_mode=thinking_mode,
|
||||
thinking_mode=thinking_mode, reasoning_effort=reasoning_effort,
|
||||
)
|
||||
cached_response = _get_cached_response(cache_key)
|
||||
if cached_response:
|
||||
@@ -2343,6 +2358,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
|
||||
)
|
||||
if provider == "mistral" and _supports_thinking(model):
|
||||
payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
|
||||
_apply_explicit_nonstream_reasoning_effort(payload, reasoning_effort)
|
||||
try:
|
||||
note_model_activity(target_url, model)
|
||||
r = httpx_post_kimi_aware(target_url, h, json=payload, timeout=timeout)
|
||||
@@ -2465,14 +2481,30 @@ def llm_call_with_fallback(candidates, messages, **kwargs) -> str:
|
||||
|
||||
|
||||
async def llm_call_async_with_fallback(candidates, messages, **kwargs) -> str:
|
||||
"""Async variant of `llm_call_with_fallback` — same semantics."""
|
||||
"""Async variant of `llm_call_with_fallback` — same semantics.
|
||||
|
||||
An optional ``candidate_request_factory(index, url, model, headers)`` may
|
||||
return ``{"messages": ..., "kwargs": {...}}`` to adjust the request per
|
||||
candidate (same contract as `llm_call_async_with_route_fallback`).
|
||||
"""
|
||||
candidate_request_factory = kwargs.pop("candidate_request_factory", None)
|
||||
cands = dedupe_model_candidates(candidates)
|
||||
if not cands:
|
||||
raise HTTPException(503, "No model endpoint configured")
|
||||
last_err = None
|
||||
for i, (url, model, headers) in enumerate(cands):
|
||||
try:
|
||||
return await llm_call_async(url, model, messages, headers=headers, **kwargs)
|
||||
candidate_messages = messages
|
||||
candidate_kwargs = kwargs
|
||||
if candidate_request_factory is not None:
|
||||
request = candidate_request_factory(i, url, model, headers) or {}
|
||||
if hasattr(request, "__await__"):
|
||||
request = await request
|
||||
candidate_messages = request.get("messages", messages)
|
||||
candidate_kwargs = {**kwargs, **(request.get("kwargs") or {})}
|
||||
return await llm_call_async(
|
||||
url, model, candidate_messages, headers=headers, **candidate_kwargs
|
||||
)
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
tag = "primary" if i == 0 else "candidate"
|
||||
@@ -2740,6 +2772,7 @@ async def llm_call_async(
|
||||
_apply_deepseek_v4_reasoning_defaults(
|
||||
payload, target_url, model, thinking_mode
|
||||
)
|
||||
_apply_explicit_nonstream_reasoning_effort(payload, reasoning_effort)
|
||||
|
||||
if _is_host_dead(target_url):
|
||||
raise HTTPException(503, f"Upstream {_host_key(target_url)} marked unreachable (cooldown active)")
|
||||
|
||||
@@ -160,6 +160,10 @@ DEFAULT_SETTINGS = {
|
||||
# Ordered fallback chain for the Utility model (summarization, naming,
|
||||
# tidy actions, etc.).
|
||||
"utility_model_fallbacks": [],
|
||||
# Opt-in reasoning effort for utility/background LLM calls. Empty keeps the
|
||||
# provider default; a value is sent only to utility-chain candidates whose
|
||||
# model advertises that level.
|
||||
"utility_reasoning_effort": "",
|
||||
"teacher_model": "",
|
||||
"teacher_enabled": False,
|
||||
"teacher_tier2_enabled": False,
|
||||
@@ -290,6 +294,7 @@ _PER_USER_KEYS = {
|
||||
# got injected into the chat composer on first open.
|
||||
"default_endpoint_id", "default_model",
|
||||
"utility_endpoint_id", "utility_model", "utility_model_fallbacks",
|
||||
"utility_reasoning_effort",
|
||||
"research_endpoint_id", "research_model",
|
||||
}
|
||||
|
||||
|
||||
@@ -106,4 +106,11 @@ async def task_llm_call_async(
|
||||
raise RuntimeError("No LLM endpoint available for background task")
|
||||
await wait_for_interactive_quiet("background task LLM")
|
||||
kwargs.setdefault("workload", "background")
|
||||
if kwargs.get("reasoning_effort") is None:
|
||||
kwargs.pop("reasoning_effort", None)
|
||||
from src.utility_effort import candidate_effort_factory
|
||||
|
||||
factory = candidate_effort_factory(owner)
|
||||
if factory is not None:
|
||||
kwargs.setdefault("candidate_request_factory", factory)
|
||||
return await llm_call_async_with_fallback(candidates, messages=messages, **kwargs)
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Reasoning effort for utility-role LLM calls.
|
||||
|
||||
One owner for the rule. Precedence for a utility call:
|
||||
|
||||
1. an explicit caller effort (never touched here);
|
||||
2. the ``utility_reasoning_effort`` setting, sent only to routes on the
|
||||
utility chain (the Utility endpoint/model plus its fallbacks, matched by
|
||||
``(url, model)``);
|
||||
3. the chat session's own effort (``thinking_mode == "effort:<level>"``), when
|
||||
the call is made for a chat session and its route is that session's route.
|
||||
|
||||
Every level is validated against what the model advertises; a model without
|
||||
evidence gets nothing. Calls with no session only ever use (2).
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def utility_routes(owner: Optional[str] = None) -> set:
|
||||
"""The ``(url, model)`` routes that make up the utility chain."""
|
||||
from src.endpoint_resolver import resolve_endpoint, resolve_utility_fallback_candidates
|
||||
|
||||
return {
|
||||
(url, model)
|
||||
for url, model, _headers in [
|
||||
resolve_endpoint("utility", owner=owner),
|
||||
*resolve_utility_fallback_candidates(owner=owner),
|
||||
]
|
||||
if url and model
|
||||
}
|
||||
|
||||
|
||||
def configured_effort(owner: Optional[str] = None) -> str:
|
||||
"""The raw setting, lowercased; empty when unset (the default)."""
|
||||
from src.settings import get_user_setting
|
||||
|
||||
return str(get_user_setting("utility_reasoning_effort", owner or "", "") or "").strip().lower()
|
||||
|
||||
|
||||
def fallback_route_efforts(owner: Optional[str] = None) -> dict[tuple[str, str], str]:
|
||||
"""Configured effort per fallback candidate in ``utility_model_fallbacks``."""
|
||||
from src.endpoint_resolver import resolve_endpoint_by_id
|
||||
from src.settings import get_user_setting, load_settings
|
||||
|
||||
try:
|
||||
settings = load_settings()
|
||||
chain = get_user_setting(
|
||||
"utility_model_fallbacks",
|
||||
owner or "",
|
||||
settings.get("utility_model_fallbacks") or [],
|
||||
) or []
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
res = {}
|
||||
for entry in chain:
|
||||
if not isinstance(entry, dict) or "reasoning_effort" not in entry:
|
||||
continue
|
||||
effort = str(entry.get("reasoning_effort") or "").strip().lower()
|
||||
ep = resolve_endpoint_by_id(
|
||||
entry.get("endpoint_id", ""),
|
||||
entry.get("model", ""),
|
||||
owner=owner,
|
||||
)
|
||||
if ep:
|
||||
url, model, _ = ep
|
||||
if url and model:
|
||||
res[(url, model)] = effort
|
||||
return res
|
||||
|
||||
|
||||
def effort_for_route(url: str, model: str, owner: Optional[str] = None) -> Optional[str]:
|
||||
"""Validated effort for ``(url, model)``, or None when it must not be sent."""
|
||||
fb_efforts = fallback_route_efforts(owner)
|
||||
if (url, model) in fb_efforts:
|
||||
fb_effort = fb_efforts[(url, model)]
|
||||
if not fb_effort or fb_effort in {"off", "none", "default"}:
|
||||
return None
|
||||
from src.chatgpt_subscription import validate_reasoning_effort
|
||||
|
||||
return validate_reasoning_effort(model, fb_effort)
|
||||
|
||||
effort = configured_effort(owner)
|
||||
if not effort or (url, model) not in utility_routes(owner):
|
||||
return None
|
||||
from src.chatgpt_subscription import validate_reasoning_effort
|
||||
|
||||
return validate_reasoning_effort(model, effort)
|
||||
|
||||
|
||||
def session_effort(session) -> Optional[str]:
|
||||
"""The raw effort level the chat session is set to, or None."""
|
||||
mode = str(getattr(session, "thinking_mode", "") or "")
|
||||
if not mode.startswith("effort:"):
|
||||
return None
|
||||
return mode[len("effort:"):].strip().lower() or None
|
||||
|
||||
|
||||
def _is_session_route(url: str, model: str, session) -> bool:
|
||||
from src.endpoint_resolver import same_endpoint_base
|
||||
|
||||
return bool(
|
||||
model
|
||||
and model == getattr(session, "model", None)
|
||||
and same_endpoint_base(url, getattr(session, "endpoint_url", None))
|
||||
)
|
||||
|
||||
|
||||
def effort_for_call(url: str, model: str, owner: Optional[str] = None, session=None) -> Optional[str]:
|
||||
"""Effort an owner-aware utility call should send to ``(url, model)``.
|
||||
|
||||
``session`` is the chat session the call is made for, or None for
|
||||
session-less work (scheduled tasks, pollers, auto-sort). Never raises:
|
||||
a lookup failure means "send nothing".
|
||||
"""
|
||||
try:
|
||||
raw = configured_effort(owner)
|
||||
if raw in {"off", "none", "default"}:
|
||||
return None
|
||||
effort = effort_for_route(url, model, owner)
|
||||
if effort or session is None:
|
||||
return effort
|
||||
level = session_effort(session)
|
||||
if not level or not _is_session_route(url, model, session):
|
||||
return None
|
||||
from src.chatgpt_subscription import validate_reasoning_effort
|
||||
|
||||
return validate_reasoning_effort(model, level)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def candidate_effort_factory(owner=None):
|
||||
"""Per-candidate factory for `llm_call_async_with_fallback`.
|
||||
|
||||
Returns None when neither the setting nor any fallback effort is configured.
|
||||
Otherwise a factory sending the effort only to utility-chain candidates
|
||||
(owner-scoped) whose model advertises it.
|
||||
"""
|
||||
raw = configured_effort(owner)
|
||||
has_primary = bool(raw and raw not in {"off", "none", "default"})
|
||||
fb_efforts = fallback_route_efforts(owner)
|
||||
has_fb = any(eff and eff not in {"off", "none", "default"} for eff in fb_efforts.values())
|
||||
if not has_primary and not has_fb:
|
||||
return None
|
||||
|
||||
def factory(_index, url, model, _headers):
|
||||
effort = effort_for_route(url, model, owner)
|
||||
return {"kwargs": {"reasoning_effort": effort}} if effort else {}
|
||||
|
||||
return factory
|
||||
@@ -445,6 +445,7 @@
|
||||
flex-shrink: 0; color: var(--fg); opacity: 0.85;
|
||||
}
|
||||
.adm-provider-logo svg { width: 14px; height: 14px; }
|
||||
.adm-model-logo { margin-right: 8px; }
|
||||
/* Pull the settings-select 4px closer to its logo chip so the row
|
||||
reads as logo+select instead of having an extra gap. */
|
||||
.adm-model-logo + .settings-select { margin-left: -4px; }
|
||||
@@ -454,6 +455,8 @@
|
||||
select shrinks to its content and the trash button rides up
|
||||
tight to it. */
|
||||
.settings-fallback-row > .settings-select { flex: 1 1 0; min-width: 0; }
|
||||
.settings-fallback-row > .settings-select:first-of-type { flex: 0 1 128px; max-width: 34%; }
|
||||
.settings-fallback-row > .settings-select.settings-fallback-effort { flex: 0 1 130px; max-width: 32%; }
|
||||
/* Constrain logo SVGs in settings dropdown chips. Many provider
|
||||
icons in providers.js declare only viewBox (no width/height),
|
||||
so without this rule the browser falls back to 300x150 and
|
||||
|
||||
+12
-2
@@ -989,7 +989,6 @@ a.chat-link[href^="#research-"] {
|
||||
.settings-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.settings-col {
|
||||
display: flex;
|
||||
@@ -998,7 +997,13 @@ a.chat-link[href^="#research-"] {
|
||||
}
|
||||
.settings-label {
|
||||
font-size: 12px;
|
||||
min-width: 70px;
|
||||
min-width: 85px;
|
||||
}
|
||||
#set-utilityEffortLabel { flex: 0 0 85px; line-height: 1.2; }
|
||||
.settings-label-sub {
|
||||
display: block;
|
||||
font-size: 10px;
|
||||
opacity: 0.6;
|
||||
}
|
||||
.settings-select {
|
||||
flex: 1;
|
||||
@@ -1117,6 +1122,11 @@ input.settings-select::placeholder { color: color-mix(in srgb, var(--fg) 35%, tr
|
||||
width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
.settings-fallback-row .settings-select.settings-fallback-effort {
|
||||
flex: 0 1 130px;
|
||||
width: 130px;
|
||||
max-width: 32%;
|
||||
}
|
||||
details.hwfit-serve-advanced > .hwfit-serve-row,
|
||||
details.hwfit-serve-advanced > .hwfit-serve-checks,
|
||||
details.hwfit-serve-advanced > .hwfit-serve-cmd-wrap,
|
||||
|
||||
@@ -1612,6 +1612,12 @@
|
||||
<span class="adm-model-logo" id="set-utilityModelSelect-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span>
|
||||
<select id="set-utilityModelSelect" class="settings-select"><option value="">—</option></select>
|
||||
</div>
|
||||
<div class="settings-row" id="set-utilityEffortRow" style="display:none;">
|
||||
<label class="settings-label" id="set-utilityEffortLabel">Effort<span class="settings-label-sub" id="set-utilityEffortQualifier">(when supported)</span></label>
|
||||
<span class="adm-model-logo" style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;flex-shrink:0;opacity:0.9;color:var(--fg);"></span>
|
||||
<select id="set-utilityEffortSelect" class="settings-select"><option value="">Provider default</option></select>
|
||||
</div>
|
||||
<div class="admin-toggle-sub" id="set-utilityEffortHelp" style="display:none;margin-top:-2px;">Applied to utility and background calls, not chat. Use “Same as chat” to match active chat effort and potentially ensure cache hits to speed up responses. Reasoning tokens count against each call's output limit, so higher efforts can truncate or empty short outputs such as auto-generated titles.</div>
|
||||
<div class="settings-row" style="align-items:flex-start;">
|
||||
<label class="settings-label" style="margin-top:6px;">Fallbacks</label>
|
||||
<div style="flex:1;display:flex;flex-direction:column;gap:6px;">
|
||||
|
||||
+165
-10
@@ -212,9 +212,11 @@ function _bindFallbackWidget(opts) {
|
||||
var endpointsRef = opts.endpoints; // mutable list reference
|
||||
var modelsFilter = opts.modelsFilter || function() { return true; };
|
||||
var settingKey = opts.settingKey;
|
||||
var current = opts.initial || []; // [{endpoint_id, model}]
|
||||
var current = opts.initial || []; // [{endpoint_id, model, reasoning_effort}]
|
||||
var supportsEffort = Boolean(opts.supportsEffort);
|
||||
var getEffortLevels = opts.getEffortLevels;
|
||||
|
||||
if (!fbContainer || !addBtn) return { setEndpoints: function() {}, setInitial: function() {} };
|
||||
if (!fbContainer || !addBtn) return { setEndpoints: function() {}, setInitial: function() {}, refresh: function() {} };
|
||||
|
||||
function enabledEps() { return (endpointsRef() || []).filter(function(e) { return e.is_enabled; }); }
|
||||
|
||||
@@ -233,8 +235,36 @@ function _bindFallbackWidget(opts) {
|
||||
if (selected) selectEl.value = selected;
|
||||
}
|
||||
|
||||
function fillEffort(selectEl, epId, modelId, selected) {
|
||||
if (!selectEl) return;
|
||||
while (selectEl.options.length) selectEl.remove(0);
|
||||
var levels = getEffortLevels ? getEffortLevels(epId, modelId) : [];
|
||||
var base = document.createElement('option');
|
||||
base.value = '';
|
||||
base.textContent = 'Provider default';
|
||||
selectEl.appendChild(base);
|
||||
levels.forEach(function(lvl) {
|
||||
var val = typeof lvl === 'string' ? lvl : (lvl && lvl.effort);
|
||||
if (val) {
|
||||
var o = document.createElement('option');
|
||||
o.value = val.toLowerCase();
|
||||
o.textContent = val;
|
||||
selectEl.appendChild(o);
|
||||
}
|
||||
});
|
||||
var validValues = [''].concat(levels.map(function(l) { return (typeof l === 'string' ? l : l.effort).toLowerCase(); }));
|
||||
selectEl.value = validValues.indexOf((selected || '').toLowerCase()) >= 0 ? (selected || '').toLowerCase() : '';
|
||||
selectEl.style.display = levels.length > 0 ? '' : 'none';
|
||||
}
|
||||
|
||||
async function save() {
|
||||
var clean = current.filter(function(f) { return f.endpoint_id && f.model; });
|
||||
var clean = current.filter(function(f) { return f.endpoint_id && f.model; }).map(function(f) {
|
||||
var item = { endpoint_id: f.endpoint_id, model: f.model };
|
||||
if (supportsEffort && typeof f.reasoning_effort !== 'undefined') {
|
||||
item.reasoning_effort = f.reasoning_effort || '';
|
||||
}
|
||||
return item;
|
||||
});
|
||||
var body = {};
|
||||
body[settingKey] = clean;
|
||||
try {
|
||||
@@ -270,13 +300,36 @@ function _bindFallbackWidget(opts) {
|
||||
fb.endpoint_id = epS.value;
|
||||
fb.model = mS.value;
|
||||
|
||||
var effS = null;
|
||||
if (supportsEffort) {
|
||||
effS = document.createElement('select');
|
||||
effS.className = 'settings-select settings-fallback-effort';
|
||||
fillEffort(effS, epS.value, mS.value, fb.reasoning_effort);
|
||||
fb.reasoning_effort = effS.value;
|
||||
effS.addEventListener('change', function() {
|
||||
fb.reasoning_effort = effS.value;
|
||||
save();
|
||||
});
|
||||
}
|
||||
|
||||
epS.addEventListener('change', function() {
|
||||
fb.endpoint_id = epS.value;
|
||||
fillModels(mS, epS.value, '');
|
||||
fb.model = mS.value;
|
||||
if (effS) {
|
||||
fillEffort(effS, epS.value, mS.value, '');
|
||||
fb.reasoning_effort = effS.value;
|
||||
}
|
||||
save();
|
||||
});
|
||||
mS.addEventListener('change', function() {
|
||||
fb.model = mS.value;
|
||||
if (effS) {
|
||||
fillEffort(effS, epS.value, mS.value, fb.reasoning_effort);
|
||||
fb.reasoning_effort = effS.value;
|
||||
}
|
||||
save();
|
||||
});
|
||||
mS.addEventListener('change', function() { fb.model = mS.value; save(); });
|
||||
|
||||
var rm = document.createElement('button');
|
||||
rm.type = 'button';
|
||||
@@ -292,6 +345,7 @@ function _bindFallbackWidget(opts) {
|
||||
row.appendChild(num);
|
||||
row.appendChild(epS);
|
||||
row.appendChild(mS);
|
||||
if (effS) row.appendChild(effS);
|
||||
row.appendChild(rm);
|
||||
fbContainer.appendChild(row);
|
||||
});
|
||||
@@ -299,7 +353,7 @@ function _bindFallbackWidget(opts) {
|
||||
|
||||
addBtn.addEventListener('click', function() {
|
||||
var first = enabledEps()[0];
|
||||
current.push({ endpoint_id: first ? first.id : '', model: '' });
|
||||
current.push({ endpoint_id: first ? first.id : '', model: '', reasoning_effort: '' });
|
||||
render();
|
||||
save();
|
||||
});
|
||||
@@ -371,9 +425,14 @@ async function initDefaultChat() {
|
||||
async function initUtilityModel() {
|
||||
var epSel = el('set-utilityEpSelect');
|
||||
var modelSel = el('set-utilityModelSelect');
|
||||
var effortSel = el('set-utilityEffortSelect');
|
||||
var effortRow = el('set-utilityEffortRow');
|
||||
var effortHelp = el('set-utilityEffortHelp');
|
||||
var effortQualifier = el('set-utilityEffortQualifier');
|
||||
var msg = el('set-utilityChatMsg');
|
||||
var _endpoints = [];
|
||||
var fallbackWidget = null;
|
||||
var _levelsByEndpoint = {};
|
||||
if (epSel && epSel.options[0]) epSel.options[0].textContent = 'Same as chat';
|
||||
if (modelSel && modelSel.options[0]) modelSel.options[0].textContent = 'Same as chat';
|
||||
|
||||
@@ -382,24 +441,110 @@ async function initUtilityModel() {
|
||||
_fillEndpointSelect(epSel, _endpoints, epSel.value, true);
|
||||
} catch (e) { console.warn('Failed to load endpoints for utility model', e); }
|
||||
|
||||
// Effort levels come from /api/models `models_metadata` (same evidence the
|
||||
// chat composer uses); /api/model-endpoints does not carry them.
|
||||
async function loadEffortLevels() {
|
||||
try {
|
||||
var r = await fetch('/api/models', { credentials: 'same-origin' });
|
||||
if (!r.ok) return;
|
||||
var data = await r.json();
|
||||
var items = Array.isArray(data) ? data : (data.models || data.items || []);
|
||||
_levelsByEndpoint = {};
|
||||
items.forEach(function(item) {
|
||||
if (item && item.endpoint_id) _levelsByEndpoint[item.endpoint_id] = item.models_metadata || {};
|
||||
});
|
||||
} catch (e) { console.warn('Failed to load reasoning levels for utility model', e); }
|
||||
}
|
||||
|
||||
function refreshEffort(selected) {
|
||||
if (!effortSel) return;
|
||||
var targetEp = epSel.value;
|
||||
var targetModel = modelSel.value;
|
||||
if (!targetEp) {
|
||||
var defEp = el('set-defaultEpSelect');
|
||||
var defModel = el('set-defaultModelSelect');
|
||||
if (defEp) targetEp = defEp.value;
|
||||
if (defModel) targetModel = defModel.value;
|
||||
}
|
||||
var meta = (_levelsByEndpoint[targetEp] || {})[targetModel] || {};
|
||||
var levels = Array.isArray(meta.supported_reasoning_levels) ? meta.supported_reasoning_levels.slice() : [];
|
||||
var isInherited = !epSel.value;
|
||||
if (isInherited && levels.length === 0) {
|
||||
var allLevels = {};
|
||||
Object.keys(_levelsByEndpoint).forEach(function(epId) {
|
||||
var models = _levelsByEndpoint[epId] || {};
|
||||
Object.keys(models).forEach(function(mId) {
|
||||
var mLevels = (models[mId] || {}).supported_reasoning_levels;
|
||||
if (Array.isArray(mLevels)) {
|
||||
mLevels.forEach(function(lvl) {
|
||||
var val = typeof lvl === 'string' ? lvl : (lvl && lvl.effort);
|
||||
if (val) allLevels[val.toLowerCase()] = true;
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
levels = Object.keys(allLevels);
|
||||
if (levels.length === 0) {
|
||||
levels = ['low', 'medium', 'high'];
|
||||
}
|
||||
}
|
||||
while (effortSel.options.length) effortSel.remove(0);
|
||||
var base = document.createElement('option');
|
||||
base.value = '';
|
||||
base.textContent = isInherited ? 'Same as chat' : 'Provider default';
|
||||
if (effortQualifier) effortQualifier.style.display = isInherited ? '' : 'none';
|
||||
effortSel.appendChild(base);
|
||||
if (isInherited) {
|
||||
var offOpt = document.createElement('option');
|
||||
offOpt.value = 'off';
|
||||
offOpt.textContent = 'Provider default';
|
||||
effortSel.appendChild(offOpt);
|
||||
}
|
||||
levels.forEach(function(level) {
|
||||
var opt = document.createElement('option');
|
||||
opt.value = level;
|
||||
opt.textContent = level;
|
||||
effortSel.appendChild(opt);
|
||||
});
|
||||
var validValues = [''].concat(isInherited ? ['off'] : []).concat(levels);
|
||||
effortSel.value = validValues.indexOf(selected) >= 0 ? selected : '';
|
||||
var show = isInherited || levels.length > 0;
|
||||
if (effortRow) effortRow.style.display = show ? '' : 'none';
|
||||
if (effortHelp) effortHelp.style.display = show ? '' : 'none';
|
||||
}
|
||||
|
||||
function refreshModels(selectedModel) {
|
||||
var epId = epSel.value;
|
||||
var ep = _endpoints.find(function(e) { return e.id === epId; });
|
||||
_fillModelSelect(modelSel, ep ? ep.models : [], selectedModel, true);
|
||||
}
|
||||
|
||||
await loadEffortLevels();
|
||||
|
||||
try {
|
||||
var res = await fetch('/api/auth/settings', { credentials: 'same-origin' });
|
||||
var settings = await res.json();
|
||||
if (settings.utility_endpoint_id) epSel.value = settings.utility_endpoint_id;
|
||||
refreshModels(settings.utility_model || '');
|
||||
refreshEffort(settings.utility_reasoning_effort || '');
|
||||
fallbackWidget = _bindFallbackWidget({
|
||||
containerId: 'set-utilityFallbacks',
|
||||
addBtnId: 'set-utilityAddFallback',
|
||||
endpoints: function() { return _endpoints; },
|
||||
settingKey: 'utility_model_fallbacks',
|
||||
supportsEffort: true,
|
||||
getEffortLevels: function(epId, modelId) {
|
||||
var meta = (_levelsByEndpoint[epId] || {})[modelId] || {};
|
||||
return Array.isArray(meta.supported_reasoning_levels) ? meta.supported_reasoning_levels : [];
|
||||
},
|
||||
initial: Array.isArray(settings.utility_model_fallbacks)
|
||||
? settings.utility_model_fallbacks.map(function(f) { return { endpoint_id: (f && f.endpoint_id) || '', model: (f && f.model) || '' }; })
|
||||
? settings.utility_model_fallbacks.map(function(f) {
|
||||
return {
|
||||
endpoint_id: (f && f.endpoint_id) || '',
|
||||
model: (f && f.model) || '',
|
||||
reasoning_effort: (f && f.reasoning_effort) || '',
|
||||
};
|
||||
})
|
||||
: [],
|
||||
});
|
||||
} catch (e) { console.warn('Failed to load utility model settings', e); }
|
||||
@@ -411,20 +556,30 @@ async function initUtilityModel() {
|
||||
try {
|
||||
await _postSettings({
|
||||
utility_endpoint_id: epSel.value || '',
|
||||
utility_model: modelSel.value || ''
|
||||
utility_model: modelSel.value || '',
|
||||
utility_reasoning_effort: effortSel ? (effortSel.value || '') : ''
|
||||
});
|
||||
msg.textContent = 'Saved'; msg.style.color = 'var(--fg)';
|
||||
setTimeout(function() { msg.textContent = ''; }, 1500);
|
||||
} catch (e) { msg.textContent = e.status === 403 ? 'Admin access is required to change these settings.' : 'Failed to save'; msg.style.color = 'var(--red)'; }
|
||||
}
|
||||
|
||||
epSel.addEventListener('change', function() { refreshModels(''); saveUtility(); });
|
||||
modelSel.addEventListener('change', saveUtility);
|
||||
// A new endpoint/model may not advertise the saved effort; refreshEffort
|
||||
// resets it to the provider default, and saveUtility persists that.
|
||||
epSel.addEventListener('change', function() { refreshModels(''); refreshEffort(''); saveUtility(); });
|
||||
modelSel.addEventListener('change', function() { refreshEffort(effortSel ? effortSel.value : ''); saveUtility(); });
|
||||
if (effortSel) effortSel.addEventListener('change', saveUtility);
|
||||
var defEp = el('set-defaultEpSelect');
|
||||
var defModel = el('set-defaultModelSelect');
|
||||
if (defEp) defEp.addEventListener('change', function() { if (!epSel.value) refreshEffort(effortSel ? effortSel.value : ''); });
|
||||
if (defModel) defModel.addEventListener('change', function() { if (!epSel.value) refreshEffort(effortSel ? effortSel.value : ''); });
|
||||
|
||||
_registerAiEndpointRefresh(function(endpoints) {
|
||||
_registerAiEndpointRefresh(async function(endpoints) {
|
||||
_endpoints = endpoints;
|
||||
_fillEndpointSelect(epSel, _endpoints, epSel.value, true);
|
||||
refreshModels(modelSel.value);
|
||||
await loadEffortLevels();
|
||||
refreshEffort(effortSel ? effortSel.value : '');
|
||||
if (fallbackWidget && fallbackWidget.refresh) fallbackWidget.refresh();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Opt-in ``utility_reasoning_effort`` for utility/background LLM calls.
|
||||
|
||||
The setting is resolved in ``task_llm_call_async`` and sent only to candidates
|
||||
on the utility chain whose model advertises the level. Explicit caller values
|
||||
win; with the setting empty nothing changes.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from src import llm_core, task_endpoint
|
||||
from src.chatgpt_subscription import validate_reasoning_effort
|
||||
|
||||
UTIL = ("https://api.openai.example/v1", "gpt-5.5", {})
|
||||
UTIL_FALLBACK = ("https://fallback.example/v1", "gpt-5.5-mini", {})
|
||||
OVERRIDE = ("https://override.example/v1", "gpt-5.5", {})
|
||||
NO_EVIDENCE = ("https://local.example/v1", "tiny-local-model", {})
|
||||
|
||||
|
||||
def _stub_resolvers(monkeypatch, utility):
|
||||
"""Stub the resolvers in both modules that look the utility chain up."""
|
||||
from src import endpoint_resolver
|
||||
|
||||
resolve = lambda prefix, *a, **k: { # noqa: E731
|
||||
"utility": utility, "task": OVERRIDE, "default": NO_EVIDENCE,
|
||||
}[prefix]
|
||||
fallbacks = lambda owner=None: [UTIL_FALLBACK] # noqa: E731
|
||||
for mod in (task_endpoint, endpoint_resolver):
|
||||
monkeypatch.setattr(mod, "resolve_endpoint", resolve)
|
||||
monkeypatch.setattr(mod, "resolve_utility_fallback_candidates", fallbacks)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chain(monkeypatch):
|
||||
"""Stub the resolvers and capture what reaches llm_call_async per model."""
|
||||
_stub_resolvers(monkeypatch, UTIL)
|
||||
|
||||
async def _quiet(*_a, **_k):
|
||||
return None
|
||||
monkeypatch.setattr(task_endpoint, "wait_for_interactive_quiet", _quiet)
|
||||
|
||||
calls = {}
|
||||
|
||||
async def fake_call(url, model, messages, **kw):
|
||||
calls[url] = kw.get("reasoning_effort")
|
||||
if url != UTIL_FALLBACK[0]:
|
||||
raise RuntimeError("force walk down the chain")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
|
||||
return calls
|
||||
|
||||
|
||||
def _set_effort(monkeypatch, value):
|
||||
monkeypatch.setattr(
|
||||
"src.settings.get_user_setting",
|
||||
lambda key, owner="", default=None: value if key == "utility_reasoning_effort" else default,
|
||||
)
|
||||
|
||||
|
||||
def _run(**kw):
|
||||
return asyncio.run(task_endpoint.task_llm_call_async(
|
||||
[{"role": "user", "content": "hi"}], **kw,
|
||||
))
|
||||
|
||||
|
||||
def test_setting_applies_only_to_utility_chain(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "high")
|
||||
_run()
|
||||
# The Background Tasks route shares the model id with the utility model but
|
||||
# is a different (url, model) route, so it gets no effort.
|
||||
assert chain[OVERRIDE[0]] is None
|
||||
assert chain[UTIL[0]] == "high"
|
||||
assert chain[UTIL_FALLBACK[0]] == "high"
|
||||
|
||||
|
||||
def test_utility_candidate_gets_validated_effort(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "HIGH")
|
||||
_run()
|
||||
assert chain[UTIL[0]] == "high"
|
||||
|
||||
|
||||
def test_effort_without_evidence_is_dropped(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "high")
|
||||
_stub_resolvers(monkeypatch, ("https://local.example/v1", "tiny-local-model", {}))
|
||||
_run()
|
||||
assert chain["https://local.example/v1"] is None
|
||||
|
||||
|
||||
def test_unsupported_level_is_dropped(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "bogus")
|
||||
_run()
|
||||
assert chain[UTIL[0]] is None
|
||||
|
||||
|
||||
def test_empty_setting_changes_nothing(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "")
|
||||
_run()
|
||||
assert chain and all(effort is None for effort in chain.values())
|
||||
|
||||
|
||||
def test_explicit_caller_effort_wins(chain, monkeypatch):
|
||||
_set_effort(monkeypatch, "high")
|
||||
_run(reasoning_effort="low")
|
||||
assert chain and all(effort == "low" for effort in chain.values())
|
||||
|
||||
|
||||
def test_fallback_factory_can_override_messages_and_kwargs(monkeypatch):
|
||||
seen = []
|
||||
|
||||
async def fake_call(url, model, messages, **kw):
|
||||
seen.append((model, messages, kw.get("reasoning_effort"), kw.get("temperature")))
|
||||
if len(seen) == 1:
|
||||
raise RuntimeError("first fails")
|
||||
return "ok"
|
||||
|
||||
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
|
||||
|
||||
def factory(index, url, model, headers):
|
||||
return {"kwargs": {"reasoning_effort": f"e{index}"}}
|
||||
|
||||
out = asyncio.run(llm_core.llm_call_async_with_fallback(
|
||||
[("http://a/v1", "m1", {}), ("http://b/v1", "m2", {})],
|
||||
[{"role": "user", "content": "x"}],
|
||||
temperature=0.1,
|
||||
candidate_request_factory=factory,
|
||||
))
|
||||
assert out == "ok"
|
||||
assert [(s[0], s[2], s[3]) for s in seen] == [("m1", "e0", 0.1), ("m2", "e1", 0.1)]
|
||||
|
||||
|
||||
def test_validate_reasoning_effort_needs_evidence():
|
||||
assert validate_reasoning_effort("gpt-5.5", "high") == "high"
|
||||
assert validate_reasoning_effort("tiny-local-model", "high") is None
|
||||
assert validate_reasoning_effort("gpt-5.5", "") is None
|
||||
|
||||
|
||||
def test_chatgpt_wire_carries_effort_and_default_omits_it():
|
||||
sent = llm_core._build_chatgpt_responses_payload(
|
||||
"gpt-5.5", [{"role": "user", "content": "hi"}], 0.3, 64, reasoning_effort="high",
|
||||
)
|
||||
assert sent["reasoning"] == {"effort": "high"}
|
||||
plain = llm_core._build_chatgpt_responses_payload(
|
||||
"gpt-5.5", [{"role": "user", "content": "hi"}], 0.3, 64,
|
||||
)
|
||||
assert "reasoning" not in plain
|
||||
|
||||
|
||||
def test_sync_llm_call_cache_key_includes_effort(monkeypatch):
|
||||
keys = []
|
||||
|
||||
def fake_cached(key):
|
||||
keys.append(key)
|
||||
return "cached"
|
||||
|
||||
monkeypatch.setattr(llm_core, "_get_cached_response", fake_cached)
|
||||
msgs = [{"role": "user", "content": "hi"}]
|
||||
llm_core.llm_call("http://x/v1", "m", msgs)
|
||||
llm_core.llm_call("http://x/v1", "m", msgs, reasoning_effort="high")
|
||||
llm_core.llm_call("http://x/v1", "m", msgs, reasoning_effort="HIGH")
|
||||
assert keys[0] != keys[1]
|
||||
assert keys[1] == keys[2]
|
||||
|
||||
|
||||
def test_setting_is_registered_default_and_per_user():
|
||||
from src.settings import DEFAULT_SETTINGS, _PER_USER_KEYS
|
||||
|
||||
assert DEFAULT_SETTINGS["utility_reasoning_effort"] == ""
|
||||
assert "utility_reasoning_effort" in _PER_USER_KEYS
|
||||
|
||||
|
||||
# --- effort_for_call: setting, then the chat session's own effort ---
|
||||
|
||||
CHAT = ("https://chat.example/v1", "gpt-5.5", {})
|
||||
|
||||
|
||||
class _Sess:
|
||||
def __init__(self, thinking_mode="effort:low", url=CHAT[0], model=CHAT[1], owner="alice"):
|
||||
self.thinking_mode = thinking_mode
|
||||
self.endpoint_url = url
|
||||
self.model = model
|
||||
self.owner = owner
|
||||
self.headers = {}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def util(monkeypatch):
|
||||
"""Utility chain = {UTIL}; the setting is controlled via ``settings[owner]``."""
|
||||
from src import utility_effort
|
||||
|
||||
settings = {}
|
||||
monkeypatch.setattr(utility_effort, "utility_routes", lambda owner=None: {(UTIL[0], UTIL[1])})
|
||||
monkeypatch.setattr(
|
||||
"src.settings.get_user_setting",
|
||||
lambda key, owner="", default=None: (
|
||||
settings.get(owner, "") if key == "utility_reasoning_effort" else default
|
||||
),
|
||||
)
|
||||
util_mod = utility_effort
|
||||
util_mod.settings = settings
|
||||
return util_mod
|
||||
|
||||
|
||||
def test_setting_applies_to_utility_route_and_wins_over_chat_effort(util):
|
||||
util.settings["alice"] = "medium"
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess("effort:low")) == "medium"
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "alice") == "medium"
|
||||
|
||||
|
||||
def test_setting_is_per_user(util):
|
||||
util.settings["alice"] = "high"
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "alice") == "high"
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "bob") is None
|
||||
|
||||
|
||||
def test_session_effort_inherited_on_the_session_route(util):
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) == "low"
|
||||
# Endpoint spelling differences (trailing slash) are still the same route.
|
||||
assert util.effort_for_call(CHAT[0] + "/", CHAT[1], "alice", _Sess("effort:low")) == "low"
|
||||
|
||||
|
||||
def test_session_effort_not_inherited_on_another_route(util):
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess("effort:low")) is None
|
||||
assert util.effort_for_call(CHAT[0], "gpt-5.5-mini", "alice", _Sess("effort:low")) is None
|
||||
|
||||
|
||||
def test_session_effort_off_or_missing_inherits_nothing(util):
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("off")) is None
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("")) is None
|
||||
|
||||
|
||||
def test_no_session_means_no_inheritance(util):
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice") is None
|
||||
|
||||
|
||||
def test_explicit_off_setting_blocks_session_effort_inheritance(util):
|
||||
util.settings["alice"] = "off"
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) is None
|
||||
assert util.candidate_effort_factory("alice") is None
|
||||
|
||||
util.settings["alice"] = "default"
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:low")) is None
|
||||
assert util.candidate_effort_factory("alice") is None
|
||||
|
||||
|
||||
def test_inherited_effort_is_validated_against_the_model(util):
|
||||
assert util.effort_for_call(
|
||||
NO_EVIDENCE[0], NO_EVIDENCE[1], "alice",
|
||||
_Sess("effort:low", url=NO_EVIDENCE[0], model=NO_EVIDENCE[1]),
|
||||
) is None
|
||||
assert util.effort_for_call(CHAT[0], CHAT[1], "alice", _Sess("effort:bogus")) is None
|
||||
|
||||
|
||||
def test_effort_for_call_never_raises(util, monkeypatch):
|
||||
def boom(*_a, **_k):
|
||||
raise RuntimeError("db down")
|
||||
|
||||
monkeypatch.setattr(util, "effort_for_route", boom)
|
||||
assert util.effort_for_call(UTIL[0], UTIL[1], "alice", _Sess()) is None
|
||||
|
||||
|
||||
# --- no global fill: callers that are not utility call sites never get an effort ---
|
||||
|
||||
def test_llm_call_async_adds_no_default_effort(util, monkeypatch):
|
||||
# The owner-less helpers resolve at global scope; set it there and per user.
|
||||
util.settings[""] = util.settings["alice"] = "high"
|
||||
seen = []
|
||||
|
||||
def fake_key(url, model, messages, temperature, max_tokens, headers=None,
|
||||
thinking_mode=None, reasoning_effort=None):
|
||||
seen.append(reasoning_effort)
|
||||
return "k"
|
||||
|
||||
monkeypatch.setattr(llm_core, "_get_cache_key", fake_key)
|
||||
monkeypatch.setattr(llm_core, "_get_cached_response", lambda key: "cached")
|
||||
msgs = [{"role": "user", "content": "hi"}]
|
||||
# Same (url, model) as the utility route, but not a utility call site.
|
||||
asyncio.run(llm_core.llm_call_async(UTIL[0], UTIL[1], msgs))
|
||||
llm_core.llm_call(UTIL[0], UTIL[1], msgs)
|
||||
asyncio.run(llm_core.llm_call_async(UTIL[0], UTIL[1], msgs, reasoning_effort="low"))
|
||||
assert seen == [None, None, "low"]
|
||||
|
||||
|
||||
# --- call sites pass the effort explicitly ---
|
||||
|
||||
def _capture_llm(monkeypatch):
|
||||
sent = []
|
||||
|
||||
async def fake_call(url, model, messages, **kw):
|
||||
sent.append((url, model, kw.get("reasoning_effort")))
|
||||
return "A title"
|
||||
|
||||
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
|
||||
return sent
|
||||
|
||||
|
||||
class _History:
|
||||
role = "user"
|
||||
content = "hello there"
|
||||
|
||||
|
||||
def _title_session(**kw):
|
||||
sess = _Sess(**kw)
|
||||
sess.id = "s1"
|
||||
sess.history = [_History()]
|
||||
return sess
|
||||
|
||||
|
||||
class _Manager:
|
||||
def update_session_name(self, *_a):
|
||||
pass
|
||||
|
||||
|
||||
def test_auto_title_inherits_chat_effort_when_utility_is_blank(util, monkeypatch):
|
||||
from routes import chat_helpers
|
||||
|
||||
sent = _capture_llm(monkeypatch)
|
||||
# Blank utility/task resolves to the session's own route.
|
||||
monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (CHAT[0], CHAT[1], {}))
|
||||
asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
|
||||
assert sent == [(CHAT[0], CHAT[1], "low")]
|
||||
|
||||
|
||||
def test_auto_title_setting_beats_chat_effort(util, monkeypatch):
|
||||
from routes import chat_helpers
|
||||
|
||||
util.settings["alice"] = "medium"
|
||||
sent = _capture_llm(monkeypatch)
|
||||
monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
|
||||
asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
|
||||
assert sent == [(UTIL[0], UTIL[1], "medium")]
|
||||
|
||||
|
||||
def test_auto_title_on_a_different_utility_model_gets_no_chat_effort(util, monkeypatch):
|
||||
from routes import chat_helpers
|
||||
|
||||
sent = _capture_llm(monkeypatch)
|
||||
monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
|
||||
asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
|
||||
assert sent == [(UTIL[0], UTIL[1], None)]
|
||||
|
||||
|
||||
def test_memory_extraction_passes_chat_effort(util, monkeypatch):
|
||||
from services.memory import memory_extractor
|
||||
|
||||
sent = []
|
||||
|
||||
async def fake_call(url, model, messages, **kw):
|
||||
sent.append(kw.get("reasoning_effort"))
|
||||
return "[]"
|
||||
|
||||
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
|
||||
|
||||
class Sess(_Sess):
|
||||
def get_context_messages(self):
|
||||
return [{"role": "user", "content": "I live in Oslo"},
|
||||
{"role": "assistant", "content": "Noted"}]
|
||||
|
||||
asyncio.run(memory_extractor.extract_and_store(
|
||||
Sess(), object(), object(), CHAT[0], CHAT[1], {},
|
||||
))
|
||||
assert sent and sent[0] == "low"
|
||||
|
||||
|
||||
def test_email_summary_sends_setting_effort_for_owner_only(util, monkeypatch):
|
||||
from routes.email import email_helpers
|
||||
|
||||
util.settings["alice"] = "medium"
|
||||
sent = []
|
||||
|
||||
async def fake_call(**kw):
|
||||
sent.append(kw.get("reasoning_effort"))
|
||||
return "- point"
|
||||
|
||||
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
|
||||
for owner in ("alice", "bob", None):
|
||||
asyncio.run(email_helpers._generate_email_summary(
|
||||
UTIL[0], UTIL[1], "a@x", "s", "body", owner=owner,
|
||||
))
|
||||
assert sent == ["medium", None, None]
|
||||
|
||||
|
||||
def test_candidate_effort_factory_scopes_to_utility_route(util):
|
||||
from src.utility_effort import candidate_effort_factory
|
||||
|
||||
assert candidate_effort_factory("alice") is None
|
||||
util.settings["alice"] = "medium"
|
||||
factory = candidate_effort_factory("alice")
|
||||
assert factory(0, UTIL[0], UTIL[1], {}) == {"kwargs": {"reasoning_effort": "medium"}}
|
||||
assert factory(1, NO_EVIDENCE[0], NO_EVIDENCE[1], {}) == {}
|
||||
assert candidate_effort_factory("bob") is None
|
||||
|
||||
|
||||
def test_fallback_route_efforts_and_factory(monkeypatch):
|
||||
from src.utility_effort import candidate_effort_factory, effort_for_route
|
||||
|
||||
def fake_resolve_id(ep_id, model, owner=None):
|
||||
if ep_id == "ep-fallback":
|
||||
return ("https://fallback.example/v1", model, {})
|
||||
return None
|
||||
|
||||
monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint_by_id", fake_resolve_id)
|
||||
monkeypatch.setattr(
|
||||
"src.settings.load_settings",
|
||||
lambda: {"utility_model_fallbacks": [{"endpoint_id": "ep-fallback", "model": "gpt-5.5-mini", "reasoning_effort": "low"}]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.settings.get_user_setting",
|
||||
lambda key, owner="", default=None: default,
|
||||
)
|
||||
|
||||
factory = candidate_effort_factory("alice")
|
||||
assert factory is not None
|
||||
assert effort_for_route("https://api.openai.example/v1", "gpt-5.5", "alice") is None
|
||||
assert effort_for_route("https://fallback.example/v1", "gpt-5.5-mini", "alice") == "low"
|
||||
assert factory(0, "https://fallback.example/v1", "gpt-5.5-mini", {}) == {"kwargs": {"reasoning_effort": "low"}}
|
||||
|
||||
|
||||
def test_sync_openai_compatible_wire_transmits_explicit_effort(monkeypatch):
|
||||
import httpx
|
||||
|
||||
sent = []
|
||||
|
||||
def fake_post(url, headers, **kwargs):
|
||||
sent.append(dict(kwargs["json"]))
|
||||
return httpx.Response(
|
||||
200, json={"choices": [{"message": {"content": "ok"}}]}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(llm_core, "_get_cached_response", lambda *_: None)
|
||||
monkeypatch.setattr(llm_core, "_set_cached_response", lambda *a, **k: None)
|
||||
monkeypatch.setattr(llm_core, "note_model_activity", lambda *a: None)
|
||||
monkeypatch.setattr(llm_core, "httpx_post_kimi_aware", fake_post)
|
||||
|
||||
url = "https://example.test/v1"
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
llm_core.llm_call(
|
||||
url, "gpt-5.5", messages, max_tokens=64,
|
||||
reasoning_effort="HIGH",
|
||||
)
|
||||
llm_core.llm_call(url, "gpt-5.5", messages, max_tokens=64)
|
||||
|
||||
assert sent[0]["reasoning_effort"] == "high"
|
||||
assert "reasoning_effort" not in sent[1]
|
||||
|
||||
|
||||
def test_async_openai_compatible_wire_transmits_explicit_effort(monkeypatch):
|
||||
import asyncio
|
||||
import httpx
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
sent = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_slot(*args, **kwargs):
|
||||
yield
|
||||
|
||||
async def fake_post(client, url, headers, **kwargs):
|
||||
sent.append(dict(kwargs["json"]))
|
||||
return httpx.Response(
|
||||
200, json={"choices": [{"message": {"content": "ok"}}]}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(llm_core, "_get_cached_response", lambda *_: None)
|
||||
monkeypatch.setattr(llm_core, "_set_cached_response", lambda *a, **k: None)
|
||||
monkeypatch.setattr(llm_core, "_get_http_client", lambda: object())
|
||||
monkeypatch.setattr(llm_core, "_is_host_dead", lambda *_: False)
|
||||
monkeypatch.setattr(llm_core, "_local_model_slot", fake_slot)
|
||||
monkeypatch.setattr(llm_core, "note_model_activity", lambda *a: None)
|
||||
monkeypatch.setattr(llm_core, "httpx_post_kimi_aware_async", fake_post)
|
||||
|
||||
async def exercise():
|
||||
url = "https://example.test/v1"
|
||||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
await llm_core.llm_call_async(
|
||||
url, "gpt-5.5", messages, max_tokens=0,
|
||||
max_retries=1, reasoning_effort="medium",
|
||||
)
|
||||
await llm_core.llm_call_async(
|
||||
url, "gpt-5.5", messages, max_tokens=0,
|
||||
max_retries=1,
|
||||
)
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
assert sent[0]["reasoning_effort"] == "medium"
|
||||
assert "reasoning_effort" not in sent[1]
|
||||
Reference in New Issue
Block a user