fix(search): keep reasoning out of web search queries and stop misrouting named targets (#7054)

* fix(llm): keep reasoning out of answers when thinking is off

Small-budget utility calls on reasoning models spend every token on
reasoning, and llm_call then returned reasoning_content as the answer. Web
search extraction sent that text to SearXNG as the query, which found
nothing, and session titles fell back as unusable.

Treat thinking_mode="off" as "only the answer": non-streaming calls now
return empty instead of substituting reasoning, matching the streaming path.
Callers that do not turn thinking off keep the existing fallback.

Turn thinking off for search query extraction, session titles without a
configured effort, scheduled task names, and deep research classification,
and strip inline think blocks from the extracted query.

* fix(routing): require computer task wording for named hosts

"on <word>" and "from <word>" counted as a named computer on their own, so
"Who is Jensen Huang from NVIDIA?" and "What happened on Monday?" routed to
shell tools and the workspace toolset.

A target now only counts next to wording that describes work on a machine,
such as "check disk usage on gpu-box" or "copy the logs from gpu-box". The
intent classifier and the agent loop's local computer check share the pattern.

* fix(chat): resolve follow-up references in web search queries

Query extraction only saw the latest message, so "Does he work with Lisa
Su?" lost who "he" was. Include the last four turns, capped per turn, and
instruct the model to use them only to resolve references. Incognito turns
do not use saved session history.

* test(llm): stop reasoning fallback tests hanging in isolation

The module replaced src.agent_tools and core.database with MagicMock before
importing agent_loop. strip_tool_blocks then returned a mock, and
_strip_think_blocks looped forever because a mock's find() never returns -1.
This hung whenever the file ran alone or first on a worker. Import the real
modules instead.

* test(search): remove extra blank line at EOF

---------

Co-authored-by: Alexandre Teixeira <general@alexandreteixeira.dev>
This commit is contained in:
Yu-Wei Chang
2026-11-06 02:11:20 +00:00
committed by GitHub
co-authored by Alexandre Teixeira
parent 1844b084bc
commit e4cc8efa19
14 changed files with 416 additions and 59 deletions
+41
View File
@@ -1,4 +1,7 @@
import pytest
from src.action_intents import classify_tool_intent, message_needs_tools
from src.turn_contract import requested_capabilities
def test_calendar_entry_request_promotes_to_agent():
@@ -97,6 +100,44 @@ def test_page_references_are_not_mistaken_for_named_computers():
assert intent.needs_tools and intent.category == "workspace"
@pytest.mark.parametrize(
"prompt",
[
"Who is Jensen Huang from NVIDIA?",
"Who is Tim Cook from Apple?",
"Who is Satya Nadella from Microsoft?",
"What happened on Monday?",
"What is the best book on Rome?",
"Tell me a story from Japan",
"What is the best GPU from NVIDIA?",
"Who is running from the police?",
],
)
def test_people_and_topics_are_not_mistaken_for_named_computers(prompt):
assert classify_tool_intent(prompt).reason != "named computer task request"
assert "shell_files" not in requested_capabilities(prompt)
@pytest.mark.parametrize(
"prompt",
[
"check disk usage on gpu-box",
"how much free disk on gpu-box?",
"what's the uptime on nas01",
"list running processes on gpu-box",
"is nginx running on gpu-box",
"is ffmpeg installed on workstation",
"what is the gpu usage on gpu-box",
"copy the logs from gpu-box",
"fetch the config file from prod-db",
"download the checkpoint from gpu-box",
],
)
def test_computer_tasks_on_a_named_host_keep_shell_tools(prompt):
assert classify_tool_intent(prompt).category == "workspace"
assert requested_capabilities(prompt) == {"shell_files"}
def test_direct_code_requests_promote_to_workspace_agent():
prompts = [
"write a Python function that parses CSV",
+122
View File
@@ -1,4 +1,5 @@
from unittest.mock import MagicMock
import httpx
from types import SimpleNamespace
from src.chat_processor import ChatProcessor
@@ -93,3 +94,124 @@ def test_build_context_preface_web_search_query_sanitization(monkeypatch):
called_query = mock_web_search.call_args[0][0]
assert len(called_query) <= 150
assert " " not in called_query
def _run_web_preface(monkeypatch, message):
mock_web_search = MagicMock(return_value=("Search Results", []))
monkeypatch.setattr("src.chat_processor.comprehensive_web_search", mock_web_search)
processor = ChatProcessor(memory_manager=MagicMock(), personal_docs_manager=MagicMock())
session = SimpleNamespace(
endpoint_url="http://127.0.0.1:8081/v1/chat/completions", model="gemma4-26b", headers={},
)
processor.build_context_preface(
message=message,
session=session,
use_web=True,
use_rag=False,
use_memory=False,
use_skills=False,
)
return mock_web_search
def test_truncated_reasoning_never_becomes_the_web_search_query(monkeypatch):
"""A reasoning model that exhausts max_tokens falls back to the user's line."""
sent = []
def fake_post(_url, **kwargs):
sent.append(kwargs["json"])
return httpx.Response(
200,
request=httpx.Request("POST", "http://127.0.0.1:8081/v1/chat/completions"),
json={"choices": [{
"finish_reason": "length",
"message": {
"content": "",
"reasoning_content": '* User message: "Who is Jensen?" * Goal: Extract a query.',
},
}]},
)
monkeypatch.setattr("src.llm_core.httpx.post", fake_post)
mock_web_search = _run_web_preface(monkeypatch, "Does he work with Lisa Su?\nThanks")
mock_web_search.assert_called_with(
"Does he work with Lisa Su?", time_filter=None, return_sources=True,
)
assert sent[0]["chat_template_kwargs"] == {"enable_thinking": False}
def test_inline_think_block_never_becomes_the_web_search_query(monkeypatch):
"""Without a reasoning parser, thinking arrives as ordinary content."""
monkeypatch.setattr(
"src.llm_core.llm_call",
MagicMock(return_value="<think>The user is asking about"),
)
mock_web_search = _run_web_preface(monkeypatch, "Who is Jensen Huang?")
mock_web_search.assert_called_with("Who is Jensen Huang?", time_filter=None, return_sources=True)
def _turn(role, content):
return SimpleNamespace(role=role, content=content)
def _extraction_messages(monkeypatch, message, history, **kwargs):
mock_llm_call = MagicMock(return_value="query")
monkeypatch.setattr("src.llm_core.llm_call", mock_llm_call)
monkeypatch.setattr(
"src.chat_processor.comprehensive_web_search",
MagicMock(return_value=("Search Results", [])),
)
processor = ChatProcessor(memory_manager=MagicMock(), personal_docs_manager=MagicMock())
processor.build_context_preface(
message=message,
session=SimpleNamespace(endpoint_url="http://local", model="test", headers={}, history=history),
use_web=True,
use_rag=False,
use_memory=False,
use_skills=False,
**kwargs,
)
return mock_llm_call.call_args.args[2]
def test_follow_up_query_extraction_sees_recent_turns(monkeypatch):
followup = "Does he work with Lisa Su?"
messages = _extraction_messages(monkeypatch, followup, [
_turn("user", "Who is Jensen Huang from NVIDIA?"),
_turn("assistant", "<think>Let me look.</think>Jensen Huang leads NVIDIA."),
_turn("user", followup),
])
assert messages[1]["content"] == (
"Earlier conversation:\n"
"user: Who is Jensen Huang from NVIDIA?\n"
"assistant: Jensen Huang leads NVIDIA.\n\n"
f"Latest message:\n{followup}"
)
def test_first_message_query_extraction_is_only_the_message(monkeypatch):
messages = _extraction_messages(monkeypatch, "Who is Jensen?", [_turn("user", "Who is Jensen?")])
assert messages[1] == {"role": "user", "content": "Who is Jensen?"}
def test_query_extraction_history_is_bounded(monkeypatch):
history = [_turn("user", f"old-{i} " + "x" * 2000) for i in range(10)]
history.append(_turn("assistant", "reply"))
prompt = _extraction_messages(monkeypatch, "next", history)[1]["content"]
assert "old-6" not in prompt
assert "old-7" in prompt and "old-9" in prompt
assert len(prompt) < 2200
def test_incognito_query_extraction_ignores_saved_history(monkeypatch):
messages = _extraction_messages(
monkeypatch, "Does he code?", [_turn("user", "Saved chat"), _turn("assistant", "Reply")],
incognito=True,
)
assert messages[1] == {"role": "user", "content": "Does he code?"}
+19
View File
@@ -93,3 +93,22 @@ def test_small_model_detection_uses_parameter_size_in_model_title():
assert DeepResearcher._looks_like_small_local_model("model-9B")
assert DeepResearcher._looks_like_small_local_model("model-10B")
assert not DeepResearcher._looks_like_small_local_model("model-11B")
def test_category_classification_turns_thinking_off(monkeypatch):
"""Reasoning lists every category name, which would defeat the label scan."""
sent = {}
async def fake_call(**kwargs):
sent.update(kwargs)
return "product"
monkeypatch.setattr("src.llm_core.llm_call_async", fake_call)
r = DeepResearcher.__new__(DeepResearcher)
r.llm_endpoint = "http://x"
r.llm_model = "m"
r.llm_headers = {}
assert asyncio.run(r._classify_category("Best ThinkPad to buy?")) == "product"
assert sent["thinking_mode"] == "off"
assert sent["max_tokens"] == 20
@@ -87,6 +87,58 @@ def test_llm_call_content_wins_over_reasoning_content(monkeypatch):
assert result == "Normal answer"
# ---------------------------------------------------------------------------
# Thinking off: reasoning never stands in for the answer
# ---------------------------------------------------------------------------
def test_llm_call_with_thinking_off_returns_empty_instead_of_reasoning(monkeypatch):
monkeypatch.setattr(
llm_core.httpx, "post",
lambda *a, **kw: _sync_response(_openai_msg("", "* User message: ...")),
)
result = llm_core.llm_call(
"http://test/v1", "qwen3-8b",
[{"role": "user", "content": "extract"}],
thinking_mode="off",
)
assert result == ""
def test_llm_call_async_with_thinking_off_returns_empty_instead_of_reasoning(monkeypatch):
class _FakeAsyncClient:
async def post(self, *a, **kw):
req = httpx.Request("POST", "http://test-async/v1/chat/completions")
return httpx.Response(200, request=req,
json=_openai_msg("", "* User message: ..."))
monkeypatch.setattr(llm_core, "_get_http_client",
lambda: _FakeAsyncClient())
result = asyncio.run(llm_core.llm_call_async(
"http://test-async/v1", "qwen3-8b",
[{"role": "user", "content": "extract"}],
thinking_mode="off",
))
assert result == ""
def test_llm_call_with_thinking_off_drops_mistral_structured_thinking(monkeypatch):
content = [
{"type": "thinking", "thinking": [{"type": "text", "text": "reasoning"}]},
{"type": "text", "text": "answer"},
]
monkeypatch.setattr(
llm_core.httpx, "post",
lambda *a, **kw: _sync_response(_openai_msg(content)),
)
result = llm_core.llm_call(
"http://test/v1", "some-model",
[{"role": "user", "content": "extract"}],
thinking_mode="off",
)
assert result == "answer"
# ---------------------------------------------------------------------------
# Streaming agent path tests (4 and 5)
# These import and test _empty_response_fallback — the real production helper
@@ -94,21 +146,7 @@ def test_llm_call_content_wins_over_reasoning_content(monkeypatch):
# changed, these tests will fail.
# ---------------------------------------------------------------------------
import sys
from unittest.mock import MagicMock
# Mock heavy DB/tool deps before importing agent_loop
for _mod in [
"sqlalchemy", "sqlalchemy.orm", "sqlalchemy.ext",
"sqlalchemy.ext.declarative", "sqlalchemy.ext.hybrid",
"sqlalchemy.sql", "sqlalchemy.sql.expression",
"src.database", "src.agent_tools",
"core.models", "core.database",
]:
if _mod not in sys.modules:
sys.modules[_mod] = MagicMock()
from src.agent_loop import _empty_response_fallback # noqa: E402
from src.agent_loop import _empty_response_fallback
# ---------------------------------------------------------------------------
+30
View File
@@ -17,6 +17,7 @@ clear_fake_database_modules()
import core.database as cdb
import routes.task_routes as task_routes
from core.database import ScheduledTask
from routes.task.task_routes import TaskCreate
_TMPDB = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
_ENGINE = create_engine(
@@ -125,3 +126,32 @@ def test_scheduler_cycle_guard_treats_cross_owner_chain_as_unsafe():
assert scheduler._has_chain_cycle(db, "bob-target-cycle", owner="alice") is True
finally:
db.close()
@pytest.mark.asyncio
async def test_generated_task_name_turns_thinking_off(monkeypatch):
sent = {}
async def fake_call(**kwargs):
sent.update(kwargs)
return "Daily AI News"
monkeypatch.setattr("src.llm_core.llm_call_async", fake_call)
db = _TS()
try:
db.add(cdb.Session(
id="task-name-session", name="s", owner="alice",
endpoint_url="http://127.0.0.1:8081/v1", model="gemma4-26b",
))
db.commit()
finally:
db.close()
create_task = _endpoint("POST", "/api/tasks")
created = await create_task(
SimpleNamespace(state=SimpleNamespace(current_user="alice"), headers={}),
TaskCreate(prompt="Summarize AI news", task_type="llm", trigger_type="webhook"),
)
assert sent["thinking_mode"] == "off"
assert sent["max_tokens"] == 20
assert created["name"] == "Daily AI News"
+7
View File
@@ -196,6 +196,13 @@ def test_network_prompts_are_classified_as_local_computer_work():
assert _looks_like_local_computer_request(prompt), prompt
def test_named_targets_need_computer_task_wording_to_be_local_computer_work():
assert _looks_like_local_computer_request("check disk usage on gpu-box")
assert _looks_like_local_computer_request("copy the logs from gpu-box")
assert not _looks_like_local_computer_request("Who is Jensen Huang from NVIDIA?")
assert not _looks_like_local_computer_request("What is the best book on Rome?")
def test_network_lookup_replaces_ping_with_address_evidence():
normalized = _tui_normalize_network_host_command(
"ping -c 4 ajax.local",
+30
View File
@@ -393,6 +393,36 @@ def test_auto_title_on_a_different_utility_model_gets_no_chat_effort(util, monke
assert sent == [(UTIL[0], UTIL[1], None)]
def _capture_title_thinking(monkeypatch):
sent = []
async def fake_call(url, model, messages, **kw):
sent.append((kw.get("reasoning_effort"), kw.get("thinking_mode")))
return "A title"
monkeypatch.setattr(llm_core, "llm_call_async", fake_call)
return sent
def test_auto_title_turns_thinking_off_without_a_configured_effort(util, monkeypatch):
from routes import chat_helpers
sent = _capture_title_thinking(monkeypatch)
monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
assert sent == [(None, "off")]
def test_auto_title_keeps_thinking_default_with_a_configured_effort(util, monkeypatch):
from routes import chat_helpers
util.settings["alice"] = "medium"
sent = _capture_title_thinking(monkeypatch)
monkeypatch.setattr(task_endpoint, "resolve_task_endpoint", lambda *a, **k: (UTIL[0], UTIL[1], {}))
asyncio.run(chat_helpers.auto_name_session(_Manager(), _title_session()))
assert sent == [("medium", None)]
def test_memory_extraction_passes_chat_effort(util, monkeypatch):
from services.memory import memory_extractor