jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-27 08:12:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge current dev and refresh PR #6738 from the 7011 integration

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-10-26 16:21:22 +00:00
parent 30430acfea 4d0610a1eb 39ca749f94
commit 444b60d14e
450 changed files with 60479 additions and 11016 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_clean_agent_preview.py
+185 -354
View File
@@ -109,37 +109,12 @@ def test_deepseek_flash_keeps_tools_but_drops_unsupported_forced_choice():
}
compatible = provider_compatible_tool_choice_request(request, 'deepseek-flash')
# A forced named choice must not hide the other offered tools.
assert [tool['function']['name'] for tool in compatible['tools']] == [
'inspect_media', 'python'
]
assert [tool['function']['name'] for tool in compatible['tools']] == ['inspect_media']
assert 'tool_choice' not in compatible
assert request['tool_choice']['function']['name'] == 'inspect_media'
assert provider_compatible_tool_choice_request(request, 'qwen3.5-9b') is request
def test_deepseek_forced_read_does_not_hide_other_offered_tools():
# Regression: a required calendar read forced tool_choice=manage_calendar on
# a turn that also offered the email tools; the DeepSeek compatibility path
# used to shrink `tools` to manage_calendar only, so the model reported the
# email tools the user asked for were unavailable.
request = {
'model': 'deepseek-v4-flash',
'tools': [
{'type': 'function', 'function': {'name': 'manage_calendar'}},
{'type': 'function', 'function': {'name': 'mcp__email__search_emails'}},
{'type': 'function', 'function': {'name': 'mcp__email__read_email'}},
],
'tool_choice': {'type': 'function', 'function': {'name': 'manage_calendar'}},
}
compatible = provider_compatible_tool_choice_request(request, 'deepseek-v4-flash')
names = [tool['function']['name'] for tool in compatible['tools']]
assert names == [
'manage_calendar', 'mcp__email__search_emails', 'mcp__email__read_email'
]
assert 'tool_choice' not in compatible
def test_deepseek_v4_flash_drops_named_choice_for_thinking_mode():
request = {
'tools': [{'type': 'function', 'function': {'name': 'write_file'}}],
@@ -161,10 +136,8 @@ def test_kimi_k3_drops_named_choice_incompatible_with_thinking_mode():
compatible = provider_compatible_tool_choice_request(request, 'kimi-k3')
# The full tool surface is preserved; only the unsupported forced choice
# is dropped.
assert [tool['function']['name'] for tool in compatible['tools']] == [
'inspect_media', 'write_file'
'inspect_media'
]
assert 'tool_choice' not in compatible
@@ -237,8 +210,8 @@ def test_private_browser_covered_click_does_not_advance_dom_revision():
def test_compact_notes_preserves_create_vs_edit_and_replacement_semantics():
schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'manage_notes')
compact = compact_schemas([schema])[0]['function']
assert 'ONE add call' in compact['description']
assert 'update with id edits an existing note, never add' in compact['description']
assert 'add creates a new note' in compact['description']
assert 'update with id' in compact['description']
assert 'replaces the whole checklist' in compact['parameters']['properties']['checklist_items']['description']
@@ -4400,20 +4373,21 @@ def test_v3_schema_uses_configured_versioned_contract_root(tmp_path, monkeypatch
(contract_root / 'eval_alltools_unseen_compare.py').write_text(
"def tools_for_mode(tools, mode):\n"
" assert mode == 'compact_contract_v5'\n"
" tools[0]['function']['parameters']['properties']['action']['description'] = 'versioned-contract-loaded'\n"
" tools[0]['function']['description'] = 'versioned-contract-loaded'\n"
" return tools\n",
encoding='utf-8',
)
monkeypatch.setenv('ODYSSEUS_TOOL_CONTRACT_ROOT', str(contract_root))
module.contract_builder.cache_clear()
try:
notes = next(
# Probe a tool whose compact description the harness does not
# replace; manage_notes now carries a full harness-owned override.
search = next(
s for s in FUNCTION_TOOL_SCHEMAS
if s['function']['name'] == 'manage_notes'
if s['function']['name'] == 'web_search'
)
compact = module.compact_schemas([notes])[0]
assert compact['function']['parameters']['properties']['action']['description'] == 'versioned-contract-loaded'
assert 'update with id edits an existing note, never add' in compact['function']['description']
compact = module.compact_schemas([search])[0]
assert compact['function']['description'].startswith('versioned-contract-loaded')
finally:
module.contract_builder.cache_clear()
@@ -4425,7 +4399,6 @@ def test_v3_document_edit_schema_has_one_unambiguous_structured_form():
assert parameters['required'] == ['edits']
assert set(parameters['properties']) == {'edits', 'more'}
assert parameters['properties']['more']['type'] == 'boolean'
assert 'more' not in parameters['required']
def test_v3_ui_schema_advertises_only_policy_executable_client_local_actions():
@@ -4560,7 +4533,7 @@ async def test_stream_emits_incremental_text_and_persistable_history(monkeypatch
raw = [chunk async for chunk in stream_preview(endpoint_url='http://test', model='test', messages=[{'role': 'user', 'content': 'hi'}], headers={}, turn_contract=contract, session_id='test', owner='test', disabled_tools=set(), tool_policy=ToolPolicy())]
events = [json.loads(s[6:]) for s in raw if '[DONE]' not in s]
assert [e['delta'] for e in events if 'delta' in e] == ['Hello', ' there']
assert requests[0]['messages'][0]['content'].startswith('You are Ajax, the assistant in Odysseus.')
assert requests[0]['messages'][0]['content'].startswith('You are Odysseus.')
metrics = next(e['data'] for e in events if e.get('type') == 'metrics')
assert metrics['clean_v3_turn'] == [{'role': 'assistant', 'content': 'Hello there'}]
assert raw[-1] == 'data: [DONE]\n\n'
@@ -4596,7 +4569,9 @@ async def test_preview_provider_stream_error_is_terminal_not_empty_answer(monkey
disabled_tools=set(), tool_policy=ToolPolicy(),
)]
assert raw[-1].startswith('event: error\ndata: ')
assert 'Qwen3_5MTPDraftModel' in raw[-1]
assert 'selected model provider failed' in raw[-1]
assert 'provider_stream_error' in raw[-1]
assert 'Qwen3_5MTPDraftModel' not in raw[-1]
assert all('returned no answer' not in chunk for chunk in raw)
assert all('"type": "metrics"' not in chunk for chunk in raw)
@@ -4696,74 +4671,6 @@ def test_ajax_c375_tool_surface_disables_progressive_thinking():
)
@pytest.mark.parametrize('name,args,reason', [
('python', {}, 'code'),
('apply_patch', {}, 'code'),
('edit_document', {}, 'document_edit'),
('draft_email_reply', {}, 'email_draft'),
('manage_tasks', {'action': 'create'}, 'task_design'),
('manage_skills', {'action': 'publish'}, 'skill_evaluation'),
('manage_tasks', {'action': 'list'}, ''),
('manage_tasks', {'action': 'pause'}, ''),
('manage_skills', {'action': 'view'}, ''),
('read_file', {}, ''),
('web_search', {}, ''),
('manage_notes', {'action': 'add'}, ''),
])
def test_tool_deliberation_is_operation_specific(name, args, reason):
from src.clean_agent_preview import tool_deliberation_reason, progressive_thinking_for_turn
assert tool_deliberation_reason(name, args) == reason
schemas = [{'function': {'name': name}}]
assert progressive_thinking_for_turn('ajax_c375', schemas, deliberate=bool(reason)) == bool(reason)
assert not progressive_thinking_for_turn('other-model', schemas, deliberate=True)
@pytest.mark.asyncio
async def test_task_creation_deliberates_before_executing_once(monkeypatch):
import src.clean_agent_preview as module
requests, executions = [], []
arguments = {'action': 'create', 'name': 'Morning news', 'prompt': 'Summarize news', 'schedule': 'daily', 'time': '09:00'}
proposal = {'choices': [{'delta': {'tool_calls': [{'index': 0, 'id': 'task-call',
'function': {'name': 'manage_tasks', 'arguments': json.dumps(arguments)}}]}}]}
responses = iter([proposal, proposal, {'choices': [{'delta': {'content': 'Created the task.'}}]}])
class Response:
def __init__(self, payload): self.payload = payload
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def raise_for_status(self): pass
async def aiter_lines(self):
yield 'data: ' + json.dumps(self.payload)
yield 'data: [DONE]'
class Client:
def __init__(self, **kwargs): pass
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def stream(self, *args, **kwargs):
requests.append(kwargs['json'])
return Response(next(responses))
async def execute(block, **kwargs):
executions.append(block)
return 'manage_tasks', {'output': 'Created task task-1', 'exit_code': 0}
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
monkeypatch.setattr(module, 'execute_tool_block', execute)
schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'manage_tasks']
contract = resolve_full_inventory_contract(schemas=schemas, policy=ToolPolicy())
raw = [chunk async for chunk in stream_preview(
endpoint_url='http://test', model='ajax_c375',
messages=[{'role': 'user', 'content': 'Create a daily task at 9am to summarize news.'}],
headers={}, turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
)]
assert len(executions) == 1, '\n'.join(raw)
assert [r['chat_template_kwargs']['enable_thinking'] for r in requests] == [False, True, False]
assert not any(m.get('tool_calls') for m in requests[1]['messages'])
assert not any('event: error' in chunk for chunk in raw)
@pytest.mark.asyncio
async def test_calendar_list_structured_result_owns_linked_terminal_render(monkeypatch):
import src.clean_agent_preview as module
@@ -5527,9 +5434,7 @@ async def test_model_choice_budget_preserves_evidence_for_final_answer_without_e
@pytest.mark.asyncio
@pytest.mark.parametrize('thinking_mode', ['auto', 'off', 'on'])
@pytest.mark.parametrize('token_limit', [None, 1024])
async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkeypatch, thinking_mode, token_limit):
async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkeypatch):
from dataclasses import replace
import src.clean_agent_preview as module
@@ -5591,17 +5496,13 @@ async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkey
routing_experiment='recent_model_choice',
)
raw = [chunk async for chunk in stream_preview(
endpoint_url='http://test', model='ajax_c375', thinking_mode=thinking_mode,
endpoint_url='http://test', model='test',
messages=[{'role': 'user', 'content': 'Tell me more about that Reuters report.'}],
headers={}, turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4, max_tokens=token_limit,
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
)]
assert executions == ['web_fetch', 'private_browser', 'web_fetch']
thinking = [r['chat_template_kwargs']['enable_thinking'] for r in requests]
# Compact owns the protocol, including for old sessions with On/Off saved.
assert thinking == [False, True, True, False]
assert requests[1]['max_tokens'] == (token_limit or 4096)
assert all(
schema['function']['name'] != 'web_fetch'
for schema in requests[1].get('tools', [])
@@ -5678,8 +5579,7 @@ async def test_duplicate_fetch_does_not_hide_distinct_pagination_request(monkeyp
@pytest.mark.asyncio
@pytest.mark.parametrize('empty_challenge', [False, True])
async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatch, empty_challenge):
async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatch):
import src.clean_agent_preview as module
responses = iter([
@@ -5701,8 +5601,10 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
},
}]}}]},
{'choices': [{'delta': {'content': (
'I found the [AI update](https://example.com/ai) page. '
'The browser search was blocked, so I used native search to locate it.'
'Recent AI developments include new model releases and policy updates. '
'The search results identify the relevant primary sources and dates for each item, '
'including publication details, concrete findings, and links readers can inspect '
'for the full context behind each development.'
)}}]},
])
requests = []
@@ -5731,9 +5633,6 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
if block.tool_type == 'private_browser':
return 'private_browser', {
'output': json.dumps({
'url': 'https://www.google.com/sorry/index?continue=search',
'title': 'Client Challenge', 'snapshot': '(empty page)',
} if empty_challenge else {
'url': 'https://www.google.com/sorry/index?continue=search',
'text': 'Our systems have detected unusual traffic. Complete the reCAPTCHA.',
}),
@@ -5759,7 +5658,7 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
)
raw = [chunk async for chunk in stream_preview(
endpoint_url='http://test', model='test',
messages=[{'role': 'user', 'content': 'Open browser and find the AI update page on example.com. Give me its link.'}],
messages=[{'role': 'user', 'content': 'Open browser and find an AI model release.'}],
headers={}, turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
)]
@@ -5767,7 +5666,6 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
assert executions == ['private_browser', 'web_search']
assert requests[1]['tool_choice'] == 'required'
assert [s['function']['name'] for s in requests[1]['tools']] == ['web_search']
assert not any(chunk.startswith('event: error') for chunk in raw), raw
events = [json.loads(chunk[6:]) for chunk in raw if '[DONE]' not in chunk]
assert any(
event.get('type') == 'tool_loop_recovery'
@@ -5830,10 +5728,9 @@ async def test_native_stream_reserves_remaining_budget_for_required_artifact(mon
async def execute(block, **kwargs):
executed.append(block.tool_type)
if block.tool_type == 'python' and '/tmp_workspace/results/out.md' in block.content:
return block.tool_type, {'output': 'ok', 'exit_code': 0,
'materialized_artifacts': ['/tmp_workspace/results']}
return block.tool_type, {"output": "ok", "exit_code": 0}
return block.tool_type, {"output": "ok", "exit_code": 0,
"materialized_artifacts": ["/tmp_workspace/results"]
if block.tool_type == "python" and "out.md" in block.content else []}
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
monkeypatch.setattr(module, "execute_tool_block", execute)
@@ -5945,10 +5842,9 @@ async def test_native_stream_reserves_wall_time_for_required_artifact(monkeypatc
executed.append(block.tool_type)
if block.tool_type == "web_search":
now[0] = 450.0
if block.tool_type == 'python' and '/tmp_workspace/results/out.md' in block.content:
return block.tool_type, {'output': 'ok', 'exit_code': 0,
'materialized_artifacts': ['/tmp_workspace/results']}
return block.tool_type, {"output": "ok", "exit_code": 0}
return block.tool_type, {"output": "ok", "exit_code": 0,
"materialized_artifacts": ["/tmp_workspace/results"]
if block.tool_type == "python" and "out.md" in block.content else []}
monkeypatch.setattr(module.time, "monotonic", lambda: now[0])
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
@@ -6540,15 +6436,14 @@ async def test_native_stream_permanently_withholds_tool_when_model_ignores_dupli
@pytest.mark.asyncio
@pytest.mark.parametrize('ignores_final_instruction', [False, True])
async def test_native_stream_terminates_after_calling_a_permanently_suppressed_tool(monkeypatch, ignores_final_instruction):
async def test_native_stream_terminates_after_calling_a_permanently_suppressed_tool(monkeypatch):
import src.clean_agent_preview as module
arguments = json.dumps({"path": "/workspace/fixtures/video.mp4"})
responses = iter([
{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": f"inspect-{index}", "function": {
"name": "inspect_media", "arguments": arguments,
}}]}}]}
for index in range(1, 5 if ignores_final_instruction else 4)
for index in range(1, 4)
] + [{"choices": [{"delta": {"content": "Final answer from existing evidence."}}]}])
class Response:
@@ -6600,15 +6495,9 @@ async def test_native_stream_terminates_after_calling_a_permanently_suppressed_t
assert 'tools' not in requests[3]
assert 'best concise final answer' in requests[3]['messages'][-1]['content'].lower()
final = [event for event in events if event.get("type") == "final_response"]
assert final == []
metrics = next(event['data'] for event in events if event.get('type') == 'metrics')
if ignores_final_instruction:
assert len(final) == 1
assert 'model repeated a tool call' in final[0]['content']
assert metrics['clean_v3_turn'][-1]['content'] == final[0]['content']
else:
assert final == []
assert metrics['clean_v3_turn'][-1]['content'] == 'Final answer from existing evidence.'
assert raw[-1] == 'data: [DONE]\n\n'
assert metrics['clean_v3_turn'][-1]['content'] == 'Final answer from existing evidence.'
@pytest.mark.asyncio
@@ -7403,175 +7292,6 @@ async def test_tool_round_leadin_is_replaced_by_terminal_synthesis(monkeypatch):
'The most epic option is the DYVLINGE swivel chair.',
]
@pytest.mark.asyncio
async def test_denied_call_after_successful_evidence_recovers_instead_of_refusing(monkeypatch):
"""A blocked call must not discard evidence an earlier call already produced."""
import src.clean_agent_preview as module
def call(index, name, arguments):
return {'index': index, 'id': f'call-{index}', 'function': {
'name': name, 'arguments': json.dumps(arguments),
}}
packets = iter([
{'choices': [{'delta': {'tool_calls': [
call(0, 'web_search', {'query': 'EEG headset prices'})]}}]},
{'choices': [{'delta': {'tool_calls': [
call(0, 'web_fetch', {'url': 'https://vendor.example/eeg'})]}}]},
{'choices': [{'delta': {'content': (
'Based on the search result, the EEG headset starts at $299 from the '
'vendor. I could not open the vendor page because that operation was '
'not permitted.'
)}}]},
])
requests = []
class Response:
def __init__(self, payload): self.payload = payload
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def raise_for_status(self): pass
async def aiter_lines(self):
yield 'data: ' + json.dumps(self.payload)
yield 'data: [DONE]'
class Client:
def __init__(self, **kwargs): pass
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def stream(self, *args, **kwargs):
requests.append(kwargs['json'])
return Response(next(packets))
async def execute(block, **kwargs):
if block.tool_type == 'web_search':
return 'web_search', {
'output': '[1] EEG Headset Prices\n https://vendor.example/eeg',
'exit_code': 0,
'evidence_status': 'available',
}
return 'web_fetch', {
'output': 'The exact operation is outside server request authority.',
'exit_code': 1,
'blocked': True,
'failure_kind': 'request_authority_denied',
}
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
monkeypatch.setattr(module, 'execute_tool_block', execute)
schemas = [
schema for schema in FUNCTION_TOOL_SCHEMAS
if schema['function']['name'] in {'web_search', 'web_fetch'}
]
contract = resolve_full_inventory_contract(schemas=schemas, policy=ToolPolicy())
raw = [chunk async for chunk in stream_preview(
endpoint_url='http://test', model='test',
messages=[{'role': 'user', 'content': (
'Search the web for the latest EEG headset prices and open the vendor page.'
)}],
headers={}, turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=5,
)]
events = [json.loads(chunk[6:]) for chunk in raw if '[DONE]' not in chunk]
assert any(
event.get('type') == 'completion_recovery'
and event.get('reason') == 'denial_after_successful_evidence'
for event in events
)
visible = ''.join(
str(e.get('delta') or e.get('content') or '') for e in events
)
assert '$299' in visible
assert denied_response() not in visible
# The recovery round must synthesize without tools.
assert requests[-1].get('tools') in (None, [])
@pytest.mark.asyncio
@pytest.mark.parametrize('thinking_mode, expected_thinking', [
('on', {'type': 'enabled'}),
(None, {'type': 'enabled'}),
('off', {'type': 'disabled'}),
])
async def test_deepseek_v4_preview_reserves_reasoning_budget_and_honors_toggle(
monkeypatch, thinking_mode, expected_thinking,
):
import src.clean_agent_preview as module
requests = []
class Response:
def __init__(self, payload): self.payload = payload
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def raise_for_status(self): pass
async def aiter_lines(self):
yield 'data: ' + json.dumps(self.payload)
yield 'data: [DONE]'
class Client:
def __init__(self, **kwargs): pass
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def stream(self, *args, **kwargs):
requests.append(kwargs['json'])
return Response({'choices': [{'delta': {
'reasoning_content': 'Let me reason about this.',
'content': 'The answer is 42.',
}}]})
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
contract = resolve_full_inventory_contract(schemas=[], policy=ToolPolicy())
kwargs = {} if thinking_mode is None else {'thinking_mode': thinking_mode}
raw = [chunk async for chunk in stream_preview(
endpoint_url='https://api.deepseek.com/v1/chat/completions',
model='deepseek-v4-flash',
messages=[{'role': 'user', 'content': 'What is the answer?'}],
headers={}, turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(), **kwargs,
)]
assert requests, 'a provider request should have been sent'
sent = requests[0]
if expected_thinking == {'type': 'disabled'}:
assert sent.get('reasoning_effort') == 'none'
else:
# Reasoning-capable models must reserve budget above the 768 default.
assert sent['max_tokens'] >= 4096
assert sent.get('reasoning_effort')
assert sent.get('thinking') == expected_thinking
assert any('The answer is 42.' in chunk for chunk in raw)
def test_workspace_process_denial_uses_an_honest_refusal(tmp_path):
from src.clean_agent_preview import (
workspace_process_denial,
WORKSPACE_REQUIRED_REFUSAL,
denied_response,
)
# A resource-identity / sealed-workspace denial is recognised from any of
# the tool-event fields (error / output JSON / failure_kind).
assert workspace_process_denial([
{'tool': 'bash', 'exit_code': 1, 'blocked': True,
'output': '{"error":"Process creation requires a sealed workspace and launch scope",'
'"failure_kind":"resource_identity_denied"}'},
])
assert workspace_process_denial([{'failure_kind': 'resource_identity_denied'}])
# A plain request-authority denial is not a workspace denial.
assert not workspace_process_denial([
{'tool': 'bash', 'blocked': True, 'failure_kind': 'request_authority_denied',
'error': 'The exact operation is outside server request authority.'},
])
assert not workspace_process_denial([])
assert 'workspace' in WORKSPACE_REQUIRED_REFUSAL.lower()
assert WORKSPACE_REQUIRED_REFUSAL != denied_response()
# The terminal-denial branch picks the workspace message when appropriate.
source = (Path(__file__).resolve().parents[1] / 'src/clean_agent_preview.py').read_text()
assert 'if workspace_process_denial(executions)' in source
assert 'WORKSPACE_REQUIRED_REFUSAL' in source
def test_spec_router_think_decision_and_reason(monkeypatch):
import sys
import types
@@ -7622,55 +7342,166 @@ def test_spec_router_think_decision_and_reason(monkeypatch):
assert 'spec_router_think_reason(' in source
def test_qwen_think_split_streams_thinking_then_answer_without_buffering():
from src.clean_agent_preview import qwen_think_split_events
def test_spec_direct_no_tools_decision(monkeypatch):
import sys
import types
from src.clean_agent_preview import spec_direct_no_tools_p
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
events = []
content = ''
for piece in ['<think>', 'abc', '</think>', 'Hello']:
content += piece
events.extend(qwen_think_split_events(content, state))
fake = types.ModuleType('src.spec_router')
fake.LAST_DECISIONS = [
{'message': 'Yo', 'choice': 'answer_directly', 'answer_directly': 0.971},
]
monkeypatch.setitem(sys.modules, 'src.spec_router', fake)
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'spec', raising=False,
)
thinking = ''.join(e['delta'] for e in events if e.get('thinking'))
answer = ''.join(e['delta'] for e in events if not e.get('thinking'))
assert thinking == 'abc'
assert answer == 'Hello'
# every emitted delta is non-empty and typed
assert all(e.get('delta') for e in events)
assert [e.get('thinking') for e in events] == [True, None]
# --- default OFF -> no override, identical behaviour --------------------
monkeypatch.delenv('ODYSSEUS_SPEC_DIRECT_NOTOOLS', raising=False)
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
# the stored answer stays identical to visible_content_after_qwen_thinking
from src.clean_agent_preview import visible_content_after_qwen_thinking
assert visible_content_after_qwen_thinking(content) == 'Hello'
# --- ON + answer_directly p >= 0.9 -> returns the confidence ------------
monkeypatch.setenv('ODYSSEUS_SPEC_DIRECT_NOTOOLS', '1')
monkeypatch.delenv('ODYSSEUS_SPEC_DIRECT_MIN_P', raising=False)
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) == 0.971
# --- low confidence -> no override --------------------------------------
fake.LAST_DECISIONS = [
{'message': 'Yo', 'choice': 'answer_directly', 'answer_directly': 0.5},
]
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
def test_qwen_think_split_holds_back_a_partial_close_tag():
from src.clean_agent_preview import qwen_think_split_events
# --- MIN_P override lowers the bar --------------------------------------
monkeypatch.setenv('ODYSSEUS_SPEC_DIRECT_MIN_P', '0.4')
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) == 0.5
monkeypatch.delenv('ODYSSEUS_SPEC_DIRECT_MIN_P', raising=False)
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
events = []
content = ''
for piece in ['<think>a', '</thi', 'nk>', 'B']:
content += piece
events.extend(qwen_think_split_events(content, state))
# --- a non answer_directly choice -> no override ------------------------
fake.LAST_DECISIONS = [
{'message': 'Yo', 'choice': 'web_search', 'answer_directly': 0.971},
]
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
thinking = ''.join(e['delta'] for e in events if e.get('thinking'))
answer = ''.join(e['delta'] for e in events if not e.get('thinking'))
assert thinking == 'a' # the partial '</thi' is never leaked
assert answer == 'B'
assert all(e.get('delta') for e in events)
# --- stale decision for a different message -> no override --------------
fake.LAST_DECISIONS = [
{'message': 'something else entirely', 'choice': 'answer_directly',
'answer_directly': 0.99},
]
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
# --- non-spec surface -> no override ------------------------------------
fake.LAST_DECISIONS = [
{'message': 'Yo', 'choice': 'answer_directly', 'answer_directly': 0.971},
]
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'native', raising=False,
)
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
def test_qwen_think_split_is_inert_without_a_think_block():
from src.clean_agent_preview import qwen_think_split_events
# --- router error -> fail open (None) -----------------------------------
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'spec', raising=False,
)
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
assert qwen_think_split_events('plain answer', state) == []
assert qwen_think_split_events('', state) == []
assert state == {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
class _Boom:
@property
def LAST_DECISIONS(self):
raise RuntimeError('router down')
monkeypatch.setitem(sys.modules, 'src.spec_router', _Boom())
assert spec_direct_no_tools_p('Yo', 'u', 'm', 'o', {}) is None
# source-level: stream_preview wires the incremental split
# --- wiring present in stream_preview -----------------------------------
source = (Path(__file__).resolve().parents[1] / 'src/clean_agent_preview.py').read_text()
assert 'qwen_think_split_events(' in source
assert "think_answer_streamed" in source
assert 'spec_direct_no_tools_p(' in source
assert '[spec-harness] direct answer' in source
def test_action_promise_ignores_fenced_code_and_knows_browse_verbs():
import src.clean_agent_preview as module
gym_reply = (
'I will browse Google Maps to find the closest gym near Todoroki.\n\n'
'```python\n'
'import requests\n'
'from bs4 import BeautifulSoup\n'
'resp = requests.get("https://maps.google.com/search?q=gym+todoroki")\n'
'soup = BeautifulSoup(resp.text, "html.parser")\n'
'for row in soup.select(".section-result"):\n'
' print(row.get_text())\n'
'```\n'
)
assert module.action_promise_response(gym_reply)
assert module.action_promise_response('I will fetch the page and scrape the listing.')
assert module.action_promise_response('Let me search for the nearest gym now.')
assert module.action_promise_response("I'll go check Google Maps for you.")
# A real answer that includes a code block is not an action promise.
assert not module.action_promise_response(
'The closest gym to Todoroki is Gold\u2019s Gym, about 600 m away.\n\n'
'```python\n'
'print("done")\n'
'```\n'
)
assert not module.action_promise_response('Here is the script you asked for:\n```\nprint(1)\n```')
@pytest.mark.parametrize('prompt', [
'please explain how a student should break down work about customer feedback; this is general advice only, do not create or schedule anything.',
'Give general advice about customer feedback and task planning; do not create anything.',
'Explain feedback tasks without scheduling anything.',
])
def test_feedback_advice_does_not_claim_write_authority(prompt):
assert not requests_mutation(prompt)
def test_feedback_subject_does_not_hide_real_write_request():
assert requests_mutation('Schedule a customer feedback task for tomorrow.')
assert requests_mutation('Do not create anything; schedule a customer feedback task tomorrow.')
@pytest.mark.asyncio
async def test_direct_feedback_advice_is_not_replaced_with_preview_denial(monkeypatch):
import src.clean_agent_preview as module
answer = 'Set a goal, group the customer comments by theme, and summarize what you learned.'
class Response:
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def raise_for_status(self): pass
async def aiter_lines(self):
yield 'data: ' + json.dumps({'choices': [{'delta': {'content': answer}}]})
yield 'data: [DONE]'
class Client:
def __init__(self, **kwargs): pass
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def stream(self, *args, **kwargs):
assert 'tools' not in kwargs['json']
return Response()
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
contract = resolve_full_inventory_contract(schemas=[], policy=ToolPolicy())
prompt = ('please explain how a student should break down work about customer feedback; '
'this is general advice only, do not create or schedule anything.')
chunks = [chunk async for chunk in stream_preview(
endpoint_url='http://test', model='ajax-ablated500-base',
messages=[{'role': 'user', 'content': prompt}], headers={},
turn_contract=contract, session_id='test', owner='test',
disabled_tools=set(), tool_policy=ToolPolicy(),
)]
events = [json.loads(chunk[6:]) for chunk in chunks if '[DONE]' not in chunk]
assert ''.join(event.get('delta', '') for event in events) == answer
assert not any(event.get('type') == 'final_response' and event.get('content') == denied_response()
for event in events)