mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-16 03:42:20 +02:00
feat(harness): authority, turn contract, tools, clean_agent_preview
Chat/agent harness work: request-authority model, turn contract, tool
policy/parsing/execution, clean_agent_preview streaming/thinking, ask_user
persistence, session probes, and scheduler.
Source: vendor/photocraft (ad1cff44) harness files, 3-way merged onto
upstream/dev (a8c147b2). Harness core files were independently developed
in both lineages; resolution policy documented in pr_work/C_conflicts.md
(vendor kept where ahead, upstream kept where vendor is behind; full delta
preserved in pr_work/C_upstream_delta.patch).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a8c147b238
commit
364229f2dc
@@ -109,12 +109,37 @@ def test_deepseek_flash_keeps_tools_but_drops_unsupported_forced_choice():
|
||||
}
|
||||
|
||||
compatible = provider_compatible_tool_choice_request(request, 'deepseek-flash')
|
||||
assert [tool['function']['name'] for tool in compatible['tools']] == ['inspect_media']
|
||||
# A forced named choice must not hide the other offered tools.
|
||||
assert [tool['function']['name'] for tool in compatible['tools']] == [
|
||||
'inspect_media', 'python'
|
||||
]
|
||||
assert 'tool_choice' not in compatible
|
||||
assert request['tool_choice']['function']['name'] == 'inspect_media'
|
||||
assert provider_compatible_tool_choice_request(request, 'qwen3.5-9b') is request
|
||||
|
||||
|
||||
def test_deepseek_forced_read_does_not_hide_other_offered_tools():
|
||||
# Regression: a required calendar read forced tool_choice=manage_calendar on
|
||||
# a turn that also offered the email tools; the DeepSeek compatibility path
|
||||
# used to shrink `tools` to manage_calendar only, so the model reported the
|
||||
# email tools the user asked for were unavailable.
|
||||
request = {
|
||||
'model': 'deepseek-v4-flash',
|
||||
'tools': [
|
||||
{'type': 'function', 'function': {'name': 'manage_calendar'}},
|
||||
{'type': 'function', 'function': {'name': 'mcp__email__search_emails'}},
|
||||
{'type': 'function', 'function': {'name': 'mcp__email__read_email'}},
|
||||
],
|
||||
'tool_choice': {'type': 'function', 'function': {'name': 'manage_calendar'}},
|
||||
}
|
||||
compatible = provider_compatible_tool_choice_request(request, 'deepseek-v4-flash')
|
||||
names = [tool['function']['name'] for tool in compatible['tools']]
|
||||
assert names == [
|
||||
'manage_calendar', 'mcp__email__search_emails', 'mcp__email__read_email'
|
||||
]
|
||||
assert 'tool_choice' not in compatible
|
||||
|
||||
|
||||
def test_deepseek_v4_flash_drops_named_choice_for_thinking_mode():
|
||||
request = {
|
||||
'tools': [{'type': 'function', 'function': {'name': 'write_file'}}],
|
||||
@@ -136,8 +161,10 @@ def test_kimi_k3_drops_named_choice_incompatible_with_thinking_mode():
|
||||
|
||||
compatible = provider_compatible_tool_choice_request(request, 'kimi-k3')
|
||||
|
||||
# The full tool surface is preserved; only the unsupported forced choice
|
||||
# is dropped.
|
||||
assert [tool['function']['name'] for tool in compatible['tools']] == [
|
||||
'inspect_media'
|
||||
'inspect_media', 'write_file'
|
||||
]
|
||||
assert 'tool_choice' not in compatible
|
||||
|
||||
@@ -210,8 +237,8 @@ def test_private_browser_covered_click_does_not_advance_dom_revision():
|
||||
def test_compact_notes_preserves_create_vs_edit_and_replacement_semantics():
|
||||
schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'manage_notes')
|
||||
compact = compact_schemas([schema])[0]['function']
|
||||
assert 'add creates a new note' in compact['description']
|
||||
assert 'update with id' in compact['description']
|
||||
assert 'ONE add call' in compact['description']
|
||||
assert 'update with id edits an existing note, never add' in compact['description']
|
||||
assert 'replaces the whole checklist' in compact['parameters']['properties']['checklist_items']['description']
|
||||
|
||||
|
||||
@@ -3511,6 +3538,10 @@ def test_every_compactly_offered_preview_tool_has_valid_policy_permitted_call():
|
||||
'web_fetch': ({'url': 'https://example.com'}, 'read this page'),
|
||||
'web_search': ({'query': 'current AI news'}, 'search the web'),
|
||||
'youtube_tool': ({'action': 'metadata', 'video_url': 'https://youtube.com/watch?v=x'}, 'read YouTube metadata'),
|
||||
'plan_edit': ({'prompt': 'make it black and white'}, 'make it black and white'),
|
||||
'run_editor_command': ({'op': 'document.info'}, 'inspect the document'),
|
||||
'get_editor_screenshot': ({}, 'take a screenshot of the editor'),
|
||||
'ai_edit_image': ({'op': 'remove-bg'}, 'remove the background'),
|
||||
}
|
||||
schemas = {
|
||||
schema['function']['name']: schema
|
||||
@@ -3537,6 +3568,9 @@ def test_every_compactly_offered_preview_tool_has_valid_policy_permitted_call():
|
||||
}
|
||||
else {'image_editing'} if name == 'edit_image'
|
||||
else {'image_generation'} if name == 'generate_image'
|
||||
else {'photo_editor'} if name in {
|
||||
'plan_edit', 'run_editor_command', 'get_editor_screenshot', 'ai_edit_image',
|
||||
}
|
||||
else frozenset()
|
||||
),
|
||||
contract_required_tools={name},
|
||||
@@ -4366,21 +4400,20 @@ def test_v3_schema_uses_configured_versioned_contract_root(tmp_path, monkeypatch
|
||||
(contract_root / 'eval_alltools_unseen_compare.py').write_text(
|
||||
"def tools_for_mode(tools, mode):\n"
|
||||
" assert mode == 'compact_contract_v5'\n"
|
||||
" tools[0]['function']['description'] = 'versioned-contract-loaded'\n"
|
||||
" tools[0]['function']['parameters']['properties']['action']['description'] = 'versioned-contract-loaded'\n"
|
||||
" return tools\n",
|
||||
encoding='utf-8',
|
||||
)
|
||||
monkeypatch.setenv('ODYSSEUS_TOOL_CONTRACT_ROOT', str(contract_root))
|
||||
module.contract_builder.cache_clear()
|
||||
try:
|
||||
# Probe a tool whose compact description the harness does not
|
||||
# replace; manage_notes now carries a full harness-owned override.
|
||||
search = next(
|
||||
notes = next(
|
||||
s for s in FUNCTION_TOOL_SCHEMAS
|
||||
if s['function']['name'] == 'web_search'
|
||||
if s['function']['name'] == 'manage_notes'
|
||||
)
|
||||
compact = module.compact_schemas([search])[0]
|
||||
assert compact['function']['description'].startswith('versioned-contract-loaded')
|
||||
compact = module.compact_schemas([notes])[0]
|
||||
assert compact['function']['parameters']['properties']['action']['description'] == 'versioned-contract-loaded'
|
||||
assert 'update with id edits an existing note, never add' in compact['function']['description']
|
||||
finally:
|
||||
module.contract_builder.cache_clear()
|
||||
|
||||
@@ -4392,6 +4425,7 @@ def test_v3_document_edit_schema_has_one_unambiguous_structured_form():
|
||||
assert parameters['required'] == ['edits']
|
||||
assert set(parameters['properties']) == {'edits', 'more'}
|
||||
assert parameters['properties']['more']['type'] == 'boolean'
|
||||
assert 'more' not in parameters['required']
|
||||
|
||||
|
||||
def test_v3_ui_schema_advertises_only_policy_executable_client_local_actions():
|
||||
@@ -4526,7 +4560,7 @@ async def test_stream_emits_incremental_text_and_persistable_history(monkeypatch
|
||||
raw = [chunk async for chunk in stream_preview(endpoint_url='http://test', model='test', messages=[{'role': 'user', 'content': 'hi'}], headers={}, turn_contract=contract, session_id='test', owner='test', disabled_tools=set(), tool_policy=ToolPolicy())]
|
||||
events = [json.loads(s[6:]) for s in raw if '[DONE]' not in s]
|
||||
assert [e['delta'] for e in events if 'delta' in e] == ['Hello', ' there']
|
||||
assert requests[0]['messages'][0]['content'].startswith('You are Odysseus.')
|
||||
assert requests[0]['messages'][0]['content'].startswith('You are Ajax, the assistant in Odysseus.')
|
||||
metrics = next(e['data'] for e in events if e.get('type') == 'metrics')
|
||||
assert metrics['clean_v3_turn'] == [{'role': 'assistant', 'content': 'Hello there'}]
|
||||
assert raw[-1] == 'data: [DONE]\n\n'
|
||||
@@ -4562,9 +4596,7 @@ async def test_preview_provider_stream_error_is_terminal_not_empty_answer(monkey
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(),
|
||||
)]
|
||||
assert raw[-1].startswith('event: error\ndata: ')
|
||||
assert 'selected model provider failed' in raw[-1]
|
||||
assert 'provider_stream_error' in raw[-1]
|
||||
assert 'Qwen3_5MTPDraftModel' not in raw[-1]
|
||||
assert 'Qwen3_5MTPDraftModel' in raw[-1]
|
||||
assert all('returned no answer' not in chunk for chunk in raw)
|
||||
assert all('"type": "metrics"' not in chunk for chunk in raw)
|
||||
|
||||
@@ -4664,6 +4696,74 @@ def test_ajax_c375_tool_surface_disables_progressive_thinking():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('name,args,reason', [
|
||||
('python', {}, 'code'),
|
||||
('apply_patch', {}, 'code'),
|
||||
('edit_document', {}, 'document_edit'),
|
||||
('draft_email_reply', {}, 'email_draft'),
|
||||
('manage_tasks', {'action': 'create'}, 'task_design'),
|
||||
('manage_skills', {'action': 'publish'}, 'skill_evaluation'),
|
||||
('manage_tasks', {'action': 'list'}, ''),
|
||||
('manage_tasks', {'action': 'pause'}, ''),
|
||||
('manage_skills', {'action': 'view'}, ''),
|
||||
('read_file', {}, ''),
|
||||
('web_search', {}, ''),
|
||||
('manage_notes', {'action': 'add'}, ''),
|
||||
])
|
||||
def test_tool_deliberation_is_operation_specific(name, args, reason):
|
||||
from src.clean_agent_preview import tool_deliberation_reason, progressive_thinking_for_turn
|
||||
assert tool_deliberation_reason(name, args) == reason
|
||||
schemas = [{'function': {'name': name}}]
|
||||
assert progressive_thinking_for_turn('ajax_c375', schemas, deliberate=bool(reason)) == bool(reason)
|
||||
assert not progressive_thinking_for_turn('other-model', schemas, deliberate=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_creation_deliberates_before_executing_once(monkeypatch):
|
||||
import src.clean_agent_preview as module
|
||||
requests, executions = [], []
|
||||
arguments = {'action': 'create', 'name': 'Morning news', 'prompt': 'Summarize news', 'schedule': 'daily', 'time': '09:00'}
|
||||
proposal = {'choices': [{'delta': {'tool_calls': [{'index': 0, 'id': 'task-call',
|
||||
'function': {'name': 'manage_tasks', 'arguments': json.dumps(arguments)}}]}}]}
|
||||
responses = iter([proposal, proposal, {'choices': [{'delta': {'content': 'Created the task.'}}]}])
|
||||
|
||||
class Response:
|
||||
def __init__(self, payload): self.payload = payload
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def raise_for_status(self): pass
|
||||
async def aiter_lines(self):
|
||||
yield 'data: ' + json.dumps(self.payload)
|
||||
yield 'data: [DONE]'
|
||||
|
||||
class Client:
|
||||
def __init__(self, **kwargs): pass
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def stream(self, *args, **kwargs):
|
||||
requests.append(kwargs['json'])
|
||||
return Response(next(responses))
|
||||
|
||||
async def execute(block, **kwargs):
|
||||
executions.append(block)
|
||||
return 'manage_tasks', {'output': 'Created task task-1', 'exit_code': 0}
|
||||
|
||||
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
|
||||
monkeypatch.setattr(module, 'execute_tool_block', execute)
|
||||
schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'manage_tasks']
|
||||
contract = resolve_full_inventory_contract(schemas=schemas, policy=ToolPolicy())
|
||||
raw = [chunk async for chunk in stream_preview(
|
||||
endpoint_url='http://test', model='ajax_c375',
|
||||
messages=[{'role': 'user', 'content': 'Create a daily task at 9am to summarize news.'}],
|
||||
headers={}, turn_contract=contract, session_id='test', owner='test',
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
|
||||
)]
|
||||
assert len(executions) == 1, '\n'.join(raw)
|
||||
assert [r['chat_template_kwargs']['enable_thinking'] for r in requests] == [False, True, False]
|
||||
assert not any(m.get('tool_calls') for m in requests[1]['messages'])
|
||||
assert not any('event: error' in chunk for chunk in raw)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calendar_list_structured_result_owns_linked_terminal_render(monkeypatch):
|
||||
import src.clean_agent_preview as module
|
||||
@@ -5427,7 +5527,9 @@ async def test_model_choice_budget_preserves_evidence_for_final_answer_without_e
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkeypatch):
|
||||
@pytest.mark.parametrize('thinking_mode', ['auto', 'off', 'on'])
|
||||
@pytest.mark.parametrize('token_limit', [None, 1024])
|
||||
async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkeypatch, thinking_mode, token_limit):
|
||||
from dataclasses import replace
|
||||
import src.clean_agent_preview as module
|
||||
|
||||
@@ -5489,13 +5591,17 @@ async def test_failed_static_fetch_recovers_once_through_rendered_browser(monkey
|
||||
routing_experiment='recent_model_choice',
|
||||
)
|
||||
raw = [chunk async for chunk in stream_preview(
|
||||
endpoint_url='http://test', model='test',
|
||||
endpoint_url='http://test', model='ajax_c375', thinking_mode=thinking_mode,
|
||||
messages=[{'role': 'user', 'content': 'Tell me more about that Reuters report.'}],
|
||||
headers={}, turn_contract=contract, session_id='test', owner='test',
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4, max_tokens=token_limit,
|
||||
)]
|
||||
|
||||
assert executions == ['web_fetch', 'private_browser', 'web_fetch']
|
||||
thinking = [r['chat_template_kwargs']['enable_thinking'] for r in requests]
|
||||
# Compact owns the protocol, including for old sessions with On/Off saved.
|
||||
assert thinking == [False, True, True, False]
|
||||
assert requests[1]['max_tokens'] == (token_limit or 4096)
|
||||
assert all(
|
||||
schema['function']['name'] != 'web_fetch'
|
||||
for schema in requests[1].get('tools', [])
|
||||
@@ -5572,7 +5678,8 @@ async def test_duplicate_fetch_does_not_hide_distinct_pagination_request(monkeyp
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatch):
|
||||
@pytest.mark.parametrize('empty_challenge', [False, True])
|
||||
async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatch, empty_challenge):
|
||||
import src.clean_agent_preview as module
|
||||
|
||||
responses = iter([
|
||||
@@ -5594,10 +5701,8 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
|
||||
},
|
||||
}]}}]},
|
||||
{'choices': [{'delta': {'content': (
|
||||
'Recent AI developments include new model releases and policy updates. '
|
||||
'The search results identify the relevant primary sources and dates for each item, '
|
||||
'including publication details, concrete findings, and links readers can inspect '
|
||||
'for the full context behind each development.'
|
||||
'I found the [AI update](https://example.com/ai) page. '
|
||||
'The browser search was blocked, so I used native search to locate it.'
|
||||
)}}]},
|
||||
])
|
||||
requests = []
|
||||
@@ -5626,6 +5731,9 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
|
||||
if block.tool_type == 'private_browser':
|
||||
return 'private_browser', {
|
||||
'output': json.dumps({
|
||||
'url': 'https://www.google.com/sorry/index?continue=search',
|
||||
'title': 'Client Challenge', 'snapshot': '(empty page)',
|
||||
} if empty_challenge else {
|
||||
'url': 'https://www.google.com/sorry/index?continue=search',
|
||||
'text': 'Our systems have detected unusual traffic. Complete the reCAPTCHA.',
|
||||
}),
|
||||
@@ -5651,7 +5759,7 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
|
||||
)
|
||||
raw = [chunk async for chunk in stream_preview(
|
||||
endpoint_url='http://test', model='test',
|
||||
messages=[{'role': 'user', 'content': 'Open browser and find an AI model release.'}],
|
||||
messages=[{'role': 'user', 'content': 'Open browser and find the AI update page on example.com. Give me its link.'}],
|
||||
headers={}, turn_contract=contract, session_id='test', owner='test',
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=4,
|
||||
)]
|
||||
@@ -5659,6 +5767,7 @@ async def test_blocked_search_engine_browser_forces_native_web_search(monkeypatc
|
||||
assert executions == ['private_browser', 'web_search']
|
||||
assert requests[1]['tool_choice'] == 'required'
|
||||
assert [s['function']['name'] for s in requests[1]['tools']] == ['web_search']
|
||||
assert not any(chunk.startswith('event: error') for chunk in raw), raw
|
||||
events = [json.loads(chunk[6:]) for chunk in raw if '[DONE]' not in chunk]
|
||||
assert any(
|
||||
event.get('type') == 'tool_loop_recovery'
|
||||
@@ -5721,9 +5830,10 @@ async def test_native_stream_reserves_remaining_budget_for_required_artifact(mon
|
||||
|
||||
async def execute(block, **kwargs):
|
||||
executed.append(block.tool_type)
|
||||
return block.tool_type, {"output": "ok", "exit_code": 0,
|
||||
"materialized_artifacts": ["/tmp_workspace/results"]
|
||||
if block.tool_type == "python" and "out.md" in block.content else []}
|
||||
if block.tool_type == 'python' and '/tmp_workspace/results/out.md' in block.content:
|
||||
return block.tool_type, {'output': 'ok', 'exit_code': 0,
|
||||
'materialized_artifacts': ['/tmp_workspace/results']}
|
||||
return block.tool_type, {"output": "ok", "exit_code": 0}
|
||||
|
||||
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
|
||||
monkeypatch.setattr(module, "execute_tool_block", execute)
|
||||
@@ -5835,9 +5945,10 @@ async def test_native_stream_reserves_wall_time_for_required_artifact(monkeypatc
|
||||
executed.append(block.tool_type)
|
||||
if block.tool_type == "web_search":
|
||||
now[0] = 450.0
|
||||
return block.tool_type, {"output": "ok", "exit_code": 0,
|
||||
"materialized_artifacts": ["/tmp_workspace/results"]
|
||||
if block.tool_type == "python" and "out.md" in block.content else []}
|
||||
if block.tool_type == 'python' and '/tmp_workspace/results/out.md' in block.content:
|
||||
return block.tool_type, {'output': 'ok', 'exit_code': 0,
|
||||
'materialized_artifacts': ['/tmp_workspace/results']}
|
||||
return block.tool_type, {"output": "ok", "exit_code": 0}
|
||||
|
||||
monkeypatch.setattr(module.time, "monotonic", lambda: now[0])
|
||||
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
|
||||
@@ -6429,14 +6540,15 @@ async def test_native_stream_permanently_withholds_tool_when_model_ignores_dupli
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_stream_terminates_after_calling_a_permanently_suppressed_tool(monkeypatch):
|
||||
@pytest.mark.parametrize('ignores_final_instruction', [False, True])
|
||||
async def test_native_stream_terminates_after_calling_a_permanently_suppressed_tool(monkeypatch, ignores_final_instruction):
|
||||
import src.clean_agent_preview as module
|
||||
arguments = json.dumps({"path": "/workspace/fixtures/video.mp4"})
|
||||
responses = iter([
|
||||
{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": f"inspect-{index}", "function": {
|
||||
"name": "inspect_media", "arguments": arguments,
|
||||
}}]}}]}
|
||||
for index in range(1, 4)
|
||||
for index in range(1, 5 if ignores_final_instruction else 4)
|
||||
] + [{"choices": [{"delta": {"content": "Final answer from existing evidence."}}]}])
|
||||
|
||||
class Response:
|
||||
@@ -6488,9 +6600,15 @@ async def test_native_stream_terminates_after_calling_a_permanently_suppressed_t
|
||||
assert 'tools' not in requests[3]
|
||||
assert 'best concise final answer' in requests[3]['messages'][-1]['content'].lower()
|
||||
final = [event for event in events if event.get("type") == "final_response"]
|
||||
assert final == []
|
||||
metrics = next(event['data'] for event in events if event.get('type') == 'metrics')
|
||||
assert metrics['clean_v3_turn'][-1]['content'] == 'Final answer from existing evidence.'
|
||||
if ignores_final_instruction:
|
||||
assert len(final) == 1
|
||||
assert 'model repeated a tool call' in final[0]['content']
|
||||
assert metrics['clean_v3_turn'][-1]['content'] == final[0]['content']
|
||||
else:
|
||||
assert final == []
|
||||
assert metrics['clean_v3_turn'][-1]['content'] == 'Final answer from existing evidence.'
|
||||
assert raw[-1] == 'data: [DONE]\n\n'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -7284,3 +7402,275 @@ async def test_tool_round_leadin_is_replaced_by_terminal_synthesis(monkeypatch):
|
||||
"I'll browse IKEA and look at their chairs.",
|
||||
'The most epic option is the DYVLINGE swivel chair.',
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_denied_call_after_successful_evidence_recovers_instead_of_refusing(monkeypatch):
|
||||
"""A blocked call must not discard evidence an earlier call already produced."""
|
||||
import src.clean_agent_preview as module
|
||||
|
||||
def call(index, name, arguments):
|
||||
return {'index': index, 'id': f'call-{index}', 'function': {
|
||||
'name': name, 'arguments': json.dumps(arguments),
|
||||
}}
|
||||
|
||||
packets = iter([
|
||||
{'choices': [{'delta': {'tool_calls': [
|
||||
call(0, 'web_search', {'query': 'EEG headset prices'})]}}]},
|
||||
{'choices': [{'delta': {'tool_calls': [
|
||||
call(0, 'web_fetch', {'url': 'https://vendor.example/eeg'})]}}]},
|
||||
{'choices': [{'delta': {'content': (
|
||||
'Based on the search result, the EEG headset starts at $299 from the '
|
||||
'vendor. I could not open the vendor page because that operation was '
|
||||
'not permitted.'
|
||||
)}}]},
|
||||
])
|
||||
requests = []
|
||||
|
||||
class Response:
|
||||
def __init__(self, payload): self.payload = payload
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def raise_for_status(self): pass
|
||||
async def aiter_lines(self):
|
||||
yield 'data: ' + json.dumps(self.payload)
|
||||
yield 'data: [DONE]'
|
||||
|
||||
class Client:
|
||||
def __init__(self, **kwargs): pass
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def stream(self, *args, **kwargs):
|
||||
requests.append(kwargs['json'])
|
||||
return Response(next(packets))
|
||||
|
||||
async def execute(block, **kwargs):
|
||||
if block.tool_type == 'web_search':
|
||||
return 'web_search', {
|
||||
'output': '[1] EEG Headset Prices\n https://vendor.example/eeg',
|
||||
'exit_code': 0,
|
||||
'evidence_status': 'available',
|
||||
}
|
||||
return 'web_fetch', {
|
||||
'output': 'The exact operation is outside server request authority.',
|
||||
'exit_code': 1,
|
||||
'blocked': True,
|
||||
'failure_kind': 'request_authority_denied',
|
||||
}
|
||||
|
||||
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
|
||||
monkeypatch.setattr(module, 'execute_tool_block', execute)
|
||||
schemas = [
|
||||
schema for schema in FUNCTION_TOOL_SCHEMAS
|
||||
if schema['function']['name'] in {'web_search', 'web_fetch'}
|
||||
]
|
||||
contract = resolve_full_inventory_contract(schemas=schemas, policy=ToolPolicy())
|
||||
raw = [chunk async for chunk in stream_preview(
|
||||
endpoint_url='http://test', model='test',
|
||||
messages=[{'role': 'user', 'content': (
|
||||
'Search the web for the latest EEG headset prices and open the vendor page.'
|
||||
)}],
|
||||
headers={}, turn_contract=contract, session_id='test', owner='test',
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), max_rounds=5,
|
||||
)]
|
||||
events = [json.loads(chunk[6:]) for chunk in raw if '[DONE]' not in chunk]
|
||||
assert any(
|
||||
event.get('type') == 'completion_recovery'
|
||||
and event.get('reason') == 'denial_after_successful_evidence'
|
||||
for event in events
|
||||
)
|
||||
visible = ''.join(
|
||||
str(e.get('delta') or e.get('content') or '') for e in events
|
||||
)
|
||||
assert '$299' in visible
|
||||
assert denied_response() not in visible
|
||||
# The recovery round must synthesize without tools.
|
||||
assert requests[-1].get('tools') in (None, [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('thinking_mode, expected_thinking', [
|
||||
('on', {'type': 'enabled'}),
|
||||
(None, {'type': 'enabled'}),
|
||||
('off', {'type': 'disabled'}),
|
||||
])
|
||||
async def test_deepseek_v4_preview_reserves_reasoning_budget_and_honors_toggle(
|
||||
monkeypatch, thinking_mode, expected_thinking,
|
||||
):
|
||||
import src.clean_agent_preview as module
|
||||
|
||||
requests = []
|
||||
|
||||
class Response:
|
||||
def __init__(self, payload): self.payload = payload
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def raise_for_status(self): pass
|
||||
async def aiter_lines(self):
|
||||
yield 'data: ' + json.dumps(self.payload)
|
||||
yield 'data: [DONE]'
|
||||
|
||||
class Client:
|
||||
def __init__(self, **kwargs): pass
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *args): pass
|
||||
def stream(self, *args, **kwargs):
|
||||
requests.append(kwargs['json'])
|
||||
return Response({'choices': [{'delta': {
|
||||
'reasoning_content': 'Let me reason about this.',
|
||||
'content': 'The answer is 42.',
|
||||
}}]})
|
||||
|
||||
monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
|
||||
contract = resolve_full_inventory_contract(schemas=[], policy=ToolPolicy())
|
||||
kwargs = {} if thinking_mode is None else {'thinking_mode': thinking_mode}
|
||||
raw = [chunk async for chunk in stream_preview(
|
||||
endpoint_url='https://api.deepseek.com/v1/chat/completions',
|
||||
model='deepseek-v4-flash',
|
||||
messages=[{'role': 'user', 'content': 'What is the answer?'}],
|
||||
headers={}, turn_contract=contract, session_id='test', owner='test',
|
||||
disabled_tools=set(), tool_policy=ToolPolicy(), **kwargs,
|
||||
)]
|
||||
|
||||
assert requests, 'a provider request should have been sent'
|
||||
sent = requests[0]
|
||||
if expected_thinking == {'type': 'disabled'}:
|
||||
assert sent.get('reasoning_effort') == 'none'
|
||||
else:
|
||||
# Reasoning-capable models must reserve budget above the 768 default.
|
||||
assert sent['max_tokens'] >= 4096
|
||||
assert sent.get('reasoning_effort')
|
||||
assert sent.get('thinking') == expected_thinking
|
||||
assert any('The answer is 42.' in chunk for chunk in raw)
|
||||
|
||||
|
||||
def test_workspace_process_denial_uses_an_honest_refusal(tmp_path):
|
||||
from src.clean_agent_preview import (
|
||||
workspace_process_denial,
|
||||
WORKSPACE_REQUIRED_REFUSAL,
|
||||
denied_response,
|
||||
)
|
||||
# A resource-identity / sealed-workspace denial is recognised from any of
|
||||
# the tool-event fields (error / output JSON / failure_kind).
|
||||
assert workspace_process_denial([
|
||||
{'tool': 'bash', 'exit_code': 1, 'blocked': True,
|
||||
'output': '{"error":"Process creation requires a sealed workspace and launch scope",'
|
||||
'"failure_kind":"resource_identity_denied"}'},
|
||||
])
|
||||
assert workspace_process_denial([{'failure_kind': 'resource_identity_denied'}])
|
||||
# A plain request-authority denial is not a workspace denial.
|
||||
assert not workspace_process_denial([
|
||||
{'tool': 'bash', 'blocked': True, 'failure_kind': 'request_authority_denied',
|
||||
'error': 'The exact operation is outside server request authority.'},
|
||||
])
|
||||
assert not workspace_process_denial([])
|
||||
assert 'workspace' in WORKSPACE_REQUIRED_REFUSAL.lower()
|
||||
assert WORKSPACE_REQUIRED_REFUSAL != denied_response()
|
||||
# The terminal-denial branch picks the workspace message when appropriate.
|
||||
source = (Path(__file__).resolve().parents[1] / 'src/clean_agent_preview.py').read_text()
|
||||
assert 'if workspace_process_denial(executions)' in source
|
||||
assert 'WORKSPACE_REQUIRED_REFUSAL' in source
|
||||
|
||||
|
||||
def test_spec_router_think_decision_and_reason(monkeypatch):
|
||||
import sys
|
||||
import types
|
||||
from src.clean_agent_preview import (
|
||||
spec_router_thinking_decision,
|
||||
spec_router_think_reason,
|
||||
)
|
||||
|
||||
# --- router disabled -> no decision, no reason (default OFF) -------------
|
||||
fake = types.ModuleType('src.spec_router')
|
||||
fake.think_router_enabled = lambda: False
|
||||
fake.route_thinking = lambda text: {'think': True, 'p_think': 0.9, 'ms': 1.0}
|
||||
monkeypatch.setitem(sys.modules, 'src.spec_router', fake)
|
||||
assert spec_router_thinking_decision('u', 'm', 'o', {}, 'hi') is None
|
||||
assert spec_router_think_reason(1, None, '') == ''
|
||||
|
||||
# --- enabled but not a spec surface -> no decision ----------------------
|
||||
fake.think_router_enabled = lambda: True
|
||||
monkeypatch.setattr(
|
||||
'src.agent_loop._configured_model_tool_surface_raw',
|
||||
lambda *a, **k: 'native', raising=False,
|
||||
)
|
||||
assert spec_router_thinking_decision('u', 'm', 'o', {}, 'hi') is None
|
||||
|
||||
# --- enabled on spec + think=True -> reason only on round 1 when no other
|
||||
monkeypatch.setattr(
|
||||
'src.agent_loop._configured_model_tool_surface_raw',
|
||||
lambda *a, **k: 'spec', raising=False,
|
||||
)
|
||||
decision = spec_router_thinking_decision('u', 'm', 'o', {}, 'plan the refactor')
|
||||
assert decision == {'think': True, 'p_think': 0.9, 'ms': 1.0}
|
||||
assert spec_router_think_reason(1, decision, '') == 'router_think'
|
||||
# existing rule wins; only round 1; think=False never fires
|
||||
assert spec_router_think_reason(1, decision, 'code') == ''
|
||||
assert spec_router_think_reason(2, decision, '') == ''
|
||||
assert spec_router_think_reason(1, {'think': False}, '') == ''
|
||||
|
||||
# --- route_thinking raises -> fail open (None), rules only --------------
|
||||
def _boom(text):
|
||||
raise RuntimeError('router down')
|
||||
fake.route_thinking = _boom
|
||||
assert spec_router_thinking_decision('u', 'm', 'o', {}, 'hi') is None
|
||||
assert spec_router_think_reason(1, None, '') == ''
|
||||
|
||||
# --- wiring present in stream_preview -----------------------------------
|
||||
source = (Path(__file__).resolve().parents[1] / 'src/clean_agent_preview.py').read_text()
|
||||
assert 'spec_router_thinking_decision(' in source
|
||||
assert 'spec_router_think_reason(' in source
|
||||
|
||||
|
||||
def test_qwen_think_split_streams_thinking_then_answer_without_buffering():
|
||||
from src.clean_agent_preview import qwen_think_split_events
|
||||
|
||||
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
|
||||
events = []
|
||||
content = ''
|
||||
for piece in ['<think>', 'abc', '</think>', 'Hello']:
|
||||
content += piece
|
||||
events.extend(qwen_think_split_events(content, state))
|
||||
|
||||
thinking = ''.join(e['delta'] for e in events if e.get('thinking'))
|
||||
answer = ''.join(e['delta'] for e in events if not e.get('thinking'))
|
||||
assert thinking == 'abc'
|
||||
assert answer == 'Hello'
|
||||
# every emitted delta is non-empty and typed
|
||||
assert all(e.get('delta') for e in events)
|
||||
assert [e.get('thinking') for e in events] == [True, None]
|
||||
|
||||
# the stored answer stays identical to visible_content_after_qwen_thinking
|
||||
from src.clean_agent_preview import visible_content_after_qwen_thinking
|
||||
assert visible_content_after_qwen_thinking(content) == 'Hello'
|
||||
|
||||
|
||||
def test_qwen_think_split_holds_back_a_partial_close_tag():
|
||||
from src.clean_agent_preview import qwen_think_split_events
|
||||
|
||||
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
|
||||
events = []
|
||||
content = ''
|
||||
for piece in ['<think>a', '</thi', 'nk>', 'B']:
|
||||
content += piece
|
||||
events.extend(qwen_think_split_events(content, state))
|
||||
|
||||
thinking = ''.join(e['delta'] for e in events if e.get('thinking'))
|
||||
answer = ''.join(e['delta'] for e in events if not e.get('thinking'))
|
||||
assert thinking == 'a' # the partial '</thi' is never leaked
|
||||
assert answer == 'B'
|
||||
assert all(e.get('delta') for e in events)
|
||||
|
||||
|
||||
def test_qwen_think_split_is_inert_without_a_think_block():
|
||||
from src.clean_agent_preview import qwen_think_split_events
|
||||
|
||||
state = {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
|
||||
assert qwen_think_split_events('plain answer', state) == []
|
||||
assert qwen_think_split_events('', state) == []
|
||||
assert state == {'think_emitted': 0, 'answer_emitted': 0, 'closed': False}
|
||||
|
||||
# source-level: stream_preview wires the incremental split
|
||||
source = (Path(__file__).resolve().parents[1] / 'src/clean_agent_preview.py').read_text()
|
||||
assert 'qwen_think_split_events(' in source
|
||||
assert "think_answer_streamed" in source
|
||||
|
||||
Reference in New Issue
Block a user