import json
import sys
import types
from contextlib import asynccontextmanager
from pathlib import Path
import pytest
from src import clean_agent_preview as runner
from src.tool_policy import ToolPolicy
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
from src.turn_contract import (
requested_capabilities,
resolve_turn_contract,
selected_tools_for_request,
)
PROMPT = 'Search the web for the Ajax population.'
def _contract(prompt=PROMPT):
policy = ToolPolicy()
selected = selected_tools_for_request(prompt)
return resolve_turn_contract(
capabilities=requested_capabilities(prompt),
schemas=FUNCTION_TOOL_SCHEMAS,
policy=policy,
selected_tools=selected,
message=prompt,
), policy
def _install_driver_router(monkeypatch, decision, picks=None):
"""Fake src.spec_router exposing spec_route_tools + LAST_DECISIONS."""
fake = types.ModuleType('src.spec_router')
fake.think_router_enabled = lambda: False
fake.LAST_DECISIONS = [decision]
def _spec_route_tools(user_message, schemas, disabled_tools=None,
messages=None, return_decision=False):
if picks is not None:
chosen = picks
elif decision.get('tool_name'):
chosen = [decision['tool_name']]
else:
chosen = list(decision.get('tools') or [])
return (chosen, True) if return_decision else chosen
fake.spec_route_tools = _spec_route_tools
monkeypatch.setitem(sys.modules, 'src.spec_router', fake)
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'spec', raising=False,
)
def _fake_response(captured, scripts):
pending = list(scripts)
class Response:
def __init__(self, script=None):
self._s = script or {}
def raise_for_status(self):
pass
async def aiter_lines(self):
calls = self._s.get('tool_calls')
if calls:
for i, c in enumerate(calls):
yield 'data: ' + json.dumps({'choices': [{'delta': {'tool_calls': [
{'index': i, 'id': f'c{i}', 'type': 'function',
'function': {'name': c['name'],
'arguments': c.get('arguments') or '{}'}}]}}]})
else:
content = self._s.get('content', 'Hi.')
yield 'data: ' + json.dumps({'choices': [{'delta': {'content': content}}]})
yield 'data: [DONE]'
@asynccontextmanager
async def response(*args, **kwargs):
captured.append(args[3] if len(args) > 3 else kwargs.get('request'))
yield Response(pending.pop(0) if pending else None)
return response
async def _run(monkeypatch, captured, model='odysseus-agentic-v4', scripts=None,
max_rounds=1):
monkeypatch.setattr(
runner, 'preview_model_response', _fake_response(captured, scripts or []))
contract, policy = _contract()
events = []
async for chunk in runner.stream_preview(
endpoint_url='http://fixture', model=model, headers={},
messages=[{'role': 'user', 'content': PROMPT}], turn_contract=contract,
session_id='spec-driver', owner='fixture', disabled_tools=set(),
tool_policy=policy, thinking_mode='off', max_rounds=max_rounds,
):
if chunk.startswith('data: ') and '[DONE]' not in chunk:
events.append(json.loads(chunk[6:]))
return contract, events
def _install_recorder(monkeypatch):
executed = []
async def _rec(block, *args, **kwargs):
executed.append(getattr(block, 'tool_type', None))
return 'ok', {'exit_code': 0, 'output': 'ok'}
monkeypatch.setattr(runner, 'execute_tool_block', _rec)
return executed
def test_spec_driver_active_gate(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'spec', raising=False)
assert runner.spec_driver_active('http://x', 'm', 'o', {}) is True
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'strict')
assert runner.spec_driver_active('http://x', 'm', 'o', {}) is False
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
monkeypatch.setattr(
'src.agent_loop._configured_model_tool_surface_raw',
lambda *a, **k: 'compact', raising=False)
assert runner.spec_driver_active('http://x', 'm', 'o', {}) is False
def test_spec_driver_pick_shapes(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
# tool pick at any confidence
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't3', 'tools': ['web_fetch'],
'top': [('web_fetch', 0.12)], 'answer_directly': 0.2, 'ask_user': 0.1,
})
pick = runner.spec_driver_pick([{'role': 'user', 'content': PROMPT}], set(), PROMPT)
assert pick == {'choice': 'tool', 'tool_name': 'web_fetch', 'p': 0.12}
# tool pick below MIN_CONF: the router return is empty but top-1 is a tool
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't5', 'tools': [],
'top': [('web_fetch', 0.12)], 'answer_directly': 0.2, 'ask_user': 0.1,
}, picks=[])
pick = runner.spec_driver_pick([], set(), PROMPT)
assert pick == {'choice': 'tool', 'tool_name': 'web_fetch', 'p': 0.12}
# answer_directly
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 'answer_directly', 'tools': [],
'top': [('answer_directly', 0.4)], 'answer_directly': 0.4, 'ask_user': 0.1,
})
pick = runner.spec_driver_pick([], set(), PROMPT)
assert pick['choice'] == 'answer_directly' and pick['p'] == 0.4
# ask_user
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 'ask_user', 'tools': [],
'top': [('ask_user', 0.3)], 'answer_directly': 0.1, 'ask_user': 0.3,
})
pick = runner.spec_driver_pick([], set(), PROMPT)
assert pick['choice'] == 'ask_user'
# router error / empty decision -> fall back to the normal menu
_install_driver_router(monkeypatch, {'message': PROMPT, 'error': 'timeout'})
assert runner.spec_driver_pick([], set(), PROMPT) is None
@pytest.mark.asyncio
async def test_spec_driver_tool_pick_sends_only_that_tool(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't1', 'tools': ['web_fetch'],
'top': [('web_fetch', 0.11)], 'answer_directly': 0.2, 'ask_user': 0.1,
}, picks=['web_fetch'])
captured = []
await _run(monkeypatch, captured)
assert captured, 'stream_preview made a provider round'
first = captured[0]
names = [(s.get('function') or {}).get('name') for s in (first.get('tools') or [])]
assert names == ['web_fetch']
assert first.get('tool_choice') == {
'type': 'function', 'function': {'name': 'web_fetch'}}
@pytest.mark.asyncio
async def test_spec_driver_answer_directly_omits_tools(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 'answer_directly', 'tools': [],
'top': [('answer_directly', 0.35)], 'answer_directly': 0.35, 'ask_user': 0.05,
}, picks=[])
captured = []
await _run(monkeypatch, captured)
assert captured
assert 'tools' not in captured[0] # vLLM rejects an empty array
@pytest.mark.asyncio
async def test_spec_driver_router_error_keeps_the_normal_menu(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': None, 'tools': [],
'top': [], 'answer_directly': 0.0, 'ask_user': 0.0, 'error': 'boom',
}, picks=[])
captured = []
contract, _events = await _run(monkeypatch, captured)
assert captured
sent = {(s.get('function') or {}).get('name')
for s in (captured[0].get('tools') or [])}
assert sent == set(contract.offered) # unchanged normal menu
assert sent
@pytest.mark.asyncio
async def test_spec_driver_pick_uses_the_full_spec_surface(monkeypatch):
"""m3518: Spec picks from the FULL surface (not the compact round menu) and
the pick is admitted/executable that round (sent as the single tool)."""
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't1', 'tools': ['manage_calendar'],
'top': [('manage_calendar', 0.9)], 'answer_directly': 0.05, 'ask_user': 0.05,
}, picks=['manage_calendar'])
captured = []
await _run(monkeypatch, captured)
assert captured
names = [(s.get('function') or {}).get('name')
for s in (captured[0].get('tools') or [])]
assert names == ['manage_calendar']
@pytest.mark.asyncio
async def test_spec_driver_pick_not_on_the_surface_keeps_the_normal_menu(monkeypatch):
"""Only a pick that is truly not executable falls back for the round."""
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't1', 'tools': ['totally_missing_tool'],
'top': [('totally_missing_tool', 0.9)], 'answer_directly': 0.05, 'ask_user': 0.05,
}, picks=['totally_missing_tool'])
captured = []
contract, _events = await _run(monkeypatch, captured)
assert captured
sent = {(s.get('function') or {}).get('name')
for s in (captured[0].get('tools') or [])}
assert 'totally_missing_tool' not in sent
assert sent == set(contract.offered) # normal menu retained
@pytest.mark.asyncio
async def test_spec_driver_drops_off_pick_native_call(monkeypatch):
"""m3517: execute ONLY the Spec pick; drop any other native call."""
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 't1', 'tools': ['web_fetch'],
'top': [('web_fetch', 0.7)], 'answer_directly': 0.1, 'ask_user': 0.1,
}, picks=['web_fetch'])
executed = _install_recorder(monkeypatch)
scripts = [
{'tool_calls': [
{'name': 'web_search', 'arguments': '{"query": "x"}'},
{'name': 'web_fetch', 'arguments': '{"url": "http://x"}'},
]},
{'content': 'done'},
]
captured = []
await _run(monkeypatch, captured, scripts=scripts, max_rounds=2)
assert executed == ['web_fetch'], executed
@pytest.mark.asyncio
async def test_spec_driver_answer_directly_strips_markup_and_executes_nothing(monkeypatch):
monkeypatch.setenv('ODYSSEUS_SPEC_MODE', 'driver')
_install_driver_router(monkeypatch, {
'message': PROMPT, 'choice': 'answer_directly', 'tools': [],
'top': [('answer_directly', 0.9)], 'answer_directly': 0.9, 'ask_user': 0.05,
}, picks=[])
executed = _install_recorder(monkeypatch)
scripts = [{
'content': 'Hello {"name": "web_search", "arguments": {}} world',
}]
captured = []
_contract_, events = await _run(monkeypatch, captured, scripts=scripts)
assert executed == []
finals = [e.get('content', '') for e in events if e.get('type') == 'final_response']
assert finals, events
assert 'Hello' in finals[-1] and '