preserve parallel tool result ordering before visual evidence

This commit is contained in:
pewdiepie-archdaemon
2026-09-17 23:14:26 +00:00
parent 1eb1afc88b
commit 3e82f0429f
2 changed files with 80 additions and 10 deletions
+17 -10
View File
@@ -4667,6 +4667,12 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac
terminal_budget_violation = False
structured_terminal_response = ''
round_recovery_messages = []
# Keep an OpenAI-compatible tool-call batch contiguous. A
# multimodal user message inserted between sibling tool
# results makes providers such as DeepSeek reject the next
# request with HTTP 400. Collect visual evidence while the
# batch executes and append it only after every tool result.
round_visual_blocks = []
for call in proposed:
name = offered_tool_alias(call['function']['name'], round_offered)
arguments = call['function']['arguments']
@@ -5411,6 +5417,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac
visual_blocks = bounded_visual_result_blocks(result, max_images=3)
if visual_blocks:
tool_event['screenshot'] = visual_blocks[0]['image_url']['url']
round_visual_blocks.extend(visual_blocks)
record_tool_execution(executions, tool_event)
yield event(tool_event)
history.append({'role': 'tool', 'tool_call_id': call['id'], 'content': output})
@@ -5596,18 +5603,18 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac
"I couldn't check the inbox because one or more email accounts are "
"currently unavailable. No reliable empty-inbox result was returned."
)
if visual_blocks:
visual_message = untrusted_context_message(
'tool visual evidence',
'Visual evidence returned by tool execution.',
)
visual_message['content'] = [
{'type': 'text', 'text': visual_message['content']},
*visual_blocks,
]
history.append(visual_message)
if policy_denied:
terminal_denial = True
if round_visual_blocks:
visual_message = untrusted_context_message(
'tool visual evidence',
'Visual evidence returned by tool execution.',
)
visual_message['content'] = [
{'type': 'text', 'text': visual_message['content']},
*round_visual_blocks[:3],
]
history.append(visual_message)
if artifact_body_handoff_target:
force_no_tools_next_round = True
replace_streamed_draft_on_finish = True
+63
View File
@@ -5607,3 +5607,66 @@ async def test_native_stream_marks_masked_shell_pipeline_failure_as_error(monkey
assert output['error'] is True
assert output['exit_code'] == 1
assert 'No such file or directory' in output['output']
@pytest.mark.asyncio
async def test_parallel_tool_results_precede_visual_evidence(monkeypatch):
import src.clean_agent_preview as module
responses = iter([
{"choices": [{"delta": {"tool_calls": [
{"index": 0, "id": "inspect", "function": {
"name": "inspect_media", "arguments": json.dumps({"path": "/workspace/input.png"})}},
{"index": 1, "id": "list", "function": {
"name": "ls", "arguments": json.dumps({"path": "/workspace"})}},
]}}]},
{"choices": [{"delta": {"content": "Done."}}]},
])
class Response:
def __init__(self, payload): self.payload = payload
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def raise_for_status(self): pass
async def aiter_lines(self):
yield "data: " + json.dumps(self.payload)
yield "data: [DONE]"
requests = []
class Client:
def __init__(self, **kwargs): pass
async def __aenter__(self): return self
async def __aexit__(self, *args): pass
def stream(self, *args, **kwargs):
requests.append(kwargs["json"])
return Response(next(responses))
async def execute(block, **kwargs):
if block.tool_type == "inspect_media":
return "inspect_media", {
"output": "image evidence", "exit_code": 0,
"images": [{"mimeType": "image/png", "data": "aQ=="}],
}
return "ls", {"output": "input.png", "exit_code": 0}
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
monkeypatch.setattr(module, "execute_tool_block", execute)
schemas = [
next(item for item in FUNCTION_TOOL_SCHEMAS if item["function"]["name"] == name)
for name in ("inspect_media", "ls")
]
contract = resolve_full_inventory_contract(schemas=schemas, policy=ToolPolicy())
_ = [chunk async for chunk in stream_preview(
endpoint_url="http://test", model="test", headers={},
messages=[{"role": "user", "content": "Inspect and list."}],
turn_contract=contract, session_id="test", owner="test",
disabled_tools=set(), tool_policy=ToolPolicy(), workspace="/tmp/workspace",
client_runtime_context={"surface": "odysseus-native", "terminal_agent": True,
"unattended_mode": True}, max_rounds=2,
)]
messages = requests[1]["messages"]
assistant_index = max(i for i, message in enumerate(messages) if message["role"] == "assistant")
assert [message["role"] for message in messages[assistant_index + 1:]] == ["tool", "tool", "user"]
assert messages[-1]["content"][1]["type"] == "image_url"