mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-27 02:22:21 +02:00
Squash Odysseus development history
This commit is contained in:
pewdiepie-archdaemon
@@ -0,0 +1,798 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import src.agent_loop as agent_loop
|
||||
from src.tool_parsing import ToolBlock
|
||||
from src.tool_capabilities import ToolGateDecision
|
||||
|
||||
|
||||
def _events(chunks):
|
||||
parsed = []
|
||||
for chunk in chunks:
|
||||
if not chunk.startswith("data: ") or chunk.startswith("data: [DONE]"):
|
||||
continue
|
||||
parsed.append(json.loads(chunk[6:]))
|
||||
return parsed
|
||||
|
||||
|
||||
def _patch_loop(monkeypatch, responses, captured_kwargs=None):
|
||||
monkeypatch.setattr(agent_loop, "get_setting", lambda key, default=None: default)
|
||||
monkeypatch.setattr(agent_loop, "get_mcp_manager", lambda: None)
|
||||
monkeypatch.setattr(agent_loop, "blocked_tools_for_owner", lambda owner: set())
|
||||
monkeypatch.setattr(agent_loop, "estimate_tokens", lambda *args, **kwargs: 10)
|
||||
monkeypatch.setattr(agent_loop, "tool_result_should_arm_gate", lambda *args, **kwargs: False)
|
||||
monkeypatch.setattr(
|
||||
agent_loop.ToolRunSecurityContext,
|
||||
"decision_for",
|
||||
lambda self, *args, **kwargs: ToolGateDecision(True),
|
||||
)
|
||||
|
||||
async def execute(block, *args, **kwargs):
|
||||
return block.tool_type, {
|
||||
"output": f"executed {block.tool_type}",
|
||||
"exit_code": 0,
|
||||
}
|
||||
|
||||
call_index = 0
|
||||
|
||||
async def stream(_candidates, messages, **kwargs):
|
||||
nonlocal call_index
|
||||
if captured_kwargs is not None:
|
||||
captured_kwargs.append(kwargs)
|
||||
response = responses[min(call_index, len(responses) - 1)]
|
||||
call_index += 1
|
||||
yield f'data: {json.dumps({"delta": response})}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
|
||||
monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream)
|
||||
return lambda: call_index
|
||||
|
||||
|
||||
def _run(instruction, *, max_rounds=4, relevant_tools=None, runtime_context=None):
|
||||
async def collect():
|
||||
return [
|
||||
chunk
|
||||
async for chunk in agent_loop.stream_agent_loop(
|
||||
"http://unused.test/v1",
|
||||
"test-model",
|
||||
[{"role": "user", "content": instruction}],
|
||||
max_rounds=max_rounds,
|
||||
relevant_tools=relevant_tools or {"write_file"},
|
||||
owner="pewds",
|
||||
client_runtime_context=runtime_context or {"terminal_agent": True},
|
||||
)
|
||||
]
|
||||
|
||||
return _events(asyncio.run(collect()))
|
||||
|
||||
|
||||
def test_failed_workspace_mutation_attempts_are_not_hidden_by_successful_probe():
|
||||
failed = ToolBlock("python", 'open("/workspace/answer.png", "wb")')
|
||||
probe = ToolBlock("inspect_media", '{"path":"/workspace/input.mp4"}')
|
||||
records = [
|
||||
{"tool_name": "python", "result": {"error": "conversion failed", "exit_code": 1}},
|
||||
{"tool_name": "inspect_media", "result": {"output": "evidence", "exit_code": 0}},
|
||||
]
|
||||
|
||||
assert agent_loop._failed_workspace_mutation_attempts([failed, probe], records) == 1
|
||||
|
||||
|
||||
def test_terminal_completion_repairs_missing_artifact_at_most_twice(monkeypatch):
|
||||
calls = _patch_loop(monkeypatch, ["Done without writing anything."])
|
||||
|
||||
events = _run("Write answer.json", max_rounds=4)
|
||||
|
||||
blocked = [event for event in events if event.get("type") == "completion_blocked"]
|
||||
assert [event["attempt"] for event in blocked] == [1, 2]
|
||||
assert calls() == 3
|
||||
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
|
||||
assert decision["status"] == "blocked"
|
||||
assert decision["missing_artifacts"] == ["answer.json"]
|
||||
|
||||
|
||||
def test_failed_trailing_tool_with_planning_prose_continues_artifact_task(monkeypatch):
|
||||
calls = _patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
"I found the source and will inspect it now.\n```bash\nfalse\n```",
|
||||
"Done without creating the required artifact.",
|
||||
],
|
||||
)
|
||||
|
||||
async def fail_execute(block, *args, **kwargs):
|
||||
return block.tool_type, {
|
||||
"output": "no matches",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute)
|
||||
|
||||
events = _run(
|
||||
"Create answer.json after inspecting the source",
|
||||
max_rounds=3,
|
||||
relevant_tools={"bash", "write_file"},
|
||||
)
|
||||
|
||||
assert calls() > 1
|
||||
assert any(
|
||||
event.get("type") == "completion_blocked"
|
||||
and event.get("decision", {}).get("missing_artifacts") == ["answer.json"]
|
||||
for event in events
|
||||
)
|
||||
|
||||
|
||||
def test_exact_failed_call_is_blocked_across_planning_and_intervening_failure(monkeypatch):
|
||||
repeated = "python3 /tmp_workspace/classify_and_copy.py"
|
||||
executed = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
f'I will run the classifier now.\n<tool_call><invoke name="bash"><parameter name="command">{repeated}</parameter></invoke></tool_call>',
|
||||
"I will repair the variable first.\n<tool_call><invoke name=\"bash\"><parameter name=\"command\">python3 -c 'print(classifications)'</parameter></invoke></tool_call>",
|
||||
f'I will retry the classifier.\n<tool_call><invoke name="bash"><parameter name="command">{repeated}</parameter></invoke></tool_call>',
|
||||
"The work is complete.",
|
||||
],
|
||||
)
|
||||
|
||||
async def fail_execute(block, *args, **kwargs):
|
||||
executed.append(block.content)
|
||||
return block.tool_type, {"output": f"failed: {block.content}", "exit_code": 1}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute)
|
||||
|
||||
events = _run(
|
||||
"Create /tmp_workspace/results after classifying the files",
|
||||
max_rounds=4,
|
||||
relevant_tools={"bash", "write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["/tmp_workspace/results"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert executed.count(repeated) == 1, events
|
||||
blocked = [event for event in events if event.get("type") == "tool_retry_blocked"]
|
||||
assert len(blocked) == 1
|
||||
assert blocked[0]["previous_round"] == 1
|
||||
assert blocked[0]["command"] == repeated
|
||||
|
||||
|
||||
def test_exact_failed_call_can_retry_after_successful_workspace_mutation(monkeypatch):
|
||||
repeated = "python3 /tmp_workspace/classify_and_copy.py"
|
||||
executed = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
f'<tool_call><invoke name="bash"><parameter name="command">{repeated}</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="write_file"><parameter name="path">/tmp_workspace/repair.py</parameter><parameter name="content">fixed = True</parameter></invoke></tool_call>',
|
||||
f'<tool_call><invoke name="bash"><parameter name="command">{repeated}</parameter></invoke></tool_call>',
|
||||
"Repair attempted.",
|
||||
],
|
||||
)
|
||||
|
||||
async def execute(block, *args, **kwargs):
|
||||
executed.append((block.tool_type, block.content))
|
||||
if block.tool_type == "write_file":
|
||||
return block.tool_type, {"output": "written", "exit_code": 0}
|
||||
return block.tool_type, {"output": "classifier failed", "exit_code": 1}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
|
||||
|
||||
events = _run(
|
||||
"Create /tmp_workspace/results after repairing and running the classifier",
|
||||
max_rounds=4,
|
||||
relevant_tools={"bash", "write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["/tmp_workspace/results"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert sum(content == repeated for _, content in executed) == 2
|
||||
assert not any(event.get("type") == "tool_retry_blocked" for event in events)
|
||||
|
||||
|
||||
def test_terminal_artifact_task_repairs_after_consecutive_failed_batches(monkeypatch):
|
||||
executed = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'<tool_call><invoke name="python"><parameter name="code">print(missing_one)</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="python"><parameter name="code">print(missing_two)</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="python"><parameter name="code">print(missing_three)</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="python"><parameter name="code">print(missing_four)</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="python"><parameter name="code">print(missing_five)</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="write_file"><parameter name="path">answer.json</parameter><parameter name="content">{\"ok\": true}</parameter></invoke></tool_call>',
|
||||
"Created answer.json.",
|
||||
],
|
||||
)
|
||||
|
||||
async def execute(block, *args, **kwargs):
|
||||
executed.append((block.tool_type, block.content))
|
||||
if block.tool_type == "write_file":
|
||||
return block.tool_type, {"output": "written", "exit_code": 0}
|
||||
return block.tool_type, {
|
||||
"error": f"NameError from {block.content}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
|
||||
|
||||
events = _run(
|
||||
"Create answer.json and verify it",
|
||||
max_rounds=7,
|
||||
relevant_tools={"python", "write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"failed_tool_round_limit": 5,
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["answer.json"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
repairs = [event for event in events if event.get("type") == "artifact_repair_required"]
|
||||
assert len(repairs) == 1
|
||||
assert repairs[0]["attempt"] == 1
|
||||
assert "missing_five" in repairs[0]["last_failure"]
|
||||
assert not any(
|
||||
event.get("type") == "loop_breaker_triggered"
|
||||
and event.get("reason") == "consecutive_tool_failures"
|
||||
for event in events
|
||||
)
|
||||
assert any(tool == "write_file" for tool, _ in executed)
|
||||
decision = next(
|
||||
event["data"]
|
||||
for event in events
|
||||
if event.get("type") == "completion_decision"
|
||||
)
|
||||
assert decision["can_complete"] is True
|
||||
|
||||
|
||||
def test_varied_failed_artifact_mutations_have_cumulative_cap(monkeypatch):
|
||||
executed = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'<tool_call><invoke name="python"><parameter name="code">'
|
||||
f'open("/workspace/answer.json", "w").write(missing_{index})'
|
||||
'</parameter></invoke></tool_call>'
|
||||
for index in range(1, 9)
|
||||
],
|
||||
)
|
||||
|
||||
async def execute(block, *args, **kwargs):
|
||||
executed.append((block.tool_type, block.content))
|
||||
return block.tool_type, {
|
||||
"error": f"NameError from {block.content}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
|
||||
|
||||
events = _run(
|
||||
"Create answer.json and verify it",
|
||||
max_rounds=8,
|
||||
relevant_tools={"python", "write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"failed_tool_round_limit": 99,
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["answer.json"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
guard = next(
|
||||
event for event in events
|
||||
if event.get("type") == "loop_breaker_triggered"
|
||||
and event.get("reason") == "cumulative_artifact_mutation_failures"
|
||||
)
|
||||
assert guard["failed_batches"] == 6
|
||||
assert len(executed) == 6
|
||||
|
||||
|
||||
def test_exact_successful_read_is_blocked_until_workspace_changes(monkeypatch):
|
||||
inspection = "find /tmp_workspace/results -type f | wc -l"
|
||||
executed = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
f'I found partial output and will count it.\n<tool_call><invoke name="bash"><parameter name="command">{inspection}</parameter></invoke></tool_call>',
|
||||
f'I should check the count once more.\n<tool_call><invoke name="bash"><parameter name="command">{inspection}</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="write_file"><parameter name="path">answer.json</parameter><parameter name="content">{\"ok\": true}</parameter></invoke></tool_call>',
|
||||
"Created answer.json.",
|
||||
],
|
||||
)
|
||||
|
||||
async def execute(block, *args, **kwargs):
|
||||
executed.append((block.tool_type, block.content))
|
||||
if block.tool_type == "write_file":
|
||||
return block.tool_type, {"output": "written", "exit_code": 0}
|
||||
return block.tool_type, {"output": "7", "exit_code": 0}
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
|
||||
|
||||
events = _run(
|
||||
"Create answer.json from the inspected workspace",
|
||||
max_rounds=4,
|
||||
relevant_tools={"bash", "write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"completion_requirements": {"required_artifacts": ["answer.json"]},
|
||||
},
|
||||
)
|
||||
|
||||
assert sum(content == inspection for _, content in executed) == 1
|
||||
blocked = [
|
||||
event for event in events
|
||||
if event.get("type") == "tool_retry_blocked"
|
||||
and event.get("reason") == "repeated_read_only_call"
|
||||
]
|
||||
assert len(blocked) == 1
|
||||
assert blocked[0]["previous_round"] == 1
|
||||
assert any(tool == "write_file" for tool, _ in executed)
|
||||
|
||||
|
||||
def test_terminal_completion_recovers_fenced_body_after_two_repairs(monkeypatch):
|
||||
executed = []
|
||||
calls = _patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
"Done without writing anything.",
|
||||
"Still done without writing anything.",
|
||||
'```json\n{"ok": true}\n```',
|
||||
"Created answer.json.",
|
||||
],
|
||||
)
|
||||
original_execute = agent_loop.execute_tool_block
|
||||
|
||||
async def record_execute(block, *args, **kwargs):
|
||||
executed.append(block)
|
||||
return await original_execute(block, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(agent_loop, "execute_tool_block", record_execute)
|
||||
|
||||
events = _run("Write answer.json", max_rounds=5)
|
||||
|
||||
assert [(block.tool_type, block.content) for block in executed] == [
|
||||
("write_file", 'answer.json\n{"ok": true}'),
|
||||
]
|
||||
decision = next(
|
||||
event["data"]
|
||||
for event in events
|
||||
if event.get("type") == "completion_decision"
|
||||
)
|
||||
assert decision["status"] == "satisfied"
|
||||
assert decision["can_complete"] is True
|
||||
|
||||
|
||||
def test_successful_artifact_write_emits_satisfied_completion(monkeypatch):
|
||||
calls = _patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'```write_file\nanswer.json\n{"ok": true}\n```',
|
||||
"Done. Created answer.json.",
|
||||
],
|
||||
)
|
||||
|
||||
events = _run("Write answer.json")
|
||||
|
||||
assert calls() == 2, events
|
||||
assert not any(event.get("type") == "completion_blocked" for event in events)
|
||||
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
|
||||
assert decision["status"] == "satisfied"
|
||||
assert decision["can_complete"] is True
|
||||
metrics = next(event["data"] for event in events if event.get("type") == "metrics")
|
||||
assert metrics["completion_decision"] == decision
|
||||
assert metrics["completion_requirements"] == {
|
||||
"required_artifacts": ["answer.json"],
|
||||
"verifier_required": False,
|
||||
"executable_verifier_available": False,
|
||||
"verifier_commands": [],
|
||||
"workspace_root": "",
|
||||
}
|
||||
assert any(
|
||||
evidence["kind"] == "artifact_mutation"
|
||||
for evidence in metrics["evidence_events"]
|
||||
)
|
||||
|
||||
|
||||
def test_current_artifact_inspection_requires_post_mutation_matching_path():
|
||||
write = {
|
||||
"tool": "write_file", "command": "/workspace/output.html\n<body/>",
|
||||
"exit_code": 0,
|
||||
}
|
||||
source_inspection = {
|
||||
"tool": "private_browser", "command": '{"url":"file:///workspace/source.html"}',
|
||||
"exit_code": 0,
|
||||
}
|
||||
output_inspection = {
|
||||
"tool": "private_browser", "command": '{"url":"file:///workspace/output.html"}',
|
||||
}
|
||||
|
||||
assert not agent_loop._artifact_has_current_inspection(
|
||||
[source_inspection, write], ["/workspace/output.html"]
|
||||
)
|
||||
assert agent_loop._artifact_has_current_inspection(
|
||||
[write, output_inspection], ["/workspace/output.html"]
|
||||
)
|
||||
assert not agent_loop._artifact_has_current_inspection(
|
||||
[write, output_inspection, write], ["/workspace/output.html"]
|
||||
)
|
||||
|
||||
|
||||
def test_verified_artifact_survives_provider_error_during_finish_round(monkeypatch):
|
||||
calls = 0
|
||||
_patch_loop(monkeypatch, [""])
|
||||
|
||||
async def stream(_candidates, messages, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
tool_calls = [
|
||||
{
|
||||
"name": "write_file",
|
||||
"arguments": json.dumps({
|
||||
"path": "/workspace/output.html",
|
||||
"content": "<main>complete</main>",
|
||||
}),
|
||||
},
|
||||
{
|
||||
"name": "private_browser",
|
||||
"arguments": json.dumps({
|
||||
"action": "open",
|
||||
"url": "file:///workspace/output.html",
|
||||
}),
|
||||
},
|
||||
]
|
||||
yield f'data: {json.dumps({"type": "tool_calls", "calls": tool_calls})}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
return
|
||||
yield 'event: error\ndata: {"status": 504, "error": "stream timeout"}\n\n'
|
||||
|
||||
monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream)
|
||||
|
||||
events = _run(
|
||||
"Create /workspace/output.html",
|
||||
max_rounds=5,
|
||||
relevant_tools={"write_file", "private_browser"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"surface": "odysseus-native",
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["/workspace/output.html"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert calls == 2
|
||||
assert not any(event.get("type") == "agent_terminal" for event in events)
|
||||
final = next(event for event in events if event.get("type") == "final_response")
|
||||
assert "output.html" in final["content"]
|
||||
assert "verified" in final["content"].lower()
|
||||
|
||||
|
||||
def test_uninspected_artifact_still_fails_on_provider_error(monkeypatch):
|
||||
calls = 0
|
||||
_patch_loop(monkeypatch, [""])
|
||||
|
||||
async def stream(_candidates, messages, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
tool_call = {
|
||||
"name": "write_file",
|
||||
"arguments": json.dumps({
|
||||
"path": "/workspace/answer.json",
|
||||
"content": '{"status": "unverified"}',
|
||||
}),
|
||||
}
|
||||
yield f'data: {json.dumps({"type": "tool_calls", "calls": [tool_call]})}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
return
|
||||
yield 'event: error\ndata: {"status": 504, "error": "stream timeout"}\n\n'
|
||||
|
||||
monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream)
|
||||
|
||||
events = _run(
|
||||
"Create /workspace/answer.json",
|
||||
max_rounds=4,
|
||||
relevant_tools={"write_file"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"surface": "odysseus-native",
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["/workspace/answer.json"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert calls == 2
|
||||
terminal = next(
|
||||
(event for event in events if event.get("type") == "agent_terminal"),
|
||||
None,
|
||||
)
|
||||
assert terminal is not None, events
|
||||
assert terminal["data"]["failed"] is True
|
||||
assert terminal["data"]["failure"]["status"] == 504
|
||||
|
||||
|
||||
def test_verified_artifact_gets_only_one_finish_nudge(monkeypatch):
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'```write_file\noutput.html\n<body>done</body>\n```',
|
||||
'<tool_call><invoke name="private_browser"><parameter name="action">open</parameter><parameter name="url">file:///workspace/output.html</parameter></invoke></tool_call>',
|
||||
'<tool_call><invoke name="read_file"><parameter name="path">output.html</parameter></invoke></tool_call>',
|
||||
"Done. Created and checked output.html.",
|
||||
],
|
||||
)
|
||||
|
||||
events = _run(
|
||||
"Create output.html",
|
||||
max_rounds=5,
|
||||
relevant_tools={"write_file", "private_browser", "read_file"},
|
||||
)
|
||||
|
||||
nudges = [event for event in events if event.get("type") == "artifact_finish_nudge"]
|
||||
assert len(nudges) == 1
|
||||
assert nudges[0]["reason"] == "artifact_complete_and_currently_inspected"
|
||||
|
||||
|
||||
def test_verified_artifact_finish_round_preserves_correction_budget(monkeypatch):
|
||||
captured = []
|
||||
_patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'```write_file\noutput.html\n<body>done</body>\n```',
|
||||
'<tool_call><invoke name="private_browser"><parameter name="action">open</parameter><parameter name="url">file:///workspace/output.html</parameter></invoke></tool_call>',
|
||||
"Done. Created and checked output.html.",
|
||||
],
|
||||
captured_kwargs=captured,
|
||||
)
|
||||
events = _run(
|
||||
"Create output.html",
|
||||
max_rounds=5,
|
||||
relevant_tools={"write_file", "private_browser"},
|
||||
)
|
||||
|
||||
assert any(event.get("type") == "artifact_finish_nudge" for event in events)
|
||||
assert captured[-1]["candidate_request_factory"]
|
||||
# Request kwargs are resolved lazily for each endpoint; exercise the
|
||||
# primary factory exactly as the fallback streamer does.
|
||||
request = asyncio.run(captured[-1]["candidate_request_factory"](0, "http://unused.test/v1", "test-model", {}))
|
||||
# The first post-inspection response may need to rewrite the complete
|
||||
# artifact. Only the response after that correction is verified is
|
||||
# bounded to a short final answer.
|
||||
assert request["kwargs"]["max_tokens"] > 2048
|
||||
|
||||
|
||||
def test_finish_nudge_does_not_accept_unfinished_correction_promise(monkeypatch):
|
||||
calls = _patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'```write_file\n/workspace/output.html\n<body>draft</body>\n```',
|
||||
'<tool_call><invoke name="private_browser"><parameter name="action">open</parameter><parameter name="url">file:///workspace/output.html</parameter></invoke></tool_call>',
|
||||
"The preview revealed a defect. I should complete output.html by adding labels.",
|
||||
'```write_file\n/workspace/output.html\n<body>corrected</body>\n```',
|
||||
"Done. Corrected and checked output.html.",
|
||||
],
|
||||
)
|
||||
|
||||
events = _run(
|
||||
"Create /workspace/output.html",
|
||||
max_rounds=6,
|
||||
relevant_tools={"write_file", "private_browser"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"surface": "odysseus-native",
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["/workspace/output.html"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert calls() == 5, events
|
||||
assert len([
|
||||
event for event in events if event.get("type") == "artifact_finish_nudge"
|
||||
]) == 1
|
||||
assert len([
|
||||
event for event in events
|
||||
if event.get("type") == "artifact_finish_after_verified_correction"
|
||||
]) == 1
|
||||
decision = next(
|
||||
event["data"] for event in events
|
||||
if event.get("type") == "completion_decision"
|
||||
)
|
||||
assert decision["can_complete"] is True
|
||||
|
||||
|
||||
def test_discovered_verifier_runs_in_same_batch_after_mutation(monkeypatch):
|
||||
calls = _patch_loop(
|
||||
monkeypatch,
|
||||
[
|
||||
'```write_file\nanswer.json\n{"ok": true}\n```',
|
||||
"Done without running the task check.",
|
||||
"Done again without selecting a tool.",
|
||||
],
|
||||
)
|
||||
|
||||
events = _run(
|
||||
"Write answer.json",
|
||||
max_rounds=6,
|
||||
relevant_tools={"write_file", "bash"},
|
||||
runtime_context={
|
||||
"terminal_agent": True,
|
||||
"completion_requirements": {
|
||||
"required_artifacts": ["answer.json"],
|
||||
"verifier_required": True,
|
||||
"executable_verifier_available": True,
|
||||
"verifier_commands": ["./test.sh"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
blocked = [event for event in events if event.get("type") == "completion_blocked"]
|
||||
assert blocked == []
|
||||
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
|
||||
assert decision["status"] == "verified"
|
||||
assert calls() == 1
|
||||
assert any(
|
||||
event.get("type") == "tool_start"
|
||||
and event.get("tool") == "bash"
|
||||
and event.get("command") == "./test.sh"
|
||||
for event in events
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_shell_wrapped_write_file_is_recovered_without_shell_execution():
|
||||
recovered = agent_loop._recover_shell_wrapped_file_tool(
|
||||
ToolBlock("bash", 'write_file /workspace/app.py "VALUE = 2"')
|
||||
)
|
||||
|
||||
assert recovered.tool_type == "write_file"
|
||||
assert recovered.content == "/workspace/app.py\nVALUE = 2"
|
||||
|
||||
|
||||
def test_terminal_multiline_shell_wrapped_write_file_is_recovered():
|
||||
recovered = agent_loop._recover_shell_wrapped_file_tool(
|
||||
ToolBlock("bash", "write_file\n/workspace/app.py\nVALUE = '<ok>'\n")
|
||||
)
|
||||
|
||||
assert recovered.tool_type == "write_file"
|
||||
assert recovered.content == "/workspace/app.py\nVALUE = '<ok>'"
|
||||
|
||||
|
||||
def test_terminal_shell_wrapped_edit_file_json_is_recovered_with_literal_operators():
|
||||
recovered = agent_loop._recover_shell_wrapped_file_tool(
|
||||
ToolBlock(
|
||||
"bash",
|
||||
'edit_file {"path": "/workspace/app.py", "old_string": "x >> y", '
|
||||
'"new_string": "x < y"}',
|
||||
)
|
||||
)
|
||||
|
||||
assert recovered.tool_type == "edit_file"
|
||||
assert json.loads(recovered.content) == {
|
||||
"path": "/workspace/app.py",
|
||||
"old_string": "x >> y",
|
||||
"new_string": "x < y",
|
||||
"replace_all": False,
|
||||
}
|
||||
|
||||
|
||||
def test_ambiguous_shell_wrapped_file_tool_is_not_recovered():
|
||||
original = ToolBlock("bash", "write_file app.py value && ./test.sh")
|
||||
|
||||
assert agent_loop._recover_shell_wrapped_file_tool(original) == original
|
||||
|
||||
|
||||
def test_terminal_adjacent_fenced_write_body_is_recovered_for_required_artifact():
|
||||
recovered = agent_loop._recover_adjacent_fenced_write_file(
|
||||
"""```bash
|
||||
write_file /workspace/report.md
|
||||
```
|
||||
|
||||
```markdown
|
||||
# Report
|
||||
|
||||
Verified result.
|
||||
```""",
|
||||
["/workspace/report.md"],
|
||||
)
|
||||
|
||||
assert recovered == ToolBlock(
|
||||
"write_file",
|
||||
"/workspace/report.md\n# Report\n\nVerified result.",
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_fenced_body_before_name_only_write_uses_single_required_artifact():
|
||||
recovered = agent_loop._recover_adjacent_fenced_write_file(
|
||||
"""```html
|
||||
<main>Complete output</main>
|
||||
```
|
||||
|
||||
```bash
|
||||
write_file
|
||||
```""",
|
||||
["/workspace/output.html"],
|
||||
)
|
||||
|
||||
assert recovered == ToolBlock(
|
||||
"write_file",
|
||||
"/workspace/output.html\n<main>Complete output</main>",
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_adjacent_fenced_write_rejects_unrequired_path():
|
||||
recovered = agent_loop._recover_adjacent_fenced_write_file(
|
||||
"""```bash
|
||||
write_file /workspace/unrequested.md
|
||||
```
|
||||
```text
|
||||
content
|
||||
```""",
|
||||
["/workspace/report.md"],
|
||||
)
|
||||
|
||||
assert recovered is None
|
||||
|
||||
|
||||
def test_terminal_recovers_unlabeled_ffmpeg_fence_for_required_media_artifact():
|
||||
recovered = agent_loop._recover_fenced_media_shell_command(
|
||||
"""I will create the requested video now.
|
||||
```
|
||||
ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4
|
||||
```""",
|
||||
["/workspace/merged.mp4"],
|
||||
)
|
||||
|
||||
assert recovered == ToolBlock(
|
||||
"bash",
|
||||
"ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4",
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_unlabeled_shell_fence_must_target_required_media_artifact():
|
||||
assert agent_loop._recover_fenced_media_shell_command(
|
||||
"""```
|
||||
ffmpeg -y -i /workspace/in.mp4 /workspace/other.mp4
|
||||
```""",
|
||||
["/workspace/merged.mp4"],
|
||||
) is None
|
||||
|
||||
|
||||
def test_terminal_unlabeled_non_media_command_remains_inert():
|
||||
assert agent_loop._recover_fenced_media_shell_command(
|
||||
"""```
|
||||
python -c 'print(1)'
|
||||
```""",
|
||||
["/workspace/result.txt"],
|
||||
) is None
|
||||
|
||||
|
||||
def test_explicit_run_the_named_script_is_a_verification_command():
|
||||
request = (
|
||||
"Write a Python script named /workspace/calculate_ratio.py that loads "
|
||||
"the JSON and saves its result. Then run the script."
|
||||
)
|
||||
|
||||
assert agent_loop._requested_verification_command(request) == (
|
||||
"python /workspace/calculate_ratio.py"
|
||||
)
|
||||
assert agent_loop._requested_post_edit_verification(request)
|
||||
|
||||
|
||||
def test_run_the_script_is_not_inferred_when_multiple_scripts_are_named():
|
||||
request = (
|
||||
"Write /workspace/prepare.py and /workspace/calculate.py, then run the script."
|
||||
)
|
||||
|
||||
assert agent_loop._requested_verification_command(request) == ""
|
||||
Reference in New Issue
Block a user