import asyncio
import json
import src.agent_loop as agent_loop
from src.tool_parsing import ToolBlock
from src.tool_capabilities import ToolGateDecision
def _events(chunks):
parsed = []
for chunk in chunks:
if not chunk.startswith("data: ") or chunk.startswith("data: [DONE]"):
continue
parsed.append(json.loads(chunk[6:]))
return parsed
def _patch_loop(monkeypatch, responses, captured_kwargs=None):
monkeypatch.setattr(agent_loop, "get_setting", lambda key, default=None: default)
monkeypatch.setattr(agent_loop, "get_mcp_manager", lambda: None)
monkeypatch.setattr(agent_loop, "blocked_tools_for_owner", lambda owner: set())
monkeypatch.setattr(agent_loop, "estimate_tokens", lambda *args, **kwargs: 10)
monkeypatch.setattr(agent_loop, "tool_result_should_arm_gate", lambda *args, **kwargs: False)
monkeypatch.setattr(
agent_loop.ToolRunSecurityContext,
"decision_for",
lambda self, *args, **kwargs: ToolGateDecision(True),
)
async def execute(block, *args, **kwargs):
return block.tool_type, {
"output": f"executed {block.tool_type}",
"exit_code": 0,
}
call_index = 0
async def stream(_candidates, messages, **kwargs):
nonlocal call_index
if captured_kwargs is not None:
captured_kwargs.append(kwargs)
response = responses[min(call_index, len(responses) - 1)]
call_index += 1
yield f'data: {json.dumps({"delta": response})}\n\n'
yield "data: [DONE]\n\n"
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream)
return lambda: call_index
def _run(instruction, *, max_rounds=4, relevant_tools=None, runtime_context=None):
async def collect():
return [
chunk
async for chunk in agent_loop.stream_agent_loop(
"http://unused.test/v1",
"test-model",
[{"role": "user", "content": instruction}],
max_rounds=max_rounds,
relevant_tools=relevant_tools or {"write_file"},
owner="pewds",
client_runtime_context=runtime_context or {"terminal_agent": True},
)
]
return _events(asyncio.run(collect()))
def test_failed_workspace_mutation_attempts_are_not_hidden_by_successful_probe():
failed = ToolBlock("python", 'open("/workspace/answer.png", "wb")')
probe = ToolBlock("inspect_media", '{"path":"/workspace/input.mp4"}')
records = [
{"tool_name": "python", "result": {"error": "conversion failed", "exit_code": 1}},
{"tool_name": "inspect_media", "result": {"output": "evidence", "exit_code": 0}},
]
assert agent_loop._failed_workspace_mutation_attempts([failed, probe], records) == 1
def test_terminal_completion_repairs_missing_artifact_at_most_twice(monkeypatch):
calls = _patch_loop(monkeypatch, ["Done without writing anything."])
events = _run("Write answer.json", max_rounds=4)
blocked = [event for event in events if event.get("type") == "completion_blocked"]
assert [event["attempt"] for event in blocked] == [1, 2]
assert calls() == 3
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
assert decision["status"] == "blocked"
assert decision["missing_artifacts"] == ["answer.json"]
def test_failed_trailing_tool_with_planning_prose_continues_artifact_task(monkeypatch):
calls = _patch_loop(
monkeypatch,
[
"I found the source and will inspect it now.\n```bash\nfalse\n```",
"Done without creating the required artifact.",
],
)
async def fail_execute(block, *args, **kwargs):
return block.tool_type, {
"output": "no matches",
"exit_code": 1,
}
monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute)
events = _run(
"Create answer.json after inspecting the source",
max_rounds=3,
relevant_tools={"bash", "write_file"},
)
assert calls() > 1
assert any(
event.get("type") == "completion_blocked"
and event.get("decision", {}).get("missing_artifacts") == ["answer.json"]
for event in events
)
def test_exact_failed_call_is_blocked_across_planning_and_intervening_failure(monkeypatch):
repeated = "python3 /tmp_workspace/classify_and_copy.py"
executed = []
_patch_loop(
monkeypatch,
[
f'I will run the classifier now.\n{repeated}',
"I will repair the variable first.\npython3 -c 'print(classifications)'",
f'I will retry the classifier.\n{repeated}',
"The work is complete.",
],
)
async def fail_execute(block, *args, **kwargs):
executed.append(block.content)
return block.tool_type, {"output": f"failed: {block.content}", "exit_code": 1}
monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute)
events = _run(
"Create /tmp_workspace/results after classifying the files",
max_rounds=4,
relevant_tools={"bash", "write_file"},
runtime_context={
"terminal_agent": True,
"completion_requirements": {
"required_artifacts": ["/tmp_workspace/results"],
},
},
)
assert executed.count(repeated) == 1, events
blocked = [event for event in events if event.get("type") == "tool_retry_blocked"]
assert len(blocked) == 1
assert blocked[0]["previous_round"] == 1
assert blocked[0]["command"] == repeated
def test_exact_failed_call_can_retry_after_successful_workspace_mutation(monkeypatch):
repeated = "python3 /tmp_workspace/classify_and_copy.py"
executed = []
_patch_loop(
monkeypatch,
[
f'{repeated}',
'/tmp_workspace/repair.pyfixed = True',
f'{repeated}',
"Repair attempted.",
],
)
async def execute(block, *args, **kwargs):
executed.append((block.tool_type, block.content))
if block.tool_type == "write_file":
return block.tool_type, {"output": "written", "exit_code": 0}
return block.tool_type, {"output": "classifier failed", "exit_code": 1}
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
events = _run(
"Create /tmp_workspace/results after repairing and running the classifier",
max_rounds=4,
relevant_tools={"bash", "write_file"},
runtime_context={
"terminal_agent": True,
"completion_requirements": {
"required_artifacts": ["/tmp_workspace/results"],
},
},
)
assert sum(content == repeated for _, content in executed) == 2
assert not any(event.get("type") == "tool_retry_blocked" for event in events)
def test_terminal_artifact_task_repairs_after_consecutive_failed_batches(monkeypatch):
executed = []
_patch_loop(
monkeypatch,
[
'print(missing_one)',
'print(missing_two)',
'print(missing_three)',
'print(missing_four)',
'print(missing_five)',
'answer.json{\"ok\": true}',
"Created answer.json.",
],
)
async def execute(block, *args, **kwargs):
executed.append((block.tool_type, block.content))
if block.tool_type == "write_file":
return block.tool_type, {"output": "written", "exit_code": 0}
return block.tool_type, {
"error": f"NameError from {block.content}",
"exit_code": 1,
}
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
events = _run(
"Create answer.json and verify it",
max_rounds=7,
relevant_tools={"python", "write_file"},
runtime_context={
"terminal_agent": True,
"failed_tool_round_limit": 5,
"completion_requirements": {
"required_artifacts": ["answer.json"],
},
},
)
repairs = [event for event in events if event.get("type") == "artifact_repair_required"]
assert len(repairs) == 1
assert repairs[0]["attempt"] == 1
assert "missing_five" in repairs[0]["last_failure"]
assert not any(
event.get("type") == "loop_breaker_triggered"
and event.get("reason") == "consecutive_tool_failures"
for event in events
)
assert any(tool == "write_file" for tool, _ in executed)
decision = next(
event["data"]
for event in events
if event.get("type") == "completion_decision"
)
assert decision["can_complete"] is True
def test_varied_failed_artifact_mutations_have_cumulative_cap(monkeypatch):
executed = []
_patch_loop(
monkeypatch,
[
''
f'open("/workspace/answer.json", "w").write(missing_{index})'
''
for index in range(1, 9)
],
)
async def execute(block, *args, **kwargs):
executed.append((block.tool_type, block.content))
return block.tool_type, {
"error": f"NameError from {block.content}",
"exit_code": 1,
}
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
events = _run(
"Create answer.json and verify it",
max_rounds=8,
relevant_tools={"python", "write_file"},
runtime_context={
"terminal_agent": True,
"failed_tool_round_limit": 99,
"completion_requirements": {
"required_artifacts": ["answer.json"],
},
},
)
guard = next(
event for event in events
if event.get("type") == "loop_breaker_triggered"
and event.get("reason") == "cumulative_artifact_mutation_failures"
)
assert guard["failed_batches"] == 6
assert len(executed) == 6
def test_exact_successful_read_is_blocked_until_workspace_changes(monkeypatch):
inspection = "find /tmp_workspace/results -type f | wc -l"
executed = []
_patch_loop(
monkeypatch,
[
f'I found partial output and will count it.\n{inspection}',
f'I should check the count once more.\n{inspection}',
'answer.json{\"ok\": true}',
"Created answer.json.",
],
)
async def execute(block, *args, **kwargs):
executed.append((block.tool_type, block.content))
if block.tool_type == "write_file":
return block.tool_type, {"output": "written", "exit_code": 0}
return block.tool_type, {"output": "7", "exit_code": 0}
monkeypatch.setattr(agent_loop, "execute_tool_block", execute)
events = _run(
"Create answer.json from the inspected workspace",
max_rounds=4,
relevant_tools={"bash", "write_file"},
runtime_context={
"terminal_agent": True,
"completion_requirements": {"required_artifacts": ["answer.json"]},
},
)
assert sum(content == inspection for _, content in executed) == 1
blocked = [
event for event in events
if event.get("type") == "tool_retry_blocked"
and event.get("reason") == "repeated_read_only_call"
]
assert len(blocked) == 1
assert blocked[0]["previous_round"] == 1
assert any(tool == "write_file" for tool, _ in executed)
def test_terminal_completion_recovers_fenced_body_after_two_repairs(monkeypatch):
executed = []
calls = _patch_loop(
monkeypatch,
[
"Done without writing anything.",
"Still done without writing anything.",
'```json\n{"ok": true}\n```',
"Created answer.json.",
],
)
original_execute = agent_loop.execute_tool_block
async def record_execute(block, *args, **kwargs):
executed.append(block)
return await original_execute(block, *args, **kwargs)
monkeypatch.setattr(agent_loop, "execute_tool_block", record_execute)
events = _run("Write answer.json", max_rounds=5)
assert [(block.tool_type, block.content) for block in executed] == [
("write_file", 'answer.json\n{"ok": true}'),
]
decision = next(
event["data"]
for event in events
if event.get("type") == "completion_decision"
)
assert decision["status"] == "satisfied"
assert decision["can_complete"] is True
def test_successful_artifact_write_emits_satisfied_completion(monkeypatch):
calls = _patch_loop(
monkeypatch,
[
'```write_file\nanswer.json\n{"ok": true}\n```',
"Done. Created answer.json.",
],
)
events = _run("Write answer.json")
assert calls() == 2, events
assert not any(event.get("type") == "completion_blocked" for event in events)
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
assert decision["status"] == "satisfied"
assert decision["can_complete"] is True
metrics = next(event["data"] for event in events if event.get("type") == "metrics")
assert metrics["completion_decision"] == decision
assert metrics["completion_requirements"] == {
"required_artifacts": ["answer.json"],
"verifier_required": False,
"executable_verifier_available": False,
"verifier_commands": [],
"workspace_root": "",
}
assert any(
evidence["kind"] == "artifact_mutation"
for evidence in metrics["evidence_events"]
)
def test_current_artifact_inspection_requires_post_mutation_matching_path():
write = {
"tool": "write_file", "command": "/workspace/output.html\n
done\n```',
'openfile:///workspace/output.html',
'output.html',
"Done. Created and checked output.html.",
],
)
events = _run(
"Create output.html",
max_rounds=5,
relevant_tools={"write_file", "private_browser", "read_file"},
)
nudges = [event for event in events if event.get("type") == "artifact_finish_nudge"]
assert len(nudges) == 1
assert nudges[0]["reason"] == "artifact_complete_and_currently_inspected"
def test_verified_artifact_finish_round_preserves_correction_budget(monkeypatch):
captured = []
_patch_loop(
monkeypatch,
[
'```write_file\noutput.html\ndone\n```',
'openfile:///workspace/output.html',
"Done. Created and checked output.html.",
],
captured_kwargs=captured,
)
events = _run(
"Create output.html",
max_rounds=5,
relevant_tools={"write_file", "private_browser"},
)
assert any(event.get("type") == "artifact_finish_nudge" for event in events)
assert captured[-1]["candidate_request_factory"]
# Request kwargs are resolved lazily for each endpoint; exercise the
# primary factory exactly as the fallback streamer does.
request = asyncio.run(captured[-1]["candidate_request_factory"](0, "http://unused.test/v1", "test-model", {}))
# The first post-inspection response may need to rewrite the complete
# artifact. Only the response after that correction is verified is
# bounded to a short final answer.
assert request["kwargs"]["max_tokens"] > 2048
def test_finish_nudge_does_not_accept_unfinished_correction_promise(monkeypatch):
calls = _patch_loop(
monkeypatch,
[
'```write_file\n/workspace/output.html\ndraft\n```',
'openfile:///workspace/output.html',
"The preview revealed a defect. I should complete output.html by adding labels.",
'```write_file\n/workspace/output.html\ncorrected\n```',
"Done. Corrected and checked output.html.",
],
)
events = _run(
"Create /workspace/output.html",
max_rounds=6,
relevant_tools={"write_file", "private_browser"},
runtime_context={
"terminal_agent": True,
"surface": "odysseus-native",
"completion_requirements": {
"required_artifacts": ["/workspace/output.html"],
},
},
)
assert calls() == 5, events
assert len([
event for event in events if event.get("type") == "artifact_finish_nudge"
]) == 1
assert len([
event for event in events
if event.get("type") == "artifact_finish_after_verified_correction"
]) == 1
decision = next(
event["data"] for event in events
if event.get("type") == "completion_decision"
)
assert decision["can_complete"] is True
def test_discovered_verifier_runs_in_same_batch_after_mutation(monkeypatch):
calls = _patch_loop(
monkeypatch,
[
'```write_file\nanswer.json\n{"ok": true}\n```',
"Done without running the task check.",
"Done again without selecting a tool.",
],
)
events = _run(
"Write answer.json",
max_rounds=6,
relevant_tools={"write_file", "bash"},
runtime_context={
"terminal_agent": True,
"completion_requirements": {
"required_artifacts": ["answer.json"],
"verifier_required": True,
"executable_verifier_available": True,
"verifier_commands": ["./test.sh"],
},
},
)
blocked = [event for event in events if event.get("type") == "completion_blocked"]
assert blocked == []
decision = next(event["data"] for event in events if event.get("type") == "completion_decision")
assert decision["status"] == "verified"
assert calls() == 1
assert any(
event.get("type") == "tool_start"
and event.get("tool") == "bash"
and event.get("command") == "./test.sh"
for event in events
)
def test_terminal_shell_wrapped_write_file_is_recovered_without_shell_execution():
recovered = agent_loop._recover_shell_wrapped_file_tool(
ToolBlock("bash", 'write_file /workspace/app.py "VALUE = 2"')
)
assert recovered.tool_type == "write_file"
assert recovered.content == "/workspace/app.py\nVALUE = 2"
def test_terminal_multiline_shell_wrapped_write_file_is_recovered():
recovered = agent_loop._recover_shell_wrapped_file_tool(
ToolBlock("bash", "write_file\n/workspace/app.py\nVALUE = ''\n")
)
assert recovered.tool_type == "write_file"
assert recovered.content == "/workspace/app.py\nVALUE = ''"
def test_terminal_shell_wrapped_edit_file_json_is_recovered_with_literal_operators():
recovered = agent_loop._recover_shell_wrapped_file_tool(
ToolBlock(
"bash",
'edit_file {"path": "/workspace/app.py", "old_string": "x >> y", '
'"new_string": "x < y"}',
)
)
assert recovered.tool_type == "edit_file"
assert json.loads(recovered.content) == {
"path": "/workspace/app.py",
"old_string": "x >> y",
"new_string": "x < y",
"replace_all": False,
}
def test_ambiguous_shell_wrapped_file_tool_is_not_recovered():
original = ToolBlock("bash", "write_file app.py value && ./test.sh")
assert agent_loop._recover_shell_wrapped_file_tool(original) == original
def test_terminal_adjacent_fenced_write_body_is_recovered_for_required_artifact():
recovered = agent_loop._recover_adjacent_fenced_write_file(
"""```bash
write_file /workspace/report.md
```
```markdown
# Report
Verified result.
```""",
["/workspace/report.md"],
)
assert recovered == ToolBlock(
"write_file",
"/workspace/report.md\n# Report\n\nVerified result.",
)
def test_terminal_fenced_body_before_name_only_write_uses_single_required_artifact():
recovered = agent_loop._recover_adjacent_fenced_write_file(
"""```html
Complete output
```
```bash
write_file
```""",
["/workspace/output.html"],
)
assert recovered == ToolBlock(
"write_file",
"/workspace/output.html\nComplete output",
)
def test_terminal_adjacent_fenced_write_rejects_unrequired_path():
recovered = agent_loop._recover_adjacent_fenced_write_file(
"""```bash
write_file /workspace/unrequested.md
```
```text
content
```""",
["/workspace/report.md"],
)
assert recovered is None
def test_terminal_recovers_unlabeled_ffmpeg_fence_for_required_media_artifact():
recovered = agent_loop._recover_fenced_media_shell_command(
"""I will create the requested video now.
```
ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4
```""",
["/workspace/merged.mp4"],
)
assert recovered == ToolBlock(
"bash",
"ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4",
)
def test_terminal_unlabeled_shell_fence_must_target_required_media_artifact():
assert agent_loop._recover_fenced_media_shell_command(
"""```
ffmpeg -y -i /workspace/in.mp4 /workspace/other.mp4
```""",
["/workspace/merged.mp4"],
) is None
def test_terminal_unlabeled_non_media_command_remains_inert():
assert agent_loop._recover_fenced_media_shell_command(
"""```
python -c 'print(1)'
```""",
["/workspace/result.txt"],
) is None
def test_explicit_run_the_named_script_is_a_verification_command():
request = (
"Write a Python script named /workspace/calculate_ratio.py that loads "
"the JSON and saves its result. Then run the script."
)
assert agent_loop._requested_verification_command(request) == (
"python /workspace/calculate_ratio.py"
)
assert agent_loop._requested_post_edit_verification(request)
def test_run_the_script_is_not_inferred_when_multiple_scripts_are_named():
request = (
"Write /workspace/prepare.py and /workspace/calculate.py, then run the script."
)
assert agent_loop._requested_verification_command(request) == ""