import asyncio import json import src.agent_loop as agent_loop from src.tool_parsing import ToolBlock from src.tool_capabilities import ToolGateDecision def _events(chunks): parsed = [] for chunk in chunks: if not chunk.startswith("data: ") or chunk.startswith("data: [DONE]"): continue parsed.append(json.loads(chunk[6:])) return parsed def _patch_loop(monkeypatch, responses, captured_kwargs=None): monkeypatch.setattr(agent_loop, "get_setting", lambda key, default=None: default) monkeypatch.setattr(agent_loop, "get_mcp_manager", lambda: None) monkeypatch.setattr(agent_loop, "blocked_tools_for_owner", lambda owner: set()) monkeypatch.setattr(agent_loop, "estimate_tokens", lambda *args, **kwargs: 10) monkeypatch.setattr(agent_loop, "tool_result_should_arm_gate", lambda *args, **kwargs: False) monkeypatch.setattr( agent_loop.ToolRunSecurityContext, "decision_for", lambda self, *args, **kwargs: ToolGateDecision(True), ) async def execute(block, *args, **kwargs): return block.tool_type, { "output": f"executed {block.tool_type}", "exit_code": 0, } call_index = 0 async def stream(_candidates, messages, **kwargs): nonlocal call_index if captured_kwargs is not None: captured_kwargs.append(kwargs) response = responses[min(call_index, len(responses) - 1)] call_index += 1 yield f'data: {json.dumps({"delta": response})}\n\n' yield "data: [DONE]\n\n" monkeypatch.setattr(agent_loop, "execute_tool_block", execute) monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream) return lambda: call_index def _run(instruction, *, max_rounds=4, relevant_tools=None, runtime_context=None): async def collect(): return [ chunk async for chunk in agent_loop.stream_agent_loop( "http://unused.test/v1", "test-model", [{"role": "user", "content": instruction}], max_rounds=max_rounds, relevant_tools=relevant_tools or {"write_file"}, owner="pewds", client_runtime_context=runtime_context or {"terminal_agent": True}, ) ] return _events(asyncio.run(collect())) def test_failed_workspace_mutation_attempts_are_not_hidden_by_successful_probe(): failed = ToolBlock("python", 'open("/workspace/answer.png", "wb")') probe = ToolBlock("inspect_media", '{"path":"/workspace/input.mp4"}') records = [ {"tool_name": "python", "result": {"error": "conversion failed", "exit_code": 1}}, {"tool_name": "inspect_media", "result": {"output": "evidence", "exit_code": 0}}, ] assert agent_loop._failed_workspace_mutation_attempts([failed, probe], records) == 1 def test_terminal_completion_repairs_missing_artifact_at_most_twice(monkeypatch): calls = _patch_loop(monkeypatch, ["Done without writing anything."]) events = _run("Write answer.json", max_rounds=4) blocked = [event for event in events if event.get("type") == "completion_blocked"] assert [event["attempt"] for event in blocked] == [1, 2] assert calls() == 3 decision = next(event["data"] for event in events if event.get("type") == "completion_decision") assert decision["status"] == "blocked" assert decision["missing_artifacts"] == ["answer.json"] def test_failed_trailing_tool_with_planning_prose_continues_artifact_task(monkeypatch): calls = _patch_loop( monkeypatch, [ "I found the source and will inspect it now.\n```bash\nfalse\n```", "Done without creating the required artifact.", ], ) async def fail_execute(block, *args, **kwargs): return block.tool_type, { "output": "no matches", "exit_code": 1, } monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute) events = _run( "Create answer.json after inspecting the source", max_rounds=3, relevant_tools={"bash", "write_file"}, ) assert calls() > 1 assert any( event.get("type") == "completion_blocked" and event.get("decision", {}).get("missing_artifacts") == ["answer.json"] for event in events ) def test_exact_failed_call_is_blocked_across_planning_and_intervening_failure(monkeypatch): repeated = "python3 /tmp_workspace/classify_and_copy.py" executed = [] _patch_loop( monkeypatch, [ f'I will run the classifier now.\n{repeated}', "I will repair the variable first.\npython3 -c 'print(classifications)'", f'I will retry the classifier.\n{repeated}', "The work is complete.", ], ) async def fail_execute(block, *args, **kwargs): executed.append(block.content) return block.tool_type, {"output": f"failed: {block.content}", "exit_code": 1} monkeypatch.setattr(agent_loop, "execute_tool_block", fail_execute) events = _run( "Create /tmp_workspace/results after classifying the files", max_rounds=4, relevant_tools={"bash", "write_file"}, runtime_context={ "terminal_agent": True, "completion_requirements": { "required_artifacts": ["/tmp_workspace/results"], }, }, ) assert executed.count(repeated) == 1, events blocked = [event for event in events if event.get("type") == "tool_retry_blocked"] assert len(blocked) == 1 assert blocked[0]["previous_round"] == 1 assert blocked[0]["command"] == repeated def test_exact_failed_call_can_retry_after_successful_workspace_mutation(monkeypatch): repeated = "python3 /tmp_workspace/classify_and_copy.py" executed = [] _patch_loop( monkeypatch, [ f'{repeated}', '/tmp_workspace/repair.pyfixed = True', f'{repeated}', "Repair attempted.", ], ) async def execute(block, *args, **kwargs): executed.append((block.tool_type, block.content)) if block.tool_type == "write_file": return block.tool_type, {"output": "written", "exit_code": 0} return block.tool_type, {"output": "classifier failed", "exit_code": 1} monkeypatch.setattr(agent_loop, "execute_tool_block", execute) events = _run( "Create /tmp_workspace/results after repairing and running the classifier", max_rounds=4, relevant_tools={"bash", "write_file"}, runtime_context={ "terminal_agent": True, "completion_requirements": { "required_artifacts": ["/tmp_workspace/results"], }, }, ) assert sum(content == repeated for _, content in executed) == 2 assert not any(event.get("type") == "tool_retry_blocked" for event in events) def test_terminal_artifact_task_repairs_after_consecutive_failed_batches(monkeypatch): executed = [] _patch_loop( monkeypatch, [ 'print(missing_one)', 'print(missing_two)', 'print(missing_three)', 'print(missing_four)', 'print(missing_five)', 'answer.json{\"ok\": true}', "Created answer.json.", ], ) async def execute(block, *args, **kwargs): executed.append((block.tool_type, block.content)) if block.tool_type == "write_file": return block.tool_type, {"output": "written", "exit_code": 0} return block.tool_type, { "error": f"NameError from {block.content}", "exit_code": 1, } monkeypatch.setattr(agent_loop, "execute_tool_block", execute) events = _run( "Create answer.json and verify it", max_rounds=7, relevant_tools={"python", "write_file"}, runtime_context={ "terminal_agent": True, "failed_tool_round_limit": 5, "completion_requirements": { "required_artifacts": ["answer.json"], }, }, ) repairs = [event for event in events if event.get("type") == "artifact_repair_required"] assert len(repairs) == 1 assert repairs[0]["attempt"] == 1 assert "missing_five" in repairs[0]["last_failure"] assert not any( event.get("type") == "loop_breaker_triggered" and event.get("reason") == "consecutive_tool_failures" for event in events ) assert any(tool == "write_file" for tool, _ in executed) decision = next( event["data"] for event in events if event.get("type") == "completion_decision" ) assert decision["can_complete"] is True def test_varied_failed_artifact_mutations_have_cumulative_cap(monkeypatch): executed = [] _patch_loop( monkeypatch, [ '' f'open("/workspace/answer.json", "w").write(missing_{index})' '' for index in range(1, 9) ], ) async def execute(block, *args, **kwargs): executed.append((block.tool_type, block.content)) return block.tool_type, { "error": f"NameError from {block.content}", "exit_code": 1, } monkeypatch.setattr(agent_loop, "execute_tool_block", execute) events = _run( "Create answer.json and verify it", max_rounds=8, relevant_tools={"python", "write_file"}, runtime_context={ "terminal_agent": True, "failed_tool_round_limit": 99, "completion_requirements": { "required_artifacts": ["answer.json"], }, }, ) guard = next( event for event in events if event.get("type") == "loop_breaker_triggered" and event.get("reason") == "cumulative_artifact_mutation_failures" ) assert guard["failed_batches"] == 6 assert len(executed) == 6 def test_exact_successful_read_is_blocked_until_workspace_changes(monkeypatch): inspection = "find /tmp_workspace/results -type f | wc -l" executed = [] _patch_loop( monkeypatch, [ f'I found partial output and will count it.\n{inspection}', f'I should check the count once more.\n{inspection}', 'answer.json{\"ok\": true}', "Created answer.json.", ], ) async def execute(block, *args, **kwargs): executed.append((block.tool_type, block.content)) if block.tool_type == "write_file": return block.tool_type, {"output": "written", "exit_code": 0} return block.tool_type, {"output": "7", "exit_code": 0} monkeypatch.setattr(agent_loop, "execute_tool_block", execute) events = _run( "Create answer.json from the inspected workspace", max_rounds=4, relevant_tools={"bash", "write_file"}, runtime_context={ "terminal_agent": True, "completion_requirements": {"required_artifacts": ["answer.json"]}, }, ) assert sum(content == inspection for _, content in executed) == 1 blocked = [ event for event in events if event.get("type") == "tool_retry_blocked" and event.get("reason") == "repeated_read_only_call" ] assert len(blocked) == 1 assert blocked[0]["previous_round"] == 1 assert any(tool == "write_file" for tool, _ in executed) def test_terminal_completion_recovers_fenced_body_after_two_repairs(monkeypatch): executed = [] calls = _patch_loop( monkeypatch, [ "Done without writing anything.", "Still done without writing anything.", '```json\n{"ok": true}\n```', "Created answer.json.", ], ) original_execute = agent_loop.execute_tool_block async def record_execute(block, *args, **kwargs): executed.append(block) return await original_execute(block, *args, **kwargs) monkeypatch.setattr(agent_loop, "execute_tool_block", record_execute) events = _run("Write answer.json", max_rounds=5) assert [(block.tool_type, block.content) for block in executed] == [ ("write_file", 'answer.json\n{"ok": true}'), ] decision = next( event["data"] for event in events if event.get("type") == "completion_decision" ) assert decision["status"] == "satisfied" assert decision["can_complete"] is True def test_successful_artifact_write_emits_satisfied_completion(monkeypatch): calls = _patch_loop( monkeypatch, [ '```write_file\nanswer.json\n{"ok": true}\n```', "Done. Created answer.json.", ], ) events = _run("Write answer.json") assert calls() == 2, events assert not any(event.get("type") == "completion_blocked" for event in events) decision = next(event["data"] for event in events if event.get("type") == "completion_decision") assert decision["status"] == "satisfied" assert decision["can_complete"] is True metrics = next(event["data"] for event in events if event.get("type") == "metrics") assert metrics["completion_decision"] == decision assert metrics["completion_requirements"] == { "required_artifacts": ["answer.json"], "verifier_required": False, "executable_verifier_available": False, "verifier_commands": [], "workspace_root": "", } assert any( evidence["kind"] == "artifact_mutation" for evidence in metrics["evidence_events"] ) def test_current_artifact_inspection_requires_post_mutation_matching_path(): write = { "tool": "write_file", "command": "/workspace/output.html\n", "exit_code": 0, } source_inspection = { "tool": "private_browser", "command": '{"url":"file:///workspace/source.html"}', "exit_code": 0, } output_inspection = { "tool": "private_browser", "command": '{"url":"file:///workspace/output.html"}', } assert not agent_loop._artifact_has_current_inspection( [source_inspection, write], ["/workspace/output.html"] ) assert agent_loop._artifact_has_current_inspection( [write, output_inspection], ["/workspace/output.html"] ) assert not agent_loop._artifact_has_current_inspection( [write, output_inspection, write], ["/workspace/output.html"] ) def test_verified_artifact_survives_provider_error_during_finish_round(monkeypatch): calls = 0 _patch_loop(monkeypatch, [""]) async def stream(_candidates, messages, **kwargs): nonlocal calls calls += 1 if calls == 1: tool_calls = [ { "name": "write_file", "arguments": json.dumps({ "path": "/workspace/output.html", "content": "
complete
", }), }, { "name": "private_browser", "arguments": json.dumps({ "action": "open", "url": "file:///workspace/output.html", }), }, ] yield f'data: {json.dumps({"type": "tool_calls", "calls": tool_calls})}\n\n' yield "data: [DONE]\n\n" return yield 'event: error\ndata: {"status": 504, "error": "stream timeout"}\n\n' monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream) events = _run( "Create /workspace/output.html", max_rounds=5, relevant_tools={"write_file", "private_browser"}, runtime_context={ "terminal_agent": True, "surface": "odysseus-native", "completion_requirements": { "required_artifacts": ["/workspace/output.html"], }, }, ) assert calls == 2 assert not any(event.get("type") == "agent_terminal" for event in events) final = next(event for event in events if event.get("type") == "final_response") assert "output.html" in final["content"] assert "verified" in final["content"].lower() def test_uninspected_artifact_still_fails_on_provider_error(monkeypatch): calls = 0 _patch_loop(monkeypatch, [""]) async def stream(_candidates, messages, **kwargs): nonlocal calls calls += 1 if calls == 1: tool_call = { "name": "write_file", "arguments": json.dumps({ "path": "/workspace/answer.json", "content": '{"status": "unverified"}', }), } yield f'data: {json.dumps({"type": "tool_calls", "calls": [tool_call]})}\n\n' yield "data: [DONE]\n\n" return yield 'event: error\ndata: {"status": 504, "error": "stream timeout"}\n\n' monkeypatch.setattr(agent_loop, "stream_llm_with_fallback", stream) events = _run( "Create /workspace/answer.json", max_rounds=4, relevant_tools={"write_file"}, runtime_context={ "terminal_agent": True, "surface": "odysseus-native", "completion_requirements": { "required_artifacts": ["/workspace/answer.json"], }, }, ) assert calls == 2 terminal = next( (event for event in events if event.get("type") == "agent_terminal"), None, ) assert terminal is not None, events assert terminal["data"]["failed"] is True assert terminal["data"]["failure"]["status"] == 504 def test_verified_artifact_gets_only_one_finish_nudge(monkeypatch): _patch_loop( monkeypatch, [ '```write_file\noutput.html\ndone\n```', 'openfile:///workspace/output.html', 'output.html', "Done. Created and checked output.html.", ], ) events = _run( "Create output.html", max_rounds=5, relevant_tools={"write_file", "private_browser", "read_file"}, ) nudges = [event for event in events if event.get("type") == "artifact_finish_nudge"] assert len(nudges) == 1 assert nudges[0]["reason"] == "artifact_complete_and_currently_inspected" def test_verified_artifact_finish_round_preserves_correction_budget(monkeypatch): captured = [] _patch_loop( monkeypatch, [ '```write_file\noutput.html\ndone\n```', 'openfile:///workspace/output.html', "Done. Created and checked output.html.", ], captured_kwargs=captured, ) events = _run( "Create output.html", max_rounds=5, relevant_tools={"write_file", "private_browser"}, ) assert any(event.get("type") == "artifact_finish_nudge" for event in events) assert captured[-1]["candidate_request_factory"] # Request kwargs are resolved lazily for each endpoint; exercise the # primary factory exactly as the fallback streamer does. request = asyncio.run(captured[-1]["candidate_request_factory"](0, "http://unused.test/v1", "test-model", {})) # The first post-inspection response may need to rewrite the complete # artifact. Only the response after that correction is verified is # bounded to a short final answer. assert request["kwargs"]["max_tokens"] > 2048 def test_finish_nudge_does_not_accept_unfinished_correction_promise(monkeypatch): calls = _patch_loop( monkeypatch, [ '```write_file\n/workspace/output.html\ndraft\n```', 'openfile:///workspace/output.html', "The preview revealed a defect. I should complete output.html by adding labels.", '```write_file\n/workspace/output.html\ncorrected\n```', "Done. Corrected and checked output.html.", ], ) events = _run( "Create /workspace/output.html", max_rounds=6, relevant_tools={"write_file", "private_browser"}, runtime_context={ "terminal_agent": True, "surface": "odysseus-native", "completion_requirements": { "required_artifacts": ["/workspace/output.html"], }, }, ) assert calls() == 5, events assert len([ event for event in events if event.get("type") == "artifact_finish_nudge" ]) == 1 assert len([ event for event in events if event.get("type") == "artifact_finish_after_verified_correction" ]) == 1 decision = next( event["data"] for event in events if event.get("type") == "completion_decision" ) assert decision["can_complete"] is True def test_discovered_verifier_runs_in_same_batch_after_mutation(monkeypatch): calls = _patch_loop( monkeypatch, [ '```write_file\nanswer.json\n{"ok": true}\n```', "Done without running the task check.", "Done again without selecting a tool.", ], ) events = _run( "Write answer.json", max_rounds=6, relevant_tools={"write_file", "bash"}, runtime_context={ "terminal_agent": True, "completion_requirements": { "required_artifacts": ["answer.json"], "verifier_required": True, "executable_verifier_available": True, "verifier_commands": ["./test.sh"], }, }, ) blocked = [event for event in events if event.get("type") == "completion_blocked"] assert blocked == [] decision = next(event["data"] for event in events if event.get("type") == "completion_decision") assert decision["status"] == "verified" assert calls() == 1 assert any( event.get("type") == "tool_start" and event.get("tool") == "bash" and event.get("command") == "./test.sh" for event in events ) def test_terminal_shell_wrapped_write_file_is_recovered_without_shell_execution(): recovered = agent_loop._recover_shell_wrapped_file_tool( ToolBlock("bash", 'write_file /workspace/app.py "VALUE = 2"') ) assert recovered.tool_type == "write_file" assert recovered.content == "/workspace/app.py\nVALUE = 2" def test_terminal_multiline_shell_wrapped_write_file_is_recovered(): recovered = agent_loop._recover_shell_wrapped_file_tool( ToolBlock("bash", "write_file\n/workspace/app.py\nVALUE = ''\n") ) assert recovered.tool_type == "write_file" assert recovered.content == "/workspace/app.py\nVALUE = ''" def test_terminal_shell_wrapped_edit_file_json_is_recovered_with_literal_operators(): recovered = agent_loop._recover_shell_wrapped_file_tool( ToolBlock( "bash", 'edit_file {"path": "/workspace/app.py", "old_string": "x >> y", ' '"new_string": "x < y"}', ) ) assert recovered.tool_type == "edit_file" assert json.loads(recovered.content) == { "path": "/workspace/app.py", "old_string": "x >> y", "new_string": "x < y", "replace_all": False, } def test_ambiguous_shell_wrapped_file_tool_is_not_recovered(): original = ToolBlock("bash", "write_file app.py value && ./test.sh") assert agent_loop._recover_shell_wrapped_file_tool(original) == original def test_terminal_adjacent_fenced_write_body_is_recovered_for_required_artifact(): recovered = agent_loop._recover_adjacent_fenced_write_file( """```bash write_file /workspace/report.md ``` ```markdown # Report Verified result. ```""", ["/workspace/report.md"], ) assert recovered == ToolBlock( "write_file", "/workspace/report.md\n# Report\n\nVerified result.", ) def test_terminal_fenced_body_before_name_only_write_uses_single_required_artifact(): recovered = agent_loop._recover_adjacent_fenced_write_file( """```html
Complete output
``` ```bash write_file ```""", ["/workspace/output.html"], ) assert recovered == ToolBlock( "write_file", "/workspace/output.html\n
Complete output
", ) def test_terminal_adjacent_fenced_write_rejects_unrequired_path(): recovered = agent_loop._recover_adjacent_fenced_write_file( """```bash write_file /workspace/unrequested.md ``` ```text content ```""", ["/workspace/report.md"], ) assert recovered is None def test_terminal_recovers_unlabeled_ffmpeg_fence_for_required_media_artifact(): recovered = agent_loop._recover_fenced_media_shell_command( """I will create the requested video now. ``` ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4 ```""", ["/workspace/merged.mp4"], ) assert recovered == ToolBlock( "bash", "ffmpeg -y -i /workspace/in.mp4 -vf scale=640:360 /workspace/merged.mp4", ) def test_terminal_unlabeled_shell_fence_must_target_required_media_artifact(): assert agent_loop._recover_fenced_media_shell_command( """``` ffmpeg -y -i /workspace/in.mp4 /workspace/other.mp4 ```""", ["/workspace/merged.mp4"], ) is None def test_terminal_unlabeled_non_media_command_remains_inert(): assert agent_loop._recover_fenced_media_shell_command( """``` python -c 'print(1)' ```""", ["/workspace/result.txt"], ) is None def test_explicit_run_the_named_script_is_a_verification_command(): request = ( "Write a Python script named /workspace/calculate_ratio.py that loads " "the JSON and saves its result. Then run the script." ) assert agent_loop._requested_verification_command(request) == ( "python /workspace/calculate_ratio.py" ) assert agent_loop._requested_post_edit_verification(request) def test_run_the_script_is_not_inferred_when_multiple_scripts_are_named(): request = ( "Write /workspace/prepare.py and /workspace/calculate.py, then run the script." ) assert agent_loop._requested_verification_command(request) == ""