fix(runtime): scope completion claims to execution obligations

tests/test_agent_evidence_loop.py

@@ -135,6 +135,40 @@ def test_terminal_completion_missing_artifact_does_not_add_model_rounds(monkeypa
     assert decision["missing_artifacts"] == ["answer.json"]
 
 
+def test_slice2_explanatory_request_does_not_add_verification_or_model_rounds(monkeypatch):
+    calls = _patch_loop(monkeypatch, ['Tests pass when the command exits zero.'])
+    events = _run('Explain how to write code and then test it.', max_rounds=1)
+    assert calls() == 1
+    decision = next(event['data'] for event in events if event.get('type') == 'completion_decision')
+    assert decision['can_complete'] is True
+    assert 'The task is incomplete' not in json.dumps(events)
+    metrics = next(event['data'] for event in events if event.get('type') == 'metrics')
+    assert not metrics['completion_requirements']['verifier_required']
+    assert metrics['completion_gate']['additional_provider_calls'] == 0
+
+
+def test_slice2_fabricated_execution_on_conversational_turn_does_not_add_rounds(monkeypatch):
+    calls = _patch_loop(monkeypatch, ['I ran pytest and all tests passed.'])
+    events = _run('Explain what pytest does.', max_rounds=1)
+    assert calls() == 1
+    final = next(event['content'] for event in events if event.get('type') == 'final_response')
+    assert 'I ran pytest' not in final
+    assert 'The task is incomplete' not in final
+    metrics = next(event['data'] for event in events if event.get('type') == 'metrics')
+    assert metrics['completion_gate']['additional_provider_calls'] == 0
+
+
+def test_slice2_unsupported_test_report_does_not_add_model_rounds(monkeypatch):
+    calls = _patch_loop(monkeypatch, ['I ran pytest and all 42 tests passed.'])
+    events = _run('Run pytest.', max_rounds=4, relevant_tools={'bash'})
+    assert calls() == 1
+    decision = next(event['data'] for event in events if event.get('type') == 'completion_decision')
+    assert decision['can_complete'] is False
+    final = next(event['content'] for event in events if event.get('type') == 'final_response')
+    assert final.startswith('The task is incomplete:')
+    assert '42' not in final
+
+
 def test_failed_trailing_tool_with_planning_prose_continues_artifact_task(monkeypatch):
     calls = _patch_loop(
         monkeypatch,

tests/test_completion_boundary.py

@@ -6,6 +6,8 @@ import json
 import pytest
 
 from src.agent_runtime.completion import with_completion_gate
+from src.agent_runtime.completion import completion_answer
+from src.agent_evidence import CompletionRequirements, EvidenceLedger, infer_completion_requirements
 from src.agent_runtime.journal import current_journal
 from src.tool_types import ToolBlock
 from tests.runtime_evidence_helpers import authoritative_executor
@@ -225,3 +227,132 @@ async def test_cancellation_closes_inner_stream_without_releasing_completion(aft
     assert _labels(chunks) == ['tool_start']
     assert closed == [True]
     assert current_journal() is None
+
+
+@pytest.mark.parametrize('prose', [
+    'Tests pass when the command exits zero.',
+    'If all tests are passing, merge the branch.',
+    'Tests passed if the command exited zero.',
+    'The documentation says "5 passed".',
+    'The documentation says "Tests: FAIL" or "Tests: PASS".',
+    'You can run pytest to verify this.',
+    'A successful test run should show no failures.',
+    'For example, I created the file and updated config.py.',
+    'If I updated config.py, I would run pytest.',
+    'Imagine I ran the tests and all 42 passed.',
+    'Done is the label for a finished item.',
+    '```text\nI ran pytest and all 42 passed.\n```',
+    'Run pytest until there are no failures.',
+])
+def test_slice2_explanatory_prose_is_not_a_current_run_claim(prose):
+    ledger = EvidenceLedger()
+    answer, reason = completion_answer(prose, ledger, ledger.evaluate())
+    assert answer == prose
+    assert not reason
+
+
+@pytest.mark.parametrize('instruction', [
+    'Explain how to write code and then test it.',
+    'Summarise this and check for typos.',
+    'Explain how to update config.py and then verify it.',
+    'Show an example of creating answer.json and checking it.',
+    'The documentation says "run pytest and create answer.json".',
+    'If you run pytest, the tests should pass.',
+])
+def test_slice2_explanatory_request_has_no_execution_requirements(instruction):
+    requirements = infer_completion_requirements(instruction)
+    assert requirements.required_artifacts == ()
+    assert not requirements.verifier_required
+    assert not requirements.executable_verifier_available
+
+
+@pytest.mark.parametrize('instruction', [
+    'Run the tests.', 'Please run pytest.', 'Can you run the test suite?',
+])
+def test_slice2_explicit_test_execution_requires_a_verifier(instruction):
+    requirements = infer_completion_requirements(instruction)
+    assert requirements.verifier_required
+    assert not EvidenceLedger(requirements).evaluate().can_complete
+
+
+@pytest.mark.parametrize('claim', [
+    'I ran the tests.', 'The tests passed.', '42 tests passed.',
+    'I created the file.', 'I updated config.py successfully.',
+])
+def test_slice2_execution_obligation_rejects_unsupported_claims(claim):
+    ledger = EvidenceLedger(CompletionRequirements(required_artifacts=('config.py',)))
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert reason
+    assert answer.startswith('The task is incomplete:')
+    assert claim not in answer
+
+
+@pytest.mark.parametrize('claim', [
+    'I ran pytest to see if the tests passed.',
+    'I updated config.py as an example.',
+    'I ran pytest and should update config.py next.',
+    'config.py was updated successfully.',
+])
+def test_slice2_subordinate_explanation_cannot_hide_a_direct_execution_report(claim):
+    ledger = EvidenceLedger()
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert reason
+    assert claim not in answer
+    assert 'The task is incomplete' not in answer
+
+
+@pytest.mark.asyncio
+async def test_slice2_client_dictionary_cannot_attest_execution():
+    @with_completion_gate
+    async def stream(messages, client_runtime_context=None):
+        yield _event({'delta': 'I ran pytest and all tests passed.'})
+        yield DONE
+
+    context = {'execution_obligation': True, 'execution_verified': True,
+               'evidence_events': [{'tool': 'bash', 'command': 'pytest', 'exit_code': 0}]}
+    chunks = [chunk async for chunk in stream(
+        [{'role': 'user', 'content': 'Explain test output.'}], client_runtime_context=context)]
+    final = next(data['content'] for event, data in _frames(chunks)
+                 if event == 'message' and data.get('type') == 'final_response')
+    assert 'I ran pytest' not in final
+    assert 'The task is incomplete' not in final
+    assert _decision(chunks)['can_complete'] is True
+
+
+@pytest.mark.asyncio
+async def test_slice2_conversational_fabrication_is_corrected_without_execution_incomplete():
+    invocations = []
+
+    @with_completion_gate
+    async def stream(messages):
+        invocations.append(1)
+        yield _event({'delta': 'The function returns a boolean. I ran pytest and all tests passed.'})
+        yield _event({'type': 'metrics', 'data': {}})
+        yield DONE
+
+    chunks = [chunk async for chunk in stream([{'role': 'user', 'content': 'Explain the function.'}])]
+    final = next(data['content'] for event, data in _frames(chunks)
+                 if event == 'message' and data.get('type') == 'final_response')
+    assert 'The function returns a boolean.' in final
+    assert 'I ran pytest' not in final
+    assert 'The task is incomplete' not in final
+    assert _decision(chunks)['can_complete'] is True
+    assert invocations == [1]
+    metrics = next(data['data'] for event, data in _frames(chunks)
+                   if event == 'message' and data.get('type') == 'metrics')
+    assert metrics['completion_gate']['additional_provider_calls'] == 0
+
+
+@pytest.mark.asyncio
+async def test_slice2_quoted_example_does_not_hide_an_unsupported_report():
+    @with_completion_gate
+    async def stream(messages):
+        yield _event({'delta': 'The docs say "5 passed". I ran pytest.'})
+        yield DONE
+
+    chunks = [chunk async for chunk in stream([{'role': 'user', 'content': 'Explain pytest output.'}])]
+    final = next(data['content'] for event, data in _frames(chunks)
+                 if event == 'message' and data.get('type') == 'final_response')
+    assert 'The docs say "5 passed".' in final
+    assert 'I ran pytest' not in final
+    assert 'The task is incomplete' not in final

tests/test_runtime_evidence_contract.py

@@ -128,7 +128,7 @@ def test_readback_does_not_substitute_for_required_executable_tests():
                                   'Test suite ran successfully', 'No failures.', 'Done.',
                                   'I executed the command.', 'Successfully created the file.'])
 def test_no_execution_receipts_cannot_support_adversarial_success_claims(claim):
-    ledger = EvidenceLedger()
+    ledger = EvidenceLedger(CompletionRequirements(verifier_required=True))
     answer, reason = completion_answer(claim, ledger, ledger.evaluate())
     assert reason
     assert answer.startswith('The task is incomplete:')
@@ -178,7 +178,7 @@ async def test_mixed_thinking_delta_cannot_publish_success_before_gate(thinking)
         yield 'data: {"type":"tool_start","tool":"bash"}\n\n'
         yield 'data: {"type":"metrics","data":{"thinking":"All tests passed."}}\n\n'
         yield 'data: [DONE]\n\n'
-    events = decode([chunk async for chunk in stream([])])
+    events = decode([chunk async for chunk in stream([{'role': 'user', 'content': 'Run the tests.'}])])
     assert events[0] == {'type': 'tool_start', 'tool': 'bash'}
     assert events[1]['type'] == 'completion_decision'
     assert not events[1]['data']['can_complete']
@@ -398,3 +398,79 @@ async def test_unknown_tool_never_creates_dispatch_identity(monkeypatch):
         await execute_tool_block(ToolBlock('unknown_nonexistent_tool', '{}'), security_context=NO_TOOL_SECURITY_CONTEXT)
     assert journal.actions[0].execution_id is None
     assert not journal.actions[0].outcome['authoritative']
+
+
+@pytest.mark.parametrize('tool,command,claim', [
+    ('read_file', 'README.md', 'I ran the tests.'),
+    ('bash', 'printf observation', 'The tests passed.'),
+    ('read_file', 'README.md', 'I updated config.py.'),
+    ('bash', 'printf observation', 'I created the file.'),
+    ('write_file', '{"path":"other.py"}', 'I updated config.py.'),
+    ('write_file', '{"path":"nested/config.py"}', 'I updated config.py.'),
+    ('bash', 'python -m unittest', 'I ran pytest and the tests passed.'),
+    ('bash', 'pytest tests/test_other.py', 'I ran pytest tests/test_config.py.'),
+    ('bash', 'pytest', 'I updated config.py and the tests passed.'),
+    ('write_file', '{"path":"config.py"}', 'I updated config.py and ran pytest.'),
+    ('bash', 'python -m unittest pytest', 'I ran pytest.'),
+    ('write_file', '{"path":"config.py"}', 'I updated "settings.py".'),
+    ('bash', 'pytest', 'Created config.py and ran pytest.'),
+])
+def test_slice2_unrelated_receipt_cannot_support_claim(tool, command, claim):
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': tool, 'command': command, 'exit_code': 0},
+    ])
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert reason
+    assert claim not in answer
+
+
+@pytest.mark.parametrize('claim', ['I ran pytest.', 'The tests passed.', 'Tests: PASS'])
+def test_slice2_matching_verifier_supports_test_claim(claim):
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': 'bash', 'command': 'python3 -m pytest -q', 'exit_code': 0},
+    ])
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert claim in answer
+    assert not reason
+
+
+@pytest.mark.parametrize('claim', ['I updated config.py.', 'I updated `./config.py` successfully.',
+                                  'I updated "config.py".'])
+def test_slice2_matching_mutation_supports_artifact_claim(claim):
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': 'edit_file', 'command': '{"path":"config.py"}', 'exit_code': 0},
+    ], CompletionRequirements(required_artifacts=('config.py',)))
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert claim in answer
+    assert not reason
+
+
+def test_slice2_one_matching_path_does_not_support_multiple_artifact_claims():
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': 'edit_file', 'command': '{"path":"config.py"}', 'exit_code': 0},
+    ])
+    claim = 'I updated config.py and settings.py.'
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert reason
+    assert claim not in answer
+
+
+def test_slice2_verifier_before_mutation_cannot_support_current_test_success():
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': 'bash', 'command': 'pytest', 'exit_code': 0},
+        {'tool': 'edit_file', 'command': '{"path":"config.py"}', 'exit_code': 0},
+    ], CompletionRequirements(required_artifacts=('config.py',)))
+    answer, reason = completion_answer('The tests passed.', ledger, ledger.evaluate())
+    assert reason
+    assert 'The tests passed.' not in answer
+
+
+def test_slice2_matching_artifact_and_verifier_support_combined_claim():
+    ledger = EvidenceLedger.from_tool_events([
+        {'tool': 'edit_file', 'command': '{"path":"config.py"}', 'exit_code': 0},
+        {'tool': 'bash', 'command': 'pytest tests/test_config.py', 'exit_code': 0},
+    ], CompletionRequirements(required_artifacts=('config.py',)))
+    claim = 'I updated config.py and ran pytest tests/test_config.py.'
+    answer, reason = completion_answer(claim, ledger, ledger.evaluate())
+    assert claim in answer
+    assert not reason