fix(runtime): reject unobserved verification and preserve safe reasoning

This commit is contained in:
Alexandre Teixeira
2026-09-26 14:24:29 +01:00
parent b241bb3a7b
commit cea8ed297e
3 changed files with 54 additions and 1 deletions
+5 -1
View File
@@ -574,6 +574,7 @@ class EvidenceLedger:
self.requirements = requirements or CompletionRequirements()
self.events: list[EvidenceEvent] = []
self._verification_versions: dict[str, str] = {}
self._verification_versions_captured = False
@classmethod
def from_tool_events(
@@ -716,6 +717,7 @@ class EvidenceLedger:
if authoritative:
versions = event.get('artifact_versions')
self._verification_versions = dict(versions) if isinstance(versions, Mapping) else {}
self._verification_versions_captured = isinstance(versions, Mapping)
self._append(
kind=EvidenceKind.VERIFIER_RESULT,
success=success,
@@ -791,7 +793,9 @@ class EvidenceLedger:
for path in self.requirements.required_artifacts:
identity = artifact_identity(path, self.requirements.workspace_root)
expected = self._verification_versions.get(identity)
if expected in {'unobserved', 'missing-or-unreadable'}:
if expected in {'unobserved', 'missing-or-unreadable'} or (
expected is None and self._verification_versions_captured
):
return CompletionDecision(CompletionStatus.BLOCKED, False,
'artifact version could not be established for verification',
(latest_verifier.event_id,))
+11
View File
@@ -155,6 +155,10 @@ def with_completion_gate(func):
if isinstance(declared, dict):
requirements = requirements_from_runtime_context({'completion_requirements': declared})
requirements = replace(requirements, workspace_root=trusted_workspace or '')
# New obligations affect future receipts only. Never
# backfill historical versions with present bytes.
journal.observed_artifacts = tuple(dict.fromkeys(
(*journal.observed_artifacts, *requirements.required_artifacts)))
continue
if kind == 'ask_user':
awaiting = True
@@ -215,6 +219,13 @@ def with_completion_gate(func):
yield _event({'type': 'completion_decision', 'data': decision.to_dict()})
replaced_answer = bool(reason or unsafe_draft or safe_answer != answer)
if replaced_answer:
reasoning = [event for event in answer_events if event.get('thinking') is True]
_, unsafe_reasoning = completion_answer(
''.join(str(event.get('delta') or '') for event in reasoning), ledger,
replace(presentation_decision, can_complete=True))
if not unsafe_reasoning:
for event in reasoning:
yield _event(event)
yield _event({'type': 'final_response', 'content': safe_answer})
else:
for event in answer_events:
+38
View File
@@ -219,6 +219,44 @@ async def successful_backend(block):
return block.tool_type, {'exit_code': 0, 'output': 'OK'}
@pytest.mark.asyncio
async def test_corrected_answer_preserves_safe_reasoning():
@with_completion_gate
async def stream(messages):
yield 'data: {"delta":"Considering blank rows.","thinking":true}\n\n'
yield 'data: {"delta":"All tests passed."}\n\n'
yield 'data: [DONE]\n\n'
events = decode([chunk async for chunk in stream([])])
assert events[0]['type'] == 'completion_decision'
assert events[1] == {'delta': 'Considering blank rows.', 'thinking': True}
assert events[2]['type'] == 'final_response'
assert 'All tests passed.' not in json.dumps(events)
@pytest.mark.asyncio
@pytest.mark.parametrize('declare_before_verification', [True, False])
async def test_late_artifact_obligations_cannot_reuse_unobserved_versions(tmp_path, declare_before_verification):
(tmp_path / 'app.py').write_text('original')
declaration = 'data: ' + json.dumps({'type': 'metrics', 'data': {
'completion_requirements': {'required_artifacts': ['app.py']}}}) + '\n\n'
@with_completion_gate
async def stream(messages, workspace=None):
if declare_before_verification:
yield declaration
await successful_backend(ToolBlock('write_file', '{"path":"app.py"}'))
await successful_backend(ToolBlock('bash', 'python -m unittest'))
(tmp_path / 'app.py').write_text('changed after verification')
if not declare_before_verification:
yield declaration
yield 'data: {"delta":"Tests passed."}\n\n'
yield 'data: [DONE]\n\n'
events = decode([chunk async for chunk in stream([], workspace=str(tmp_path))])
decision = next(e['data'] for e in events if e.get('type') == 'completion_decision')
assert not decision['can_complete']
assert decision['status'] == 'blocked'
assert 'Tests passed.' not in json.dumps(events)
@pytest.mark.asyncio
async def test_normalization_preserves_provider_arguments_and_replay_identity():
journal = ActionJournal(run_id='known')