fix: close Wave 1.1 completion-gate audit findings

- Headless consumers (task scheduler, background follow-up) now treat a
  completion-gate final_response as the authoritative answer instead of
  collecting deltas only. A gated replacement no longer leaves scheduled
  output empty, which used to trigger an extra, ungated grace-summary
  model call.
- The scheduler closes the agent stream with contextlib.aclosing, so the
  approval-pause break unwinds the gate's journal and teacher-takeover
  context in its own task. Chained runs no longer inherit a stale
  parent_run_id, and later finalization no longer raises ContextVar
  reset errors.
- On provider error, the completion gate applies the live answer's
  statement filter to persisted round_texts. Diagnostics and the failure
  note survive; claims rejected by the gate cannot reappear on reload.
This commit is contained in:
Alexandre Teixeira
2026-10-01 14:49:32 +01:00
parent f4793696f4
commit d49071bbec
5 changed files with 300 additions and 64 deletions
+39 -1
View File
@@ -219,7 +219,9 @@ async def test_failed_terminal_diagnostics_survive_answer_replacement(terminal_k
chunks = [chunk async for chunk in stream([])]
terminal = next(data['data'] for event, data in _frames(chunks)
if event == 'message' and data.get('type') == terminal_kind)
assert terminal['round_texts'] == diagnostics
# Diagnostics and the failure note survive; the rejected claim does not,
# because round_texts are rendered again when the turn is reloaded.
assert terminal['round_texts'] == ['Earlier tool failure and retry', '[Agent stopped: HTTP 504]']
assert terminal['round_models'] == ['first-model', 'failed-model']
assert terminal['failure'] == {'status': 504, 'message': 'Model request failed'}
assert terminal['failed'] is True
@@ -229,6 +231,42 @@ async def test_failed_terminal_diagnostics_survive_answer_replacement(terminal_k
assert _labels(chunks).index(terminal_kind) < _labels(chunks).index('error')
@pytest.mark.asyncio
async def test_provider_failure_round_texts_cannot_replay_removed_claim_after_reload():
claim = 'I created report.md and all tests passed.'
note = '[Agent stopped: Model request failed (HTTP 504)]'
@with_completion_gate
async def stream(messages):
yield _event({'type': 'tool_start', 'tool': 'read_file'})
yield _event({'delta': 'Inspected the layout. ' + claim})
yield ERROR
yield _event({'type': 'agent_terminal', 'data': {
'failed': True, 'failure': {'status': 504, 'message': 'Model request failed'},
'tool_events': [{'round': 1, 'tool': 'read_file'}],
'round_texts': ['Inspected the layout. ' + claim, 'Retrying the build.\n\n' + note],
}})
yield DONE
chunks = [chunk async for chunk in stream([{'role': 'user', 'content': 'create report.md and run the tests'}])]
assert _labels(chunks) == [
'tool_start', 'final_response', 'completion_decision', 'agent_terminal', 'error',
], _labels(chunks)
assert DONE not in chunks
live = next(data['content'] for event, data in _frames(chunks)
if event == 'message' and data.get('type') == 'final_response')
terminal = next(data['data'] for event, data in _frames(chunks)
if event == 'message' and data.get('type') == 'agent_terminal')
# The chat route persists this metadata and the renderer rebuilds one bubble
# per round from it, so every persisted round is presentation.
persisted = terminal['round_texts']
assert persisted == ['Inspected the layout.', 'Retrying the build.\n\n' + note]
for text in [live, *persisted]:
assert 'tests passed' not in text and 'created report.md' not in text
assert 'Inspected the layout.' in live
assert terminal['completion_decision']['status'] == 'failed'
@pytest.mark.asyncio
async def test_error_boundary_is_independent_of_network_chunking():
@with_completion_gate
+158
View File
@@ -0,0 +1,158 @@
"""Headless consumers present the completion gate's answer and close its stream."""
import asyncio
import json
import sys
import types
from types import SimpleNamespace
import pytest
from src.agent_runtime.completion import with_completion_gate
from src.agent_runtime.journal import current_journal
from src.teacher_escalation import with_teacher_takeover
CLAIM = 'I created report.md and all tests passed.'
def _event(payload):
return 'data: ' + json.dumps(payload) + '\n\n'
def _task():
return SimpleNamespace(
crew_member_id=None, endpoint_url='http://ep/v1', model='m',
session_id='s', owner='admin', prompt='create report.md and run the tests',
name='job', max_steps=5, character_id=None,
)
def _gated_loop(released):
"""Real gate and takeover adapters around a loop that over-claims."""
@with_teacher_takeover
@with_completion_gate
async def stream_agent_loop(*args, messages=None, client_runtime_context=None, **kwargs):
yield _event({'delta': 'Inspected the layout. ' + CLAIM})
yield _event({'type': 'metrics', 'data': {}})
yield 'data: [DONE]\n\n'
async def recording(*args, **kwargs):
async for chunk in stream_agent_loop(*args, **kwargs):
if chunk.startswith('data: {') and '"final_response"' in chunk:
released.append(json.loads(chunk[6:])['content'])
yield chunk
return recording
async def test_scheduler_result_is_the_gated_replacement_without_grace_call(monkeypatch):
from src.task_scheduler import TaskScheduler
released = []
grace_calls = []
async def grace(*args, **kwargs):
grace_calls.append(kwargs)
return 'ungated summary: all tests passed'
monkeypatch.setattr('src.agent_loop.stream_agent_loop', _gated_loop(released))
monkeypatch.setattr('src.task_endpoint.resolve_task_candidates', lambda **kwargs: [])
monkeypatch.setattr('src.task_endpoint.task_llm_call_async', grace)
result = await TaskScheduler(session_manager=None)._run_agent_loop(
'http://ep/v1', 'model', _task(), 's')
assert len(released) == 1
assert result == released[0].strip()
assert 'Inspected the layout.' in result
assert 'tests passed' not in result
assert grace_calls == []
def test_background_followup_prose_is_the_gated_replacement(monkeypatch):
from src import bg_monitor
released = []
agent_loop = types.ModuleType('src.agent_loop')
agent_loop.stream_agent_loop = _gated_loop(released)
monkeypatch.setitem(sys.modules, 'src.agent_loop', agent_loop)
sess = SimpleNamespace(endpoint_url='http://example.test', model='model',
headers=None, context_length=0, id='s1', owner='owner')
full, _ = asyncio.run(bg_monitor._drain_agent(
sess, [{'role': 'user', 'content': 'create report.md and run the tests'}]))
assert len(released) == 1
assert full == released[0]
assert 'tests passed' not in full
@pytest.mark.parametrize('consumer', ['scheduler', 'background'])
def test_later_answer_supersedes_earlier_replacement(monkeypatch, consumer):
async def stream_agent_loop(*args, **kwargs):
yield _event({'type': 'final_response', 'content': 'Earlier summary.'})
yield _event({'delta': 'Final '})
yield _event({'delta': 'answer.'})
yield 'data: [DONE]\n\n'
if consumer == 'scheduler':
from src.task_scheduler import TaskScheduler
monkeypatch.setattr('src.agent_loop.stream_agent_loop', stream_agent_loop)
monkeypatch.setattr('src.task_endpoint.resolve_task_candidates', lambda **kwargs: [])
result = asyncio.run(TaskScheduler(session_manager=None)._run_agent_loop(
'http://ep/v1', 'model', _task(), 's'))
else:
from src import bg_monitor
agent_loop = types.ModuleType('src.agent_loop')
agent_loop.stream_agent_loop = stream_agent_loop
monkeypatch.setitem(sys.modules, 'src.agent_loop', agent_loop)
sess = SimpleNamespace(endpoint_url='http://example.test', model='model',
headers=None, context_length=0, id='s1')
result, _ = asyncio.run(bg_monitor._drain_agent(sess, []))
assert result == 'Final answer.'
async def test_scheduler_approval_pause_closes_gated_stream_in_its_own_context(monkeypatch):
from src.task_scheduler import TaskScheduler
closed = []
lineage = []
@with_teacher_takeover
@with_completion_gate
async def paused_loop(*args, messages=None, client_runtime_context=None, **kwargs):
try:
yield _event({'type': 'tool_output', 'tool': 'bash', 'output': 'Waiting for an exact user approval.',
'ask_user': {'kind': 'tool_approval', 'approval_id': 'missing'}})
yield _event({'delta': 'not reached'})
finally:
closed.append(current_journal() is not None)
@with_teacher_takeover
@with_completion_gate
async def later_loop(*args, messages=None, client_runtime_context=None, **kwargs):
yield _event({'delta': 'Later run.'})
yield _event({'type': 'metrics', 'data': {}})
yield 'data: [DONE]\n\n'
async def later_run():
async for chunk in later_loop(messages=[{'role': 'user', 'content': 'x'}]):
if chunk.startswith('data: {') and '"metrics"' in chunk:
lineage.append(json.loads(chunk[6:])['data']['parent_run_id'])
monkeypatch.setattr('src.agent_loop.stream_agent_loop', paused_loop)
monkeypatch.setattr('src.task_endpoint.resolve_task_candidates', lambda **kwargs: [])
result = await TaskScheduler(session_manager=None)._run_agent_loop(
'http://ep/v1', 'model', _task(), 's')
assert 'paused safely' in result
# Closed during the pause, while its own journal was still bound.
assert closed == [True]
assert current_journal() is None
# Neither a chained task (which copies this context) nor a later run in
# this task inherits the paused run's journal as its parent.
chained = asyncio.create_task(later_run())
await chained
await later_run()
assert lineage == [None, None]