diff --git a/scripts/verify_clean_v3_search_quality.mjs b/scripts/verify_clean_v3_search_quality.mjs index a0e9026ea..a3b432e41 100644 --- a/scripts/verify_clean_v3_search_quality.mjs +++ b/scripts/verify_clean_v3_search_quality.mjs @@ -88,6 +88,9 @@ async function send(page, prompt) { prompt, seconds: (performance.now() - started) / 1000, streamed_text_chunks: events.filter(event => typeof event.delta === 'string' && event.delta.length).length, final_replacement_count: events.filter(event => event.type === 'final_response').length, + event_order: events.filter(event => event.type === 'tool_start' || event.type === 'final_response' || event.delta) + .map(event => event.type === 'tool_start' ? `tool:${event.tool}` : event.type === 'final_response' ? 'final' : 'text') + .filter((value, index, array) => index === 0 || value !== array[index - 1]), rendered_answers: renderedAnswers, rounds: metrics?.agent_rounds ?? null, tool_execution_timings: metrics?.tool_execution_timings || [], @@ -121,6 +124,7 @@ try { const cases = [ ['news-typo', ['latset ai neews?', 'more about the second story, with sources'], true], ['country-casual', ['whats new in japan rn'], true], + ['country-sweden', ['Latest news in sweden'], true], ['software-typo', ['latest pythno verison? official source pls'], true], ['manual', ['find official english manual for Sony WH-1000XM5'], true], ['comparison', ['compare current firefox and chrome privacy features with sources'], true], diff --git a/src/clean_agent_preview.py b/src/clean_agent_preview.py index 761c9ad5b..46a55b586 100644 --- a/src/clean_agent_preview.py +++ b/src/clean_agent_preview.py @@ -4089,6 +4089,26 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac for round_number in range(1, round_limit + 1): rounds_used = round_number yield event({'type': 'agent_step', 'round': round_number}) + # Enforce a known research prerequisite before asking the model + # for another response, not after streaming a premature answer. + if ( + broad_current_web_request(direct_user_text) + and successful_web_searches == 1 and web_search_attempts < 2 + and not breadth_recovery_attempted and not search_completion_attempted + and not force_no_tools_next_round and not required_artifacts + and round_number < round_limit + and any(canonical(s['function']['name']) == 'web_search' for s in offered) + and 'web_search' not in permanently_suppressed_tools + ): + breadth_recovery_attempted = True + force_web_search_next_round = True + history.append({'role': 'user', '_harness_control': True, 'content': ( + 'Continue research before drafting the answer. Use the findings from the ' + 'first search to choose one materially different follow-up query that ' + 'fills a gap or corroborates the strongest findings. Then assess the ' + 'source evidence and prepare the briefing. Do not repeat the same query.' + )}) + yield event({'type': 'completion_recovery', 'reason': 'research_before_synthesis'}) if ( required_artifacts and not successful_write @@ -4260,6 +4280,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if ( not prior_summary_answer and not progressive_thinking + and request.get('tool_choice') in (None, 'auto', 'none') ): text_event = {'delta': text} if replace_streamed_draft_on_finish and not streamed_round_text: @@ -4385,9 +4406,12 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'Completion check: the draft is still too shallow and does not ' 'answer the broad current-information request. Using the Web ' 'evidence already gathered, provide a complete useful briefing ' - 'of at least several substantive paragraphs or equivalent bullets, ' + 'with a short opening and clearly separated **bold topic labels** ' + 'or Markdown headings, followed by substantive paragraphs or bullets, ' 'with the main findings, context, source links, and any evidence ' - 'limitations. Do not call another tool or return another one-sentence summary.' + 'limitations. Use descriptive Markdown links next to the supported ' + 'findings rather than raw URLs. Do not call another tool or return ' + 'another one-sentence summary.' ), }) yield event({ @@ -5632,7 +5656,9 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac recovery = ( 'Research budget reached: no more tools will be offered. Using only ' 'the search evidence already returned, provide the complete final ' - 'answer now with useful detail and source URLs. Do not emit a tool call.' + 'answer now with useful detail. For a multi-topic briefing, use bold ' + 'topic labels or headings, separated paragraphs or bullets, and ' + 'descriptive source links alongside supported findings. Do not emit a tool call.' ) if history and history[-1].get('_harness_control'): history[-1]['content'] = ( diff --git a/tests/test_clean_agent_preview.py b/tests/test_clean_agent_preview.py index a2f60118b..451ca0f78 100644 --- a/tests/test_clean_agent_preview.py +++ b/tests/test_clean_agent_preview.py @@ -1200,7 +1200,7 @@ async def test_stream_bounds_research_to_two_searches_fetch_then_synthesis(monke @pytest.mark.parametrize('embedded_article', [False, True]) @pytest.mark.parametrize('empty_second_search', [False, True]) @pytest.mark.parametrize('sources_requested', [False, True]) -async def test_stream_retries_an_obviously_truncated_broad_web_answer(monkeypatch, embedded_article, empty_second_search, sources_requested): +async def test_stream_research_prerequisite_precedes_broad_web_answer(monkeypatch, embedded_article, empty_second_search, sources_requested): """Broad current research expands, retrieves evidence, then synthesizes.""" import src.clean_agent_preview as module @@ -1211,7 +1211,6 @@ async def test_stream_retries_an_obviously_truncated_broad_web_answer(monkeypatc 'arguments': json.dumps({'query': 'latest AI news'}), }, }]}}]}, - {'choices': [{'delta': {'content': 'Current AI news includes reports about U.'}}]}, {'choices': [{'delta': {'tool_calls': [{ 'index': 0, 'id': 'search-2', 'function': { 'name': 'web_search', @@ -1240,7 +1239,7 @@ async def test_stream_retries_an_obviously_truncated_broad_web_answer(monkeypatc ] article = packets[-1]['choices'][0]['delta']['content'] if embedded_article or empty_second_search: - packets.pop(3) + packets.pop(2) packets = iter(packets) requests = [] @@ -1293,27 +1292,27 @@ async def test_stream_retries_an_obviously_truncated_broad_web_answer(monkeypatc )] events = [json.loads(chunk[6:]) for chunk in raw if '[DONE]' not in chunk] - assert len(requests) == (4 if embedded_article or empty_second_search else 5) - assert requests[2]['tool_choice'] == 'required' - assert [s['function']['name'] for s in requests[2]['tools']] == ['web_search'] + assert len(requests) == (3 if embedded_article or empty_second_search else 4) + assert requests[1]['tool_choice'] == 'required' + assert [s['function']['name'] for s in requests[1]['tools']] == ['web_search'] if not embedded_article and not empty_second_search: - assert requests[3]['tool_choice'] == { + assert requests[2]['tool_choice'] == { 'type': 'function', 'function': {'name': 'web_fetch'}, } assert any(s['function']['name'] == 'web_fetch' for s in requests[-1]['tools']) assert requests[-1].get('tool_choice') != 'none' assert any( event.get('type') == 'completion_recovery' - and event.get('reason') == 'insufficient_research_breadth' + and event.get('reason') == 'research_before_synthesis' for event in events ) - assert sum(event.get('reason') == 'insufficient_research_breadth' for event in events) == 1 + assert sum(event.get('reason') == 'research_before_synthesis' for event in events) == 1 assert any( event.get('type') == 'final_response' and 'fuller evidence-based briefing' in event.get('content', '') for event in events ) - assert any( + assert not any( event.get('delta') == 'Current AI news includes reports about U.' for event in events ) @@ -1324,7 +1323,7 @@ async def test_stream_retries_an_obviously_truncated_broad_web_answer(monkeypatc replacement = next(event for event in events if event.get('type') == 'final_response') assert replacement['replacement_scope'] == 'turn' assert replacement['render_owner'] == 'streamed' - assert any(event.get('delta') and event.get('replacement_scope') == 'turn' for event in events) + assert any(event.get('delta') for event in events) def test_task_renderer_honors_few_and_filters_confirmed_morning_schedule():