From 6d8c21b334cfdb5f53c4b4b3de63b2ef20789bd7 Mon Sep 17 00:00:00 2001 From: pewdiepie-archdaemon Date: Thu, 17 Sep 2026 21:21:59 +0000 Subject: [PATCH] Recognize explicit link instructions and expose tool latency in search audit --- scripts/verify_clean_v3_search_quality.mjs | 3 +++ src/clean_agent_preview.py | 8 ++++++++ tests/test_search_observation_budget.py | 16 ++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/scripts/verify_clean_v3_search_quality.mjs b/scripts/verify_clean_v3_search_quality.mjs index c01b3967e..b6e69acb8 100644 --- a/scripts/verify_clean_v3_search_quality.mjs +++ b/scripts/verify_clean_v3_search_quality.mjs @@ -82,6 +82,9 @@ async function send(page, prompt) { const observation = { prompt, seconds: (performance.now() - started) / 1000, rounds: metrics?.agent_rounds ?? null, + tool_execution_timings: metrics?.tool_execution_timings || [], + runtime_seconds: metrics?.response_time ?? null, + output_tokens: metrics?.output_tokens ?? null, actual_model: metrics?.model ?? null, selection_mode: contract?.selection_mode ?? null, policy_decisions: metrics?.policy_decisions || [], diff --git a/src/clean_agent_preview.py b/src/clean_agent_preview.py index 98ca99bd5..bafb35a88 100644 --- a/src/clean_agent_preview.py +++ b/src/clean_agent_preview.py @@ -3251,6 +3251,7 @@ def requested_web_source_links(user_text): r'|\b(?:\d+|one|two|three|four|five)\s+(?:official\s+)?(?:source\s+)?links?\b' r'|\bofficial\s+source\b' r'|\b(?:with|include|provide|cite|show|give|find)\s+(?:the\s+)?(?:official\s+)?(?:sources|citations)\b' + r'|\blink\s+(?:to\s+)?(?:the\s+|your\s+)?(?:original\s+|official\s+)?(?:instructions|sources|documentation|articles?|reports?|studies|manuals?|guides?)\b' r'|\b(?:find|locate|get|download)\b.{0,60}\bofficial\b.{0,60}\b(?:manual|guide|handbook|pdf|documentation)\b' r'|\b(?:find|locate|get|download)\b.{0,80}\b(?:manual|guide|handbook|pdf)\b.{0,40}\b(?:online|official)\b', str(user_text or ''), @@ -3798,6 +3799,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if is_odysseus_merged_tools_model(model): temperature = 0.0 started = time.monotonic() + tool_execution_timings = [] model_choice_experiment = getattr(turn_contract, 'routing_experiment', 'baseline') != 'baseline' direct_user_text = next( (_conversation_user_text(m.get('content', '')) for m in reversed(messages) @@ -4871,6 +4873,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac else: from src.tool_routing_experiment import note_fixture_scope fixture_token = note_fixture_scope.set(experiment_fixture_ids or None) + tool_started = time.monotonic() try: execution_attempted = True desc, result = await execute_tool_block( @@ -4886,6 +4889,10 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac ): result = ui_toggle_state_result(client_runtime_context) finally: + tool_execution_timings.append({ + 'tool': canonical(block.tool_type), 'round': round_number, + 'seconds': round(time.monotonic() - tool_started, 3), + }) note_fixture_scope.reset(fixture_token) if (tool_type == 'private_browser' and not (result or {}).get('blocked') @@ -5692,6 +5699,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'temperature': temperature, 'max_output_tokens': request_max_tokens, 'tool_calls': calls, + 'tool_execution_timings': tool_execution_timings, 'tool_events': executions, 'clean_v3_turn': text_only_clean_trace(history[initial_length:]), 'policy_decisions': policy_decisions, 'schema_mode': 'compact_contract_v5', 'clean_v3_preview': True, diff --git a/tests/test_search_observation_budget.py b/tests/test_search_observation_budget.py index 7843ce4bc..ba7a222ea 100644 --- a/tests/test_search_observation_budget.py +++ b/tests/test_search_observation_budget.py @@ -4,6 +4,22 @@ import json import pytest +@pytest.mark.parametrize('prompt', [ + 'Explain the settings and link the instructions, not just the homepage.', + 'Can you link to the original studies?', + 'Summarize this and link your sources.', +]) +def test_link_as_a_verb_requests_source_completion(prompt): + from src.clean_agent_preview import requested_web_source_links + assert requested_web_source_links(prompt) + + +@pytest.mark.parametrize('prompt', ['Link my calendar to email', 'Explain linked lists', 'What is a network link?']) +def test_non_source_link_intent_does_not_require_citations(prompt): + from src.clean_agent_preview import requested_web_source_links + assert not requested_web_source_links(prompt) + + @pytest.mark.parametrize('prompt', [ 'fix spelling: i recieved the calender invte', 'fix typos only: serch teh web for latset ai neews',