From 69a1b97c96b2e515a99cd959e5efd68308ca0614 Mon Sep 17 00:00:00 2001 From: pewdiepie-archdaemon Date: Thu, 17 Sep 2026 18:49:59 +0000 Subject: [PATCH] recover rendered pages from fetch boilerplate --- src/clean_agent_preview.py | 50 +++++++++++++++++++++++++++++++ tests/test_clean_agent_preview.py | 13 +++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/clean_agent_preview.py b/src/clean_agent_preview.py index d452b8af2..e733fe37b 100644 --- a/src/clean_agent_preview.py +++ b/src/clean_agent_preview.py @@ -3412,6 +3412,23 @@ def browser_observation_access_blocked(raw): )) +def web_fetch_observation_is_boilerplate(raw): + """Detect a nominally successful fetch containing repeated site chrome only.""" + text = re.sub(r'\s+', ' ', str(raw or '')).strip() + words = re.findall(r"[A-Za-z0-9][A-Za-z0-9'’-]*", text.casefold()) + if len(words) < 100: + return False + width = 12 + shingles = [tuple(words[index:index + width]) for index in range(len(words) - width + 1)] + if not shingles: + return False + counts = {} + for shingle in shingles: + counts[shingle] = counts.get(shingle, 0) + 1 + repeated = sum(count - 1 for count in counts.values() if count > 1) + return max(counts.values(), default=0) >= 3 and repeated / len(shingles) >= 0.25 + + def bounded_visual_result_blocks(result, *, max_images=3): """Return all inline tool pixels packed within the model image limit.""" images = result.get('images') if isinstance(result, dict) else None @@ -3772,6 +3789,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac answer_recovery_attempted = False force_no_tools_next_round = False force_web_search_next_round = False + force_private_browser_next_round = False suggestion_retry_required = False suggestion_retry_attempted = False media_detail_nudge_sent = False @@ -3955,6 +3973,20 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'function': {'name': web_search_name}, } force_web_search_next_round = False + if force_private_browser_next_round: + private_browser_name = next( + ( + schema['function']['name'] for schema in round_offered + if canonical(schema['function']['name']) == 'private_browser' + ), + None, + ) + if private_browser_name: + request['tool_choice'] = { + 'type': 'function', + 'function': {'name': private_browser_name}, + } + force_private_browser_next_round = False pending, content = {}, '' async with preview_model_response(client, endpoint_url, headers, request, context_recovery) as response: response.raise_for_status() @@ -4672,6 +4704,24 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac result.get('error') or result.get('exit_code') not in (None, 0) ) + if ( + not failed + and canonical(actual_tool) == 'web_fetch' + and web_fetch_observation_is_boilerplate(output) + ): + recovery = ( + 'The static fetch returned repeated navigation or site chrome without ' + 'substantive page content. Treat it as unreadable and use the rendered ' + 'private browser for the same URL.' + ) + result = {**result, 'error': recovery, 'exit_code': 1} + output = recovery + failed = True + force_private_browser_next_round = True + yield event({ + 'type': 'tool_loop_recovery', + 'reason': 'web_fetch_boilerplate_fallback', + }) if not failed and canonical(actual_tool) == 'web_search': successful_web_searches += 1 if successful_web_searches == 2 and not required_artifacts: diff --git a/tests/test_clean_agent_preview.py b/tests/test_clean_agent_preview.py index 6ec687020..242773db2 100644 --- a/tests/test_clean_agent_preview.py +++ b/tests/test_clean_agent_preview.py @@ -5,7 +5,7 @@ import jsonschema import pytest import re -from src.clean_agent_preview import conversation, readonly_call, preview_call_allowed, evaluate_preview_call, authorized_write_families, compact_schemas, normalize_preview_function_args, normalize_preview_call_args, private_browser_dom_batch, private_browser_state_transition, private_browser_success_repeat_limit, stream_preview, denied_response, execution_has_write_effect, requests_mutation, claims_completion, recent_successful_write_families, scope_preview_contract, multimodal_image_count, attachment_reference_count, active_document_context_message, active_email_context_message, targets_active_editor, active_editor_whole_draft_request, active_editor_suggestion_request, scope_active_editor_contract, native_execution_limits, interactive_execution_limit, runtime_required_artifacts, document_suggestions_event, document_suggestion_quality_error, required_read_tool_choice, required_active_editor_tool_choice, sealed_read_arguments, email_identifier_error, requested_item_limit, contract_item_limit, notes_terminal_response, documents_terminal_response, shell_listing_terminal_response, shell_output_terminal_response, ui_panel_terminal_response, ui_toggle_state_result, calendar_terminal_response, memory_terminal_response, tasks_terminal_response, task_list_requires_synthesis, skills_terminal_response, cookbook_servers_terminal_response, prior_short_answer_for_no_tool_summary, prior_collection_repeat_answer, prior_failed_operation_answer, prior_cookbook_server_answer, prior_workspace_path_answer, prior_web_source_answer, inherit_referential_read_arguments, normalized_search_intent, requested_web_source_links, web_source_links, requested_web_link_limit, preserve_requested_web_recency, ground_referenced_note_content, note_search_result_empty, note_referent_error, research_referent_error, private_browser_open_url, private_browser_effective_url, record_tool_execution, align_structured_tool_history, provider_request_messages, offered_tool_alias, dependent_write_prerequisite_error, bounded_research_tool_policy, retrieved_source_urls, serialize_required_email_attachment_chain +from src.clean_agent_preview import conversation, readonly_call, preview_call_allowed, evaluate_preview_call, authorized_write_families, compact_schemas, normalize_preview_function_args, normalize_preview_call_args, private_browser_dom_batch, private_browser_state_transition, private_browser_success_repeat_limit, stream_preview, denied_response, execution_has_write_effect, requests_mutation, claims_completion, recent_successful_write_families, scope_preview_contract, multimodal_image_count, attachment_reference_count, active_document_context_message, active_email_context_message, targets_active_editor, active_editor_whole_draft_request, active_editor_suggestion_request, scope_active_editor_contract, native_execution_limits, interactive_execution_limit, runtime_required_artifacts, document_suggestions_event, document_suggestion_quality_error, required_read_tool_choice, required_active_editor_tool_choice, sealed_read_arguments, email_identifier_error, requested_item_limit, contract_item_limit, notes_terminal_response, documents_terminal_response, shell_listing_terminal_response, shell_output_terminal_response, ui_panel_terminal_response, ui_toggle_state_result, calendar_terminal_response, memory_terminal_response, tasks_terminal_response, task_list_requires_synthesis, skills_terminal_response, cookbook_servers_terminal_response, prior_short_answer_for_no_tool_summary, prior_collection_repeat_answer, prior_failed_operation_answer, prior_cookbook_server_answer, prior_workspace_path_answer, prior_web_source_answer, inherit_referential_read_arguments, normalized_search_intent, requested_web_source_links, web_source_links, requested_web_link_limit, preserve_requested_web_recency, ground_referenced_note_content, note_search_result_empty, note_referent_error, research_referent_error, private_browser_open_url, private_browser_effective_url, web_fetch_observation_is_boilerplate, record_tool_execution, align_structured_tool_history, provider_request_messages, offered_tool_alias, dependent_write_prerequisite_error, bounded_research_tool_policy, retrieved_source_urls, serialize_required_email_attachment_chain from src.tool_capabilities import capabilities_for_tool @@ -1816,6 +1816,17 @@ def test_browser_access_gate_is_not_treated_as_page_evidence(): ) +def test_repeated_navigation_only_fetch_is_not_treated_as_page_evidence(): + navigation = ( + 'World SECTIONS Politics Tech TOP STORIES Newsletter Sign In Subscribe ' + 'The Morning Wire See All Newsletters Asia Pacific Europe Africa ' + ) + assert web_fetch_observation_is_boilerplate(navigation * 5) + assert not web_fetch_observation_is_boilerplate( + 'The article reports that four crew members were aboard. ' * 8 + ) + + def test_preview_allows_only_reversible_client_local_ui_control(): assert preview_call_allowed( 'ui_control', {'action': 'open_panel', 'name': 'gallery'}, 'open gallery'