Squash Odysseus development history

2026-10-17 04:12:24 +02:00 · 2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
@@ -1,19 +1,20 @@
 """Webpage content fetching with caching, PDF extraction, and summarization helpers."""

+import copy
 import io
-import ipaddress
 import json
 import os
 import re
 import logging
-import socket
 from datetime import datetime, timedelta
 from typing import List
-from urllib.parse import urljoin, urlparse

 import httpx
 from bs4 import BeautifulSoup

+from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
+from src import outbound_fetch as _outbound_fetch
+
 from .analytics import RateLimitError, error_logger
 from .cache import (
    CONTENT_CACHE_DIR,
@@ -24,73 +25,39 @@ from .cache import (

 logger = logging.getLogger(__name__)

-_PRIVATE_NETWORKS = (
-    ipaddress.ip_network("0.0.0.0/8"),
-    ipaddress.ip_network("10.0.0.0/8"),
-    ipaddress.ip_network("127.0.0.0/8"),
-    ipaddress.ip_network("169.254.0.0/16"),
-    ipaddress.ip_network("172.16.0.0/12"),
-    ipaddress.ip_network("192.168.0.0/16"),
-    ipaddress.ip_network("::1/128"),
-    ipaddress.ip_network("fc00::/7"),
-    ipaddress.ip_network("fe80::/10"),
-)
+def _is_private_address(addr):
+    return _outbound_fetch._is_private_address(addr)


-def _is_private_address(addr: ipaddress._BaseAddress) -> bool:
-    return addr.is_private or addr.is_loopback or addr.is_link_local or any(addr in net for net in _PRIVATE_NETWORKS)
+def _resolve_hostname_ips(hostname):
+    return _outbound_fetch._resolve_hostname_ips(hostname)


-def _resolve_hostname_ips(hostname: str) -> list[ipaddress._BaseAddress]:
-    try:
-        infos = socket.getaddrinfo(hostname, None)
-    except Exception:
-        return []
-    out = []
-    for info in infos:
-        try:
-            out.append(ipaddress.ip_address(info[4][0]))
-        except Exception:
-            continue
-    return out
+def _public_http_url(url):
+    return _outbound_fetch._public_http_url(url, resolver=_resolve_hostname_ips)


-def _public_http_url(url: str) -> bool:
-    try:
-        parsed = urlparse(url)
-        if parsed.scheme not in ("http", "https"):
-            return False
-        host = (parsed.hostname or "").strip()
-        if not host:
-            return False
-        lower = host.lower()
-        if lower in ("localhost", "metadata", "metadata.google.internal"):
-            return False
-        if lower.endswith((".local", ".localhost", ".internal", ".lan", ".intranet")):
-            return False
-        try:
-            return not _is_private_address(ipaddress.ip_address(host))
-        except ValueError:
-            pass
-        addrs = _resolve_hostname_ips(host)
-        return bool(addrs) and not any(_is_private_address(a) for a in addrs)
-    except Exception:
-        return False
+def _resolve_public_ips(url):
+    return _outbound_fetch._resolve_public_ips(url, resolver=_resolve_hostname_ips)


-def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response:
-    current = url
-    for _ in range(max_redirects + 1):
-        if not _public_http_url(current):
-            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
-        response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False)
-        if response.status_code not in (301, 302, 303, 307, 308):
-            return response
-        location = response.headers.get("location")
-        if not location:
-            return response
-        current = urljoin(str(response.url), location)
-    raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
+_PinnedBackend = _outbound_fetch._PinnedBackend
+_PinnedTransport = _outbound_fetch._PinnedTransport
+BodyTooLargeError = _outbound_fetch.BodyTooLargeError
+_CappedFetch = _outbound_fetch._CappedFetch
+
+
+def _get_public_url(url, headers, timeout, max_redirects=5, max_bytes=None):
+    return _outbound_fetch._get_public_url(
+        url,
+        headers=headers,
+        timeout=timeout,
+        max_redirects=max_redirects,
+        max_bytes=max_bytes,
+        resolve_public_ips=_resolve_public_ips,
+        transport_factory=_PinnedTransport,
+    )
+

 # PDF extraction (optional dependency)
 try:
@@ -98,6 +65,49 @@ try:
 except ImportError:
    pdf_extract_text = None  # type: ignore

+try:
+    from pypdf import PdfReader
+except ImportError:
+    PdfReader = None  # type: ignore
+
+
+def _extract_pdf_text(pdf_bytes: bytes, url: str = "") -> str:
+    """Extract PDF text with available permissive dependencies."""
+    # Prefer pypdf's layout mode. Plain text extraction and pdfminer often
+    # collapse table columns into an ambiguous number stream, which makes a
+    # correct source passage easy for the model to misread.
+    if PdfReader is not None:
+        try:
+            reader = PdfReader(io.BytesIO(pdf_bytes))
+            pages: List[str] = []
+            for idx, page in enumerate(reader.pages):
+                try:
+                    try:
+                        page_text = page.extract_text(extraction_mode="layout") or ""
+                    except TypeError:
+                        page_text = page.extract_text() or ""
+                except Exception as e:
+                    logger.warning(f"pypdf extraction failed for {url} page {idx + 1}: {e}")
+                    page_text = ""
+                if page_text.strip():
+                    pages.append(f"[Page {idx + 1}]\n{page_text.strip()}")
+            if pages:
+                return "\n\n".join(pages)
+        except Exception as e:
+            logger.warning(f"pypdf extraction failed for {url}: {e}")
+
+    if pdf_extract_text is not None:
+        try:
+            text = pdf_extract_text(io.BytesIO(pdf_bytes)) or ""
+            if text.strip():
+                return text
+        except Exception as e:
+            logger.warning(f"pdfminer extraction failed for {url}: {e}")
+
+    if PdfReader is None and pdf_extract_text is None:
+        logger.error("No PDF text extractor installed; install pdfminer.six or pypdf.")
+    return ""
+

 # ----------------------------------------------------------------------
 # HTML extraction helpers
@@ -115,6 +125,28 @@ def _extract_meta(soup: BeautifulSoup) -> dict:
    return {"description": description, "keywords": keywords}


+def _extract_og_image(soup: BeautifulSoup) -> str:
+    """Extract the best representative image URL from meta tags.
+
+    Only returns absolute http(s) URLs -- skips relative paths and data URIs.
+    """
+    candidates = []
+    for prop in ("og:image", "og:image:url", "og:image:secure_url"):
+        tag = soup.find("meta", attrs={"property": prop})
+        if tag and tag.get("content", "").strip():
+            candidates.append(tag["content"].strip())
+    tag = soup.find("meta", attrs={"name": "twitter:image"})
+    if tag and tag.get("content", "").strip():
+        candidates.append(tag["content"].strip())
+    tag = soup.find("meta", attrs={"name": "thumbnail"})
+    if tag and tag.get("content", "").strip():
+        candidates.append(tag["content"].strip())
+    for url in candidates:
+        if url.startswith(("https://", "http://")) and not url.endswith((".svg", ".ico")):
+            return url
+    return ""
+
+
 def _extract_lists(soup: BeautifulSoup) -> List[List[str]]:
    """Return a list of lists, each inner list representing a <ul>/<ol>."""
    all_lists = []
@@ -189,9 +221,19 @@ def _empty_result(url: str, error: str = "") -> dict:
 # ----------------------------------------------------------------------
 # Main content fetcher
 # ----------------------------------------------------------------------
-def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict:
-    """Fetch and extract meaningful content from a webpage with caching."""
-    cache_key = generate_cache_key(url)
+def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
+                          max_bytes: int = None) -> dict:
+    """Fetch and extract meaningful content from a webpage with caching.
+
+    ``max_bytes`` raises the download budget per call (clamped to the hard
+    cap); the default is the soft cap. When the body is cut short the result
+    carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can
+    tell the model the content is partial (#3812).
+    """
+    effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
+    # The cap is part of the cache identity: a truncated soft-cap fetch must
+    # not be served to a later full-budget request for the same URL.
+    cache_key = generate_cache_key(f"{url}#cap={effective_cap}")
    cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache"

    # Check cache
@@ -214,18 +256,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
    # Fetch
    try:
        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "User-Agent": WEB_FETCH_USER_AGENT,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate",
            "Connection": "keep-alive",
        }
-        response = _get_public_url(url, headers=headers, timeout=timeout)
+        response = _get_public_url(url, headers=headers, timeout=timeout,
+                                   max_bytes=effective_cap)

        if response.status_code == 429:
            raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})")

        response.raise_for_status()
+    except BodyTooLargeError as e:
+        error_logger.warning(f"Refused oversized body for {url}: {e}")
+        return _empty_result(url, f"TooLarge: {e}")
+    except httpx.HTTPStatusError as e:
+        error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}")
+        return _empty_result(url, f"HTTP {e.response.status_code}: {e}")
    except httpx.RequestError as e:
        error_logger.error(f"NetworkError fetching {url} (attempt {retry_attempt}): {e}")
        return _empty_result(url, f"NetworkError: {e}")
@@ -233,19 +281,54 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
        error_logger.error(str(e))
        return _empty_result(url, str(e))

+    # Size bookkeeping shared by every content branch below. getattr keeps
+    # plain httpx.Response stand-ins (tests) working without the cap fields.
+    _size_fields = {
+        "truncated": getattr(response, "truncated", False),
+        "fetched_bytes": len(response.content),
+        "total_bytes": getattr(response, "declared_bytes", None),
+    }
+
    # PDF handling
    content_type = response.headers.get("Content-Type", "").lower()
    if "application/pdf" in content_type or url.lower().endswith(".pdf"):
-        if pdf_extract_text is None:
-            logger.error("pdfminer.six is not installed; cannot extract PDF text.")
-            pdf_text = ""
-        else:
+        if (
+            _size_fields["truncated"]
+            and effective_cap < WEB_FETCH_HARD_MAX_BYTES
+            and (
+                _size_fields["total_bytes"] is None
+                or _size_fields["total_bytes"] <= WEB_FETCH_HARD_MAX_BYTES
+            )
+        ):
            try:
-                pdf_bytes = io.BytesIO(response.content)
-                pdf_text = pdf_extract_text(pdf_bytes)
+                response = _get_public_url(
+                    url,
+                    headers=headers,
+                    timeout=timeout,
+                    max_bytes=WEB_FETCH_HARD_MAX_BYTES,
+                )
+                _size_fields = {
+                    "truncated": getattr(response, "truncated", False),
+                    "fetched_bytes": len(response.content),
+                    "total_bytes": getattr(response, "declared_bytes", None),
+                }
+                effective_cap = WEB_FETCH_HARD_MAX_BYTES
+            except BodyTooLargeError as e:
+                error_logger.warning(f"Refused oversized PDF body for {url}: {e}")
+                return _empty_result(url, f"TooLarge: {e}")
            except Exception as e:
-                logger.warning(f"PDF extraction failed for {url}: {e}")
-                pdf_text = ""
+                logger.warning(f"Full-budget PDF retry failed for {url}: {e}")
+        if _size_fields["truncated"]:
+            # A PDF cut mid-stream is not parseable; unlike text there is no
+            # useful partial result, so report the budget problem instead.
+            _declared = _size_fields["total_bytes"]
+            error = (
+                f"TooLarge: PDF decoded body exceeded the {effective_cap:,}-byte fetch budget"
+                + (f" (declared compressed size {_declared:,} bytes)" if _declared else "")
+                + "; retry with a larger budget if it fits under the hard cap"
+            )
+            return {**_empty_result(url, error), **_size_fields}
+        pdf_text = _extract_pdf_text(response.content, url)
        result = {
            "url": url,
            "title": os.path.basename(url),
@@ -259,6 +342,42 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
            "js_message": "",
            "success": bool(pdf_text),
            "error": "" if pdf_text else "Failed to extract PDF text",
+            **_size_fields,
+        }
+        _cache_result(cache_file, cache_key, result, url)
+        return result
+
+    # Plain-text / Markdown / JSON handling. Sources like
+    # raw.githubusercontent.com serve Markdown as `text/plain`, JSON APIs and
+    # raw config files serve `application/json`, and a lot of code and tool
+    # docs live in `.md` / `.txt`. These have no HTML structure, so the HTML
+    # branch below would extract nothing and report "no readable text content".
+    # Return the body verbatim instead. The `is_html` guard keeps real HTML
+    # (including `application/xhtml+xml`) on the parsing path; the `json` check
+    # covers `application/json` and `+json` suffixes; the URL-suffix fallback
+    # catches servers that mislabel text files as `application/octet-stream`.
+    is_html = "html" in content_type
+    is_json = "json" in content_type
+    url_path = url.lower().split("?", 1)[0].split("#", 1)[0]
+    looks_like_text_file = url_path.endswith(
+        (".md", ".markdown", ".txt", ".text", ".json", ".jsonl")
+    )
+    if not is_html and (content_type.startswith("text/") or is_json or looks_like_text_file):
+        text_body = (response.text or "").strip()
+        result = {
+            "url": url,
+            "title": os.path.basename(url_path) or url,
+            "content": text_body,
+            "lists": [],
+            "tables": [],
+            "code_blocks": [],
+            "meta_description": "",
+            "meta_keywords": "",
+            "js_rendered": False,
+            "js_message": "",
+            "success": bool(text_body),
+            "error": "" if text_body else "Empty response body",
+            **_size_fields,
        }
        _cache_result(cache_file, cache_key, result, url)
        return result
@@ -275,10 +394,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
    title_tag = soup.find("title")
    title_text = title_tag.get_text(strip=True) if title_tag else ""
    meta_info = _extract_meta(soup)
+    og_image = _extract_og_image(soup)
    js_rendered = _detect_js_frameworks(soup)
    js_message = "Page appears to be rendered by a JavaScript framework; content may be incomplete." if js_rendered else ""

-    # Main textual content (heuristic)
+    # Main textual content (heuristic): prefer semantic / "content"-classed
+    # containers to skip nav/footer/boilerplate; tuned for article pages.
    main_content = ""
    content_areas = soup.find_all(
        ["main", "article", "section", "div"],
@@ -287,12 +408,23 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
    if content_areas:
        for area in content_areas[:3]:
            main_content += area.get_text(separator=" ", strip=True) + " "
-    if not main_content:
+    main_content = re.sub(r"\s+", " ", main_content).strip()
+
+    # If the heuristic finds only a tiny wrapper, fall back to body text with
+    # obvious boilerplate stripped so UI/deep-research search results do not
+    # look empty for app/landing pages.
+    THIN_CONTENT_CHARS = 600
+    if len(main_content) < THIN_CONTENT_CHARS:
        body = soup.find("body")
        if body:
-            main_content = body.get_text(separator=" ", strip=True)
-
-    main_content = re.sub(r"\s+", " ", main_content).strip()[:8000]
+            body_copy = copy.copy(body)
+            for noise in body_copy.find_all(
+                ["script", "style", "noscript", "template", "nav", "header", "footer", "aside"]
+            ):
+                noise.extract()
+            body_text = re.sub(r"\s+", " ", body_copy.get_text(separator=" ", strip=True)).strip()
+            if len(body_text) > len(main_content):
+                main_content = body_text

    result = {
        "url": url,
@@ -303,10 +435,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
        "code_blocks": _extract_code_blocks(soup),
        "meta_description": meta_info.get("description", ""),
        "meta_keywords": meta_info.get("keywords", ""),
+        "og_image": og_image,
        "js_rendered": js_rendered,
        "js_message": js_message,
        "success": True,
        "error": "",
+        **_size_fields,
    }
    _cache_result(cache_file, cache_key, result, url)
    return result
@@ -348,13 +482,18 @@ def get_tldr(text: str, max_sentences: int = 3) -> str:

 def extract_quotes(text: str) -> List[str]:
    """Return quoted excerpts that are at least 15 characters long."""
-    return [m.group(1).strip() for m in re.finditer(r'["\']([^"\']{15,}?)["\']', text)]
+    # Backreference the opening quote so the closing quote must match it —
+    # otherwise `"text'` (open double, close single) is treated as a quote.
+    return [m.group(2).strip() for m in re.finditer(r'(["\'])([^"\']{15,}?)\1', text)]


 def extract_statistics(text: str) -> List[str]:
    """Find numbers, percentages, dates and simple measurements."""
+    # Match a comma-grouped number (1,000,000) OR a plain digit run (50000) —
+    # the old `\d{1,3}(?:,\d{3})*` matched only the first 3 digits of a
+    # comma-less number, and the trailing `\b` dropped a closing `%`.
    pattern = re.compile(
-        r"\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?\b",
+        r"\b(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?",
        re.IGNORECASE,
    )
    return [m.group(0).strip() for m in pattern.finditer(text)]