Squash Odysseus development history

services/search/content.py

@@ -65,6 +65,49 @@ try:
 except ImportError:
     pdf_extract_text = None  # type: ignore
 
+try:
+    from pypdf import PdfReader
+except ImportError:
+    PdfReader = None  # type: ignore
+
+
+def _extract_pdf_text(pdf_bytes: bytes, url: str = "") -> str:
+    """Extract PDF text with available permissive dependencies."""
+    # Prefer pypdf's layout mode. Plain text extraction and pdfminer often
+    # collapse table columns into an ambiguous number stream, which makes a
+    # correct source passage easy for the model to misread.
+    if PdfReader is not None:
+        try:
+            reader = PdfReader(io.BytesIO(pdf_bytes))
+            pages: List[str] = []
+            for idx, page in enumerate(reader.pages):
+                try:
+                    try:
+                        page_text = page.extract_text(extraction_mode="layout") or ""
+                    except TypeError:
+                        page_text = page.extract_text() or ""
+                except Exception as e:
+                    logger.warning(f"pypdf extraction failed for {url} page {idx + 1}: {e}")
+                    page_text = ""
+                if page_text.strip():
+                    pages.append(f"[Page {idx + 1}]\n{page_text.strip()}")
+            if pages:
+                return "\n\n".join(pages)
+        except Exception as e:
+            logger.warning(f"pypdf extraction failed for {url}: {e}")
+
+    if pdf_extract_text is not None:
+        try:
+            text = pdf_extract_text(io.BytesIO(pdf_bytes)) or ""
+            if text.strip():
+                return text
+        except Exception as e:
+            logger.warning(f"pdfminer extraction failed for {url}: {e}")
+
+    if PdfReader is None and pdf_extract_text is None:
+        logger.error("No PDF text extractor installed; install pdfminer.six or pypdf.")
+    return ""
+
 
 # ----------------------------------------------------------------------
 # HTML extraction helpers
@@ -216,9 +259,6 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
             "User-Agent": WEB_FETCH_USER_AGENT,
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.5",
-            # identity so the streamed size cap in _get_public_url stays honest
-            # (a compressed body can decode to far more than Content-Length).
-            "Accept-Encoding": "identity",
             "Connection": "keep-alive",
         }
         response = _get_public_url(url, headers=headers, timeout=timeout,
@@ -252,26 +292,43 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
     # PDF handling
     content_type = response.headers.get("Content-Type", "").lower()
     if "application/pdf" in content_type or url.lower().endswith(".pdf"):
+        if (
+            _size_fields["truncated"]
+            and effective_cap < WEB_FETCH_HARD_MAX_BYTES
+            and (
+                _size_fields["total_bytes"] is None
+                or _size_fields["total_bytes"] <= WEB_FETCH_HARD_MAX_BYTES
+            )
+        ):
+            try:
+                response = _get_public_url(
+                    url,
+                    headers=headers,
+                    timeout=timeout,
+                    max_bytes=WEB_FETCH_HARD_MAX_BYTES,
+                )
+                _size_fields = {
+                    "truncated": getattr(response, "truncated", False),
+                    "fetched_bytes": len(response.content),
+                    "total_bytes": getattr(response, "declared_bytes", None),
+                }
+                effective_cap = WEB_FETCH_HARD_MAX_BYTES
+            except BodyTooLargeError as e:
+                error_logger.warning(f"Refused oversized PDF body for {url}: {e}")
+                return _empty_result(url, f"TooLarge: {e}")
+            except Exception as e:
+                logger.warning(f"Full-budget PDF retry failed for {url}: {e}")
         if _size_fields["truncated"]:
             # A PDF cut mid-stream is not parseable; unlike text there is no
             # useful partial result, so report the budget problem instead.
             _declared = _size_fields["total_bytes"]
-            return _empty_result(
-                url,
-                f"TooLarge: PDF exceeds the {effective_cap:,}-byte fetch budget"
-                + (f" (size {_declared:,} bytes)" if _declared else "")
-                + "; retry with a larger budget if it fits under the hard cap",
+            error = (
+                f"TooLarge: PDF decoded body exceeded the {effective_cap:,}-byte fetch budget"
+                + (f" (declared compressed size {_declared:,} bytes)" if _declared else "")
+                + "; retry with a larger budget if it fits under the hard cap"
             )
-        if pdf_extract_text is None:
-            logger.error("pdfminer.six is not installed; cannot extract PDF text.")
-            pdf_text = ""
-        else:
-            try:
-                pdf_bytes = io.BytesIO(response.content)
-                pdf_text = pdf_extract_text(pdf_bytes)
-            except Exception as e:
-                logger.warning(f"PDF extraction failed for {url}: {e}")
-                pdf_text = ""
+            return {**_empty_result(url, error), **_size_fields}
+        pdf_text = _extract_pdf_text(response.content, url)
         result = {
             "url": url,
             "title": os.path.basename(url),

services/search/core.py

@@ -2,11 +2,15 @@
 
 import json
 import logging
+import re
+import xml.etree.ElementTree as ET
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from typing import Dict, Any, Optional, List, Set
 from urllib.parse import urlparse
 
+import httpx
+
 from .analytics import (
     NetworkError,
     ParseError,
@@ -97,6 +101,8 @@ def _call_provider(provider_name: str, query: str, count: int, time_filter: str
     """Call a search provider by name. Returns list of results or empty list."""
     if provider_name == "searxng":
         return searxng_search_api(query, count, time_filter=time_filter)
+    elif provider_name == "searxng_yep":
+        return searxng_search_api(query, count, time_filter=time_filter, engines="yep")
     elif provider_name == "brave":
         return brave_search(query, count, time_filter)
     elif provider_name == "duckduckgo":
@@ -127,7 +133,484 @@ def _build_provider_chain(primary: str) -> List[str]:
     for fb in fallbacks:
         if fb and fb != primary and fb not in chain and fb != "disabled":
             chain.append(fb)
-    return chain
+    from .providers import provider_configured
+    configured = [provider for provider in chain if provider_configured(provider)]
+    for provider in set(chain) - set(configured):
+        logger.warning("Skipping unconfigured search provider: %s", provider)
+    if primary == "searxng" and configured == ["searxng"]:
+        # No usable configured fallback: try a separate engine on the same
+        # private metasearch instance before reporting retrieval failure.
+        configured.append("searxng_yep")
+    return configured
+
+
+_SEARCH_QUERY_FILLER = {
+    "what", "whats", "what's", "which", "when", "where", "year", "from",
+    "any", "info", "information", "details", "update", "updates",
+    "with", "this", "that", "search", "lookup", "look", "find", "tell",
+    "about", "quick", "please", "pls", "official", "links", "source",
+    "sources", "news", "headlines", "breaking", "latest", "current",
+    "newest", "recent", "today", "now",
+    "release", "releases", "version", "versions", "changelog", "github",
+    "gitlab", "weather", "forecast", "forecasts", "tomorrow", "hourly",
+    "daily", "temperature", "temperatures", "conditions", "rain", "raining",
+    "chance", "precipitation",
+    "january", "february", "march", "april", "may", "june", "july",
+    "august", "september", "october", "november", "december",
+    "the", "and", "or", "but", "are", "was", "were", "does", "did",
+    "can", "could", "should", "would", "will", "has", "have", "had",
+    "for", "into", "onto", "near", "over", "under",
+}
+
+_SHORT_QUERY_SUBJECTS = {"ai", "ar", "eu", "uk", "us", "vr"}
+
+_WEATHER_QUERY_HINTS = {
+    "weather", "forecast", "forecasts", "temperature", "temperatures",
+    "rain", "raining", "precipitation", "humid", "humidity", "wind",
+}
+_WEATHER_RESULT_HINTS = {
+    "weather", "forecast", "temperature", "temperatures", "rain",
+    "precipitation", "humidity", "wind", "accuweather", "meteoblue",
+    "weather-atlas", "weather25", "weather365", "easeweather",
+}
+
+
+def _meaningful_query_terms(query: str) -> list[str]:
+    return [
+        term
+        for term in re.findall(r"[a-z0-9]+", str(query or "").lower())
+        if (len(term) > 2 or term in _SHORT_QUERY_SUBJECTS)
+        and not term.isdigit()
+        and term not in _SEARCH_QUERY_FILLER
+    ]
+
+
+def _result_has_query_overlap(query: str, result: dict) -> bool:
+    terms = _meaningful_query_terms(query)
+    if not terms:
+        return True
+    text = " ".join(
+        str(result.get(key) or "").lower()
+        for key in ("title", "snippet", "url")
+    )
+    query_tokens = set(re.findall(r"[a-z0-9]+", str(query or "").lower()))
+    if query_tokens & _WEATHER_QUERY_HINTS:
+        return (
+            any(re.search(rf"\b{re.escape(term)}\b", text) for term in terms)
+            and any(marker in text for marker in _WEATHER_RESULT_HINTS)
+        )
+    result_tokens = set(re.findall(r"[a-z0-9]+", text))
+
+    def lexical_root(word: str) -> str:
+        for suffix in ("ation", "ition", "ence", "ance", "ment", "ents", "ent", "ant", "ing", "ed", "es", "s"):
+            if word.endswith(suffix) and len(word) - len(suffix) >= 6:
+                return word[:-len(suffix)]
+        return word
+
+    result_roots = {lexical_root(token) for token in result_tokens}
+    matched_terms = {
+        term for term in terms
+        if term in result_tokens or lexical_root(term) in result_roots
+    }
+    # A single broad token is not enough evidence for a detailed entity/event
+    # query.  For example, SearXNG may answer "Sweden 78 year old British woman
+    # deportation Brexit ..." with generic Sweden tourism pages.  Treat that as
+    # an empty provider result so the configured fallback gets a chance.
+    minimum_matches = 2 if len(set(terms)) >= 4 else 1
+    return len(matched_terms) >= minimum_matches
+
+
+def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]:
+    if not results:
+        return []
+    relevant = [result for result in results if _result_has_query_overlap(query, result)]
+    # Only reject a provider when it returned a fully off-topic page set. Mixed
+    # result pages are common; ranking can handle those.
+    return relevant if relevant else []
+
+
+_SCHOLARLY_QUERY_CUE_RE = re.compile(
+    r"\b(?:paper|preprint|arxiv|proceedings|table\s+\d+|figure\s+\d+|"
+    r"appendix\s+[a-z0-9]+|benchmark(?:s)?)\b",
+    re.IGNORECASE,
+)
+_SCHOLARLY_TITLE_FILLER = _SEARCH_QUERY_FILLER | {
+    "paper", "preprint", "arxiv", "proceedings", "table", "figure",
+    "appendix", "authors", "author", "extract", "locate", "read",
+}
+_ARXIV_IDENTIFIER_RE = re.compile(
+    r"(?i)(?:\barxiv\s*:\s*|\barxiv\.org/(?:abs|pdf|html)/)?"
+    r"(?P<identifier>\d{4}\.\d{4,5}(?:v\d+)?)\b"
+)
+_FORMAL_PUBLICATION_CUE_RE = re.compile(
+    r"\b(?:publish(?:ed|ing|cation)?|venue|conference|journal|proceedings|doi)\b",
+    re.IGNORECASE,
+)
+
+
+def _exact_arxiv_identifier_results(query: str) -> list[dict]:
+    """Return deterministic official landing pages for explicit arXiv IDs."""
+    seen: set[str] = set()
+    results: list[dict] = []
+    for match in _ARXIV_IDENTIFIER_RE.finditer(str(query or "")):
+        identifier = match.group("identifier")
+        canonical = re.sub(r"v\d+$", "", identifier, flags=re.IGNORECASE)
+        if canonical in seen:
+            continue
+        seen.add(canonical)
+        results.append({
+            "title": f"arXiv:{canonical} — exact identifier match",
+            "url": f"https://arxiv.org/abs/{canonical}",
+            "snippet": (
+                "Official arXiv landing page resolved directly from the exact "
+                "identifier in the query."
+            ),
+            "source": "arxiv",
+        })
+    return results
+
+
+def _title_before_explicit_arxiv_identifier(query: str) -> str:
+    """Extract a probable title that precedes an explicit arXiv identifier."""
+
+    text = re.sub(r"\s+", " ", str(query or "")).strip()
+    match = _ARXIV_IDENTIFIER_RE.search(text)
+    if not match or not _FORMAL_PUBLICATION_CUE_RE.search(text):
+        return ""
+    candidate = text[:match.start()].strip(" \t,;:-'\"")
+    candidate = re.sub(
+        r"\barxiv(?:\.org)?(?:\s*:\s*|\s+(?:abs|pdf|html)\s*[/ :]*)?$",
+        "",
+        candidate,
+        flags=re.IGNORECASE,
+    ).strip(" \t,;:-'\"")
+    candidate = re.sub(
+        r"^(?:(?:please\s+)?(?:find|locate|search\s+for|look\s+up|verify|check)\s+)"
+        r"(?:(?:the|this)\s+)?(?:paper\s+)?",
+        "",
+        candidate,
+        flags=re.IGNORECASE,
+    ).strip(" \t,;:-'\"")
+    return candidate if len(_normalized_title_terms(candidate)) >= 2 else ""
+
+
+def _normalized_title_terms(value: str) -> list[str]:
+    return [
+        token
+        for token in re.findall(r"[a-z0-9]+", str(value or "").lower())
+        if len(token) > 1 and token not in _SCHOLARLY_TITLE_FILLER
+    ]
+
+
+def _is_distinctive_short_scholarly_title(value: str) -> bool:
+    """Recognize compact model/report names without accepting generic phrases."""
+
+    terms = _normalized_title_terms(value)
+    if not 1 <= len(terms) <= 2:
+        return False
+    text = str(value or "").strip()
+    return bool(
+        re.search(r"\d", text)
+        or re.search(r"\b[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*\b", text)
+    )
+
+
+def _scholarly_title_from_query(query: str) -> str:
+    """Extract a probable paper title only from clearly scholarly searches."""
+
+    text = re.sub(r"\s+", " ", str(query or "")).strip()
+    if not text or not _SCHOLARLY_QUERY_CUE_RE.search(text):
+        return ""
+
+    quoted = [
+        candidate.strip()
+        for candidate in re.findall(r'["“”]([^"“”]{4,180})["“”]', text)
+        if len(_normalized_title_terms(candidate)) >= 3
+        or _is_distinctive_short_scholarly_title(candidate)
+    ]
+    if quoted:
+        return max(quoted, key=lambda candidate: len(_normalized_title_terms(candidate)))
+
+    before_paper = re.search(
+        r"(?:^|\b(?:find|locate|read|from|about)\s+)(.{4,160}?)\s+"
+        r"(?:paper|preprint)\b",
+        text,
+        re.IGNORECASE,
+    )
+    if before_paper:
+        candidate = before_paper.group(1).strip(" ,:;-'")
+        if (
+            len(_normalized_title_terms(candidate)) >= 3
+            or _is_distinctive_short_scholarly_title(candidate)
+        ):
+            return candidate
+
+    before_locator = re.match(
+        r"(.{2,80}?)\s+(?:table|figure)\s+\d+\b",
+        text,
+        re.IGNORECASE,
+    )
+    if before_locator:
+        candidate = before_locator.group(1).strip(" ,:;-'\"")
+        if _is_distinctive_short_scholarly_title(candidate):
+            return candidate
+    return ""
+
+
+def _result_strongly_matches_title(title: str, result: dict) -> bool:
+    wanted = set(_normalized_title_terms(title))
+    found = set(_normalized_title_terms(str(result.get("title") or "")))
+    if len(wanted) < 2 or not found:
+        return False
+    overlap = len(wanted & found) / len(wanted)
+    return overlap >= (1.0 if len(wanted) == 2 else 0.8)
+
+
+def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve a paper title through arXiv's public Atom API."""
+
+    try:
+        response = httpx.get(
+            "https://export.arxiv.org/api/query",
+            params={
+                "search_query": f'ti:"{title}"',
+                "start": 0,
+                "max_results": max(1, min(int(count), 5)),
+            },
+            headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
+            timeout=12.0,
+            follow_redirects=True,
+        )
+        response.raise_for_status()
+        root = ET.fromstring(response.text)
+    except Exception as exc:
+        logger.info("arXiv title lookup failed for %r: %s", title, exc)
+        return []
+
+    namespace = {"atom": "http://www.w3.org/2005/Atom"}
+    matches: list[dict] = []
+    for entry in root.findall("atom:entry", namespace):
+        result_title = " ".join(
+            (entry.findtext("atom:title", default="", namespaces=namespace) or "").split()
+        )
+        if not _result_strongly_matches_title(title, {"title": result_title}):
+            continue
+        entry_id = (entry.findtext("atom:id", default="", namespaces=namespace) or "").strip()
+        arxiv_id = entry_id.rstrip("/").rsplit("/", 1)[-1]
+        if not arxiv_id:
+            continue
+        summary = " ".join(
+            (entry.findtext("atom:summary", default="", namespaces=namespace) or "").split()
+        )
+        matches.append({
+            "title": result_title,
+            "url": f"https://arxiv.org/abs/{arxiv_id}",
+            "snippet": summary,
+            "source": "arxiv",
+        })
+    return matches
+
+
+def _openalex_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve an exact scholarly title through OpenAlex metadata."""
+
+    try:
+        # OpenAlex treats a literal question mark as query syntax and returns
+        # HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".
+        search_title = re.sub(r"[?]+", " ", str(title or "")).strip()
+        response = httpx.get(
+            "https://api.openalex.org/works",
+            params={
+                "search": search_title,
+                "per-page": max(1, min(int(count), 5)),
+                "select": (
+                    "display_name,doi,primary_location,publication_year,type"
+                ),
+            },
+            headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
+            timeout=12.0,
+            follow_redirects=True,
+        )
+        response.raise_for_status()
+        payload = response.json()
+    except Exception as exc:
+        logger.info("OpenAlex title lookup failed for %r: %s", title, exc)
+        return []
+
+    matches: list[dict] = []
+    for item in payload.get("results", []):
+        result_title = str(item.get("display_name") or "").strip()
+        if not _result_strongly_matches_title(title, {"title": result_title}):
+            continue
+        location = item.get("primary_location") or {}
+        url = str(location.get("landing_page_url") or item.get("doi") or "").strip()
+        if url.startswith("http://arxiv.org/"):
+            url = "https://" + url[len("http://"):]
+        if not url:
+            continue
+        snippet = "Exact scholarly-title match from OpenAlex metadata."
+        venue = str(location.get("raw_source_name") or "").strip()
+        year = item.get("publication_year")
+        publication_type = str(item.get("type") or "").strip()
+        version = str(location.get("version") or "").strip()
+        formal_parts: list[str] = []
+        if venue:
+            formal_parts.append(f"{venue}, {year}" if year else venue)
+        elif year:
+            formal_parts.append(str(year))
+        if publication_type:
+            formal_parts.append(f"type: {publication_type}")
+        if version:
+            formal_parts.append(f"version: {version}")
+        if formal_parts:
+            snippet += f" Formal publication: {'; '.join(formal_parts)}."
+        matches.append({
+            "title": result_title,
+            "url": url,
+            "snippet": snippet,
+            "source": "openalex",
+        })
+    return matches
+
+
+def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:
+    """Retry a noisy scholarly query as a bare title, then use arXiv API."""
+
+    try:
+        simplified = searxng_search_api(title, count=max(3, count))
+    except Exception as exc:
+        logger.info("Simplified scholarly search failed for %r: %s", title, exc)
+        simplified = []
+    exact = [
+        result for result in simplified
+        if _result_strongly_matches_title(title, result)
+    ]
+    if exact:
+        return exact[:count]
+    openalex = _openalex_title_results(title, count)
+    if openalex:
+        return openalex
+    return _arxiv_title_results(title, count)
+
+
+def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:
+    """Resolve a clear paper title without waiting on generic search providers."""
+
+    # OpenAlex typically resolves titles in under a second and often returns
+    # the official arXiv landing page. The arXiv API remains the fallback.
+    openalex = _openalex_title_results(title, count)
+    if openalex:
+        return openalex
+    return _arxiv_title_results(title, count)
+
+
+def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:
+    """Prepend an exact arXiv match when a scholarly SERP missed its title."""
+
+    current = list(results or [])
+    identifier_results = _exact_arxiv_identifier_results(query)
+    if identifier_results:
+        title = _title_before_explicit_arxiv_identifier(query)
+        formal_results: list[dict] = []
+        if title:
+            formal_results = [
+                item
+                for item in _openalex_title_results(title, min(count, 3))
+                if "arxiv.org/" not in str(item.get("url") or "").lower()
+            ]
+        exact_urls = {str(item["url"]) for item in identifier_results}
+        formal_urls = {str(item.get("url") or "") for item in formal_results}
+        return (
+            formal_results
+            + identifier_results
+            + [
+                item for item in current
+                if str(item.get("url") or "") not in exact_urls | formal_urls
+            ]
+        )[:count]
+    title = _scholarly_title_from_query(query)
+    if not title:
+        return current
+    exact_current = [
+        item for item in current
+        if _result_strongly_matches_title(title, item)
+    ]
+    if exact_current:
+        exact_ids = {id(item) for item in exact_current}
+        return (exact_current + [item for item in current if id(item) not in exact_ids])[:count]
+    arxiv_results = _scholarly_title_results(title, min(count, 3))
+    if not arxiv_results:
+        return current
+    seen = {str(item.get("url") or "") for item in arxiv_results}
+    return (arxiv_results + [item for item in current if str(item.get("url") or "") not in seen])[:count]
+
+
+def _subject_first_weather_query(query: str) -> str:
+    """Rewrite natural weather questions into the shape SearXNG handles best."""
+    text = re.sub(r"\s+", " ", str(query or "")).strip(" ?")
+    if not text:
+        return text
+    if not (set(re.findall(r"[a-z0-9]+", text.lower())) & _WEATHER_QUERY_HINTS):
+        return text
+    loc_match = re.search(
+        r"\b(?:weather|forecast)\s+(?:in|for|at)\s+(.+)$",
+        text,
+        re.IGNORECASE,
+    )
+    if not loc_match:
+        loc_match = re.search(
+            r"\b(?:weather|forecast)\b.*?\b(?:in|for|at)\s+(.+)$",
+            text,
+            re.IGNORECASE,
+        )
+    if not loc_match:
+        return text
+    location = loc_match.group(1).strip(" ?.,")
+    timing = ""
+    timing_match = re.search(
+        r"\b(today|tomorrow|tonight|this\s+week|next\s+week|now|current)\b",
+        location,
+        re.IGNORECASE,
+    )
+    if timing_match:
+        timing = timing_match.group(1).lower()
+        location = (
+            location[: timing_match.start()] + location[timing_match.end():]
+        ).strip(" ?.,")
+    if not location:
+        return text
+    return re.sub(r"\s+", " ", f"{location} weather forecast {timing}").strip()
+
+
+def _provider_friendly_query(query: str) -> str:
+    """Convert generic question grammar to keyword order without changing its topic."""
+    text = _subject_first_weather_query(query)
+    match = re.fullmatch(
+        r"(?:what|which)\s+(year|date|time)\s+(?:did|does|do|was|were|is|are)\s+(.+)",
+        text,
+        re.IGNORECASE,
+    )
+    if match:
+        return f"{match.group(2).strip()} {match.group(1).lower()}"
+    # Search providers already receive recency separately. Remove a leading
+    # conversational request shell so ranking is driven by the subject rather
+    # than words such as "any", "latest", and "information".
+    cleaned = re.sub(
+        r"^(?:can|could|would)\s+you\s+(?:find|search|look\s+up)\s+",
+        "",
+        text,
+        flags=re.IGNORECASE,
+    )
+    cleaned = re.sub(
+        r"^(?:any\s+)?(?:latest|current|recent)?\s*"
+        r"(?:news|info(?:rmation)?|updates?|details?)\s+(?:on|about)\s+",
+        "",
+        cleaned,
+        flags=re.IGNORECASE,
+    )
+    if cleaned.strip():
+        return cleaned.strip()
+    return text
 
 
 # ----------------------------------------------------------------------
@@ -135,6 +618,7 @@ def _build_provider_chain(primary: str) -> List[str]:
 # ----------------------------------------------------------------------
 def searxng_search_results(query: str, count: int = 10, time_filter: str = None) -> list[dict]:
     """Perform a web search using configured provider with caching and retry."""
+    provider_query = _provider_friendly_query(query)
     settings = _get_search_settings()
     search_provider = settings.get("search_provider", "searxng")
     result_count = _get_result_count()
@@ -142,7 +626,17 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
     if count == 10:
         count = result_count
 
-    cache_key = generate_cache_key(f"{query}|{count}|{time_filter}")
+    # A named scholarly work has a deterministic metadata path. Resolve that
+    # first instead of spending the full tool deadline retrying generic search
+    # providers; the returned official URL lets the agent proceed to PDF tools.
+    scholarly_title = _scholarly_title_from_query(provider_query)
+    if scholarly_title:
+        direct_results = _direct_scholarly_title_results(scholarly_title, count)
+        if direct_results:
+            _record_query(provider_query, True, cache_hit=False)
+            return direct_results[:count]
+
+    cache_key = generate_cache_key(f"{provider_query}|{count}|{time_filter}")
     cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
 
     # Check cache
@@ -155,8 +649,22 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
             if expiry and datetime.now() < expiry:
                 logger.debug(f"Search cache hit for query: {query}")
                 results = cached_data["data"]
-                _record_query(query, bool(results), cache_hit=True)
-                return results
+                # Ranking/relevance logic evolves independently from provider
+                # results. Re-apply it on cache hits so stale cached ordering
+                # does not preserve bad SERP choices after a harness fix.
+                results = _filter_low_relevance_results(provider_query, results)
+                if results:
+                    results = rank_search_results(provider_query, results)
+                results = _augment_scholarly_results(provider_query, results, count)
+                if results:
+                    _record_query(query, True, cache_hit=True)
+                    return results
+                logger.info(
+                    "Search cache hit for %r became empty after relevance filtering; refetching",
+                    provider_query,
+                )
+                cache_file.unlink(missing_ok=True)
+                search_cache_index.pop(cache_key, None)
             else:
                 cache_file.unlink(missing_ok=True)
                 search_cache_index.pop(cache_key, None)
@@ -178,7 +686,8 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
         for attempt in range(2):
             try:
                 logger.info(f"Attempting {provider_name} search (attempt {attempt + 1})")
-                results = _call_provider(provider_name, query, count, time_filter)
+                results = _call_provider(provider_name, provider_query, count, time_filter)
+                results = _filter_low_relevance_results(provider_query, results)
                 if results:
                     logger.info(f"{provider_name} search succeeded with {len(results)} results")
                     break
@@ -189,11 +698,14 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
         if results:
             break
 
+    results = _augment_scholarly_results(provider_query, results, count)
+
     success = bool(results)
-    _record_query(query, success, cache_hit=False)
+    _record_query(provider_query, success, cache_hit=False)
 
     if success:
-        results = rank_search_results(query, results)
+        results = rank_search_results(provider_query, results)
+        results = _augment_scholarly_results(provider_query, results, count)
         try:
             expiry = datetime.now() + _cache_duration_for_query(query)
             cache_data = {
@@ -206,10 +718,10 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
             search_cache_index[cache_key] = datetime.now()
             cleanup_cache(SEARCH_CACHE_DIR, search_cache_index, timedelta(hours=1))
         except Exception as e:
-            logger.warning(f"Failed to write search cache for {query}: {e}")
+            logger.warning(f"Failed to write search cache for {provider_query}: {e}")
 
     if not success:
-        logger.error(f"All search providers failed for query: {query}")
+        logger.error(f"All search providers failed for query: {provider_query}")
 
     return results
 
@@ -260,7 +772,8 @@ def comprehensive_web_search(
     return_sources: bool = False,
 ):
     """Perform comprehensive web search with content fetching and advanced filtering."""
-    logger.info(f"Starting comprehensive search for: {query}")
+    provider_query = _provider_friendly_query(query)
+    logger.info(f"Starting comprehensive search for: {provider_query}")
     if time_filter:
         logger.info(f"Applying time filter: {time_filter}")
 
@@ -285,7 +798,8 @@ def comprehensive_web_search(
         empty = False
         for attempt in range(2):
             try:
-                search_results = _call_provider(provider_name, query, fetch_count, time_filter)
+                search_results = _call_provider(provider_name, provider_query, fetch_count, time_filter)
+                search_results = _filter_low_relevance_results(provider_query, search_results)
                 if search_results:
                     provider_attempts[provider_name] = f"ok ({len(search_results)})"
                     logger.info(f"Comprehensive search: {provider_name} returned {len(search_results)} results")
@@ -301,6 +815,12 @@ def comprehensive_web_search(
         elif empty:
             provider_attempts[provider_name] = "empty"
 
+    search_results = _augment_scholarly_results(
+        provider_query,
+        search_results,
+        fetch_count,
+    )
+
     if not search_results:
         tally = ", ".join(f"{p}:{r}" for p, r in provider_attempts.items()) or "no providers configured"
         any_errors = any(r.startswith("error") for r in provider_attempts.values())
@@ -315,7 +835,12 @@ def comprehensive_web_search(
         logger.warning(msg)
         return (msg, []) if return_sources else msg
 
-    search_results = rank_search_results(query, search_results)
+    search_results = rank_search_results(provider_query, search_results)
+    search_results = _augment_scholarly_results(
+        provider_query,
+        search_results,
+        fetch_count,
+    )
 
     # URL filter helper
     def url_passes_filters(url: str) -> bool:
@@ -399,7 +924,7 @@ def comprehensive_web_search(
 
     output_parts.append("=" * 70)
     output_parts.append("WEB SEARCH RESULTS AND FETCHED CONTENT")
-    output_parts.append(f"Query: {query}")
+    output_parts.append(f"Query: {provider_query}")
     output_parts.append(f"Searched {len(search_results)} results, fetched {len(fetched_content)} pages")
     output_parts.append("=" * 70)
     output_parts.append("")

services/search/providers.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import re
 from typing import List, Optional
 from urllib.parse import urljoin, urlparse, parse_qs
 
@@ -33,9 +34,16 @@ def _get_search_settings() -> dict:
     """Return search settings from admin config, falling back to env defaults."""
     try:
         from src.settings import load_settings
-        return load_settings()
+        settings = dict(load_settings())
     except Exception:
-        return {}
+        settings = {}
+    # Headless/native deployments do not necessarily have an admin settings
+    # database.  Require an explicit Odysseus-prefixed override so ordinary UI
+    # configuration remains authoritative by default.
+    env_provider = os.environ.get("ODYSSEUS_SEARCH_PROVIDER", "").strip().lower()
+    if env_provider:
+        settings["search_provider"] = env_provider
+    return settings
 
 
 def _get_search_instance() -> str:
@@ -66,13 +74,18 @@ def _get_provider_key(provider: str) -> str:
     if legacy:
         return legacy
     env_map = {
-        "brave": "DATA_BRAVE_API_KEY",
-        "google_pse": "GOOGLE_API_KEY",
-        "tavily": "TAVILY_API_KEY",
-        "serper": "SERPER_API_KEY",
+        # DATA_BRAVE_API_KEY is the historical Odysseus name; BRAVE_API_KEY is
+        # the standard name used by headless runners and the Brave SDK.
+        "brave": ("DATA_BRAVE_API_KEY", "BRAVE_API_KEY"),
+        "google_pse": ("GOOGLE_API_KEY",),
+        "tavily": ("TAVILY_API_KEY",),
+        "serper": ("SERPER_API_KEY",),
     }
-    env_name = env_map.get(provider, "")
-    return (os.environ.get(env_name) or "").strip() if env_name else ""
+    for env_name in env_map.get(provider, ()):
+        value = (os.environ.get(env_name) or "").strip()
+        if value:
+            return value
+    return ""
 
 
 def _get_result_count() -> int:
@@ -84,6 +97,19 @@ def _get_result_count() -> int:
         return 5
 
 
+def provider_configured(provider: str) -> bool:
+    """Configuration readiness only; a configured engine can still fail upstream."""
+    if provider in {"searxng", "searxng_yep", "duckduckgo"}:
+        return True
+    if provider not in {"brave", "google_pse", "tavily", "serper"}:
+        return False
+    if not _get_provider_key(provider):
+        return False
+    if provider == "google_pse":
+        return bool(_get_search_settings().get("google_pse_cx") or os.environ.get("GOOGLE_PSE_CX"))
+    return True
+
+
 # Canonical SafeSearch levels: "strict" (default), "moderate", "off".
 # Each provider has its own knob name and value space -- see _safesearch_for(...).
 _SAFESEARCH_LEVELS = ("strict", "moderate", "off")
@@ -124,6 +150,24 @@ def _safesearch_for(provider: str) -> Optional[str]:
 # ── SearXNG ──
 
 _NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "idag")
+_NEWS_EVENT_HINT_RE = re.compile(
+    r"\b(?:deport(?:ation|ed|ing)?|arrest(?:ed|s)?|election(?:s)?|"
+    r"evacuat(?:e|ed|ion)|flood(?:ing|s|ed)?|sanction(?:s|ed)?)\b",
+    re.IGNORECASE,
+)
+_SOFTWARE_RELEASE_HINTS = (
+    "github",
+    "gitlab",
+    "release",
+    "releases",
+    "version",
+    "versions",
+    "changelog",
+    "change log",
+    "pypi",
+    "npm",
+    "package",
+)
 
 # Default general engines (google/duckduckgo/brave/startpage/wikipedia) are
 # routinely rate-limited / CAPTCHA-blocked on this instance and return nothing.
@@ -133,7 +177,7 @@ _GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "bing,mojeek,presea
 
 
 def searxng_search_api(query: str, count: Optional[int] = None, categories: str = "general",
-                       time_filter: Optional[str] = None) -> List[dict]:
+                       time_filter: Optional[str] = None, *, engines: Optional[str] = None) -> List[dict]:
     """Search using SearXNG JSON API. Returns list of {title, url, snippet}."""
     count = count if count is not None else _get_result_count()
     instance = _get_search_instance()
@@ -158,7 +202,19 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
         "safesearch": _safesearch_for("searxng"),
     }
     q_lc = query.lower()
-    is_news = time_filter is not None or any(h in q_lc for h in _NEWS_HINTS)
+    # Fresh software-version queries are usually better served by general
+    # search or canonical project pages than by the news vertical. For example
+    # "latest ollama release version github" can return a sparse news result
+    # that gets filtered as irrelevant, while general engines find GitHub.
+    is_software_release_query = any(h in q_lc for h in _SOFTWARE_RELEASE_HINTS)
+    is_news = (
+        not is_software_release_query
+        and (
+            time_filter is not None
+            or any(h in q_lc for h in _NEWS_HINTS)
+            or bool(_NEWS_EVENT_HINT_RE.search(query))
+        )
+    )
     if is_news and categories == "general":
         params["categories"] = "news"
         if time_filter in ("day", "week", "month", "year"):
@@ -171,6 +227,9 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
         # set returns 0 on this instance — see _GENERAL_ENGINES).
         if categories == "general" and _GENERAL_ENGINES:
             params["engines"] = _GENERAL_ENGINES
+    if engines:
+        params["categories"] = "general"
+        params["engines"] = engines
     try:
         def _parse_results(results):
             return [
@@ -178,6 +237,10 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
                     "title": r.get("title", ""),
                     "url": r.get("url", ""),
                     "snippet": r.get("content", ""),
+                    "provider": "searxng",
+                    "engines": r.get("engines", []),
+                    "published_date": r.get("publishedDate"),
+                    "query": query,
                 }
                 for r in results[:count]
                 if r.get("url")

services/search/ranking.py

@@ -67,6 +67,22 @@ _TRUSTED_NEWS_DOMAINS = {
     "www.theguardian.com", "euronews.com", "www.euronews.com",
     "dw.com", "www.dw.com", "government.se", "www.government.se",
 }
+_SOFTWARE_RELEASE_HINTS = {
+    "github", "gitlab", "release", "releases", "version", "versions",
+    "changelog", "package", "pypi", "npm",
+}
+_PRODUCT_SPEC_HINTS = {
+    "product", "hardware", "device", "phone", "laptop", "desktop", "computer",
+    "chip", "cpu", "gpu", "mac", "iphone", "ipad", "android", "camera",
+    "console", "kindle", "tesla", "car", "model", "price", "pricing", "cost",
+    "buy", "shop", "order", "preorder", "pre-order", "spec", "specs",
+    "specifications", "available", "availability", "ship", "shipping",
+    "released", "launch", "launched", "vram", "memory", "ram", "storage",
+}
+_COMMERCE_OR_SPEC_PATH_HINTS = (
+    "/shop", "/buy", "/store", "/product", "/products", "/spec", "/specs",
+    "/support", "/tech-specs", "/technical-specifications",
+)
 
 
 def _domain(url: str) -> str:
@@ -95,6 +111,8 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
     query_lc = query.lower()
     is_news_query = any(term in _NEWS_HINTS for term in query_terms)
     is_sports_query = bool(_SPORTS_HINT_RE.search(query_lc))
+    is_software_release_query = any(term in _SOFTWARE_RELEASE_HINTS for term in query_terms)
+    is_product_spec_query = any(term in _PRODUCT_SPEC_HINTS for term in query_terms)
 
     def title_score(title: str) -> float:
         if not title:
@@ -144,6 +162,41 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             adjustment -= 1.0
         return adjustment
 
+    def software_release_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_software_release_query:
+            return 0.0
+        netloc = _domain(url)
+        path = urlparse(url).path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if netloc in {"github.com", "www.github.com", "gitlab.com", "www.gitlab.com"}:
+            adjustment += 1.6
+        if "/releases" in path or "/tags" in path:
+            adjustment += 1.2
+        if any(_has_word(text, term) for term in ("release", "releases", "changelog", "version")):
+            adjustment += 0.4
+        if netloc in {"releasealert.dev", "releases.sh", "releasebot.io"}:
+            adjustment -= 0.8
+        return adjustment
+
+    def product_spec_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_product_spec_query:
+            return 0.0
+        parsed = urlparse(url)
+        netloc = parsed.netloc.lower()
+        path = parsed.path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if any(hint in path for hint in _COMMERCE_OR_SPEC_PATH_HINTS):
+            adjustment += 1.1
+        if re.search(r"\b(?:official|specs?|specifications|tech specs|buy|shop|store|price|pricing|available|ships?)\b", text):
+            adjustment += 0.5
+        if netloc.endswith(".com") and any(_has_word(netloc, term) for term in query_terms if len(term) >= 4):
+            adjustment += 0.4
+        if re.search(r"\b(?:rumor|rumour|leak|may|could|expected|reportedly|unannounced)\b", text):
+            adjustment -= 0.8
+        return adjustment
+
     ranked = []
     for result in results:
         title = result.get("title", "")
@@ -157,6 +210,8 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             + 1.5 * domain_score(url)
             + 1.0 * recency_score(age)
             + news_quality_adjustment(title, snippet, url)
+            + software_release_adjustment(title, snippet, url)
+            + product_spec_adjustment(title, snippet, url)
         )
         ranked.append((score, result))