Squash Odysseus development history

services/search/providers.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import re
 from typing import List, Optional
 from urllib.parse import urljoin, urlparse, parse_qs
 
@@ -33,9 +34,16 @@ def _get_search_settings() -> dict:
     """Return search settings from admin config, falling back to env defaults."""
     try:
         from src.settings import load_settings
-        return load_settings()
+        settings = dict(load_settings())
     except Exception:
-        return {}
+        settings = {}
+    # Headless/native deployments do not necessarily have an admin settings
+    # database.  Require an explicit Odysseus-prefixed override so ordinary UI
+    # configuration remains authoritative by default.
+    env_provider = os.environ.get("ODYSSEUS_SEARCH_PROVIDER", "").strip().lower()
+    if env_provider:
+        settings["search_provider"] = env_provider
+    return settings
 
 
 def _get_search_instance() -> str:
@@ -66,13 +74,18 @@ def _get_provider_key(provider: str) -> str:
     if legacy:
         return legacy
     env_map = {
-        "brave": "DATA_BRAVE_API_KEY",
-        "google_pse": "GOOGLE_API_KEY",
-        "tavily": "TAVILY_API_KEY",
-        "serper": "SERPER_API_KEY",
+        # DATA_BRAVE_API_KEY is the historical Odysseus name; BRAVE_API_KEY is
+        # the standard name used by headless runners and the Brave SDK.
+        "brave": ("DATA_BRAVE_API_KEY", "BRAVE_API_KEY"),
+        "google_pse": ("GOOGLE_API_KEY",),
+        "tavily": ("TAVILY_API_KEY",),
+        "serper": ("SERPER_API_KEY",),
     }
-    env_name = env_map.get(provider, "")
-    return (os.environ.get(env_name) or "").strip() if env_name else ""
+    for env_name in env_map.get(provider, ()):
+        value = (os.environ.get(env_name) or "").strip()
+        if value:
+            return value
+    return ""
 
 
 def _get_result_count() -> int:
@@ -84,6 +97,19 @@ def _get_result_count() -> int:
         return 5
 
 
+def provider_configured(provider: str) -> bool:
+    """Configuration readiness only; a configured engine can still fail upstream."""
+    if provider in {"searxng", "searxng_yep", "duckduckgo"}:
+        return True
+    if provider not in {"brave", "google_pse", "tavily", "serper"}:
+        return False
+    if not _get_provider_key(provider):
+        return False
+    if provider == "google_pse":
+        return bool(_get_search_settings().get("google_pse_cx") or os.environ.get("GOOGLE_PSE_CX"))
+    return True
+
+
 # Canonical SafeSearch levels: "strict" (default), "moderate", "off".
 # Each provider has its own knob name and value space -- see _safesearch_for(...).
 _SAFESEARCH_LEVELS = ("strict", "moderate", "off")
@@ -124,6 +150,24 @@ def _safesearch_for(provider: str) -> Optional[str]:
 # ── SearXNG ──
 
 _NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "idag")
+_NEWS_EVENT_HINT_RE = re.compile(
+    r"\b(?:deport(?:ation|ed|ing)?|arrest(?:ed|s)?|election(?:s)?|"
+    r"evacuat(?:e|ed|ion)|flood(?:ing|s|ed)?|sanction(?:s|ed)?)\b",
+    re.IGNORECASE,
+)
+_SOFTWARE_RELEASE_HINTS = (
+    "github",
+    "gitlab",
+    "release",
+    "releases",
+    "version",
+    "versions",
+    "changelog",
+    "change log",
+    "pypi",
+    "npm",
+    "package",
+)
 
 # Default general engines (google/duckduckgo/brave/startpage/wikipedia) are
 # routinely rate-limited / CAPTCHA-blocked on this instance and return nothing.
@@ -133,7 +177,7 @@ _GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "bing,mojeek,presea
 
 
 def searxng_search_api(query: str, count: Optional[int] = None, categories: str = "general",
-                       time_filter: Optional[str] = None) -> List[dict]:
+                       time_filter: Optional[str] = None, *, engines: Optional[str] = None) -> List[dict]:
     """Search using SearXNG JSON API. Returns list of {title, url, snippet}."""
     count = count if count is not None else _get_result_count()
     instance = _get_search_instance()
@@ -158,7 +202,19 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
         "safesearch": _safesearch_for("searxng"),
     }
     q_lc = query.lower()
-    is_news = time_filter is not None or any(h in q_lc for h in _NEWS_HINTS)
+    # Fresh software-version queries are usually better served by general
+    # search or canonical project pages than by the news vertical. For example
+    # "latest ollama release version github" can return a sparse news result
+    # that gets filtered as irrelevant, while general engines find GitHub.
+    is_software_release_query = any(h in q_lc for h in _SOFTWARE_RELEASE_HINTS)
+    is_news = (
+        not is_software_release_query
+        and (
+            time_filter is not None
+            or any(h in q_lc for h in _NEWS_HINTS)
+            or bool(_NEWS_EVENT_HINT_RE.search(query))
+        )
+    )
     if is_news and categories == "general":
         params["categories"] = "news"
         if time_filter in ("day", "week", "month", "year"):
@@ -171,6 +227,9 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
         # set returns 0 on this instance — see _GENERAL_ENGINES).
         if categories == "general" and _GENERAL_ENGINES:
             params["engines"] = _GENERAL_ENGINES
+    if engines:
+        params["categories"] = "general"
+        params["engines"] = engines
     try:
         def _parse_results(results):
             return [
@@ -178,6 +237,10 @@ def searxng_search_api(query: str, count: Optional[int] = None, categories: str
                     "title": r.get("title", ""),
                     "url": r.get("url", ""),
                     "snippet": r.get("content", ""),
+                    "provider": "searxng",
+                    "engines": r.get("engines", []),
+                    "published_date": r.get("publishedDate"),
+                    "query": query,
                 }
                 for r in results[:count]
                 if r.get("url")