jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-15 03:12:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squash Odysseus development history

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/search/analytics.py
+37 -25
View File
@@ -6,21 +6,29 @@ from collections import Counter
from pathlib import Path
from typing import Dict, Any
from core.constants import DATA_DIR
from .cache import cache_metrics
logger = logging.getLogger(__name__)
# Dedicated error logger with file handler
_error_log_path = Path(__file__).resolve().parent.parent / "search_engine_error.log"
_error_handler = logging.FileHandler(_error_log_path, encoding="utf-8")
_error_handler.setLevel(logging.WARNING)
_error_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
# Dedicated error logger — write to the data logs directory (writable on both
# native runs and Docker, where DATA_DIR resolves to the bind-mounted volume).
_log_dir = Path(DATA_DIR) / "logs"
_error_log_path = _log_dir / "search_engine_error.log"
error_logger = logging.getLogger("search_engine_error")
error_logger.addHandler(_error_handler)
error_logger.propagate = False
try:
_log_dir.mkdir(parents=True, exist_ok=True)
_error_handler = logging.FileHandler(_error_log_path, encoding="utf-8")
_error_handler.setLevel(logging.WARNING)
_error_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
error_logger.addHandler(_error_handler)
except Exception as _e:
logging.getLogger(__name__).warning("search_engine_error log handler unavailable: %s", _e)
# Analytics file
ANALYTICS_FILE = Path(__file__).resolve().parent.parent / "search_analytics.json"
# Analytics file — also in the writable logs volume.
ANALYTICS_FILE = _log_dir / "search_analytics.json"
# ----------------------------------------------------------------------
@@ -45,32 +53,36 @@ class RateLimitError(SearchEngineError):
# ----------------------------------------------------------------------
# Analytics helpers
# ----------------------------------------------------------------------
def _default_analytics() -> Dict[str, Any]:
return {
"total_queries": 0,
"successful_queries": 0,
"failed_queries": 0,
"cache_hits": 0,
"cache_misses": 0,
"query_patterns": {},
}
def _load_analytics() -> Dict[str, Any]:
"""Load analytics data from the JSON file, creating defaults if missing."""
if not ANALYTICS_FILE.exists():
default = {
"total_queries": 0,
"successful_queries": 0,
"failed_queries": 0,
"cache_hits": 0,
"cache_misses": 0,
"query_patterns": {},
}
default = _default_analytics()
_save_analytics(default)
return default
try:
with open(ANALYTICS_FILE, "r", encoding="utf-8") as f:
return json.load(f)
data = json.load(f)
# Merge over defaults so a file written by an older schema (or a
# partial write) still has every counter — _record_query indexes
# these keys directly and would otherwise raise KeyError.
merged = _default_analytics()
if isinstance(data, dict):
merged.update(data)
return merged
except Exception as e:
logger.warning(f"Failed to load analytics file: {e}")
return {
"total_queries": 0,
"successful_queries": 0,
"failed_queries": 0,
"cache_hits": 0,
"cache_misses": 0,
"query_patterns": {},
}
return _default_analytics()
def _save_analytics(data: Dict[str, Any]) -> None:
services/search/cache.py
+10 -4
View File
@@ -6,17 +6,23 @@ from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict
from core.constants import DATA_DIR
logger = logging.getLogger(__name__)
# Cache directories
CACHE_DIR = Path(__file__).resolve().parent.parent / "cache"
CACHE_DIR = Path(DATA_DIR) / "cache"
SEARCH_CACHE_DIR = CACHE_DIR / "search"
CONTENT_CACHE_DIR = CACHE_DIR / "content"
CACHE_MAX_ENTRIES = 1000
# Create cache directories
SEARCH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
CONTENT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
# Create cache directories. Guarded so an unwritable path (e.g. a read-only
# mount) degrades to no-disk-cache instead of crashing module import.
try:
SEARCH_CACHE_DIR.mkdir(parents=True, exist_ok=True)
CONTENT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
except OSError as _e:
logger.warning("Search cache directory unavailable (%s); disk cache disabled", _e)
# Track cache size for LRU eviction
search_cache_index: Dict[str, datetime] = {}
services/search/content.py
+222 -83
View File
@@ -1,19 +1,20 @@
"""Webpage content fetching with caching, PDF extraction, and summarization helpers."""
import copy
import io
import ipaddress
import json
import os
import re
import logging
import socket
from datetime import datetime, timedelta
from typing import List
from urllib.parse import urljoin, urlparse
import httpx
from bs4 import BeautifulSoup
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
from src import outbound_fetch as _outbound_fetch
from .analytics import RateLimitError, error_logger
from .cache import (
CONTENT_CACHE_DIR,
@@ -24,73 +25,39 @@ from .cache import (
logger = logging.getLogger(__name__)
_PRIVATE_NETWORKS = (
ipaddress.ip_network("0.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
)
def _is_private_address(addr):
return _outbound_fetch._is_private_address(addr)
def _is_private_address(addr: ipaddress._BaseAddress) -> bool:
return addr.is_private or addr.is_loopback or addr.is_link_local or any(addr in net for net in _PRIVATE_NETWORKS)
def _resolve_hostname_ips(hostname):
return _outbound_fetch._resolve_hostname_ips(hostname)
def _resolve_hostname_ips(hostname: str) -> list[ipaddress._BaseAddress]:
try:
infos = socket.getaddrinfo(hostname, None)
except Exception:
return []
out = []
for info in infos:
try:
out.append(ipaddress.ip_address(info[4][0]))
except Exception:
continue
return out
def _public_http_url(url):
return _outbound_fetch._public_http_url(url, resolver=_resolve_hostname_ips)
def _public_http_url(url: str) -> bool:
try:
parsed = urlparse(url)
if parsed.scheme not in ("http", "https"):
return False
host = (parsed.hostname or "").strip()
if not host:
return False
lower = host.lower()
if lower in ("localhost", "metadata", "metadata.google.internal"):
return False
if lower.endswith((".local", ".localhost", ".internal", ".lan", ".intranet")):
return False
try:
return not _is_private_address(ipaddress.ip_address(host))
except ValueError:
pass
addrs = _resolve_hostname_ips(host)
return bool(addrs) and not any(_is_private_address(a) for a in addrs)
except Exception:
return False
def _resolve_public_ips(url):
return _outbound_fetch._resolve_public_ips(url, resolver=_resolve_hostname_ips)
def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5) -> httpx.Response:
current = url
for _ in range(max_redirects + 1):
if not _public_http_url(current):
raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
response = httpx.get(current, headers=headers, timeout=timeout, follow_redirects=False)
if response.status_code not in (301, 302, 303, 307, 308):
return response
location = response.headers.get("location")
if not location:
return response
current = urljoin(str(response.url), location)
raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
_PinnedBackend = _outbound_fetch._PinnedBackend
_PinnedTransport = _outbound_fetch._PinnedTransport
BodyTooLargeError = _outbound_fetch.BodyTooLargeError
_CappedFetch = _outbound_fetch._CappedFetch
def _get_public_url(url, headers, timeout, max_redirects=5, max_bytes=None):
return _outbound_fetch._get_public_url(
url,
headers=headers,
timeout=timeout,
max_redirects=max_redirects,
max_bytes=max_bytes,
resolve_public_ips=_resolve_public_ips,
transport_factory=_PinnedTransport,
)
# PDF extraction (optional dependency)
try:
@@ -98,6 +65,49 @@ try:
except ImportError:
pdf_extract_text = None # type: ignore
try:
from pypdf import PdfReader
except ImportError:
PdfReader = None # type: ignore
def _extract_pdf_text(pdf_bytes: bytes, url: str = "") -> str:
"""Extract PDF text with available permissive dependencies."""
# Prefer pypdf's layout mode. Plain text extraction and pdfminer often
# collapse table columns into an ambiguous number stream, which makes a
# correct source passage easy for the model to misread.
if PdfReader is not None:
try:
reader = PdfReader(io.BytesIO(pdf_bytes))
pages: List[str] = []
for idx, page in enumerate(reader.pages):
try:
try:
page_text = page.extract_text(extraction_mode="layout") or ""
except TypeError:
page_text = page.extract_text() or ""
except Exception as e:
logger.warning(f"pypdf extraction failed for {url} page {idx + 1}: {e}")
page_text = ""
if page_text.strip():
pages.append(f"[Page {idx + 1}]\n{page_text.strip()}")
if pages:
return "\n\n".join(pages)
except Exception as e:
logger.warning(f"pypdf extraction failed for {url}: {e}")
if pdf_extract_text is not None:
try:
text = pdf_extract_text(io.BytesIO(pdf_bytes)) or ""
if text.strip():
return text
except Exception as e:
logger.warning(f"pdfminer extraction failed for {url}: {e}")
if PdfReader is None and pdf_extract_text is None:
logger.error("No PDF text extractor installed; install pdfminer.six or pypdf.")
return ""
# ----------------------------------------------------------------------
# HTML extraction helpers
@@ -115,6 +125,28 @@ def _extract_meta(soup: BeautifulSoup) -> dict:
return {"description": description, "keywords": keywords}
def _extract_og_image(soup: BeautifulSoup) -> str:
"""Extract the best representative image URL from meta tags.
Only returns absolute http(s) URLs -- skips relative paths and data URIs.
"""
candidates = []
for prop in ("og:image", "og:image:url", "og:image:secure_url"):
tag = soup.find("meta", attrs={"property": prop})
if tag and tag.get("content", "").strip():
candidates.append(tag["content"].strip())
tag = soup.find("meta", attrs={"name": "twitter:image"})
if tag and tag.get("content", "").strip():
candidates.append(tag["content"].strip())
tag = soup.find("meta", attrs={"name": "thumbnail"})
if tag and tag.get("content", "").strip():
candidates.append(tag["content"].strip())
for url in candidates:
if url.startswith(("https://", "http://")) and not url.endswith((".svg", ".ico")):
return url
return ""
def _extract_lists(soup: BeautifulSoup) -> List[List[str]]:
"""Return a list of lists, each inner list representing a <ul>/<ol>."""
all_lists = []
@@ -189,9 +221,19 @@ def _empty_result(url: str, error: str = "") -> dict:
# ----------------------------------------------------------------------
# Main content fetcher
# ----------------------------------------------------------------------
def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) -> dict:
"""Fetch and extract meaningful content from a webpage with caching."""
cache_key = generate_cache_key(url)
def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
max_bytes: int = None) -> dict:
"""Fetch and extract meaningful content from a webpage with caching.
``max_bytes`` raises the download budget per call (clamped to the hard
cap); the default is the soft cap. When the body is cut short the result
carries ``truncated``/``fetched_bytes``/``total_bytes`` so callers can
tell the model the content is partial (#3812).
"""
effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
# The cap is part of the cache identity: a truncated soft-cap fetch must
# not be served to a later full-budget request for the same URL.
cache_key = generate_cache_key(f"{url}#cap={effective_cap}")
cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache"
# Check cache
@@ -214,18 +256,24 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
# Fetch
try:
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
"User-Agent": WEB_FETCH_USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
}
response = _get_public_url(url, headers=headers, timeout=timeout)
response = _get_public_url(url, headers=headers, timeout=timeout,
max_bytes=effective_cap)
if response.status_code == 429:
raise RateLimitError(f"Rate limit hit for {url} (attempt {retry_attempt})")
response.raise_for_status()
except BodyTooLargeError as e:
error_logger.warning(f"Refused oversized body for {url}: {e}")
return _empty_result(url, f"TooLarge: {e}")
except httpx.HTTPStatusError as e:
error_logger.warning(f"HTTP {e.response.status_code} fetching {url}: {e}")
return _empty_result(url, f"HTTP {e.response.status_code}: {e}")
except httpx.RequestError as e:
error_logger.error(f"NetworkError fetching {url} (attempt {retry_attempt}): {e}")
return _empty_result(url, f"NetworkError: {e}")
@@ -233,19 +281,54 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
error_logger.error(str(e))
return _empty_result(url, str(e))
# Size bookkeeping shared by every content branch below. getattr keeps
# plain httpx.Response stand-ins (tests) working without the cap fields.
_size_fields = {
"truncated": getattr(response, "truncated", False),
"fetched_bytes": len(response.content),
"total_bytes": getattr(response, "declared_bytes", None),
}
# PDF handling
content_type = response.headers.get("Content-Type", "").lower()
if "application/pdf" in content_type or url.lower().endswith(".pdf"):
if pdf_extract_text is None:
logger.error("pdfminer.six is not installed; cannot extract PDF text.")
pdf_text = ""
else:
if (
_size_fields["truncated"]
and effective_cap < WEB_FETCH_HARD_MAX_BYTES
and (
_size_fields["total_bytes"] is None
or _size_fields["total_bytes"] <= WEB_FETCH_HARD_MAX_BYTES
)
):
try:
pdf_bytes = io.BytesIO(response.content)
pdf_text = pdf_extract_text(pdf_bytes)
response = _get_public_url(
url,
headers=headers,
timeout=timeout,
max_bytes=WEB_FETCH_HARD_MAX_BYTES,
)
_size_fields = {
"truncated": getattr(response, "truncated", False),
"fetched_bytes": len(response.content),
"total_bytes": getattr(response, "declared_bytes", None),
}
effective_cap = WEB_FETCH_HARD_MAX_BYTES
except BodyTooLargeError as e:
error_logger.warning(f"Refused oversized PDF body for {url}: {e}")
return _empty_result(url, f"TooLarge: {e}")
except Exception as e:
logger.warning(f"PDF extraction failed for {url}: {e}")
pdf_text = ""
logger.warning(f"Full-budget PDF retry failed for {url}: {e}")
if _size_fields["truncated"]:
# A PDF cut mid-stream is not parseable; unlike text there is no
# useful partial result, so report the budget problem instead.
_declared = _size_fields["total_bytes"]
error = (
f"TooLarge: PDF decoded body exceeded the {effective_cap:,}-byte fetch budget"
+ (f" (declared compressed size {_declared:,} bytes)" if _declared else "")
+ "; retry with a larger budget if it fits under the hard cap"
)
return {**_empty_result(url, error), **_size_fields}
pdf_text = _extract_pdf_text(response.content, url)
result = {
"url": url,
"title": os.path.basename(url),
@@ -259,6 +342,42 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
"js_message": "",
"success": bool(pdf_text),
"error": "" if pdf_text else "Failed to extract PDF text",
**_size_fields,
}
_cache_result(cache_file, cache_key, result, url)
return result
# Plain-text / Markdown / JSON handling. Sources like
# raw.githubusercontent.com serve Markdown as `text/plain`, JSON APIs and
# raw config files serve `application/json`, and a lot of code and tool
# docs live in `.md` / `.txt`. These have no HTML structure, so the HTML
# branch below would extract nothing and report "no readable text content".
# Return the body verbatim instead. The `is_html` guard keeps real HTML
# (including `application/xhtml+xml`) on the parsing path; the `json` check
# covers `application/json` and `+json` suffixes; the URL-suffix fallback
# catches servers that mislabel text files as `application/octet-stream`.
is_html = "html" in content_type
is_json = "json" in content_type
url_path = url.lower().split("?", 1)[0].split("#", 1)[0]
looks_like_text_file = url_path.endswith(
(".md", ".markdown", ".txt", ".text", ".json", ".jsonl")
)
if not is_html and (content_type.startswith("text/") or is_json or looks_like_text_file):
text_body = (response.text or "").strip()
result = {
"url": url,
"title": os.path.basename(url_path) or url,
"content": text_body,
"lists": [],
"tables": [],
"code_blocks": [],
"meta_description": "",
"meta_keywords": "",
"js_rendered": False,
"js_message": "",
"success": bool(text_body),
"error": "" if text_body else "Empty response body",
**_size_fields,
}
_cache_result(cache_file, cache_key, result, url)
return result
@@ -275,10 +394,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
title_tag = soup.find("title")
title_text = title_tag.get_text(strip=True) if title_tag else ""
meta_info = _extract_meta(soup)
og_image = _extract_og_image(soup)
js_rendered = _detect_js_frameworks(soup)
js_message = "Page appears to be rendered by a JavaScript framework; content may be incomplete." if js_rendered else ""
# Main textual content (heuristic)
# Main textual content (heuristic): prefer semantic / "content"-classed
# containers to skip nav/footer/boilerplate; tuned for article pages.
main_content = ""
content_areas = soup.find_all(
["main", "article", "section", "div"],
@@ -287,12 +408,23 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
if content_areas:
for area in content_areas[:3]:
main_content += area.get_text(separator=" ", strip=True) + " "
if not main_content:
main_content = re.sub(r"\s+", " ", main_content).strip()
# If the heuristic finds only a tiny wrapper, fall back to body text with
# obvious boilerplate stripped so UI/deep-research search results do not
# look empty for app/landing pages.
THIN_CONTENT_CHARS = 600
if len(main_content) < THIN_CONTENT_CHARS:
body = soup.find("body")
if body:
main_content = body.get_text(separator=" ", strip=True)
main_content = re.sub(r"\s+", " ", main_content).strip()[:8000]
body_copy = copy.copy(body)
for noise in body_copy.find_all(
["script", "style", "noscript", "template", "nav", "header", "footer", "aside"]
):
noise.extract()
body_text = re.sub(r"\s+", " ", body_copy.get_text(separator=" ", strip=True)).strip()
if len(body_text) > len(main_content):
main_content = body_text
result = {
"url": url,
@@ -303,10 +435,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0) ->
"code_blocks": _extract_code_blocks(soup),
"meta_description": meta_info.get("description", ""),
"meta_keywords": meta_info.get("keywords", ""),
"og_image": og_image,
"js_rendered": js_rendered,
"js_message": js_message,
"success": True,
"error": "",
**_size_fields,
}
_cache_result(cache_file, cache_key, result, url)
return result
@@ -348,13 +482,18 @@ def get_tldr(text: str, max_sentences: int = 3) -> str:
def extract_quotes(text: str) -> List[str]:
"""Return quoted excerpts that are at least 15 characters long."""
return [m.group(1).strip() for m in re.finditer(r'["\']([^"\']{15,}?)["\']', text)]
# Backreference the opening quote so the closing quote must match it —
# otherwise `"text'` (open double, close single) is treated as a quote.
return [m.group(2).strip() for m in re.finditer(r'(["\'])([^"\']{15,}?)\1', text)]
def extract_statistics(text: str) -> List[str]:
"""Find numbers, percentages, dates and simple measurements."""
# Match a comma-grouped number (1,000,000) OR a plain digit run (50000) —
# the old `\d{1,3}(?:,\d{3})*` matched only the first 3 digits of a
# comma-less number, and the trailing `\b` dropped a closing `%`.
pattern = re.compile(
r"\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?\b",
r"\b(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?\s*(%|percent|‰|per cent|[a-zA-Z]+)?",
re.IGNORECASE,
)
return [m.group(0).strip() for m in pattern.finditer(text)]
services/search/core.py
+592 -22
View File
@@ -2,11 +2,15 @@
import json
import logging
import re
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List, Set
from urllib.parse import urlparse
import httpx
from .analytics import (
NetworkError,
ParseError,
@@ -30,6 +34,7 @@ from .providers import (
tavily_search,
serper_search,
_get_search_settings,
_get_provider_key,
_get_result_count,
)
from .content import (
@@ -48,30 +53,56 @@ SEARCH_CONFIG: Dict[str, Any] = {
}
def _is_secret_key(name: str) -> bool:
"""True for config keys that hold a credential (e.g. ``brave_api_key``)."""
return name.endswith(("_api_key", "_key", "_token", "_secret"))
def get_search_config() -> Dict[str, Any]:
"""Get current search configuration including active provider info."""
"""Get current search configuration including active provider info.
Never returns stored API keys: callers — including the unauthenticated
``GET /api/search/config`` route — only need key *presence* via
``has_api_key``, not the secret itself (#1661).
"""
config = SEARCH_CONFIG.copy()
settings = _get_search_settings()
provider = settings.get("search_provider", "searxng")
config["active_provider"] = provider
config["has_api_key"] = bool((settings.get("search_api_key") or "").strip())
config["has_api_key"] = bool(_get_provider_key(provider))
config["result_count"] = _get_result_count()
if provider == "searxng":
from .providers import _get_search_instance
config["search_url"] = _get_search_instance()
return config
# Strip any string-valued credential so secrets never reach the response;
# the boolean has_api_key flag (presence only) is preserved.
return {
k: v for k, v in config.items()
if not (isinstance(v, str) and _is_secret_key(k))
}
def update_search_config(api_key: str = None, **kwargs):
"""Update search configuration (e.g. Brave API key)."""
if api_key:
SEARCH_CONFIG["brave_api_key"] = api_key
"""Merge non-secret search config into SEARCH_CONFIG.
Provider API keys are intentionally NOT cached here. They are read on demand
from settings/env via ``_get_provider_key`` (e.g. ``brave_search``), so the
previous ``SEARCH_CONFIG["brave_api_key"] = api_key`` cache was never used
for search and only leaked the decrypted key through ``get_search_config`` /
``GET /api/search/config`` (#1661). ``api_key`` is accepted for backward
compatibility but no longer stored.
"""
for k, v in kwargs.items():
if not _is_secret_key(k):
SEARCH_CONFIG[k] = v
def _call_provider(provider_name: str, query: str, count: int, time_filter: str = None) -> List[dict]:
"""Call a search provider by name. Returns list of results or empty list."""
if provider_name == "searxng":
return searxng_search_api(query, count, time_filter=time_filter)
elif provider_name == "searxng_yep":
return searxng_search_api(query, count, time_filter=time_filter, engines="yep")
elif provider_name == "brave":
return brave_search(query, count, time_filter)
elif provider_name == "duckduckgo":
@@ -102,7 +133,484 @@ def _build_provider_chain(primary: str) -> List[str]:
for fb in fallbacks:
if fb and fb != primary and fb not in chain and fb != "disabled":
chain.append(fb)
return chain
from .providers import provider_configured
configured = [provider for provider in chain if provider_configured(provider)]
for provider in set(chain) - set(configured):
logger.warning("Skipping unconfigured search provider: %s", provider)
if primary == "searxng" and configured == ["searxng"]:
# No usable configured fallback: try a separate engine on the same
# private metasearch instance before reporting retrieval failure.
configured.append("searxng_yep")
return configured
_SEARCH_QUERY_FILLER = {
"what", "whats", "what's", "which", "when", "where", "year", "from",
"any", "info", "information", "details", "update", "updates",
"with", "this", "that", "search", "lookup", "look", "find", "tell",
"about", "quick", "please", "pls", "official", "links", "source",
"sources", "news", "headlines", "breaking", "latest", "current",
"newest", "recent", "today", "now",
"release", "releases", "version", "versions", "changelog", "github",
"gitlab", "weather", "forecast", "forecasts", "tomorrow", "hourly",
"daily", "temperature", "temperatures", "conditions", "rain", "raining",
"chance", "precipitation",
"january", "february", "march", "april", "may", "june", "july",
"august", "september", "october", "november", "december",
"the", "and", "or", "but", "are", "was", "were", "does", "did",
"can", "could", "should", "would", "will", "has", "have", "had",
"for", "into", "onto", "near", "over", "under",
}
_SHORT_QUERY_SUBJECTS = {"ai", "ar", "eu", "uk", "us", "vr"}
_WEATHER_QUERY_HINTS = {
"weather", "forecast", "forecasts", "temperature", "temperatures",
"rain", "raining", "precipitation", "humid", "humidity", "wind",
}
_WEATHER_RESULT_HINTS = {
"weather", "forecast", "temperature", "temperatures", "rain",
"precipitation", "humidity", "wind", "accuweather", "meteoblue",
"weather-atlas", "weather25", "weather365", "easeweather",
}
def _meaningful_query_terms(query: str) -> list[str]:
return [
term
for term in re.findall(r"[a-z0-9]+", str(query or "").lower())
if (len(term) > 2 or term in _SHORT_QUERY_SUBJECTS)
and not term.isdigit()
and term not in _SEARCH_QUERY_FILLER
]
def _result_has_query_overlap(query: str, result: dict) -> bool:
terms = _meaningful_query_terms(query)
if not terms:
return True
text = " ".join(
str(result.get(key) or "").lower()
for key in ("title", "snippet", "url")
)
query_tokens = set(re.findall(r"[a-z0-9]+", str(query or "").lower()))
if query_tokens & _WEATHER_QUERY_HINTS:
return (
any(re.search(rf"\b{re.escape(term)}\b", text) for term in terms)
and any(marker in text for marker in _WEATHER_RESULT_HINTS)
)
result_tokens = set(re.findall(r"[a-z0-9]+", text))
def lexical_root(word: str) -> str:
for suffix in ("ation", "ition", "ence", "ance", "ment", "ents", "ent", "ant", "ing", "ed", "es", "s"):
if word.endswith(suffix) and len(word) - len(suffix) >= 6:
return word[:-len(suffix)]
return word
result_roots = {lexical_root(token) for token in result_tokens}
matched_terms = {
term for term in terms
if term in result_tokens or lexical_root(term) in result_roots
}
# A single broad token is not enough evidence for a detailed entity/event
# query. For example, SearXNG may answer "Sweden 78 year old British woman
# deportation Brexit ..." with generic Sweden tourism pages. Treat that as
# an empty provider result so the configured fallback gets a chance.
minimum_matches = 2 if len(set(terms)) >= 4 else 1
return len(matched_terms) >= minimum_matches
def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]:
if not results:
return []
relevant = [result for result in results if _result_has_query_overlap(query, result)]
# Only reject a provider when it returned a fully off-topic page set. Mixed
# result pages are common; ranking can handle those.
return relevant if relevant else []
_SCHOLARLY_QUERY_CUE_RE = re.compile(
r"\b(?:paper|preprint|arxiv|proceedings|table\s+\d+|figure\s+\d+|"
r"appendix\s+[a-z0-9]+|benchmark(?:s)?)\b",
re.IGNORECASE,
)
_SCHOLARLY_TITLE_FILLER = _SEARCH_QUERY_FILLER | {
"paper", "preprint", "arxiv", "proceedings", "table", "figure",
"appendix", "authors", "author", "extract", "locate", "read",
}
_ARXIV_IDENTIFIER_RE = re.compile(
r"(?i)(?:\barxiv\s*:\s*|\barxiv\.org/(?:abs|pdf|html)/)?"
r"(?P<identifier>\d{4}\.\d{4,5}(?:v\d+)?)\b"
)
_FORMAL_PUBLICATION_CUE_RE = re.compile(
r"\b(?:publish(?:ed|ing|cation)?|venue|conference|journal|proceedings|doi)\b",
re.IGNORECASE,
)
def _exact_arxiv_identifier_results(query: str) -> list[dict]:
"""Return deterministic official landing pages for explicit arXiv IDs."""
seen: set[str] = set()
results: list[dict] = []
for match in _ARXIV_IDENTIFIER_RE.finditer(str(query or "")):
identifier = match.group("identifier")
canonical = re.sub(r"v\d+$", "", identifier, flags=re.IGNORECASE)
if canonical in seen:
continue
seen.add(canonical)
results.append({
"title": f"arXiv:{canonical} — exact identifier match",
"url": f"https://arxiv.org/abs/{canonical}",
"snippet": (
"Official arXiv landing page resolved directly from the exact "
"identifier in the query."
),
"source": "arxiv",
})
return results
def _title_before_explicit_arxiv_identifier(query: str) -> str:
"""Extract a probable title that precedes an explicit arXiv identifier."""
text = re.sub(r"\s+", " ", str(query or "")).strip()
match = _ARXIV_IDENTIFIER_RE.search(text)
if not match or not _FORMAL_PUBLICATION_CUE_RE.search(text):
return ""
candidate = text[:match.start()].strip(" \t,;:-'\"")
candidate = re.sub(
r"\barxiv(?:\.org)?(?:\s*:\s*|\s+(?:abs|pdf|html)\s*[/ :]*)?$",
"",
candidate,
flags=re.IGNORECASE,
).strip(" \t,;:-'\"")
candidate = re.sub(
r"^(?:(?:please\s+)?(?:find|locate|search\s+for|look\s+up|verify|check)\s+)"
r"(?:(?:the|this)\s+)?(?:paper\s+)?",
"",
candidate,
flags=re.IGNORECASE,
).strip(" \t,;:-'\"")
return candidate if len(_normalized_title_terms(candidate)) >= 2 else ""
def _normalized_title_terms(value: str) -> list[str]:
return [
token
for token in re.findall(r"[a-z0-9]+", str(value or "").lower())
if len(token) > 1 and token not in _SCHOLARLY_TITLE_FILLER
]
def _is_distinctive_short_scholarly_title(value: str) -> bool:
"""Recognize compact model/report names without accepting generic phrases."""
terms = _normalized_title_terms(value)
if not 1 <= len(terms) <= 2:
return False
text = str(value or "").strip()
return bool(
re.search(r"\d", text)
or re.search(r"\b[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*\b", text)
)
def _scholarly_title_from_query(query: str) -> str:
"""Extract a probable paper title only from clearly scholarly searches."""
text = re.sub(r"\s+", " ", str(query or "")).strip()
if not text or not _SCHOLARLY_QUERY_CUE_RE.search(text):
return ""
quoted = [
candidate.strip()
for candidate in re.findall(r'["“”]([^"“”]{4,180})["“”]', text)
if len(_normalized_title_terms(candidate)) >= 3
or _is_distinctive_short_scholarly_title(candidate)
]
if quoted:
return max(quoted, key=lambda candidate: len(_normalized_title_terms(candidate)))
before_paper = re.search(
r"(?:^|\b(?:find|locate|read|from|about)\s+)(.{4,160}?)\s+"
r"(?:paper|preprint)\b",
text,
re.IGNORECASE,
)
if before_paper:
candidate = before_paper.group(1).strip(" ,:;-'")
if (
len(_normalized_title_terms(candidate)) >= 3
or _is_distinctive_short_scholarly_title(candidate)
):
return candidate
before_locator = re.match(
r"(.{2,80}?)\s+(?:table|figure)\s+\d+\b",
text,
re.IGNORECASE,
)
if before_locator:
candidate = before_locator.group(1).strip(" ,:;-'\"")
if _is_distinctive_short_scholarly_title(candidate):
return candidate
return ""
def _result_strongly_matches_title(title: str, result: dict) -> bool:
wanted = set(_normalized_title_terms(title))
found = set(_normalized_title_terms(str(result.get("title") or "")))
if len(wanted) < 2 or not found:
return False
overlap = len(wanted & found) / len(wanted)
return overlap >= (1.0 if len(wanted) == 2 else 0.8)
def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve a paper title through arXiv's public Atom API."""
try:
response = httpx.get(
"https://export.arxiv.org/api/query",
params={
"search_query": f'ti:"{title}"',
"start": 0,
"max_results": max(1, min(int(count), 5)),
},
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
timeout=12.0,
follow_redirects=True,
)
response.raise_for_status()
root = ET.fromstring(response.text)
except Exception as exc:
logger.info("arXiv title lookup failed for %r: %s", title, exc)
return []
namespace = {"atom": "http://www.w3.org/2005/Atom"}
matches: list[dict] = []
for entry in root.findall("atom:entry", namespace):
result_title = " ".join(
(entry.findtext("atom:title", default="", namespaces=namespace) or "").split()
)
if not _result_strongly_matches_title(title, {"title": result_title}):
continue
entry_id = (entry.findtext("atom:id", default="", namespaces=namespace) or "").strip()
arxiv_id = entry_id.rstrip("/").rsplit("/", 1)[-1]
if not arxiv_id:
continue
summary = " ".join(
(entry.findtext("atom:summary", default="", namespaces=namespace) or "").split()
)
matches.append({
"title": result_title,
"url": f"https://arxiv.org/abs/{arxiv_id}",
"snippet": summary,
"source": "arxiv",
})
return matches
def _openalex_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve an exact scholarly title through OpenAlex metadata."""
try:
# OpenAlex treats a literal question mark as query syntax and returns
# HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".
search_title = re.sub(r"[?]+", " ", str(title or "")).strip()
response = httpx.get(
"https://api.openalex.org/works",
params={
"search": search_title,
"per-page": max(1, min(int(count), 5)),
"select": (
"display_name,doi,primary_location,publication_year,type"
),
},
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
timeout=12.0,
follow_redirects=True,
)
response.raise_for_status()
payload = response.json()
except Exception as exc:
logger.info("OpenAlex title lookup failed for %r: %s", title, exc)
return []
matches: list[dict] = []
for item in payload.get("results", []):
result_title = str(item.get("display_name") or "").strip()
if not _result_strongly_matches_title(title, {"title": result_title}):
continue
location = item.get("primary_location") or {}
url = str(location.get("landing_page_url") or item.get("doi") or "").strip()
if url.startswith("http://arxiv.org/"):
url = "https://" + url[len("http://"):]
if not url:
continue
snippet = "Exact scholarly-title match from OpenAlex metadata."
venue = str(location.get("raw_source_name") or "").strip()
year = item.get("publication_year")
publication_type = str(item.get("type") or "").strip()
version = str(location.get("version") or "").strip()
formal_parts: list[str] = []
if venue:
formal_parts.append(f"{venue}, {year}" if year else venue)
elif year:
formal_parts.append(str(year))
if publication_type:
formal_parts.append(f"type: {publication_type}")
if version:
formal_parts.append(f"version: {version}")
if formal_parts:
snippet += f" Formal publication: {'; '.join(formal_parts)}."
matches.append({
"title": result_title,
"url": url,
"snippet": snippet,
"source": "openalex",
})
return matches
def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:
"""Retry a noisy scholarly query as a bare title, then use arXiv API."""
try:
simplified = searxng_search_api(title, count=max(3, count))
except Exception as exc:
logger.info("Simplified scholarly search failed for %r: %s", title, exc)
simplified = []
exact = [
result for result in simplified
if _result_strongly_matches_title(title, result)
]
if exact:
return exact[:count]
openalex = _openalex_title_results(title, count)
if openalex:
return openalex
return _arxiv_title_results(title, count)
def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve a clear paper title without waiting on generic search providers."""
# OpenAlex typically resolves titles in under a second and often returns
# the official arXiv landing page. The arXiv API remains the fallback.
openalex = _openalex_title_results(title, count)
if openalex:
return openalex
return _arxiv_title_results(title, count)
def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:
"""Prepend an exact arXiv match when a scholarly SERP missed its title."""
current = list(results or [])
identifier_results = _exact_arxiv_identifier_results(query)
if identifier_results:
title = _title_before_explicit_arxiv_identifier(query)
formal_results: list[dict] = []
if title:
formal_results = [
item
for item in _openalex_title_results(title, min(count, 3))
if "arxiv.org/" not in str(item.get("url") or "").lower()
]
exact_urls = {str(item["url"]) for item in identifier_results}
formal_urls = {str(item.get("url") or "") for item in formal_results}
return (
formal_results
+ identifier_results
+ [
item for item in current
if str(item.get("url") or "") not in exact_urls | formal_urls
]
)[:count]
title = _scholarly_title_from_query(query)
if not title:
return current
exact_current = [
item for item in current
if _result_strongly_matches_title(title, item)
]
if exact_current:
exact_ids = {id(item) for item in exact_current}
return (exact_current + [item for item in current if id(item) not in exact_ids])[:count]
arxiv_results = _scholarly_title_results(title, min(count, 3))
if not arxiv_results:
return current
seen = {str(item.get("url") or "") for item in arxiv_results}
return (arxiv_results + [item for item in current if str(item.get("url") or "") not in seen])[:count]
def _subject_first_weather_query(query: str) -> str:
"""Rewrite natural weather questions into the shape SearXNG handles best."""
text = re.sub(r"\s+", " ", str(query or "")).strip(" ?")
if not text:
return text
if not (set(re.findall(r"[a-z0-9]+", text.lower())) & _WEATHER_QUERY_HINTS):
return text
loc_match = re.search(
r"\b(?:weather|forecast)\s+(?:in|for|at)\s+(.+)$",
text,
re.IGNORECASE,
)
if not loc_match:
loc_match = re.search(
r"\b(?:weather|forecast)\b.*?\b(?:in|for|at)\s+(.+)$",
text,
re.IGNORECASE,
)
if not loc_match:
return text
location = loc_match.group(1).strip(" ?.,")
timing = ""
timing_match = re.search(
r"\b(today|tomorrow|tonight|this\s+week|next\s+week|now|current)\b",
location,
re.IGNORECASE,
)
if timing_match:
timing = timing_match.group(1).lower()
location = (
location[: timing_match.start()] + location[timing_match.end():]
).strip(" ?.,")
if not location:
return text
return re.sub(r"\s+", " ", f"{location} weather forecast {timing}").strip()
def _provider_friendly_query(query: str) -> str:
"""Convert generic question grammar to keyword order without changing its topic."""
text = _subject_first_weather_query(query)
match = re.fullmatch(
r"(?:what|which)\s+(year|date|time)\s+(?:did|does|do|was|were|is|are)\s+(.+)",
text,
re.IGNORECASE,
)
if match:
return f"{match.group(2).strip()} {match.group(1).lower()}"
# Search providers already receive recency separately. Remove a leading
# conversational request shell so ranking is driven by the subject rather
# than words such as "any", "latest", and "information".
cleaned = re.sub(
r"^(?:can|could|would)\s+you\s+(?:find|search|look\s+up)\s+",
"",
text,
flags=re.IGNORECASE,
)
cleaned = re.sub(
r"^(?:any\s+)?(?:latest|current|recent)?\s*"
r"(?:news|info(?:rmation)?|updates?|details?)\s+(?:on|about)\s+",
"",
cleaned,
flags=re.IGNORECASE,
)
if cleaned.strip():
return cleaned.strip()
return text
# ----------------------------------------------------------------------
@@ -110,6 +618,7 @@ def _build_provider_chain(primary: str) -> List[str]:
# ----------------------------------------------------------------------
def searxng_search_results(query: str, count: int = 10, time_filter: str = None) -> list[dict]:
"""Perform a web search using configured provider with caching and retry."""
provider_query = _provider_friendly_query(query)
settings = _get_search_settings()
search_provider = settings.get("search_provider", "searxng")
result_count = _get_result_count()
@@ -117,7 +626,17 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if count == 10:
count = result_count
cache_key = generate_cache_key(f"{query}|{count}|{time_filter}")
# A named scholarly work has a deterministic metadata path. Resolve that
# first instead of spending the full tool deadline retrying generic search
# providers; the returned official URL lets the agent proceed to PDF tools.
scholarly_title = _scholarly_title_from_query(provider_query)
if scholarly_title:
direct_results = _direct_scholarly_title_results(scholarly_title, count)
if direct_results:
_record_query(provider_query, True, cache_hit=False)
return direct_results[:count]
cache_key = generate_cache_key(f"{provider_query}|{count}|{time_filter}")
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
# Check cache
@@ -130,8 +649,22 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if expiry and datetime.now() < expiry:
logger.debug(f"Search cache hit for query: {query}")
results = cached_data["data"]
_record_query(query, bool(results), cache_hit=True)
return results
# Ranking/relevance logic evolves independently from provider
# results. Re-apply it on cache hits so stale cached ordering
# does not preserve bad SERP choices after a harness fix.
results = _filter_low_relevance_results(provider_query, results)
if results:
results = rank_search_results(provider_query, results)
results = _augment_scholarly_results(provider_query, results, count)
if results:
_record_query(query, True, cache_hit=True)
return results
logger.info(
"Search cache hit for %r became empty after relevance filtering; refetching",
provider_query,
)
cache_file.unlink(missing_ok=True)
search_cache_index.pop(cache_key, None)
else:
cache_file.unlink(missing_ok=True)
search_cache_index.pop(cache_key, None)
@@ -153,7 +686,8 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
for attempt in range(2):
try:
logger.info(f"Attempting {provider_name} search (attempt {attempt + 1})")
results = _call_provider(provider_name, query, count, time_filter)
results = _call_provider(provider_name, provider_query, count, time_filter)
results = _filter_low_relevance_results(provider_query, results)
if results:
logger.info(f"{provider_name} search succeeded with {len(results)} results")
break
@@ -164,11 +698,14 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if results:
break
results = _augment_scholarly_results(provider_query, results, count)
success = bool(results)
_record_query(query, success, cache_hit=False)
_record_query(provider_query, success, cache_hit=False)
if success:
results = rank_search_results(query, results)
results = rank_search_results(provider_query, results)
results = _augment_scholarly_results(provider_query, results, count)
try:
expiry = datetime.now() + _cache_duration_for_query(query)
cache_data = {
@@ -181,10 +718,10 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
search_cache_index[cache_key] = datetime.now()
cleanup_cache(SEARCH_CACHE_DIR, search_cache_index, timedelta(hours=1))
except Exception as e:
logger.warning(f"Failed to write search cache for {query}: {e}")
logger.warning(f"Failed to write search cache for {provider_query}: {e}")
if not success:
logger.error(f"All search providers failed for query: {query}")
logger.error(f"All search providers failed for query: {provider_query}")
return results
@@ -203,7 +740,10 @@ def invalidate_search_cache(query: Optional[str] = None) -> None:
search_cache_index.clear()
logger.info("All search cache entries have been cleared.")
else:
cache_key = generate_cache_key(f"{query}|10|None")
# Match the key the write path stores: searxng_search_results replaces
# the caller's default count with the configured _get_result_count()
# (default 5), so a hardcoded "|10|None" never matched a real entry.
cache_key = generate_cache_key(f"{query}|{_get_result_count()}|None")
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
if cache_file.exists():
try:
@@ -232,7 +772,8 @@ def comprehensive_web_search(
return_sources: bool = False,
):
"""Perform comprehensive web search with content fetching and advanced filtering."""
logger.info(f"Starting comprehensive search for: {query}")
provider_query = _provider_friendly_query(query)
logger.info(f"Starting comprehensive search for: {provider_query}")
if time_filter:
logger.info(f"Applying time filter: {time_filter}")
@@ -257,7 +798,8 @@ def comprehensive_web_search(
empty = False
for attempt in range(2):
try:
search_results = _call_provider(provider_name, query, fetch_count, time_filter)
search_results = _call_provider(provider_name, provider_query, fetch_count, time_filter)
search_results = _filter_low_relevance_results(provider_query, search_results)
if search_results:
provider_attempts[provider_name] = f"ok ({len(search_results)})"
logger.info(f"Comprehensive search: {provider_name} returned {len(search_results)} results")
@@ -273,6 +815,12 @@ def comprehensive_web_search(
elif empty:
provider_attempts[provider_name] = "empty"
search_results = _augment_scholarly_results(
provider_query,
search_results,
fetch_count,
)
if not search_results:
tally = ", ".join(f"{p}:{r}" for p, r in provider_attempts.items()) or "no providers configured"
any_errors = any(r.startswith("error") for r in provider_attempts.values())
@@ -287,7 +835,12 @@ def comprehensive_web_search(
logger.warning(msg)
return (msg, []) if return_sources else msg
search_results = rank_search_results(query, search_results)
search_results = rank_search_results(provider_query, search_results)
search_results = _augment_scholarly_results(
provider_query,
search_results,
fetch_count,
)
# URL filter helper
def url_passes_filters(url: str) -> bool:
@@ -328,6 +881,12 @@ def comprehensive_web_search(
for r in search_results if r.get("url")
]
# Map each URL to its [i] number in the sources list so fetched content
# blocks can be labeled with the SAME index the model cites.
_url_index = {
r["url"]: i for i, r in enumerate(search_results, 1) if r.get("url")
}
# Fetch content in parallel
fetched_content = []
with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -340,6 +899,10 @@ def comprehensive_web_search(
try:
result = future.result()
if result["success"] and result["content"] and len(result["content"]) >= min_content_length:
# Remember which source this fetch belongs to: redirects
# can change result["url"] and completion order is
# arbitrary, so the block label cannot be recomputed later.
result["source_index"] = _url_index.get(url)
fetched_content.append(result)
except Exception as e:
logger.error(f"Exception while fetching {url}: {str(e)}")
@@ -361,7 +924,7 @@ def comprehensive_web_search(
output_parts.append("=" * 70)
output_parts.append("WEB SEARCH RESULTS AND FETCHED CONTENT")
output_parts.append(f"Query: {query}")
output_parts.append(f"Query: {provider_query}")
output_parts.append(f"Searched {len(search_results)} results, fetched {len(fetched_content)} pages")
output_parts.append("=" * 70)
output_parts.append("")
@@ -380,8 +943,15 @@ def comprehensive_web_search(
output_parts.append("FETCHED PAGE CONTENT:")
output_parts.append("-" * 50)
for i, content in enumerate(fetched_content, 1):
output_parts.append(f"\n[CONTENT {i}] From: {content['url']}")
# Emit blocks in source order, numbered with the same [i] as the
# sources list, so [CONTENT 2] really is content from source [2].
# Before this, blocks were numbered 1..N in fetch COMPLETION order,
# which matched neither the sources list nor each other run to run.
fetched_content.sort(key=lambda c: c.get("source_index") or len(search_results) + 1)
for content in fetched_content:
_idx = content.get("source_index")
_label = f"[CONTENT {_idx}]" if _idx else "[CONTENT]"
output_parts.append(f"\n{_label} From: {content['url']}")
output_parts.append(f"Title: {content['title']}")
output_parts.append("-" * 30)
services/search/providers.py
+204 -27
View File
@@ -3,19 +3,19 @@
import json
import logging
import os
import re
from typing import List, Optional
from urllib.parse import urljoin, urlparse, parse_qs
import httpx
from bs4 import BeautifulSoup
from src.constants import SEARXNG_INSTANCE
from src.constants import SEARXNG_INSTANCE, REQUEST_TIMEOUT, WEB_FETCH_USER_AGENT
from .analytics import RateLimitError, error_logger
from .query import build_enhanced_query
logger = logging.getLogger(__name__)
REQUEST_TIMEOUT = 20
# Provider registry — maps setting value to (label, needs_key, needs_url)
PROVIDER_INFO = {
"searxng": ("SearXNG", False, True),
@@ -34,9 +34,16 @@ def _get_search_settings() -> dict:
"""Return search settings from admin config, falling back to env defaults."""
try:
from src.settings import load_settings
return load_settings()
settings = dict(load_settings())
except Exception:
return {}
settings = {}
# Headless/native deployments do not necessarily have an admin settings
# database. Require an explicit Odysseus-prefixed override so ordinary UI
# configuration remains authoritative by default.
env_provider = os.environ.get("ODYSSEUS_SEARCH_PROVIDER", "").strip().lower()
if env_provider:
settings["search_provider"] = env_provider
return settings
def _get_search_instance() -> str:
@@ -63,7 +70,22 @@ def _get_provider_key(provider: str) -> str:
if val:
return val
# Legacy fallback: old shared search_api_key field
return (settings.get("search_api_key") or "").strip()
legacy = (settings.get("search_api_key") or "").strip()
if legacy:
return legacy
env_map = {
# DATA_BRAVE_API_KEY is the historical Odysseus name; BRAVE_API_KEY is
# the standard name used by headless runners and the Brave SDK.
"brave": ("DATA_BRAVE_API_KEY", "BRAVE_API_KEY"),
"google_pse": ("GOOGLE_API_KEY",),
"tavily": ("TAVILY_API_KEY",),
"serper": ("SERPER_API_KEY",),
}
for env_name in env_map.get(provider, ()):
value = (os.environ.get(env_name) or "").strip()
if value:
return value
return ""
def _get_result_count() -> int:
@@ -75,9 +97,77 @@ def _get_result_count() -> int:
return 5
def provider_configured(provider: str) -> bool:
"""Configuration readiness only; a configured engine can still fail upstream."""
if provider in {"searxng", "searxng_yep", "duckduckgo"}:
return True
if provider not in {"brave", "google_pse", "tavily", "serper"}:
return False
if not _get_provider_key(provider):
return False
if provider == "google_pse":
return bool(_get_search_settings().get("google_pse_cx") or os.environ.get("GOOGLE_PSE_CX"))
return True
# Canonical SafeSearch levels: "strict" (default), "moderate", "off".
# Each provider has its own knob name and value space -- see _safesearch_for(...).
_SAFESEARCH_LEVELS = ("strict", "moderate", "off")
def _get_safesearch_level() -> str:
"""Return configured SafeSearch level normalized to a canonical value."""
settings = _get_search_settings()
raw = (settings.get("search_safesearch") or "strict").strip().lower()
if raw in _SAFESEARCH_LEVELS:
return raw
aliases = {
"on": "strict", "high": "strict", "2": "strict",
"medium": "moderate", "1": "moderate", "default": "moderate",
"none": "off", "disabled": "off", "0": "off",
}
return aliases.get(raw, "strict")
def _safesearch_for(provider: str) -> Optional[str]:
"""Translate the canonical SafeSearch level into provider-specific values."""
level = _get_safesearch_level()
if provider == "searxng":
return {"strict": "2", "moderate": "1", "off": "0"}[level]
if provider == "brave":
return level
if provider == "duckduckgo_lib":
return {"strict": "on", "moderate": "moderate", "off": "off"}[level]
if provider == "duckduckgo_html":
return {"strict": "1", "moderate": "-1", "off": "-2"}[level]
if provider == "google_pse":
return None if level == "off" else "active"
if provider == "serper":
return None if level == "off" else "active"
return None
# ── SearXNG ──
_NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "idag")
_NEWS_EVENT_HINT_RE = re.compile(
r"\b(?:deport(?:ation|ed|ing)?|arrest(?:ed|s)?|election(?:s)?|"
r"evacuat(?:e|ed|ion)|flood(?:ing|s|ed)?|sanction(?:s|ed)?)\b",
re.IGNORECASE,
)
_SOFTWARE_RELEASE_HINTS = (
"github",
"gitlab",
"release",
"releases",
"version",
"versions",
"changelog",
"change log",
"pypi",
"npm",
"package",
)
# Default general engines (google/duckduckgo/brave/startpage/wikipedia) are
# routinely rate-limited / CAPTCHA-blocked on this instance and return nothing.
@@ -86,12 +176,13 @@ _NEWS_HINTS = ("news", "nyheter", "headlines", "breaking", "latest", "today", "i
_GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "bing,mojeek,presearch")
def searxng_search_api(query: str, count: int = 10, categories: str = "general",
time_filter: Optional[str] = None) -> List[dict]:
def searxng_search_api(query: str, count: Optional[int] = None, categories: str = "general",
time_filter: Optional[str] = None, *, engines: Optional[str] = None) -> List[dict]:
"""Search using SearXNG JSON API. Returns list of {title, url, snippet}."""
count = count if count is not None else _get_result_count()
instance = _get_search_instance()
api_key = ""
headers = {"User-Agent": "Mozilla/5.0"}
headers = {"User-Agent": WEB_FETCH_USER_AGENT}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
# News/fresh queries do badly in the 'general' category — it favours
@@ -104,9 +195,26 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
# languages and brand-ambiguous terms bleed in foreign SEO pages (e.g.
# "Odyssey" → Honda Japan, "Trojan" → Japanese malware blogs, "Polyphemus"
# → Chinese math forums). The news path already did this; general didn't.
params = {"q": query, "format": "json", "language": "en"}
params = {
"q": query,
"format": "json",
"language": "en",
"safesearch": _safesearch_for("searxng"),
}
q_lc = query.lower()
is_news = time_filter is not None or any(h in q_lc for h in _NEWS_HINTS)
# Fresh software-version queries are usually better served by general
# search or canonical project pages than by the news vertical. For example
# "latest ollama release version github" can return a sparse news result
# that gets filtered as irrelevant, while general engines find GitHub.
is_software_release_query = any(h in q_lc for h in _SOFTWARE_RELEASE_HINTS)
is_news = (
not is_software_release_query
and (
time_filter is not None
or any(h in q_lc for h in _NEWS_HINTS)
or bool(_NEWS_EVENT_HINT_RE.search(query))
)
)
if is_news and categories == "general":
params["categories"] = "news"
if time_filter in ("day", "week", "month", "year"):
@@ -119,6 +227,9 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
# set returns 0 on this instance — see _GENERAL_ENGINES).
if categories == "general" and _GENERAL_ENGINES:
params["engines"] = _GENERAL_ENGINES
if engines:
params["categories"] = "general"
params["engines"] = engines
try:
def _parse_results(results):
return [
@@ -126,6 +237,10 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
"title": r.get("title", ""),
"url": r.get("url", ""),
"snippet": r.get("content", ""),
"provider": "searxng",
"engines": r.get("engines", []),
"published_date": r.get("publishedDate"),
"query": query,
}
for r in results[:count]
if r.get("url")
@@ -153,6 +268,7 @@ def searxng_search_api(query: str, count: int = 10, categories: str = "general",
"format": "json",
"language": "en",
"categories": "general",
"safesearch": _safesearch_for("searxng"),
}
if _GENERAL_ENGINES:
fallback["engines"] = _GENERAL_ENGINES
@@ -197,13 +313,13 @@ def searxng_search(query, max_results=10):
"""Search using SearXNG instance - parsing HTML."""
instance = _get_search_instance()
api_key = ""
req_headers = {"User-Agent": "Mozilla/5.0"}
req_headers = {"User-Agent": WEB_FETCH_USER_AGENT}
if api_key:
req_headers["Authorization"] = f"Bearer {api_key}"
try:
response = httpx.get(
f"{instance}/search",
params={"q": query},
params={"q": query, "safesearch": _safesearch_for("searxng")},
headers=req_headers,
timeout=10,
)
@@ -228,8 +344,9 @@ def searxng_search(query, max_results=10):
# ── Brave ──
def brave_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
def brave_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
"""Search using Brave API with key from admin settings or env var."""
count = count if count is not None else _get_result_count()
api_key = _get_provider_key("brave") or os.environ.get("DATA_BRAVE_API_KEY") or ""
return _brave_search_impl(query, count, time_filter, search_config={"brave_api_key": api_key})
@@ -248,7 +365,11 @@ def _brave_search_impl(query: str, count: int, time_filter: Optional[str] = None
return []
headers = {"X-Subscription-Token": brave_api_key, "Accept": "application/json"}
params = {"q": enhanced_query, "count": count}
params = {
"q": enhanced_query,
"count": count,
"safesearch": _safesearch_for("brave"),
}
if time_filter:
time_map = {"day": "day", "week": "week", "month": "month", "year": "year"}
if time_filter in time_map:
@@ -297,14 +418,41 @@ def _brave_search_impl(query: str, count: int, time_filter: Optional[str] = None
# ── DuckDuckGo (free, no key) ──
def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
def _is_duckduckgo_host(host: str) -> bool:
"""True only for duckduckgo.com and its subdomains."""
host = (host or "").lower()
return host == "duckduckgo.com" or host.endswith(".duckduckgo.com")
def _resolve_ddg_redirect(raw: str) -> str:
"""Resolve a DuckDuckGo /l/?uddg= redirect URL to its destination."""
if not raw:
return raw
resolved = raw
if resolved.startswith("//"):
resolved = "https:" + resolved
elif resolved.startswith("/"):
resolved = urljoin("https://html.duckduckgo.com", resolved)
try:
parsed = urlparse(resolved)
if _is_duckduckgo_host(parsed.hostname) and parsed.path.rstrip("/") == "/l":
qs = parse_qs(parsed.query)
if "uddg" in qs:
return qs["uddg"][0]
except Exception:
pass
return resolved
def duckduckgo_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
"""Search using DuckDuckGo via the duckduckgo-search library. No API key needed."""
count = count if count is not None else _get_result_count()
def _html_fallback() -> List[dict]:
try:
response = httpx.get(
"https://html.duckduckgo.com/html/",
params={"q": query},
headers={"User-Agent": "Mozilla/5.0"},
params={"q": query, "kp": _safesearch_for("duckduckgo_html")},
headers={"User-Agent": WEB_FETCH_USER_AGENT},
timeout=REQUEST_TIMEOUT,
)
response.raise_for_status()
@@ -314,7 +462,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
link = result.select_one(".result__a")
if not link:
continue
url = link.get("href", "")
url = _resolve_ddg_redirect(link.get("href", ""))
if not url:
continue
snippet_el = result.select_one(".result__snippet")
@@ -330,7 +478,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
return []
try:
from duckduckgo_search import DDGS
from ddgs import DDGS
except ImportError:
logger.warning("duckduckgo-search package not installed; using HTML fallback")
return _html_fallback()
@@ -342,7 +490,12 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
try:
ddgs = DDGS()
raw = ddgs.text(query, max_results=count, timelimit=timelimit)
raw = ddgs.text(
query,
max_results=count,
timelimit=timelimit,
safesearch=_safesearch_for("duckduckgo_lib"),
)
results = []
for item in raw:
url = item.get("href", "")
@@ -362,7 +515,7 @@ def duckduckgo_search(query: str, count: int = 10, time_filter: Optional[str] =
# ── Google Programmable Search Engine ──
def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
def google_pse_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
"""Search using Google PSE (Custom Search JSON API).
Requires two keys in settings:
@@ -370,6 +523,7 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
- google_pse_cx: Programmable Search Engine ID (cx)
Or env vars GOOGLE_API_KEY and GOOGLE_PSE_CX.
"""
count = count if count is not None else _get_result_count()
settings = _get_search_settings()
api_key = _get_provider_key("google_pse") or os.environ.get("GOOGLE_API_KEY", "")
cx = (settings.get("google_pse_cx") or "").strip() or os.environ.get("GOOGLE_PSE_CX", "")
@@ -384,6 +538,9 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
"q": query,
"num": min(count, 10), # Google PSE max is 10 per request
}
safe = _safesearch_for("google_pse")
if safe:
params["safe"] = safe
if time_filter:
# dateRestrict: d[number], w[number], m[number], y[number]
time_map = {"day": "d1", "week": "w1", "month": "m1", "year": "y1"}
@@ -399,7 +556,6 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
if response.status_code == 429:
raise RateLimitError("Google PSE rate limit hit")
response.raise_for_status()
data = response.json()
except httpx.RequestError as e:
error_logger.error(f"Google PSE search failed: {e}")
return []
@@ -407,6 +563,12 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
error_logger.error(str(e))
return []
try:
data = response.json()
except json.JSONDecodeError as e:
error_logger.error(f"Google PSE returned invalid JSON: {e}")
return []
results = []
for item in data.get("items", [])[:count]:
url = item.get("link", "")
@@ -424,8 +586,9 @@ def google_pse_search(query: str, count: int = 10, time_filter: Optional[str] =
# ── Tavily ──
def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
def tavily_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
"""Search using Tavily API. Requires search_api_key or TAVILY_API_KEY env var."""
count = count if count is not None else _get_result_count()
api_key = _get_provider_key("tavily") or os.environ.get("TAVILY_API_KEY", "")
if not api_key:
logger.warning("Tavily: no API key configured")
@@ -451,7 +614,6 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
if response.status_code == 429:
raise RateLimitError("Tavily rate limit hit")
response.raise_for_status()
data = response.json()
except httpx.RequestError as e:
error_logger.error(f"Tavily search failed: {e}")
return []
@@ -459,6 +621,12 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
error_logger.error(str(e))
return []
try:
data = response.json()
except json.JSONDecodeError as e:
error_logger.error(f"Tavily returned invalid JSON: {e}")
return []
results = []
for item in data.get("results", [])[:count]:
url = item.get("url", "")
@@ -477,8 +645,9 @@ def tavily_search(query: str, count: int = 10, time_filter: Optional[str] = None
# ── Serper.dev ──
def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None) -> List[dict]:
def serper_search(query: str, count: Optional[int] = None, time_filter: Optional[str] = None) -> List[dict]:
"""Search using Serper.dev API. Requires search_api_key or SERPER_API_KEY env var."""
count = count if count is not None else _get_result_count()
api_key = _get_provider_key("serper") or os.environ.get("SERPER_API_KEY", "")
if not api_key:
logger.warning("Serper: no API key configured")
@@ -488,6 +657,9 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
"q": query,
"num": count,
}
safe = _safesearch_for("serper")
if safe:
payload["safe"] = safe
if time_filter:
time_map = {"day": "qdr:d", "week": "qdr:w", "month": "qdr:m", "year": "qdr:y"}
if time_filter in time_map:
@@ -503,7 +675,6 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
if response.status_code == 429:
raise RateLimitError("Serper rate limit hit")
response.raise_for_status()
data = response.json()
except httpx.RequestError as e:
error_logger.error(f"Serper search failed: {e}")
return []
@@ -511,6 +682,12 @@ def serper_search(query: str, count: int = 10, time_filter: Optional[str] = None
error_logger.error(str(e))
return []
try:
data = response.json()
except json.JSONDecodeError as e:
error_logger.error(f"Serper returned invalid JSON: {e}")
return []
results = []
for item in data.get("organic", [])[:count]:
url = item.get("link", "")
services/search/query.py
+25 -4
View File
@@ -13,23 +13,36 @@ logger = logging.getLogger(__name__)
# ----------------------------------------------------------------------
def _detect_question_type(query: str) -> Optional[str]:
"""Return the leading question word if present (who, what, when, where, why, how)."""
if not isinstance(query, str):
return None
q = query.strip().lower()
for word in ("who", "what", "when", "where", "why", "how"):
if q.startswith(word):
# Require a whole-word match: a bare prefix mis-flags ordinary queries
# like "whatsapp pricing" (-> what) or "however ..." (-> how), which
# then get spurious boost terms OR-appended in enhance_query.
if q == word or q.startswith(word + " "):
return word
return None
def _extract_entities(query: str) -> Dict[str, List[str]]:
"""Lightweight entity extraction: capitalized words and date patterns."""
if not isinstance(query, str):
return {"names": [], "dates": []}
entities: Dict[str, List[str]] = {"names": [], "dates": []}
qtype = _detect_question_type(query)
cleaned = query
if qtype:
cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
entities["names"].append(token)
for year in re.findall(r"\b(19|20)\d{2}\b", cleaned):
# Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
# class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
# "São" (shredded). Keep the ASCII behaviour — the word boundary already
# excludes camelCase mid-word capitals — by requiring an all-alphabetic
# token of length > 1 whose first character is uppercase.
for token in re.findall(r"\b\w+\b", cleaned):
if len(token) > 1 and token[0].isupper() and token.isalpha():
entities["names"].append(token)
for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
entities["dates"].append(year)
month_day_year = re.findall(
r"\b(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\s+\d{1,2},?\s*\d{4}\b",
@@ -42,12 +55,16 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
def _split_multi_part(query: str) -> List[str]:
"""Split a query into sub-queries on common conjunctions."""
if not isinstance(query, str):
return []
parts = re.split(r"\s+and\s+|\s+or\s+|;", query, flags=re.I)
return [p.strip() for p in parts if p.strip()]
def _extract_site_filter(query: str) -> Tuple[str, Optional[str]]:
"""Detect a 'site:example.com' token. Returns (query_without_token, site_or_None)."""
if not isinstance(query, str):
return "", None
match = re.search(r"\bsite:([^\s]+)", query, flags=re.I)
if match:
site = match.group(1)
@@ -68,6 +85,8 @@ def _boost_entities_in_query(base_query: str, entities: Dict[str, List[str]]) ->
def enhance_query(original_query: str) -> Tuple[str, Optional[str]]:
"""Process the original query: site filter, question type boosts, entity extraction."""
if not isinstance(original_query, str):
original_query = ""
query_without_site, site = _extract_site_filter(original_query)
sub_queries = _split_multi_part(query_without_site)
@@ -117,6 +136,8 @@ def build_enhanced_query(query: str, time_filter: str = None) -> str:
def _is_news_query(query: str) -> bool:
"""Lightweight heuristic to decide if a query is news-oriented."""
news_terms = {"news", "latest", "breaking", "today", "today's", "current", "updates", "happening"}
if not isinstance(query, str):
return False
tokens = set(re.findall(r"\b\w+\b", query.lower()))
return bool(tokens & news_terms)
services/search/ranking.py
+116 -24
View File
@@ -2,17 +2,59 @@
import re
import logging
from datetime import datetime
from datetime import datetime, timezone
from typing import List, Optional
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
_AGE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S")
def _utcnow_naive() -> datetime:
"""Naive UTC 'now'. Matches the naive, UTC-style published dates parsed below,
and is safe on Python 3.14 where ``datetime.utcnow()`` is removed (#1116)."""
return datetime.now(timezone.utc).replace(tzinfo=None)
def recency_score(age_str: Optional[str], now: Optional[datetime] = None) -> float:
"""Score how recent a result is: 1.0 for <=7 days old, 0.0 for >=30 days.
The age is measured against UTC, not local time. The previous code used
``datetime.now()`` (local) against UTC-style published dates, so the age was
skewed by the host's UTC offset; it was also a latent crash once neighbouring
code moves to timezone-aware datetimes (#1116). ``now`` is injectable for tests.
"""
if not age_str:
return 0.0
dt = None
for fmt in _AGE_FORMATS:
try:
dt = datetime.strptime(age_str, fmt)
break
except Exception:
dt = None
if not dt:
return 0.0
now = now or _utcnow_naive()
days_old = (now - dt).days
if days_old <= 7:
return 1.0
if days_old >= 30:
return 0.0
return (30 - days_old) / 23
_NEWS_HINTS = {"news", "nyheter", "headlines", "breaking", "latest", "today", "idag"}
_SPORTS_HINTS = {
"sport", "sports", "soccer", "football", "hockey", "nba", "nfl", "mlb",
"fifa", "world cup", "championship", "quarterfinal", "eliminates",
}
# Word-boundary match so "sport" does not fire inside "transport"/"passport"
# and a domain like "transport.gov" is not mistaken for a sports site.
_SPORTS_HINT_RE = re.compile(
r"\b(?:" + "|".join(re.escape(h) for h in _SPORTS_HINTS) + r")\b"
)
_LOW_VALUE_NEWS_DOMAINS = {
"facebook.com", "www.facebook.com", "sports.yahoo.com", "yahoo.com",
"www.yahoo.com", "msn.com", "www.msn.com",
@@ -25,6 +67,22 @@ _TRUSTED_NEWS_DOMAINS = {
"www.theguardian.com", "euronews.com", "www.euronews.com",
"dw.com", "www.dw.com", "government.se", "www.government.se",
}
_SOFTWARE_RELEASE_HINTS = {
"github", "gitlab", "release", "releases", "version", "versions",
"changelog", "package", "pypi", "npm",
}
_PRODUCT_SPEC_HINTS = {
"product", "hardware", "device", "phone", "laptop", "desktop", "computer",
"chip", "cpu", "gpu", "mac", "iphone", "ipad", "android", "camera",
"console", "kindle", "tesla", "car", "model", "price", "pricing", "cost",
"buy", "shop", "order", "preorder", "pre-order", "spec", "specs",
"specifications", "available", "availability", "ship", "shipping",
"released", "launch", "launched", "vram", "memory", "ram", "storage",
}
_COMMERCE_OR_SPEC_PATH_HINTS = (
"/shop", "/buy", "/store", "/product", "/products", "/spec", "/specs",
"/support", "/tech-specs", "/technical-specifications",
)
def _domain(url: str) -> str:
@@ -34,25 +92,40 @@ def _domain(url: str) -> str:
return ""
def _has_word(text: str, term: str) -> bool:
"""True if ``term`` appears in ``text`` as a whole word.
Query terms are matched on word boundaries so a short term doesn't match
inside an unrelated word: "us" must not match "business"/"music", "port"
must not match "transport"/"support". This mirrors the tokenization used to
build ``query_terms`` (``\\b\\w+\\b``). #1473 converted the title and sports
checks to word boundaries; the snippet and subject-term checks below use
the same helper so the whole file stays consistent.
"""
return re.search(rf"\b{re.escape(term)}\b", text) is not None
def rank_search_results(query: str, results: List[dict]) -> List[dict]:
"""Rank search results by title relevance, snippet quality, domain authority, and recency."""
query_terms = [t.lower() for t in re.findall(r"\b\w+\b", query)]
query_lc = query.lower()
is_news_query = any(term in _NEWS_HINTS for term in query_terms)
is_sports_query = any(hint in query_lc for hint in _SPORTS_HINTS)
is_sports_query = bool(_SPORTS_HINT_RE.search(query_lc))
is_software_release_query = any(term in _SOFTWARE_RELEASE_HINTS for term in query_terms)
is_product_spec_query = any(term in _PRODUCT_SPEC_HINTS for term in query_terms)
def title_score(title: str) -> float:
if not title:
return 0.0
title_lc = title.lower()
matches = sum(1 for term in query_terms if re.search(rf"\b{re.escape(term)}\b", title_lc))
matches = sum(1 for term in query_terms if _has_word(title_lc, term))
return matches / len(query_terms) if query_terms else 0.0
def snippet_score(snippet: str) -> float:
if not snippet:
return 0.0
length_factor = min(len(snippet), 200) / 200
term_hits = sum(1 for term in query_terms if term in snippet.lower())
term_hits = sum(1 for term in query_terms if _has_word(snippet.lower(), term))
term_factor = term_hits / len(query_terms) if query_terms else 0.0
return (length_factor + term_factor) / 2
@@ -68,24 +141,6 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
return 0.7
return 0.4
def recency_score(age_str: Optional[str]) -> float:
if not age_str:
return 0.0
for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"):
try:
dt = datetime.strptime(age_str, fmt)
break
except Exception:
dt = None
if not dt:
return 0.0
days_old = (datetime.now() - dt).days
if days_old <= 7:
return 1.0
if days_old >= 30:
return 0.0
return (30 - days_old) / 23
def news_quality_adjustment(title: str, snippet: str, url: str) -> float:
if not is_news_query:
return 0.0
@@ -98,15 +153,50 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
adjustment += 0.4
if netloc in _LOW_VALUE_NEWS_DOMAINS:
adjustment -= 0.8
if not is_sports_query and any(hint in text or hint in netloc for hint in _SPORTS_HINTS):
if not is_sports_query and (_SPORTS_HINT_RE.search(text) or _SPORTS_HINT_RE.search(netloc)):
adjustment -= 1.5
# A country/news query should not rank a page whose title/snippet barely
# mentions the country above actual news pages for that country.
subject_terms = [t for t in query_terms if t not in _NEWS_HINTS]
if subject_terms and not any(t in text or t in netloc for t in subject_terms):
if subject_terms and not any(_has_word(text, t) or _has_word(netloc, t) for t in subject_terms):
adjustment -= 1.0
return adjustment
def software_release_adjustment(title: str, snippet: str, url: str) -> float:
if not is_software_release_query:
return 0.0
netloc = _domain(url)
path = urlparse(url).path.lower()
text = f"{title} {snippet} {netloc} {path}".lower()
adjustment = 0.0
if netloc in {"github.com", "www.github.com", "gitlab.com", "www.gitlab.com"}:
adjustment += 1.6
if "/releases" in path or "/tags" in path:
adjustment += 1.2
if any(_has_word(text, term) for term in ("release", "releases", "changelog", "version")):
adjustment += 0.4
if netloc in {"releasealert.dev", "releases.sh", "releasebot.io"}:
adjustment -= 0.8
return adjustment
def product_spec_adjustment(title: str, snippet: str, url: str) -> float:
if not is_product_spec_query:
return 0.0
parsed = urlparse(url)
netloc = parsed.netloc.lower()
path = parsed.path.lower()
text = f"{title} {snippet} {netloc} {path}".lower()
adjustment = 0.0
if any(hint in path for hint in _COMMERCE_OR_SPEC_PATH_HINTS):
adjustment += 1.1
if re.search(r"\b(?:official|specs?|specifications|tech specs|buy|shop|store|price|pricing|available|ships?)\b", text):
adjustment += 0.5
if netloc.endswith(".com") and any(_has_word(netloc, term) for term in query_terms if len(term) >= 4):
adjustment += 0.4
if re.search(r"\b(?:rumor|rumour|leak|may|could|expected|reportedly|unannounced)\b", text):
adjustment -= 0.8
return adjustment
ranked = []
for result in results:
title = result.get("title", "")
@@ -120,6 +210,8 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
+ 1.5 * domain_score(url)
+ 1.0 * recency_score(age)
+ news_quality_adjustment(title, snippet, url)
+ software_release_adjustment(title, snippet, url)
+ product_spec_adjustment(title, snippet, url)
)
ranked.append((score, result))
services/search/service.py
+12 -5
View File
@@ -62,17 +62,24 @@ class SearchService:
SearchResponse with results
"""
depth = depth or self.default_depth
fetch_content = fetch_content if fetch_content is not None else self.fetch_content
# Use existing search implementation
raw_results = await comprehensive_web_search(
# comprehensive_web_search is synchronous and, with return_sources=True,
# returns (context_str, [{"url", "title"}, ...]). Run it off the event
# loop so we don't block it, and use the source list as the result rows.
# `fetch_content` is accepted for API compatibility; the comprehensive
# search always fetches page content.
import asyncio
_context, raw_results = await asyncio.to_thread(
comprehensive_web_search,
query,
max_results=10 * depth,
fetch_content=fetch_content,
max_pages=10 * depth,
return_sources=True,
)
results = []
for r in raw_results:
if not isinstance(r, dict):
continue
results.append(SearchResult(
url=r.get("url", ""),
title=r.get("title", ""),