mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-15 12:42:21 +02:00
1004 lines
39 KiB
Python
1004 lines
39 KiB
Python
"""Core search orchestrators: searxng_search_results, comprehensive_web_search, config, cache invalidation."""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Any, Optional, List, Set
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
|
|
from .analytics import (
|
|
NetworkError,
|
|
ParseError,
|
|
RateLimitError,
|
|
error_logger,
|
|
_record_query,
|
|
)
|
|
from .cache import (
|
|
SEARCH_CACHE_DIR,
|
|
search_cache_index,
|
|
generate_cache_key,
|
|
cleanup_cache,
|
|
)
|
|
from .query import _cache_duration_for_query
|
|
from .ranking import rank_search_results
|
|
from .providers import (
|
|
searxng_search_api,
|
|
brave_search,
|
|
duckduckgo_search,
|
|
google_pse_search,
|
|
tavily_search,
|
|
serper_search,
|
|
_get_search_settings,
|
|
_get_provider_key,
|
|
_get_result_count,
|
|
)
|
|
from .content import (
|
|
fetch_webpage_content,
|
|
extract_key_points,
|
|
get_tldr,
|
|
extract_quotes,
|
|
extract_statistics,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ========= CONFIG =========
|
|
SEARCH_CONFIG: Dict[str, Any] = {
|
|
"primary_provider": "searxng",
|
|
}
|
|
|
|
|
|
def _is_secret_key(name: str) -> bool:
|
|
"""True for config keys that hold a credential (e.g. ``brave_api_key``)."""
|
|
return name.endswith(("_api_key", "_key", "_token", "_secret"))
|
|
|
|
|
|
def get_search_config() -> Dict[str, Any]:
|
|
"""Get current search configuration including active provider info.
|
|
|
|
Never returns stored API keys: callers — including the unauthenticated
|
|
``GET /api/search/config`` route — only need key *presence* via
|
|
``has_api_key``, not the secret itself (#1661).
|
|
"""
|
|
config = SEARCH_CONFIG.copy()
|
|
settings = _get_search_settings()
|
|
provider = settings.get("search_provider", "searxng")
|
|
config["active_provider"] = provider
|
|
config["has_api_key"] = bool(_get_provider_key(provider))
|
|
config["result_count"] = _get_result_count()
|
|
if provider == "searxng":
|
|
from .providers import _get_search_instance
|
|
config["search_url"] = _get_search_instance()
|
|
# Strip any string-valued credential so secrets never reach the response;
|
|
# the boolean has_api_key flag (presence only) is preserved.
|
|
return {
|
|
k: v for k, v in config.items()
|
|
if not (isinstance(v, str) and _is_secret_key(k))
|
|
}
|
|
|
|
|
|
def update_search_config(api_key: str = None, **kwargs):
|
|
"""Merge non-secret search config into SEARCH_CONFIG.
|
|
|
|
Provider API keys are intentionally NOT cached here. They are read on demand
|
|
from settings/env via ``_get_provider_key`` (e.g. ``brave_search``), so the
|
|
previous ``SEARCH_CONFIG["brave_api_key"] = api_key`` cache was never used
|
|
for search and only leaked the decrypted key through ``get_search_config`` /
|
|
``GET /api/search/config`` (#1661). ``api_key`` is accepted for backward
|
|
compatibility but no longer stored.
|
|
"""
|
|
for k, v in kwargs.items():
|
|
if not _is_secret_key(k):
|
|
SEARCH_CONFIG[k] = v
|
|
|
|
|
|
def _call_provider(provider_name: str, query: str, count: int, time_filter: str = None) -> List[dict]:
|
|
"""Call a search provider by name. Returns list of results or empty list."""
|
|
if provider_name == "searxng":
|
|
return searxng_search_api(query, count, time_filter=time_filter)
|
|
elif provider_name == "searxng_yep":
|
|
return searxng_search_api(query, count, time_filter=time_filter, engines="yep")
|
|
elif provider_name == "brave":
|
|
return brave_search(query, count, time_filter)
|
|
elif provider_name == "duckduckgo":
|
|
return duckduckgo_search(query, count, time_filter)
|
|
elif provider_name == "google_pse":
|
|
return google_pse_search(query, count, time_filter)
|
|
elif provider_name == "tavily":
|
|
return tavily_search(query, count, time_filter)
|
|
elif provider_name == "serper":
|
|
return serper_search(query, count, time_filter)
|
|
return []
|
|
|
|
|
|
# If the self-hosted SearXNG instance is up but all enabled engines return
|
|
# empty, fall back to the no-key provider so "search X" still works on fresh
|
|
# installs. Users can override/disable with `search_fallback_chain`.
|
|
_FALLBACK_ORDER = ["duckduckgo"]
|
|
|
|
|
|
def _build_provider_chain(primary: str) -> List[str]:
|
|
"""Build ordered list: primary first, then configured/default fallbacks."""
|
|
chain = [primary]
|
|
settings = _get_search_settings()
|
|
user_chain = settings.get("search_fallback_chain") or []
|
|
if isinstance(user_chain, str):
|
|
user_chain = [s.strip() for s in user_chain.split(",") if s.strip()]
|
|
fallbacks = user_chain if user_chain else _FALLBACK_ORDER
|
|
for fb in fallbacks:
|
|
if fb and fb != primary and fb not in chain and fb != "disabled":
|
|
chain.append(fb)
|
|
from .providers import provider_configured
|
|
configured = [provider for provider in chain if provider_configured(provider)]
|
|
for provider in set(chain) - set(configured):
|
|
logger.warning("Skipping unconfigured search provider: %s", provider)
|
|
if primary == "searxng" and configured == ["searxng"]:
|
|
# No usable configured fallback: try a separate engine on the same
|
|
# private metasearch instance before reporting retrieval failure.
|
|
configured.append("searxng_yep")
|
|
return configured
|
|
|
|
|
|
_SEARCH_QUERY_FILLER = {
|
|
"what", "whats", "what's", "which", "when", "where", "year", "from",
|
|
"any", "info", "information", "details", "update", "updates",
|
|
"with", "this", "that", "search", "lookup", "look", "find", "tell",
|
|
"about", "quick", "please", "pls", "official", "links", "source",
|
|
"sources", "news", "headlines", "breaking", "latest", "current",
|
|
"newest", "recent", "today", "now",
|
|
"release", "releases", "version", "versions", "changelog", "github",
|
|
"gitlab", "weather", "forecast", "forecasts", "tomorrow", "hourly",
|
|
"daily", "temperature", "temperatures", "conditions", "rain", "raining",
|
|
"chance", "precipitation",
|
|
"january", "february", "march", "april", "may", "june", "july",
|
|
"august", "september", "october", "november", "december",
|
|
"the", "and", "or", "but", "are", "was", "were", "does", "did",
|
|
"can", "could", "should", "would", "will", "has", "have", "had",
|
|
"for", "into", "onto", "near", "over", "under",
|
|
}
|
|
|
|
_SHORT_QUERY_SUBJECTS = {"ai", "ar", "eu", "uk", "us", "vr"}
|
|
|
|
_WEATHER_QUERY_HINTS = {
|
|
"weather", "forecast", "forecasts", "temperature", "temperatures",
|
|
"rain", "raining", "precipitation", "humid", "humidity", "wind",
|
|
}
|
|
_WEATHER_RESULT_HINTS = {
|
|
"weather", "forecast", "temperature", "temperatures", "rain",
|
|
"precipitation", "humidity", "wind", "accuweather", "meteoblue",
|
|
"weather-atlas", "weather25", "weather365", "easeweather",
|
|
}
|
|
|
|
|
|
def _meaningful_query_terms(query: str) -> list[str]:
|
|
return [
|
|
term
|
|
for term in re.findall(r"[a-z0-9]+", str(query or "").lower())
|
|
if (len(term) > 2 or term in _SHORT_QUERY_SUBJECTS)
|
|
and not term.isdigit()
|
|
and term not in _SEARCH_QUERY_FILLER
|
|
]
|
|
|
|
|
|
def _result_has_query_overlap(query: str, result: dict) -> bool:
|
|
terms = _meaningful_query_terms(query)
|
|
if not terms:
|
|
return True
|
|
text = " ".join(
|
|
str(result.get(key) or "").lower()
|
|
for key in ("title", "snippet", "url")
|
|
)
|
|
query_tokens = set(re.findall(r"[a-z0-9]+", str(query or "").lower()))
|
|
if query_tokens & _WEATHER_QUERY_HINTS:
|
|
return (
|
|
any(re.search(rf"\b{re.escape(term)}\b", text) for term in terms)
|
|
and any(marker in text for marker in _WEATHER_RESULT_HINTS)
|
|
)
|
|
result_tokens = set(re.findall(r"[a-z0-9]+", text))
|
|
|
|
def lexical_root(word: str) -> str:
|
|
for suffix in ("ation", "ition", "ence", "ance", "ment", "ents", "ent", "ant", "ing", "ed", "es", "s"):
|
|
if word.endswith(suffix) and len(word) - len(suffix) >= 6:
|
|
return word[:-len(suffix)]
|
|
return word
|
|
|
|
result_roots = {lexical_root(token) for token in result_tokens}
|
|
matched_terms = {
|
|
term for term in terms
|
|
if term in result_tokens or lexical_root(term) in result_roots
|
|
}
|
|
# A single broad token is not enough evidence for a detailed entity/event
|
|
# query. For example, SearXNG may answer "Sweden 78 year old British woman
|
|
# deportation Brexit ..." with generic Sweden tourism pages. Treat that as
|
|
# an empty provider result so the configured fallback gets a chance.
|
|
minimum_matches = 2 if len(set(terms)) >= 4 else 1
|
|
return len(matched_terms) >= minimum_matches
|
|
|
|
|
|
def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]:
|
|
if not results:
|
|
return []
|
|
relevant = [result for result in results if _result_has_query_overlap(query, result)]
|
|
# Only reject a provider when it returned a fully off-topic page set. Mixed
|
|
# result pages are common; ranking can handle those.
|
|
return relevant if relevant else []
|
|
|
|
|
|
_SCHOLARLY_QUERY_CUE_RE = re.compile(
|
|
r"\b(?:paper|preprint|arxiv|proceedings|table\s+\d+|figure\s+\d+|"
|
|
r"appendix\s+[a-z0-9]+|benchmark(?:s)?)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
_SCHOLARLY_TITLE_FILLER = _SEARCH_QUERY_FILLER | {
|
|
"paper", "preprint", "arxiv", "proceedings", "table", "figure",
|
|
"appendix", "authors", "author", "extract", "locate", "read",
|
|
}
|
|
_ARXIV_IDENTIFIER_RE = re.compile(
|
|
r"(?i)(?:\barxiv\s*:\s*|\barxiv\.org/(?:abs|pdf|html)/)?"
|
|
r"(?P<identifier>\d{4}\.\d{4,5}(?:v\d+)?)\b"
|
|
)
|
|
_FORMAL_PUBLICATION_CUE_RE = re.compile(
|
|
r"\b(?:publish(?:ed|ing|cation)?|venue|conference|journal|proceedings|doi)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _exact_arxiv_identifier_results(query: str) -> list[dict]:
|
|
"""Return deterministic official landing pages for explicit arXiv IDs."""
|
|
seen: set[str] = set()
|
|
results: list[dict] = []
|
|
for match in _ARXIV_IDENTIFIER_RE.finditer(str(query or "")):
|
|
identifier = match.group("identifier")
|
|
canonical = re.sub(r"v\d+$", "", identifier, flags=re.IGNORECASE)
|
|
if canonical in seen:
|
|
continue
|
|
seen.add(canonical)
|
|
results.append({
|
|
"title": f"arXiv:{canonical} — exact identifier match",
|
|
"url": f"https://arxiv.org/abs/{canonical}",
|
|
"snippet": (
|
|
"Official arXiv landing page resolved directly from the exact "
|
|
"identifier in the query."
|
|
),
|
|
"source": "arxiv",
|
|
})
|
|
return results
|
|
|
|
|
|
def _title_before_explicit_arxiv_identifier(query: str) -> str:
|
|
"""Extract a probable title that precedes an explicit arXiv identifier."""
|
|
|
|
text = re.sub(r"\s+", " ", str(query or "")).strip()
|
|
match = _ARXIV_IDENTIFIER_RE.search(text)
|
|
if not match or not _FORMAL_PUBLICATION_CUE_RE.search(text):
|
|
return ""
|
|
candidate = text[:match.start()].strip(" \t,;:-'\"")
|
|
candidate = re.sub(
|
|
r"\barxiv(?:\.org)?(?:\s*:\s*|\s+(?:abs|pdf|html)\s*[/ :]*)?$",
|
|
"",
|
|
candidate,
|
|
flags=re.IGNORECASE,
|
|
).strip(" \t,;:-'\"")
|
|
candidate = re.sub(
|
|
r"^(?:(?:please\s+)?(?:find|locate|search\s+for|look\s+up|verify|check)\s+)"
|
|
r"(?:(?:the|this)\s+)?(?:paper\s+)?",
|
|
"",
|
|
candidate,
|
|
flags=re.IGNORECASE,
|
|
).strip(" \t,;:-'\"")
|
|
return candidate if len(_normalized_title_terms(candidate)) >= 2 else ""
|
|
|
|
|
|
def _normalized_title_terms(value: str) -> list[str]:
|
|
return [
|
|
token
|
|
for token in re.findall(r"[a-z0-9]+", str(value or "").lower())
|
|
if len(token) > 1 and token not in _SCHOLARLY_TITLE_FILLER
|
|
]
|
|
|
|
|
|
def _is_distinctive_short_scholarly_title(value: str) -> bool:
|
|
"""Recognize compact model/report names without accepting generic phrases."""
|
|
|
|
terms = _normalized_title_terms(value)
|
|
if not 1 <= len(terms) <= 2:
|
|
return False
|
|
text = str(value or "").strip()
|
|
return bool(
|
|
re.search(r"\d", text)
|
|
or re.search(r"\b[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*\b", text)
|
|
)
|
|
|
|
|
|
def _scholarly_title_from_query(query: str) -> str:
|
|
"""Extract a probable paper title only from clearly scholarly searches."""
|
|
|
|
text = re.sub(r"\s+", " ", str(query or "")).strip()
|
|
if not text or not _SCHOLARLY_QUERY_CUE_RE.search(text):
|
|
return ""
|
|
|
|
quoted = [
|
|
candidate.strip()
|
|
for candidate in re.findall(r'["“”]([^"“”]{4,180})["“”]', text)
|
|
if len(_normalized_title_terms(candidate)) >= 3
|
|
or _is_distinctive_short_scholarly_title(candidate)
|
|
]
|
|
if quoted:
|
|
return max(quoted, key=lambda candidate: len(_normalized_title_terms(candidate)))
|
|
|
|
before_paper = re.search(
|
|
r"(?:^|\b(?:find|locate|read|from|about)\s+)(.{4,160}?)\s+"
|
|
r"(?:paper|preprint)\b",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if before_paper:
|
|
candidate = before_paper.group(1).strip(" ,:;-'")
|
|
if (
|
|
len(_normalized_title_terms(candidate)) >= 3
|
|
or _is_distinctive_short_scholarly_title(candidate)
|
|
):
|
|
return candidate
|
|
|
|
before_locator = re.match(
|
|
r"(.{2,80}?)\s+(?:table|figure)\s+\d+\b",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if before_locator:
|
|
candidate = before_locator.group(1).strip(" ,:;-'\"")
|
|
if _is_distinctive_short_scholarly_title(candidate):
|
|
return candidate
|
|
return ""
|
|
|
|
|
|
def _result_strongly_matches_title(title: str, result: dict) -> bool:
|
|
wanted = set(_normalized_title_terms(title))
|
|
found = set(_normalized_title_terms(str(result.get("title") or "")))
|
|
if len(wanted) < 2 or not found:
|
|
return False
|
|
overlap = len(wanted & found) / len(wanted)
|
|
return overlap >= (1.0 if len(wanted) == 2 else 0.8)
|
|
|
|
|
|
def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:
|
|
"""Resolve a paper title through arXiv's public Atom API."""
|
|
|
|
try:
|
|
response = httpx.get(
|
|
"https://export.arxiv.org/api/query",
|
|
params={
|
|
"search_query": f'ti:"{title}"',
|
|
"start": 0,
|
|
"max_results": max(1, min(int(count), 5)),
|
|
},
|
|
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
|
|
timeout=12.0,
|
|
follow_redirects=True,
|
|
)
|
|
response.raise_for_status()
|
|
root = ET.fromstring(response.text)
|
|
except Exception as exc:
|
|
logger.info("arXiv title lookup failed for %r: %s", title, exc)
|
|
return []
|
|
|
|
namespace = {"atom": "http://www.w3.org/2005/Atom"}
|
|
matches: list[dict] = []
|
|
for entry in root.findall("atom:entry", namespace):
|
|
result_title = " ".join(
|
|
(entry.findtext("atom:title", default="", namespaces=namespace) or "").split()
|
|
)
|
|
if not _result_strongly_matches_title(title, {"title": result_title}):
|
|
continue
|
|
entry_id = (entry.findtext("atom:id", default="", namespaces=namespace) or "").strip()
|
|
arxiv_id = entry_id.rstrip("/").rsplit("/", 1)[-1]
|
|
if not arxiv_id:
|
|
continue
|
|
summary = " ".join(
|
|
(entry.findtext("atom:summary", default="", namespaces=namespace) or "").split()
|
|
)
|
|
matches.append({
|
|
"title": result_title,
|
|
"url": f"https://arxiv.org/abs/{arxiv_id}",
|
|
"snippet": summary,
|
|
"source": "arxiv",
|
|
})
|
|
return matches
|
|
|
|
|
|
def _openalex_title_results(title: str, count: int = 3) -> list[dict]:
|
|
"""Resolve an exact scholarly title through OpenAlex metadata."""
|
|
|
|
try:
|
|
# OpenAlex treats a literal question mark as query syntax and returns
|
|
# HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".
|
|
search_title = re.sub(r"[?]+", " ", str(title or "")).strip()
|
|
response = httpx.get(
|
|
"https://api.openalex.org/works",
|
|
params={
|
|
"search": search_title,
|
|
"per-page": max(1, min(int(count), 5)),
|
|
"select": (
|
|
"display_name,doi,primary_location,publication_year,type"
|
|
),
|
|
},
|
|
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
|
|
timeout=12.0,
|
|
follow_redirects=True,
|
|
)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
except Exception as exc:
|
|
logger.info("OpenAlex title lookup failed for %r: %s", title, exc)
|
|
return []
|
|
|
|
matches: list[dict] = []
|
|
for item in payload.get("results", []):
|
|
result_title = str(item.get("display_name") or "").strip()
|
|
if not _result_strongly_matches_title(title, {"title": result_title}):
|
|
continue
|
|
location = item.get("primary_location") or {}
|
|
url = str(location.get("landing_page_url") or item.get("doi") or "").strip()
|
|
if url.startswith("http://arxiv.org/"):
|
|
url = "https://" + url[len("http://"):]
|
|
if not url:
|
|
continue
|
|
snippet = "Exact scholarly-title match from OpenAlex metadata."
|
|
venue = str(location.get("raw_source_name") or "").strip()
|
|
year = item.get("publication_year")
|
|
publication_type = str(item.get("type") or "").strip()
|
|
version = str(location.get("version") or "").strip()
|
|
formal_parts: list[str] = []
|
|
if venue:
|
|
formal_parts.append(f"{venue}, {year}" if year else venue)
|
|
elif year:
|
|
formal_parts.append(str(year))
|
|
if publication_type:
|
|
formal_parts.append(f"type: {publication_type}")
|
|
if version:
|
|
formal_parts.append(f"version: {version}")
|
|
if formal_parts:
|
|
snippet += f" Formal publication: {'; '.join(formal_parts)}."
|
|
matches.append({
|
|
"title": result_title,
|
|
"url": url,
|
|
"snippet": snippet,
|
|
"source": "openalex",
|
|
})
|
|
return matches
|
|
|
|
|
|
def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:
|
|
"""Retry a noisy scholarly query as a bare title, then use arXiv API."""
|
|
|
|
try:
|
|
simplified = searxng_search_api(title, count=max(3, count))
|
|
except Exception as exc:
|
|
logger.info("Simplified scholarly search failed for %r: %s", title, exc)
|
|
simplified = []
|
|
exact = [
|
|
result for result in simplified
|
|
if _result_strongly_matches_title(title, result)
|
|
]
|
|
if exact:
|
|
return exact[:count]
|
|
openalex = _openalex_title_results(title, count)
|
|
if openalex:
|
|
return openalex
|
|
return _arxiv_title_results(title, count)
|
|
|
|
|
|
def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:
|
|
"""Resolve a clear paper title without waiting on generic search providers."""
|
|
|
|
# OpenAlex typically resolves titles in under a second and often returns
|
|
# the official arXiv landing page. The arXiv API remains the fallback.
|
|
openalex = _openalex_title_results(title, count)
|
|
if openalex:
|
|
return openalex
|
|
return _arxiv_title_results(title, count)
|
|
|
|
|
|
def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:
|
|
"""Prepend an exact arXiv match when a scholarly SERP missed its title."""
|
|
|
|
current = list(results or [])
|
|
identifier_results = _exact_arxiv_identifier_results(query)
|
|
if identifier_results:
|
|
title = _title_before_explicit_arxiv_identifier(query)
|
|
formal_results: list[dict] = []
|
|
if title:
|
|
formal_results = [
|
|
item
|
|
for item in _openalex_title_results(title, min(count, 3))
|
|
if "arxiv.org/" not in str(item.get("url") or "").lower()
|
|
]
|
|
exact_urls = {str(item["url"]) for item in identifier_results}
|
|
formal_urls = {str(item.get("url") or "") for item in formal_results}
|
|
return (
|
|
formal_results
|
|
+ identifier_results
|
|
+ [
|
|
item for item in current
|
|
if str(item.get("url") or "") not in exact_urls | formal_urls
|
|
]
|
|
)[:count]
|
|
title = _scholarly_title_from_query(query)
|
|
if not title:
|
|
return current
|
|
exact_current = [
|
|
item for item in current
|
|
if _result_strongly_matches_title(title, item)
|
|
]
|
|
if exact_current:
|
|
exact_ids = {id(item) for item in exact_current}
|
|
return (exact_current + [item for item in current if id(item) not in exact_ids])[:count]
|
|
arxiv_results = _scholarly_title_results(title, min(count, 3))
|
|
if not arxiv_results:
|
|
return current
|
|
seen = {str(item.get("url") or "") for item in arxiv_results}
|
|
return (arxiv_results + [item for item in current if str(item.get("url") or "") not in seen])[:count]
|
|
|
|
|
|
def _subject_first_weather_query(query: str) -> str:
|
|
"""Rewrite natural weather questions into the shape SearXNG handles best."""
|
|
text = re.sub(r"\s+", " ", str(query or "")).strip(" ?")
|
|
if not text:
|
|
return text
|
|
if not (set(re.findall(r"[a-z0-9]+", text.lower())) & _WEATHER_QUERY_HINTS):
|
|
return text
|
|
loc_match = re.search(
|
|
r"\b(?:weather|forecast)\s+(?:in|for|at)\s+(.+)$",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if not loc_match:
|
|
loc_match = re.search(
|
|
r"\b(?:weather|forecast)\b.*?\b(?:in|for|at)\s+(.+)$",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if not loc_match:
|
|
return text
|
|
location = loc_match.group(1).strip(" ?.,")
|
|
timing = ""
|
|
timing_match = re.search(
|
|
r"\b(today|tomorrow|tonight|this\s+week|next\s+week|now|current)\b",
|
|
location,
|
|
re.IGNORECASE,
|
|
)
|
|
if timing_match:
|
|
timing = timing_match.group(1).lower()
|
|
location = (
|
|
location[: timing_match.start()] + location[timing_match.end():]
|
|
).strip(" ?.,")
|
|
if not location:
|
|
return text
|
|
return re.sub(r"\s+", " ", f"{location} weather forecast {timing}").strip()
|
|
|
|
|
|
def _provider_friendly_query(query: str) -> str:
|
|
"""Convert generic question grammar to keyword order without changing its topic."""
|
|
text = _subject_first_weather_query(query)
|
|
match = re.fullmatch(
|
|
r"(?:what|which)\s+(year|date|time)\s+(?:did|does|do|was|were|is|are)\s+(.+)",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if match:
|
|
return f"{match.group(2).strip()} {match.group(1).lower()}"
|
|
# Search providers already receive recency separately. Remove a leading
|
|
# conversational request shell so ranking is driven by the subject rather
|
|
# than words such as "any", "latest", and "information".
|
|
cleaned = re.sub(
|
|
r"^(?:can|could|would)\s+you\s+(?:find|search|look\s+up)\s+",
|
|
"",
|
|
text,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
cleaned = re.sub(
|
|
r"^(?:any\s+)?(?:latest|current|recent)?\s*"
|
|
r"(?:news|info(?:rmation)?|updates?|details?)\s+(?:on|about)\s+",
|
|
"",
|
|
cleaned,
|
|
flags=re.IGNORECASE,
|
|
)
|
|
if cleaned.strip():
|
|
return cleaned.strip()
|
|
return text
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Unified search with caching and retry
|
|
# ----------------------------------------------------------------------
|
|
def searxng_search_results(query: str, count: int = 10, time_filter: str = None) -> list[dict]:
|
|
"""Perform a web search using configured provider with caching and retry."""
|
|
provider_query = _provider_friendly_query(query)
|
|
settings = _get_search_settings()
|
|
search_provider = settings.get("search_provider", "searxng")
|
|
result_count = _get_result_count()
|
|
# Use configured count if caller used default
|
|
if count == 10:
|
|
count = result_count
|
|
|
|
# A named scholarly work has a deterministic metadata path. Resolve that
|
|
# first instead of spending the full tool deadline retrying generic search
|
|
# providers; the returned official URL lets the agent proceed to PDF tools.
|
|
scholarly_title = _scholarly_title_from_query(provider_query)
|
|
if scholarly_title:
|
|
direct_results = _direct_scholarly_title_results(scholarly_title, count)
|
|
if direct_results:
|
|
_record_query(provider_query, True, cache_hit=False)
|
|
return direct_results[:count]
|
|
|
|
cache_key = generate_cache_key(f"{provider_query}|{count}|{time_filter}")
|
|
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
|
|
|
|
# Check cache
|
|
if cache_file.exists():
|
|
try:
|
|
with open(cache_file, "r", encoding="utf-8") as f:
|
|
cached_data = json.load(f)
|
|
expiry_raw = cached_data.get("expiry")
|
|
expiry = datetime.fromisoformat(expiry_raw) if expiry_raw else None
|
|
if expiry and datetime.now() < expiry:
|
|
logger.debug(f"Search cache hit for query: {query}")
|
|
results = cached_data["data"]
|
|
# Ranking/relevance logic evolves independently from provider
|
|
# results. Re-apply it on cache hits so stale cached ordering
|
|
# does not preserve bad SERP choices after a harness fix.
|
|
results = _filter_low_relevance_results(provider_query, results)
|
|
if results:
|
|
results = rank_search_results(provider_query, results)
|
|
results = _augment_scholarly_results(provider_query, results, count)
|
|
if results:
|
|
_record_query(query, True, cache_hit=True)
|
|
return results
|
|
logger.info(
|
|
"Search cache hit for %r became empty after relevance filtering; refetching",
|
|
provider_query,
|
|
)
|
|
cache_file.unlink(missing_ok=True)
|
|
search_cache_index.pop(cache_key, None)
|
|
else:
|
|
cache_file.unlink(missing_ok=True)
|
|
search_cache_index.pop(cache_key, None)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to read search cache for {query}: {e}")
|
|
cache_file.unlink(missing_ok=True)
|
|
search_cache_index.pop(cache_key, None)
|
|
|
|
logger.debug(f"Search cache miss for query: {query}")
|
|
|
|
if search_provider == "disabled":
|
|
logger.info("Search is disabled via admin settings")
|
|
return []
|
|
|
|
provider_chain = _build_provider_chain(search_provider)
|
|
|
|
results: List[dict] = []
|
|
for provider_name in provider_chain:
|
|
for attempt in range(2):
|
|
try:
|
|
logger.info(f"Attempting {provider_name} search (attempt {attempt + 1})")
|
|
results = _call_provider(provider_name, provider_query, count, time_filter)
|
|
results = _filter_low_relevance_results(provider_query, results)
|
|
if results:
|
|
logger.info(f"{provider_name} search succeeded with {len(results)} results")
|
|
break
|
|
except (NetworkError, ParseError, RateLimitError) as e:
|
|
error_logger.error(f"{provider_name} search error (attempt {attempt + 1}): {e}")
|
|
except Exception as e:
|
|
error_logger.error(f"Unexpected error during {provider_name} search (attempt {attempt + 1}): {e}")
|
|
if results:
|
|
break
|
|
|
|
results = _augment_scholarly_results(provider_query, results, count)
|
|
|
|
success = bool(results)
|
|
_record_query(provider_query, success, cache_hit=False)
|
|
|
|
if success:
|
|
results = rank_search_results(provider_query, results)
|
|
results = _augment_scholarly_results(provider_query, results, count)
|
|
try:
|
|
expiry = datetime.now() + _cache_duration_for_query(query)
|
|
cache_data = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"expiry": expiry.isoformat(),
|
|
"data": results,
|
|
}
|
|
with open(cache_file, "w", encoding="utf-8") as f:
|
|
json.dump(cache_data, f)
|
|
search_cache_index[cache_key] = datetime.now()
|
|
cleanup_cache(SEARCH_CACHE_DIR, search_cache_index, timedelta(hours=1))
|
|
except Exception as e:
|
|
logger.warning(f"Failed to write search cache for {provider_query}: {e}")
|
|
|
|
if not success:
|
|
logger.error(f"All search providers failed for query: {provider_query}")
|
|
|
|
return results
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Cache invalidation
|
|
# ----------------------------------------------------------------------
|
|
def invalidate_search_cache(query: Optional[str] = None) -> None:
|
|
"""Invalidate cached search results. None clears all, otherwise just the given query."""
|
|
if query is None:
|
|
for file in SEARCH_CACHE_DIR.glob("*.cache"):
|
|
try:
|
|
file.unlink(missing_ok=True)
|
|
except Exception as e:
|
|
error_logger.warning(f"Failed to delete cache file {file}: {e}")
|
|
search_cache_index.clear()
|
|
logger.info("All search cache entries have been cleared.")
|
|
else:
|
|
# Match the key the write path stores: searxng_search_results replaces
|
|
# the caller's default count with the configured _get_result_count()
|
|
# (default 5), so a hardcoded "|10|None" never matched a real entry.
|
|
cache_key = generate_cache_key(f"{query}|{_get_result_count()}|None")
|
|
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
|
|
if cache_file.exists():
|
|
try:
|
|
cache_file.unlink(missing_ok=True)
|
|
search_cache_index.pop(cache_key, None)
|
|
logger.info(f"Cache entry for query '{query}' has been invalidated.")
|
|
except Exception as e:
|
|
error_logger.warning(f"Failed to delete cache file for query '{query}': {e}")
|
|
else:
|
|
logger.info(f"No cache entry found for query '{query}'.")
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Comprehensive web search (with advanced filtering)
|
|
# ----------------------------------------------------------------------
|
|
def comprehensive_web_search(
|
|
query: str,
|
|
max_pages: int = 3,
|
|
max_workers: int = 4,
|
|
time_filter: str = None,
|
|
domain_whitelist: Optional[Set[str]] = None,
|
|
domain_blacklist: Optional[Set[str]] = None,
|
|
content_type: Optional[str] = None,
|
|
language: Optional[str] = None,
|
|
min_content_length: int = 0,
|
|
return_sources: bool = False,
|
|
):
|
|
"""Perform comprehensive web search with content fetching and advanced filtering."""
|
|
provider_query = _provider_friendly_query(query)
|
|
logger.info(f"Starting comprehensive search for: {provider_query}")
|
|
if time_filter:
|
|
logger.info(f"Applying time filter: {time_filter}")
|
|
|
|
settings = _get_search_settings()
|
|
search_provider = settings.get("search_provider", "searxng")
|
|
result_count = _get_result_count()
|
|
|
|
if search_provider == "disabled":
|
|
logger.info("Search is disabled via admin settings")
|
|
msg = "Web search is disabled by the administrator."
|
|
return (msg, []) if return_sources else msg
|
|
|
|
# Use configured result count (at least max_pages for content fetching)
|
|
fetch_count = max(result_count, max_pages)
|
|
|
|
provider_chain = _build_provider_chain(search_provider)
|
|
|
|
search_results = []
|
|
provider_attempts = {}
|
|
for provider_name in provider_chain:
|
|
last_err = None
|
|
empty = False
|
|
for attempt in range(2):
|
|
try:
|
|
search_results = _call_provider(provider_name, provider_query, fetch_count, time_filter)
|
|
search_results = _filter_low_relevance_results(provider_query, search_results)
|
|
if search_results:
|
|
provider_attempts[provider_name] = f"ok ({len(search_results)})"
|
|
logger.info(f"Comprehensive search: {provider_name} returned {len(search_results)} results")
|
|
break
|
|
empty = True
|
|
except Exception as e:
|
|
last_err = e
|
|
logger.warning(f"Comprehensive search: {provider_name} attempt {attempt + 1} failed: {e}")
|
|
if search_results:
|
|
break
|
|
if last_err is not None:
|
|
provider_attempts[provider_name] = f"error: {last_err}"
|
|
elif empty:
|
|
provider_attempts[provider_name] = "empty"
|
|
|
|
search_results = _augment_scholarly_results(
|
|
provider_query,
|
|
search_results,
|
|
fetch_count,
|
|
)
|
|
|
|
if not search_results:
|
|
tally = ", ".join(f"{p}:{r}" for p, r in provider_attempts.items()) or "no providers configured"
|
|
any_errors = any(r.startswith("error") for r in provider_attempts.values())
|
|
if any_errors:
|
|
msg = f"Web search failed — all providers errored or returned empty. Tried: {tally}"
|
|
else:
|
|
msg = (
|
|
f"No search results found. Tried: {tally}. "
|
|
"All providers returned empty — possibly a niche query or upstream rate-limiting; "
|
|
"rephrasing or using the browser tool for a specific URL may help."
|
|
)
|
|
logger.warning(msg)
|
|
return (msg, []) if return_sources else msg
|
|
|
|
search_results = rank_search_results(provider_query, search_results)
|
|
search_results = _augment_scholarly_results(
|
|
provider_query,
|
|
search_results,
|
|
fetch_count,
|
|
)
|
|
|
|
# URL filter helper
|
|
def url_passes_filters(url: str) -> bool:
|
|
try:
|
|
netloc = urlparse(url).netloc.lower()
|
|
except Exception:
|
|
return False
|
|
if domain_whitelist is not None and netloc not in domain_whitelist:
|
|
return False
|
|
if domain_blacklist is not None and netloc in domain_blacklist:
|
|
return False
|
|
if content_type:
|
|
ct = content_type.lower()
|
|
if ct == "article":
|
|
if not any(k in url.lower() for k in ("article", "blog", "news", "post")):
|
|
return False
|
|
elif ct == "forum":
|
|
if not any(k in url.lower() for k in ("forum", "discussion", "thread", "topic")):
|
|
return False
|
|
elif ct == "academic":
|
|
if not any(k in url.lower() for k in ("pdf", "doi", "scholar", "arxiv", "journal", "research")):
|
|
return False
|
|
if language:
|
|
lang_pat = language.lower()
|
|
if not (f"/{lang_pat}/" in url.lower() or f"?lang={lang_pat}" in url.lower() or f"&lang={lang_pat}" in url.lower()):
|
|
return False
|
|
return True
|
|
|
|
filtered_urls = [r["url"] for r in search_results[:max_pages] if url_passes_filters(r["url"])]
|
|
if not filtered_urls:
|
|
logger.warning("All URLs filtered out by advanced criteria")
|
|
msg = "No suitable results after applying filters."
|
|
return (msg, []) if return_sources else msg
|
|
|
|
# Build sources list for the frontend (before content fetching)
|
|
_source_list = [
|
|
{"url": r.get("url", ""), "title": r.get("title", "")}
|
|
for r in search_results if r.get("url")
|
|
]
|
|
|
|
# Map each URL to its [i] number in the sources list so fetched content
|
|
# blocks can be labeled with the SAME index the model cites.
|
|
_url_index = {
|
|
r["url"]: i for i, r in enumerate(search_results, 1) if r.get("url")
|
|
}
|
|
|
|
# Fetch content in parallel
|
|
fetched_content = []
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
future_to_url = {
|
|
executor.submit(fetch_webpage_content, url, 8, retry_attempt=0): url
|
|
for url in filtered_urls
|
|
}
|
|
for future in as_completed(future_to_url):
|
|
url = future_to_url[future]
|
|
try:
|
|
result = future.result()
|
|
if result["success"] and result["content"] and len(result["content"]) >= min_content_length:
|
|
# Remember which source this fetch belongs to: redirects
|
|
# can change result["url"] and completion order is
|
|
# arbitrary, so the block label cannot be recomputed later.
|
|
result["source_index"] = _url_index.get(url)
|
|
fetched_content.append(result)
|
|
except Exception as e:
|
|
logger.error(f"Exception while fetching {url}: {str(e)}")
|
|
|
|
logger.info(f"Successfully fetched content from {len(fetched_content)} pages")
|
|
|
|
# Format results
|
|
output_parts = []
|
|
|
|
if search_results:
|
|
output_parts.append("```sources")
|
|
for i, result in enumerate(search_results, 1):
|
|
output_parts.append(f"[{i}] {result['title']}")
|
|
output_parts.append(f" {result['url']}")
|
|
if result.get("age"):
|
|
output_parts.append(f" {result['age']}")
|
|
output_parts.append("```")
|
|
output_parts.append("")
|
|
|
|
output_parts.append("=" * 70)
|
|
output_parts.append("WEB SEARCH RESULTS AND FETCHED CONTENT")
|
|
output_parts.append(f"Query: {provider_query}")
|
|
output_parts.append(f"Searched {len(search_results)} results, fetched {len(fetched_content)} pages")
|
|
output_parts.append("=" * 70)
|
|
output_parts.append("")
|
|
|
|
output_parts.append("SEARCH RESULTS SUMMARY:")
|
|
output_parts.append("-" * 50)
|
|
for i, result in enumerate(search_results, 1):
|
|
output_parts.append(f"\n[{i}] {result['title']}")
|
|
output_parts.append(f" URL: {result['url']}")
|
|
output_parts.append(f" Snippet: {result['snippet'][:200]}...")
|
|
if result.get("age"):
|
|
output_parts.append(f" Age: {result['age']}")
|
|
|
|
if fetched_content:
|
|
output_parts.append("\n" + "=" * 70)
|
|
output_parts.append("FETCHED PAGE CONTENT:")
|
|
output_parts.append("-" * 50)
|
|
|
|
# Emit blocks in source order, numbered with the same [i] as the
|
|
# sources list, so [CONTENT 2] really is content from source [2].
|
|
# Before this, blocks were numbered 1..N in fetch COMPLETION order,
|
|
# which matched neither the sources list nor each other run to run.
|
|
fetched_content.sort(key=lambda c: c.get("source_index") or len(search_results) + 1)
|
|
for content in fetched_content:
|
|
_idx = content.get("source_index")
|
|
_label = f"[CONTENT {_idx}]" if _idx else "[CONTENT]"
|
|
output_parts.append(f"\n{_label} From: {content['url']}")
|
|
output_parts.append(f"Title: {content['title']}")
|
|
output_parts.append("-" * 30)
|
|
|
|
text = content["content"][:3000]
|
|
if len(content["content"]) > 3000:
|
|
text += "... [truncated]"
|
|
output_parts.append(text)
|
|
|
|
key_points = extract_key_points(content["content"])
|
|
if key_points:
|
|
output_parts.append("\nKey Points:")
|
|
for pt in key_points[:5]:
|
|
output_parts.append(f"- {pt}")
|
|
|
|
tldr = get_tldr(content["content"])
|
|
if tldr:
|
|
output_parts.append("\nTL;DR:")
|
|
output_parts.append(tldr)
|
|
|
|
quotes = extract_quotes(content["content"])
|
|
if quotes:
|
|
output_parts.append("\nImportant Quotes:")
|
|
for q in quotes[:3]:
|
|
output_parts.append(f"\u201c{q}\u201d")
|
|
|
|
stats = extract_statistics(content["content"])
|
|
if stats:
|
|
output_parts.append("\nData / Statistics:")
|
|
for s in stats[:5]:
|
|
output_parts.append(f"- {s}")
|
|
|
|
output_parts.append("")
|
|
|
|
output_parts.append("=" * 70)
|
|
output_parts.append("END OF WEB SEARCH RESULTS")
|
|
output_parts.append("=" * 70)
|
|
|
|
instructions = (
|
|
"\n\nIMPORTANT INSTRUCTIONS:\n"
|
|
"1. Use the above web search results and fetched content to answer the user's question\n"
|
|
"2. Prioritize information from the FETCHED PAGE CONTENT section as it contains actual page data\n"
|
|
"3. Cross-reference multiple sources when possible\n"
|
|
"4. If the information is time-sensitive, pay attention to the age of the results\n"
|
|
"5. Be explicit if the search results don't contain sufficient information to fully answer the question"
|
|
)
|
|
output_parts.append(instructions)
|
|
|
|
result = "\n".join(output_parts)
|
|
return (result, _source_list) if return_sources else result
|