jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-16 11:52:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squash Odysseus development history

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/search/core.py
+592 -22
View File
@@ -2,11 +2,15 @@
import json
import logging
import re
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List, Set
from urllib.parse import urlparse
import httpx
from .analytics import (
NetworkError,
ParseError,
@@ -30,6 +34,7 @@ from .providers import (
tavily_search,
serper_search,
_get_search_settings,
_get_provider_key,
_get_result_count,
)
from .content import (
@@ -48,30 +53,56 @@ SEARCH_CONFIG: Dict[str, Any] = {
}
def _is_secret_key(name: str) -> bool:
"""True for config keys that hold a credential (e.g. ``brave_api_key``)."""
return name.endswith(("_api_key", "_key", "_token", "_secret"))
def get_search_config() -> Dict[str, Any]:
"""Get current search configuration including active provider info."""
"""Get current search configuration including active provider info.
Never returns stored API keys: callers — including the unauthenticated
``GET /api/search/config`` route — only need key *presence* via
``has_api_key``, not the secret itself (#1661).
"""
config = SEARCH_CONFIG.copy()
settings = _get_search_settings()
provider = settings.get("search_provider", "searxng")
config["active_provider"] = provider
config["has_api_key"] = bool((settings.get("search_api_key") or "").strip())
config["has_api_key"] = bool(_get_provider_key(provider))
config["result_count"] = _get_result_count()
if provider == "searxng":
from .providers import _get_search_instance
config["search_url"] = _get_search_instance()
return config
# Strip any string-valued credential so secrets never reach the response;
# the boolean has_api_key flag (presence only) is preserved.
return {
k: v for k, v in config.items()
if not (isinstance(v, str) and _is_secret_key(k))
}
def update_search_config(api_key: str = None, **kwargs):
"""Update search configuration (e.g. Brave API key)."""
if api_key:
SEARCH_CONFIG["brave_api_key"] = api_key
"""Merge non-secret search config into SEARCH_CONFIG.
Provider API keys are intentionally NOT cached here. They are read on demand
from settings/env via ``_get_provider_key`` (e.g. ``brave_search``), so the
previous ``SEARCH_CONFIG["brave_api_key"] = api_key`` cache was never used
for search and only leaked the decrypted key through ``get_search_config`` /
``GET /api/search/config`` (#1661). ``api_key`` is accepted for backward
compatibility but no longer stored.
"""
for k, v in kwargs.items():
if not _is_secret_key(k):
SEARCH_CONFIG[k] = v
def _call_provider(provider_name: str, query: str, count: int, time_filter: str = None) -> List[dict]:
"""Call a search provider by name. Returns list of results or empty list."""
if provider_name == "searxng":
return searxng_search_api(query, count, time_filter=time_filter)
elif provider_name == "searxng_yep":
return searxng_search_api(query, count, time_filter=time_filter, engines="yep")
elif provider_name == "brave":
return brave_search(query, count, time_filter)
elif provider_name == "duckduckgo":
@@ -102,7 +133,484 @@ def _build_provider_chain(primary: str) -> List[str]:
for fb in fallbacks:
if fb and fb != primary and fb not in chain and fb != "disabled":
chain.append(fb)
return chain
from .providers import provider_configured
configured = [provider for provider in chain if provider_configured(provider)]
for provider in set(chain) - set(configured):
logger.warning("Skipping unconfigured search provider: %s", provider)
if primary == "searxng" and configured == ["searxng"]:
# No usable configured fallback: try a separate engine on the same
# private metasearch instance before reporting retrieval failure.
configured.append("searxng_yep")
return configured
_SEARCH_QUERY_FILLER = {
"what", "whats", "what's", "which", "when", "where", "year", "from",
"any", "info", "information", "details", "update", "updates",
"with", "this", "that", "search", "lookup", "look", "find", "tell",
"about", "quick", "please", "pls", "official", "links", "source",
"sources", "news", "headlines", "breaking", "latest", "current",
"newest", "recent", "today", "now",
"release", "releases", "version", "versions", "changelog", "github",
"gitlab", "weather", "forecast", "forecasts", "tomorrow", "hourly",
"daily", "temperature", "temperatures", "conditions", "rain", "raining",
"chance", "precipitation",
"january", "february", "march", "april", "may", "june", "july",
"august", "september", "october", "november", "december",
"the", "and", "or", "but", "are", "was", "were", "does", "did",
"can", "could", "should", "would", "will", "has", "have", "had",
"for", "into", "onto", "near", "over", "under",
}
_SHORT_QUERY_SUBJECTS = {"ai", "ar", "eu", "uk", "us", "vr"}
_WEATHER_QUERY_HINTS = {
"weather", "forecast", "forecasts", "temperature", "temperatures",
"rain", "raining", "precipitation", "humid", "humidity", "wind",
}
_WEATHER_RESULT_HINTS = {
"weather", "forecast", "temperature", "temperatures", "rain",
"precipitation", "humidity", "wind", "accuweather", "meteoblue",
"weather-atlas", "weather25", "weather365", "easeweather",
}
def _meaningful_query_terms(query: str) -> list[str]:
return [
term
for term in re.findall(r"[a-z0-9]+", str(query or "").lower())
if (len(term) > 2 or term in _SHORT_QUERY_SUBJECTS)
and not term.isdigit()
and term not in _SEARCH_QUERY_FILLER
]
def _result_has_query_overlap(query: str, result: dict) -> bool:
terms = _meaningful_query_terms(query)
if not terms:
return True
text = " ".join(
str(result.get(key) or "").lower()
for key in ("title", "snippet", "url")
)
query_tokens = set(re.findall(r"[a-z0-9]+", str(query or "").lower()))
if query_tokens & _WEATHER_QUERY_HINTS:
return (
any(re.search(rf"\b{re.escape(term)}\b", text) for term in terms)
and any(marker in text for marker in _WEATHER_RESULT_HINTS)
)
result_tokens = set(re.findall(r"[a-z0-9]+", text))
def lexical_root(word: str) -> str:
for suffix in ("ation", "ition", "ence", "ance", "ment", "ents", "ent", "ant", "ing", "ed", "es", "s"):
if word.endswith(suffix) and len(word) - len(suffix) >= 6:
return word[:-len(suffix)]
return word
result_roots = {lexical_root(token) for token in result_tokens}
matched_terms = {
term for term in terms
if term in result_tokens or lexical_root(term) in result_roots
}
# A single broad token is not enough evidence for a detailed entity/event
# query. For example, SearXNG may answer "Sweden 78 year old British woman
# deportation Brexit ..." with generic Sweden tourism pages. Treat that as
# an empty provider result so the configured fallback gets a chance.
minimum_matches = 2 if len(set(terms)) >= 4 else 1
return len(matched_terms) >= minimum_matches
def _filter_low_relevance_results(query: str, results: list[dict]) -> list[dict]:
if not results:
return []
relevant = [result for result in results if _result_has_query_overlap(query, result)]
# Only reject a provider when it returned a fully off-topic page set. Mixed
# result pages are common; ranking can handle those.
return relevant if relevant else []
_SCHOLARLY_QUERY_CUE_RE = re.compile(
r"\b(?:paper|preprint|arxiv|proceedings|table\s+\d+|figure\s+\d+|"
r"appendix\s+[a-z0-9]+|benchmark(?:s)?)\b",
re.IGNORECASE,
)
_SCHOLARLY_TITLE_FILLER = _SEARCH_QUERY_FILLER | {
"paper", "preprint", "arxiv", "proceedings", "table", "figure",
"appendix", "authors", "author", "extract", "locate", "read",
}
_ARXIV_IDENTIFIER_RE = re.compile(
r"(?i)(?:\barxiv\s*:\s*|\barxiv\.org/(?:abs|pdf|html)/)?"
r"(?P<identifier>\d{4}\.\d{4,5}(?:v\d+)?)\b"
)
_FORMAL_PUBLICATION_CUE_RE = re.compile(
r"\b(?:publish(?:ed|ing|cation)?|venue|conference|journal|proceedings|doi)\b",
re.IGNORECASE,
)
def _exact_arxiv_identifier_results(query: str) -> list[dict]:
"""Return deterministic official landing pages for explicit arXiv IDs."""
seen: set[str] = set()
results: list[dict] = []
for match in _ARXIV_IDENTIFIER_RE.finditer(str(query or "")):
identifier = match.group("identifier")
canonical = re.sub(r"v\d+$", "", identifier, flags=re.IGNORECASE)
if canonical in seen:
continue
seen.add(canonical)
results.append({
"title": f"arXiv:{canonical} — exact identifier match",
"url": f"https://arxiv.org/abs/{canonical}",
"snippet": (
"Official arXiv landing page resolved directly from the exact "
"identifier in the query."
),
"source": "arxiv",
})
return results
def _title_before_explicit_arxiv_identifier(query: str) -> str:
"""Extract a probable title that precedes an explicit arXiv identifier."""
text = re.sub(r"\s+", " ", str(query or "")).strip()
match = _ARXIV_IDENTIFIER_RE.search(text)
if not match or not _FORMAL_PUBLICATION_CUE_RE.search(text):
return ""
candidate = text[:match.start()].strip(" \t,;:-'\"")
candidate = re.sub(
r"\barxiv(?:\.org)?(?:\s*:\s*|\s+(?:abs|pdf|html)\s*[/ :]*)?$",
"",
candidate,
flags=re.IGNORECASE,
).strip(" \t,;:-'\"")
candidate = re.sub(
r"^(?:(?:please\s+)?(?:find|locate|search\s+for|look\s+up|verify|check)\s+)"
r"(?:(?:the|this)\s+)?(?:paper\s+)?",
"",
candidate,
flags=re.IGNORECASE,
).strip(" \t,;:-'\"")
return candidate if len(_normalized_title_terms(candidate)) >= 2 else ""
def _normalized_title_terms(value: str) -> list[str]:
return [
token
for token in re.findall(r"[a-z0-9]+", str(value or "").lower())
if len(token) > 1 and token not in _SCHOLARLY_TITLE_FILLER
]
def _is_distinctive_short_scholarly_title(value: str) -> bool:
"""Recognize compact model/report names without accepting generic phrases."""
terms = _normalized_title_terms(value)
if not 1 <= len(terms) <= 2:
return False
text = str(value or "").strip()
return bool(
re.search(r"\d", text)
or re.search(r"\b[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*\b", text)
)
def _scholarly_title_from_query(query: str) -> str:
"""Extract a probable paper title only from clearly scholarly searches."""
text = re.sub(r"\s+", " ", str(query or "")).strip()
if not text or not _SCHOLARLY_QUERY_CUE_RE.search(text):
return ""
quoted = [
candidate.strip()
for candidate in re.findall(r'["“”]([^"“”]{4,180})["“”]', text)
if len(_normalized_title_terms(candidate)) >= 3
or _is_distinctive_short_scholarly_title(candidate)
]
if quoted:
return max(quoted, key=lambda candidate: len(_normalized_title_terms(candidate)))
before_paper = re.search(
r"(?:^|\b(?:find|locate|read|from|about)\s+)(.{4,160}?)\s+"
r"(?:paper|preprint)\b",
text,
re.IGNORECASE,
)
if before_paper:
candidate = before_paper.group(1).strip(" ,:;-'")
if (
len(_normalized_title_terms(candidate)) >= 3
or _is_distinctive_short_scholarly_title(candidate)
):
return candidate
before_locator = re.match(
r"(.{2,80}?)\s+(?:table|figure)\s+\d+\b",
text,
re.IGNORECASE,
)
if before_locator:
candidate = before_locator.group(1).strip(" ,:;-'\"")
if _is_distinctive_short_scholarly_title(candidate):
return candidate
return ""
def _result_strongly_matches_title(title: str, result: dict) -> bool:
wanted = set(_normalized_title_terms(title))
found = set(_normalized_title_terms(str(result.get("title") or "")))
if len(wanted) < 2 or not found:
return False
overlap = len(wanted & found) / len(wanted)
return overlap >= (1.0 if len(wanted) == 2 else 0.8)
def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve a paper title through arXiv's public Atom API."""
try:
response = httpx.get(
"https://export.arxiv.org/api/query",
params={
"search_query": f'ti:"{title}"',
"start": 0,
"max_results": max(1, min(int(count), 5)),
},
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
timeout=12.0,
follow_redirects=True,
)
response.raise_for_status()
root = ET.fromstring(response.text)
except Exception as exc:
logger.info("arXiv title lookup failed for %r: %s", title, exc)
return []
namespace = {"atom": "http://www.w3.org/2005/Atom"}
matches: list[dict] = []
for entry in root.findall("atom:entry", namespace):
result_title = " ".join(
(entry.findtext("atom:title", default="", namespaces=namespace) or "").split()
)
if not _result_strongly_matches_title(title, {"title": result_title}):
continue
entry_id = (entry.findtext("atom:id", default="", namespaces=namespace) or "").strip()
arxiv_id = entry_id.rstrip("/").rsplit("/", 1)[-1]
if not arxiv_id:
continue
summary = " ".join(
(entry.findtext("atom:summary", default="", namespaces=namespace) or "").split()
)
matches.append({
"title": result_title,
"url": f"https://arxiv.org/abs/{arxiv_id}",
"snippet": summary,
"source": "arxiv",
})
return matches
def _openalex_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve an exact scholarly title through OpenAlex metadata."""
try:
# OpenAlex treats a literal question mark as query syntax and returns
# HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".
search_title = re.sub(r"[?]+", " ", str(title or "")).strip()
response = httpx.get(
"https://api.openalex.org/works",
params={
"search": search_title,
"per-page": max(1, min(int(count), 5)),
"select": (
"display_name,doi,primary_location,publication_year,type"
),
},
headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},
timeout=12.0,
follow_redirects=True,
)
response.raise_for_status()
payload = response.json()
except Exception as exc:
logger.info("OpenAlex title lookup failed for %r: %s", title, exc)
return []
matches: list[dict] = []
for item in payload.get("results", []):
result_title = str(item.get("display_name") or "").strip()
if not _result_strongly_matches_title(title, {"title": result_title}):
continue
location = item.get("primary_location") or {}
url = str(location.get("landing_page_url") or item.get("doi") or "").strip()
if url.startswith("http://arxiv.org/"):
url = "https://" + url[len("http://"):]
if not url:
continue
snippet = "Exact scholarly-title match from OpenAlex metadata."
venue = str(location.get("raw_source_name") or "").strip()
year = item.get("publication_year")
publication_type = str(item.get("type") or "").strip()
version = str(location.get("version") or "").strip()
formal_parts: list[str] = []
if venue:
formal_parts.append(f"{venue}, {year}" if year else venue)
elif year:
formal_parts.append(str(year))
if publication_type:
formal_parts.append(f"type: {publication_type}")
if version:
formal_parts.append(f"version: {version}")
if formal_parts:
snippet += f" Formal publication: {'; '.join(formal_parts)}."
matches.append({
"title": result_title,
"url": url,
"snippet": snippet,
"source": "openalex",
})
return matches
def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:
"""Retry a noisy scholarly query as a bare title, then use arXiv API."""
try:
simplified = searxng_search_api(title, count=max(3, count))
except Exception as exc:
logger.info("Simplified scholarly search failed for %r: %s", title, exc)
simplified = []
exact = [
result for result in simplified
if _result_strongly_matches_title(title, result)
]
if exact:
return exact[:count]
openalex = _openalex_title_results(title, count)
if openalex:
return openalex
return _arxiv_title_results(title, count)
def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:
"""Resolve a clear paper title without waiting on generic search providers."""
# OpenAlex typically resolves titles in under a second and often returns
# the official arXiv landing page. The arXiv API remains the fallback.
openalex = _openalex_title_results(title, count)
if openalex:
return openalex
return _arxiv_title_results(title, count)
def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:
"""Prepend an exact arXiv match when a scholarly SERP missed its title."""
current = list(results or [])
identifier_results = _exact_arxiv_identifier_results(query)
if identifier_results:
title = _title_before_explicit_arxiv_identifier(query)
formal_results: list[dict] = []
if title:
formal_results = [
item
for item in _openalex_title_results(title, min(count, 3))
if "arxiv.org/" not in str(item.get("url") or "").lower()
]
exact_urls = {str(item["url"]) for item in identifier_results}
formal_urls = {str(item.get("url") or "") for item in formal_results}
return (
formal_results
+ identifier_results
+ [
item for item in current
if str(item.get("url") or "") not in exact_urls | formal_urls
]
)[:count]
title = _scholarly_title_from_query(query)
if not title:
return current
exact_current = [
item for item in current
if _result_strongly_matches_title(title, item)
]
if exact_current:
exact_ids = {id(item) for item in exact_current}
return (exact_current + [item for item in current if id(item) not in exact_ids])[:count]
arxiv_results = _scholarly_title_results(title, min(count, 3))
if not arxiv_results:
return current
seen = {str(item.get("url") or "") for item in arxiv_results}
return (arxiv_results + [item for item in current if str(item.get("url") or "") not in seen])[:count]
def _subject_first_weather_query(query: str) -> str:
"""Rewrite natural weather questions into the shape SearXNG handles best."""
text = re.sub(r"\s+", " ", str(query or "")).strip(" ?")
if not text:
return text
if not (set(re.findall(r"[a-z0-9]+", text.lower())) & _WEATHER_QUERY_HINTS):
return text
loc_match = re.search(
r"\b(?:weather|forecast)\s+(?:in|for|at)\s+(.+)$",
text,
re.IGNORECASE,
)
if not loc_match:
loc_match = re.search(
r"\b(?:weather|forecast)\b.*?\b(?:in|for|at)\s+(.+)$",
text,
re.IGNORECASE,
)
if not loc_match:
return text
location = loc_match.group(1).strip(" ?.,")
timing = ""
timing_match = re.search(
r"\b(today|tomorrow|tonight|this\s+week|next\s+week|now|current)\b",
location,
re.IGNORECASE,
)
if timing_match:
timing = timing_match.group(1).lower()
location = (
location[: timing_match.start()] + location[timing_match.end():]
).strip(" ?.,")
if not location:
return text
return re.sub(r"\s+", " ", f"{location} weather forecast {timing}").strip()
def _provider_friendly_query(query: str) -> str:
"""Convert generic question grammar to keyword order without changing its topic."""
text = _subject_first_weather_query(query)
match = re.fullmatch(
r"(?:what|which)\s+(year|date|time)\s+(?:did|does|do|was|were|is|are)\s+(.+)",
text,
re.IGNORECASE,
)
if match:
return f"{match.group(2).strip()} {match.group(1).lower()}"
# Search providers already receive recency separately. Remove a leading
# conversational request shell so ranking is driven by the subject rather
# than words such as "any", "latest", and "information".
cleaned = re.sub(
r"^(?:can|could|would)\s+you\s+(?:find|search|look\s+up)\s+",
"",
text,
flags=re.IGNORECASE,
)
cleaned = re.sub(
r"^(?:any\s+)?(?:latest|current|recent)?\s*"
r"(?:news|info(?:rmation)?|updates?|details?)\s+(?:on|about)\s+",
"",
cleaned,
flags=re.IGNORECASE,
)
if cleaned.strip():
return cleaned.strip()
return text
# ----------------------------------------------------------------------
@@ -110,6 +618,7 @@ def _build_provider_chain(primary: str) -> List[str]:
# ----------------------------------------------------------------------
def searxng_search_results(query: str, count: int = 10, time_filter: str = None) -> list[dict]:
"""Perform a web search using configured provider with caching and retry."""
provider_query = _provider_friendly_query(query)
settings = _get_search_settings()
search_provider = settings.get("search_provider", "searxng")
result_count = _get_result_count()
@@ -117,7 +626,17 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if count == 10:
count = result_count
cache_key = generate_cache_key(f"{query}|{count}|{time_filter}")
# A named scholarly work has a deterministic metadata path. Resolve that
# first instead of spending the full tool deadline retrying generic search
# providers; the returned official URL lets the agent proceed to PDF tools.
scholarly_title = _scholarly_title_from_query(provider_query)
if scholarly_title:
direct_results = _direct_scholarly_title_results(scholarly_title, count)
if direct_results:
_record_query(provider_query, True, cache_hit=False)
return direct_results[:count]
cache_key = generate_cache_key(f"{provider_query}|{count}|{time_filter}")
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
# Check cache
@@ -130,8 +649,22 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if expiry and datetime.now() < expiry:
logger.debug(f"Search cache hit for query: {query}")
results = cached_data["data"]
_record_query(query, bool(results), cache_hit=True)
return results
# Ranking/relevance logic evolves independently from provider
# results. Re-apply it on cache hits so stale cached ordering
# does not preserve bad SERP choices after a harness fix.
results = _filter_low_relevance_results(provider_query, results)
if results:
results = rank_search_results(provider_query, results)
results = _augment_scholarly_results(provider_query, results, count)
if results:
_record_query(query, True, cache_hit=True)
return results
logger.info(
"Search cache hit for %r became empty after relevance filtering; refetching",
provider_query,
)
cache_file.unlink(missing_ok=True)
search_cache_index.pop(cache_key, None)
else:
cache_file.unlink(missing_ok=True)
search_cache_index.pop(cache_key, None)
@@ -153,7 +686,8 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
for attempt in range(2):
try:
logger.info(f"Attempting {provider_name} search (attempt {attempt + 1})")
results = _call_provider(provider_name, query, count, time_filter)
results = _call_provider(provider_name, provider_query, count, time_filter)
results = _filter_low_relevance_results(provider_query, results)
if results:
logger.info(f"{provider_name} search succeeded with {len(results)} results")
break
@@ -164,11 +698,14 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
if results:
break
results = _augment_scholarly_results(provider_query, results, count)
success = bool(results)
_record_query(query, success, cache_hit=False)
_record_query(provider_query, success, cache_hit=False)
if success:
results = rank_search_results(query, results)
results = rank_search_results(provider_query, results)
results = _augment_scholarly_results(provider_query, results, count)
try:
expiry = datetime.now() + _cache_duration_for_query(query)
cache_data = {
@@ -181,10 +718,10 @@ def searxng_search_results(query: str, count: int = 10, time_filter: str = None)
search_cache_index[cache_key] = datetime.now()
cleanup_cache(SEARCH_CACHE_DIR, search_cache_index, timedelta(hours=1))
except Exception as e:
logger.warning(f"Failed to write search cache for {query}: {e}")
logger.warning(f"Failed to write search cache for {provider_query}: {e}")
if not success:
logger.error(f"All search providers failed for query: {query}")
logger.error(f"All search providers failed for query: {provider_query}")
return results
@@ -203,7 +740,10 @@ def invalidate_search_cache(query: Optional[str] = None) -> None:
search_cache_index.clear()
logger.info("All search cache entries have been cleared.")
else:
cache_key = generate_cache_key(f"{query}|10|None")
# Match the key the write path stores: searxng_search_results replaces
# the caller's default count with the configured _get_result_count()
# (default 5), so a hardcoded "|10|None" never matched a real entry.
cache_key = generate_cache_key(f"{query}|{_get_result_count()}|None")
cache_file = SEARCH_CACHE_DIR / f"{cache_key}.cache"
if cache_file.exists():
try:
@@ -232,7 +772,8 @@ def comprehensive_web_search(
return_sources: bool = False,
):
"""Perform comprehensive web search with content fetching and advanced filtering."""
logger.info(f"Starting comprehensive search for: {query}")
provider_query = _provider_friendly_query(query)
logger.info(f"Starting comprehensive search for: {provider_query}")
if time_filter:
logger.info(f"Applying time filter: {time_filter}")
@@ -257,7 +798,8 @@ def comprehensive_web_search(
empty = False
for attempt in range(2):
try:
search_results = _call_provider(provider_name, query, fetch_count, time_filter)
search_results = _call_provider(provider_name, provider_query, fetch_count, time_filter)
search_results = _filter_low_relevance_results(provider_query, search_results)
if search_results:
provider_attempts[provider_name] = f"ok ({len(search_results)})"
logger.info(f"Comprehensive search: {provider_name} returned {len(search_results)} results")
@@ -273,6 +815,12 @@ def comprehensive_web_search(
elif empty:
provider_attempts[provider_name] = "empty"
search_results = _augment_scholarly_results(
provider_query,
search_results,
fetch_count,
)
if not search_results:
tally = ", ".join(f"{p}:{r}" for p, r in provider_attempts.items()) or "no providers configured"
any_errors = any(r.startswith("error") for r in provider_attempts.values())
@@ -287,7 +835,12 @@ def comprehensive_web_search(
logger.warning(msg)
return (msg, []) if return_sources else msg
search_results = rank_search_results(query, search_results)
search_results = rank_search_results(provider_query, search_results)
search_results = _augment_scholarly_results(
provider_query,
search_results,
fetch_count,
)
# URL filter helper
def url_passes_filters(url: str) -> bool:
@@ -328,6 +881,12 @@ def comprehensive_web_search(
for r in search_results if r.get("url")
]
# Map each URL to its [i] number in the sources list so fetched content
# blocks can be labeled with the SAME index the model cites.
_url_index = {
r["url"]: i for i, r in enumerate(search_results, 1) if r.get("url")
}
# Fetch content in parallel
fetched_content = []
with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -340,6 +899,10 @@ def comprehensive_web_search(
try:
result = future.result()
if result["success"] and result["content"] and len(result["content"]) >= min_content_length:
# Remember which source this fetch belongs to: redirects
# can change result["url"] and completion order is
# arbitrary, so the block label cannot be recomputed later.
result["source_index"] = _url_index.get(url)
fetched_content.append(result)
except Exception as e:
logger.error(f"Exception while fetching {url}: {str(e)}")
@@ -361,7 +924,7 @@ def comprehensive_web_search(
output_parts.append("=" * 70)
output_parts.append("WEB SEARCH RESULTS AND FETCHED CONTENT")
output_parts.append(f"Query: {query}")
output_parts.append(f"Query: {provider_query}")
output_parts.append(f"Searched {len(search_results)} results, fetched {len(fetched_content)} pages")
output_parts.append("=" * 70)
output_parts.append("")
@@ -380,8 +943,15 @@ def comprehensive_web_search(
output_parts.append("FETCHED PAGE CONTENT:")
output_parts.append("-" * 50)
for i, content in enumerate(fetched_content, 1):
output_parts.append(f"\n[CONTENT {i}] From: {content['url']}")
# Emit blocks in source order, numbered with the same [i] as the
# sources list, so [CONTENT 2] really is content from source [2].
# Before this, blocks were numbered 1..N in fetch COMPLETION order,
# which matched neither the sources list nor each other run to run.
fetched_content.sort(key=lambda c: c.get("source_index") or len(search_results) + 1)
for content in fetched_content:
_idx = content.get("source_index")
_label = f"[CONTENT {_idx}]" if _idx else "[CONTENT]"
output_parts.append(f"\n{_label} From: {content['url']}")
output_parts.append(f"Title: {content['title']}")
output_parts.append("-" * 30)