Squash Odysseus development history

services/search/query.py

@@ -13,23 +13,36 @@ logger = logging.getLogger(__name__)
 # ----------------------------------------------------------------------
 def _detect_question_type(query: str) -> Optional[str]:
     """Return the leading question word if present (who, what, when, where, why, how)."""
+    if not isinstance(query, str):
+        return None
     q = query.strip().lower()
     for word in ("who", "what", "when", "where", "why", "how"):
-        if q.startswith(word):
+        # Require a whole-word match: a bare prefix mis-flags ordinary queries
+        # like "whatsapp pricing" (-> what) or "however ..." (-> how), which
+        # then get spurious boost terms OR-appended in enhance_query.
+        if q == word or q.startswith(word + " "):
             return word
     return None
 
 
 def _extract_entities(query: str) -> Dict[str, List[str]]:
     """Lightweight entity extraction: capitalized words and date patterns."""
+    if not isinstance(query, str):
+        return {"names": [], "dates": []}
     entities: Dict[str, List[str]] = {"names": [], "dates": []}
     qtype = _detect_question_type(query)
     cleaned = query
     if qtype:
         cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
-    for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
-        entities["names"].append(token)
-    for year in re.findall(r"\b(19|20)\d{2}\b", cleaned):
+    # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
+    # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
+    # "São" (shredded). Keep the ASCII behaviour — the word boundary already
+    # excludes camelCase mid-word capitals — by requiring an all-alphabetic
+    # token of length > 1 whose first character is uppercase.
+    for token in re.findall(r"\b\w+\b", cleaned):
+        if len(token) > 1 and token[0].isupper() and token.isalpha():
+            entities["names"].append(token)
+    for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
         entities["dates"].append(year)
     month_day_year = re.findall(
         r"\b(?:Jan|January|Feb|February|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|August|Sep|Sept|September|Oct|October|Nov|November|Dec|December)\s+\d{1,2},?\s*\d{4}\b",
@@ -42,12 +55,16 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
 
 def _split_multi_part(query: str) -> List[str]:
     """Split a query into sub-queries on common conjunctions."""
+    if not isinstance(query, str):
+        return []
     parts = re.split(r"\s+and\s+|\s+or\s+|;", query, flags=re.I)
     return [p.strip() for p in parts if p.strip()]
 
 
 def _extract_site_filter(query: str) -> Tuple[str, Optional[str]]:
     """Detect a 'site:example.com' token. Returns (query_without_token, site_or_None)."""
+    if not isinstance(query, str):
+        return "", None
     match = re.search(r"\bsite:([^\s]+)", query, flags=re.I)
     if match:
         site = match.group(1)
@@ -68,6 +85,8 @@ def _boost_entities_in_query(base_query: str, entities: Dict[str, List[str]]) ->
 
 def enhance_query(original_query: str) -> Tuple[str, Optional[str]]:
     """Process the original query: site filter, question type boosts, entity extraction."""
+    if not isinstance(original_query, str):
+        original_query = ""
     query_without_site, site = _extract_site_filter(original_query)
     sub_queries = _split_multi_part(query_without_site)
 
@@ -117,6 +136,8 @@ def build_enhanced_query(query: str, time_filter: str = None) -> str:
 def _is_news_query(query: str) -> bool:
     """Lightweight heuristic to decide if a query is news-oriented."""
     news_terms = {"news", "latest", "breaking", "today", "today's", "current", "updates", "happening"}
+    if not isinstance(query, str):
+        return False
     tokens = set(re.findall(r"\b\w+\b", query.lower()))
     return bool(tokens & news_terms)