Squash Odysseus development history

services/search/ranking.py

@@ -2,17 +2,59 @@
 
 import re
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
+_AGE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S")
+
+
+def _utcnow_naive() -> datetime:
+    """Naive UTC 'now'. Matches the naive, UTC-style published dates parsed below,
+    and is safe on Python 3.14 where ``datetime.utcnow()`` is removed (#1116)."""
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+def recency_score(age_str: Optional[str], now: Optional[datetime] = None) -> float:
+    """Score how recent a result is: 1.0 for <=7 days old, 0.0 for >=30 days.
+
+    The age is measured against UTC, not local time. The previous code used
+    ``datetime.now()`` (local) against UTC-style published dates, so the age was
+    skewed by the host's UTC offset; it was also a latent crash once neighbouring
+    code moves to timezone-aware datetimes (#1116). ``now`` is injectable for tests.
+    """
+    if not age_str:
+        return 0.0
+    dt = None
+    for fmt in _AGE_FORMATS:
+        try:
+            dt = datetime.strptime(age_str, fmt)
+            break
+        except Exception:
+            dt = None
+    if not dt:
+        return 0.0
+    now = now or _utcnow_naive()
+    days_old = (now - dt).days
+    if days_old <= 7:
+        return 1.0
+    if days_old >= 30:
+        return 0.0
+    return (30 - days_old) / 23
+
+
 _NEWS_HINTS = {"news", "nyheter", "headlines", "breaking", "latest", "today", "idag"}
 _SPORTS_HINTS = {
     "sport", "sports", "soccer", "football", "hockey", "nba", "nfl", "mlb",
     "fifa", "world cup", "championship", "quarterfinal", "eliminates",
 }
+# Word-boundary match so "sport" does not fire inside "transport"/"passport"
+# and a domain like "transport.gov" is not mistaken for a sports site.
+_SPORTS_HINT_RE = re.compile(
+    r"\b(?:" + "|".join(re.escape(h) for h in _SPORTS_HINTS) + r")\b"
+)
 _LOW_VALUE_NEWS_DOMAINS = {
     "facebook.com", "www.facebook.com", "sports.yahoo.com", "yahoo.com",
     "www.yahoo.com", "msn.com", "www.msn.com",
@@ -25,6 +67,22 @@ _TRUSTED_NEWS_DOMAINS = {
     "www.theguardian.com", "euronews.com", "www.euronews.com",
     "dw.com", "www.dw.com", "government.se", "www.government.se",
 }
+_SOFTWARE_RELEASE_HINTS = {
+    "github", "gitlab", "release", "releases", "version", "versions",
+    "changelog", "package", "pypi", "npm",
+}
+_PRODUCT_SPEC_HINTS = {
+    "product", "hardware", "device", "phone", "laptop", "desktop", "computer",
+    "chip", "cpu", "gpu", "mac", "iphone", "ipad", "android", "camera",
+    "console", "kindle", "tesla", "car", "model", "price", "pricing", "cost",
+    "buy", "shop", "order", "preorder", "pre-order", "spec", "specs",
+    "specifications", "available", "availability", "ship", "shipping",
+    "released", "launch", "launched", "vram", "memory", "ram", "storage",
+}
+_COMMERCE_OR_SPEC_PATH_HINTS = (
+    "/shop", "/buy", "/store", "/product", "/products", "/spec", "/specs",
+    "/support", "/tech-specs", "/technical-specifications",
+)
 
 
 def _domain(url: str) -> str:
@@ -34,25 +92,40 @@ def _domain(url: str) -> str:
         return ""
 
 
+def _has_word(text: str, term: str) -> bool:
+    """True if ``term`` appears in ``text`` as a whole word.
+
+    Query terms are matched on word boundaries so a short term doesn't match
+    inside an unrelated word: "us" must not match "business"/"music", "port"
+    must not match "transport"/"support". This mirrors the tokenization used to
+    build ``query_terms`` (``\\b\\w+\\b``). #1473 converted the title and sports
+    checks to word boundaries; the snippet and subject-term checks below use
+    the same helper so the whole file stays consistent.
+    """
+    return re.search(rf"\b{re.escape(term)}\b", text) is not None
+
+
 def rank_search_results(query: str, results: List[dict]) -> List[dict]:
     """Rank search results by title relevance, snippet quality, domain authority, and recency."""
     query_terms = [t.lower() for t in re.findall(r"\b\w+\b", query)]
     query_lc = query.lower()
     is_news_query = any(term in _NEWS_HINTS for term in query_terms)
-    is_sports_query = any(hint in query_lc for hint in _SPORTS_HINTS)
+    is_sports_query = bool(_SPORTS_HINT_RE.search(query_lc))
+    is_software_release_query = any(term in _SOFTWARE_RELEASE_HINTS for term in query_terms)
+    is_product_spec_query = any(term in _PRODUCT_SPEC_HINTS for term in query_terms)
 
     def title_score(title: str) -> float:
         if not title:
             return 0.0
         title_lc = title.lower()
-        matches = sum(1 for term in query_terms if re.search(rf"\b{re.escape(term)}\b", title_lc))
+        matches = sum(1 for term in query_terms if _has_word(title_lc, term))
         return matches / len(query_terms) if query_terms else 0.0
 
     def snippet_score(snippet: str) -> float:
         if not snippet:
             return 0.0
         length_factor = min(len(snippet), 200) / 200
-        term_hits = sum(1 for term in query_terms if term in snippet.lower())
+        term_hits = sum(1 for term in query_terms if _has_word(snippet.lower(), term))
         term_factor = term_hits / len(query_terms) if query_terms else 0.0
         return (length_factor + term_factor) / 2
 
@@ -68,24 +141,6 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             return 0.7
         return 0.4
 
-    def recency_score(age_str: Optional[str]) -> float:
-        if not age_str:
-            return 0.0
-        for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"):
-            try:
-                dt = datetime.strptime(age_str, fmt)
-                break
-            except Exception:
-                dt = None
-        if not dt:
-            return 0.0
-        days_old = (datetime.now() - dt).days
-        if days_old <= 7:
-            return 1.0
-        if days_old >= 30:
-            return 0.0
-        return (30 - days_old) / 23
-
     def news_quality_adjustment(title: str, snippet: str, url: str) -> float:
         if not is_news_query:
             return 0.0
@@ -98,15 +153,50 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             adjustment += 0.4
         if netloc in _LOW_VALUE_NEWS_DOMAINS:
             adjustment -= 0.8
-        if not is_sports_query and any(hint in text or hint in netloc for hint in _SPORTS_HINTS):
+        if not is_sports_query and (_SPORTS_HINT_RE.search(text) or _SPORTS_HINT_RE.search(netloc)):
             adjustment -= 1.5
         # A country/news query should not rank a page whose title/snippet barely
         # mentions the country above actual news pages for that country.
         subject_terms = [t for t in query_terms if t not in _NEWS_HINTS]
-        if subject_terms and not any(t in text or t in netloc for t in subject_terms):
+        if subject_terms and not any(_has_word(text, t) or _has_word(netloc, t) for t in subject_terms):
             adjustment -= 1.0
         return adjustment
 
+    def software_release_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_software_release_query:
+            return 0.0
+        netloc = _domain(url)
+        path = urlparse(url).path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if netloc in {"github.com", "www.github.com", "gitlab.com", "www.gitlab.com"}:
+            adjustment += 1.6
+        if "/releases" in path or "/tags" in path:
+            adjustment += 1.2
+        if any(_has_word(text, term) for term in ("release", "releases", "changelog", "version")):
+            adjustment += 0.4
+        if netloc in {"releasealert.dev", "releases.sh", "releasebot.io"}:
+            adjustment -= 0.8
+        return adjustment
+
+    def product_spec_adjustment(title: str, snippet: str, url: str) -> float:
+        if not is_product_spec_query:
+            return 0.0
+        parsed = urlparse(url)
+        netloc = parsed.netloc.lower()
+        path = parsed.path.lower()
+        text = f"{title} {snippet} {netloc} {path}".lower()
+        adjustment = 0.0
+        if any(hint in path for hint in _COMMERCE_OR_SPEC_PATH_HINTS):
+            adjustment += 1.1
+        if re.search(r"\b(?:official|specs?|specifications|tech specs|buy|shop|store|price|pricing|available|ships?)\b", text):
+            adjustment += 0.5
+        if netloc.endswith(".com") and any(_has_word(netloc, term) for term in query_terms if len(term) >= 4):
+            adjustment += 0.4
+        if re.search(r"\b(?:rumor|rumour|leak|may|could|expected|reportedly|unannounced)\b", text):
+            adjustment -= 0.8
+        return adjustment
+
     ranked = []
     for result in results:
         title = result.get("title", "")
@@ -120,6 +210,8 @@ def rank_search_results(query: str, results: List[dict]) -> List[dict]:
             + 1.5 * domain_score(url)
             + 1.0 * recency_score(age)
             + news_quality_adjustment(title, snippet, url)
+            + software_release_adjustment(title, snippet, url)
+            + product_spec_adjustment(title, snippet, url)
         )
         ranked.append((score, result))