Preserve preview harness, editor, email and task improvements

Snapshot current maintainer-preview application changes and regression fixtures for integration into lab. Excludes local runtime data, evaluation outputs and source backups. Focused Python regression selection: 140 passed; full suite not certified.

services/search/content.py

@@ -8,6 +8,7 @@ import re
 import logging
 from datetime import datetime, timedelta
 from typing import List
+from urllib.parse import urljoin, urlsplit, quote
 
 import httpx
 from bs4 import BeautifulSoup
@@ -147,6 +148,69 @@ def _extract_og_image(soup: BeautifulSoup) -> str:
     return ""
 
 
+def _linked_text(area, base_url: str) -> str:
+    """Preserve observed anchor destinations and block order without fetching links."""
+    area = copy.copy(area)
+    for anchor in area.find_all('a', href=True):
+        label = ' '.join(anchor.get_text(' ', strip=True).split())
+        href = str(anchor.get('href') or '').strip()
+        if not label or not href or href.startswith('#'):
+            continue
+        target = urljoin(base_url, href)
+        try:
+            parsed = urlsplit(target)
+            if parsed.scheme not in {'http', 'https'} or not parsed.hostname or parsed.username or parsed.password:
+                continue
+        except ValueError:
+            continue
+        label = re.sub(r'([\\\[\]])', r'\\\1', label)
+        target = quote(target, safe=":/?#[]@!$&'()*+,;=%~_-.")
+        anchor.replace_with(f'[{label}](<{target}>)')
+    for block in area.find_all(['p', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'article', 'br']):
+        block.insert_before('\n')
+        block.insert_after('\n')
+    return '\n'.join(' '.join(line.split()) for line in area.get_text(' ', strip=False).splitlines() if line.strip())
+
+
+def _page_entries(areas, base_url: str) -> list[dict]:
+    """Recognize repeated listing structures, retaining DOM order, not popularity."""
+    entries = []
+    seen = set()
+    for area in areas:
+        nodes = ([area] if area.name == 'article' else []) + area.find_all(['li', 'article', 'tr'])
+        for node in nodes:
+            anchor = None
+            if node.name == 'tr':
+                cells = node.find_all(['td', 'th'], recursive=False)
+                if cells and re.fullmatch(r'\d+[.)]?', cells[0].get_text(strip=True)):
+                    anchor = next((a for a in node.find_all('a', href=True)
+                                   if a.get_text(strip=True)), None)
+            elif node.name == 'article':
+                heading = node.find(['h1', 'h2', 'h3', 'h4'])
+                anchor = heading.find('a', href=True) if heading else None
+            elif node.parent and node.parent.name == 'ol':
+                anchor = node.find('a', href=True)
+            if not anchor:
+                continue
+            title = ' '.join(anchor.get_text(' ', strip=True).split())
+            href = str(anchor.get('href') or '').strip()
+            if not title or not href or href.startswith('#'):
+                continue
+            url = urljoin(base_url, href)
+            try:
+                parsed = urlsplit(url)
+                if parsed.scheme not in {'http', 'https'} or not parsed.hostname or parsed.username or parsed.password:
+                    continue
+            except ValueError:
+                continue
+            if (title, url) not in seen:
+                seen.add((title, url))
+                entries.append({'title': title, 'url': url})
+            if len(entries) == 100:
+                return entries
+    return entries if len(entries) >= 2 else []
+
+
 def _extract_lists(soup: BeautifulSoup) -> List[List[str]]:
     """Return a list of lists, each inner list representing a <ul>/<ol>."""
     all_lists = []
@@ -233,7 +297,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
     effective_cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     # The cap is part of the cache identity: a truncated soft-cap fetch must
     # not be served to a later full-budget request for the same URL.
-    cache_key = generate_cache_key(f"{url}#cap={effective_cap}#extract=semantic-v4")
+    cache_key = generate_cache_key(f"{url}#cap={effective_cap}#extract=semantic-links-v7")
     cache_file = CONTENT_CACHE_DIR / f"{cache_key}.cache"
 
     # Check cache
@@ -402,6 +466,12 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
     title_tag = soup.find("title")
     title_text = title_tag.get_text(strip=True) if title_tag else ""
     meta_info = _extract_meta(soup)
+    link_base = str(getattr(response, 'url', None) or url)
+    base_tag = soup.find('base', href=True)
+    if base_tag:
+        candidate_base = urljoin(link_base, str(base_tag['href']))
+        if candidate_base.startswith(('https://', 'http://')):
+            link_base = candidate_base
     og_image = _extract_og_image(soup)
     js_rendered = _detect_js_frameworks(soup)
     js_message = "Page appears to be rendered by a JavaScript framework; content may be incomplete." if js_rendered else ""
@@ -434,6 +504,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
     if content_areas:
         for area in content_areas:
             main_content += area.get_text(separator=" ", strip=True) + " "
+    linked_areas = content_areas
     main_content = re.sub(r"\s+", " ", main_content).strip()
 
     # If the heuristic finds only a tiny wrapper, fall back to body text with
@@ -451,6 +522,7 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
             body_text = re.sub(r"\s+", " ", body_copy.get_text(separator=" ", strip=True)).strip()
             if len(body_text) > len(main_content):
                 main_content = body_text
+                linked_areas = [body_copy]
 
     # HTTP 200 does not imply an article was retrieved. Classify only short
     # interstitials with both a challenge title and corroborating body text;
@@ -470,6 +542,8 @@ def fetch_webpage_content(url: str, timeout: int = 5, retry_attempt: int = 0,
         "url": url,
         "title": title_text,
         "content": main_content,
+        "linked_content": '\n'.join(_linked_text(area, link_base) for area in linked_areas),
+        "page_entries": _page_entries(linked_areas, link_base),
         "lists": _extract_lists(soup),
         "tables": _extract_tables(soup),
         "code_blocks": _extract_code_blocks(soup),

services/search/providers.py

@@ -172,11 +172,10 @@ _SOFTWARE_RELEASE_HINTS = (
     "package",
 )
 
-# Default general engines (google/duckduckgo/brave/startpage/wikipedia) are
-# routinely rate-limited / CAPTCHA-blocked on this instance and return nothing.
-# Pin engines that actually respond so non-news queries get results without any
-# third-party API fallback. Override via SEARXNG_GENERAL_ENGINES.
-_GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "bing,mojeek,presearch")
+# Verified with the pinned September SearXNG adapters. Bing can return unrelated
+# pages as successful results; do not prefer it over working general engines.
+# Deployments can override this via SEARXNG_GENERAL_ENGINES.
+_GENERAL_ENGINES = os.environ.get("SEARXNG_GENERAL_ENGINES", "google,brave,duckduckgo")
 
 
 def searxng_search_api(query: str, count: Optional[int] = None, categories: str = "general",