Squash Odysseus development history

services/research/research_handler.py

@@ -14,9 +14,12 @@ import time
 from pathlib import Path
 from typing import Optional, Dict
 
+from src.research_utils import is_low_quality
+from src.constants import DEEP_RESEARCH_DIR
+
 logger = logging.getLogger(__name__)
 
-RESEARCH_DATA_DIR = Path("data/deep_research")
+RESEARCH_DATA_DIR = Path(DEEP_RESEARCH_DIR)
 
 
 class ResearchHandler:
@@ -114,7 +117,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return {
                     "status": data.get("status", "done"),
                     "progress": {},
@@ -151,7 +154,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return data.get("result")
             except Exception:
                 pass
@@ -171,7 +174,7 @@ class ResearchHandler:
         path = RESEARCH_DATA_DIR / f"{session_id}.json"
         if path.exists():
             try:
-                data = json.loads(path.read_text())
+                data = json.loads(path.read_text(encoding="utf-8"))
                 return data.get("sources")
             except Exception:
                 pass
@@ -179,13 +182,16 @@ class ResearchHandler:
 
     @staticmethod
     def _extract_sources(findings: list) -> list:
-        """Extract deduplicated [{url, title}] from findings."""
+        """Extract deduplicated [{url, title}] from findings, filtering low-quality ones."""
         seen = set()
         sources = []
         for f in findings:
+            if not isinstance(f, dict):
+                continue
             url = f.get("url", "")
             title = f.get("title", "") or url
-            if url and url not in seen:
+            summary = f.get("summary", "") or f.get("evidence", "")
+            if url and url not in seen and not is_low_quality(summary):
                 seen.add(url)
                 sources.append({"url": url, "title": title})
         return sources
@@ -219,7 +225,7 @@ class ResearchHandler:
                 "started_at": entry["started_at"],
                 "completed_at": time.time(),
             }
-            path.write_text(json.dumps(data))
+            path.write_text(json.dumps(data), encoding="utf-8")
             logger.info(f"Research result saved to {path}")
         except Exception as e:
             logger.error(f"Failed to save research result: {e}")
@@ -281,6 +287,7 @@ class ResearchHandler:
                 query, report, stats, elapsed,
                 findings=researcher.findings,
                 evolving_report=researcher.evolving_report,
+                analyzed_urls=getattr(researcher, "analyzed_urls", None),
             )
 
         except Exception as e:
@@ -327,7 +334,8 @@ class ResearchHandler:
 
     def _format_research_report(
         self, query: str, full_report: str, stats: dict, elapsed: float,
-        findings: list = None, evolving_report: str = None,
+        findings: Optional[list] = None, evolving_report: Optional[str] = None,
+        analyzed_urls: Optional[list] = None,
     ) -> str:
         """Format research report with sources list and expandable raw findings."""
         summary_lines = [
@@ -338,19 +346,34 @@ class ResearchHandler:
         ]
         summary_text = " | ".join(summary_lines)
 
-        # Build sources list with clickable links
+        # Build sources list with clickable links. Keep the curated Sources
+        # section filtered for citation quality, but also list every unique URL
+        # the research run inspected so the "URLs Analyzed" count is auditable.
         sources_section = ""
-        if findings:
+        analyzed_urls_section = ""
+        url_items = analyzed_urls if analyzed_urls is not None else findings
+        if findings or url_items:
             seen_urls = set()
             source_lines = []
-            for f in findings:
+            analyzed_seen = set()
+            analyzed_lines = []
+            for f in findings or []:
                 url = f.get("url", "")
                 title = f.get("title", "") or url
-                if url and url not in seen_urls:
+                summary = f.get("summary", "") or f.get("evidence", "")
+                if url and url not in seen_urls and not is_low_quality(summary):
                     seen_urls.add(url)
                     source_lines.append(f"- [{title}]({url})")
+            for item in url_items or []:
+                url = item.get("url", "")
+                title = item.get("title", "") or url
+                if url and url not in analyzed_seen:
+                    analyzed_seen.add(url)
+                    analyzed_lines.append(f"{len(analyzed_lines) + 1}. [{title}]({url})")
             if source_lines:
                 sources_section = "\n### Sources\n\n" + "\n".join(source_lines) + "\n"
+            if analyzed_lines:
+                analyzed_urls_section = "\n### Analyzed URLs\n\n" + "\n".join(analyzed_lines) + "\n"
 
         # Build raw findings section (individual extractions per source)
         raw_findings_section = ""
@@ -386,6 +409,7 @@ class ResearchHandler:
 {full_report}
 
 {sources_section}
+{analyzed_urls_section}
 {collected_section}
 ---
 

services/research/service.py

@@ -1,11 +1,16 @@
 # services/research/service.py
 """Research service — deep research with LLM-in-the-loop."""
 
+import re
 from dataclasses import dataclass, field
 from typing import List, Optional, Callable
 
 from .research_handler import ResearchHandler
 
+# Markdown source links emitted by ResearchHandler._format_research_report,
+# e.g. "- [Some Title](https://example.com/page)".
+_SOURCE_LINK_RE = re.compile(r"^\s*-\s*\[(?P<title>[^\]]*)\]\((?P<url>[^)]+)\)\s*$")
+
 
 @dataclass
 class ResearchSource:
@@ -75,26 +80,71 @@ class ResearchService:
 
         duration = time.time() - start
 
-        # Parse result into structured format
-        sources = [
-            ResearchSource(
-                url=s.get("url", ""),
-                title=s.get("title", ""),
-                snippet=s.get("snippet", ""),
-                relevance=s.get("relevance", 0.0),
+        # call_research_service returns a formatted markdown report string
+        # (see ResearchHandler.call_research_service -> _format_research_report),
+        # not a dict. Treat it as such; tolerate an unexpected dict/None defensively.
+        if isinstance(result, dict):
+            sources = [
+                ResearchSource(
+                    url=s.get("url", ""),
+                    title=s.get("title", ""),
+                    snippet=s.get("snippet", ""),
+                    relevance=s.get("relevance", 0.0),
+                )
+                for s in result.get("sources", [])
+                if isinstance(s, dict)
+            ]
+            return ResearchResult(
+                query=topic,
+                summary=result.get("summary", result.get("answer", "")),
+                sources=sources,
+                sections=result.get("sections", []),
+                tokens_used=result.get("tokens_used", 0),
+                duration_seconds=duration,
             )
-            for s in result.get("sources", [])
-        ]
 
+        report = result if isinstance(result, str) else ""
         return ResearchResult(
             query=topic,
-            summary=result.get("summary", result.get("answer", "")),
-            sources=sources,
-            sections=result.get("sections", []),
-            tokens_used=result.get("tokens_used", 0),
+            summary=report,
+            sources=self._parse_sources(report),
             duration_seconds=duration,
         )
 
+    @staticmethod
+    def _parse_sources(report: str) -> List[ResearchSource]:
+        """Extract sources from the markdown ### Sources section of a report.
+
+        ResearchHandler emits one ``- [title](url)`` link per deduplicated
+        finding under a ``### Sources`` heading. Parse only that section so
+        inline links elsewhere in the body are not mistaken for sources.
+        """
+        if not report:
+            return []
+        sources: List[ResearchSource] = []
+        seen = set()
+        in_sources = False
+        for line in report.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("###") or stripped.startswith("##"):
+                in_sources = stripped.lower().lstrip("#").strip() == "sources"
+                continue
+            if not in_sources:
+                continue
+            match = _SOURCE_LINK_RE.match(line)
+            if not match:
+                continue
+            url = match.group("url").strip()
+            if not url or url in seen:
+                continue
+            seen.add(url)
+            sources.append(
+                # snippet is required on ResearchSource; markdown source links
+                # carry no snippet, so default to empty (matches the dict path).
+                ResearchSource(url=url, title=match.group("title").strip(), snippet="")
+            )
+        return sources
+
     def start_background(
         self,
         session_id: str,