fix(research): typeset math in visual reports with the shared renderer (#6929)

src/report_math.py

@@ -0,0 +1,87 @@
+"""Preserve report TeX for markdown.js's existing deferred KaTeX renderer."""
+import re
+from html.parser import HTMLParser
+from xml.etree import ElementTree
+
+from markdown.blockprocessors import BlockProcessor
+from markdown.extensions import Extension
+from markdown.inlinepatterns import InlineProcessor
+from markdown.util import AtomicString
+
+
+# Keep delimiter and currency rules aligned with mathBlocks in markdown.js.
+_ESCAPED_MATH = r"\\\[([\s\S]*?)\\\]|\\\(([^\n]*?)\\\)"
+_DOLLAR_MATH = (
+    r"\$\$([\s\S]*?)\$\$|"
+    r"(?<![\$\d])\$(?!\$)(?=\S)([^\$\n]+?)(?<=\S)\$(?!\$|\d)"
+)
+
+
+def _placeholder(tex, display):
+    node = ElementTree.Element("span", {
+        "class": "ody-math-pending", "data-display": str(display).lower(),
+    })
+    node.text = AtomicString(tex)
+    return node
+
+
+class _HtmlContext(HTMLParser):
+    """Inline HTML is still raw when math runs, including tag attributes."""
+    def __init__(self):
+        super().__init__()
+        self.protected = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in {"code", "pre", "script", "style", "textarea"}:
+            self.protected.append(tag)
+
+    def handle_endtag(self, tag):
+        if tag in self.protected:
+            del self.protected[self.protected.index(tag):]
+
+
+class _InlineMath(InlineProcessor):
+    def handleMatch(self, match, data):
+        prefix = data[:match.start()]
+        if "<" in prefix:
+            context = _HtmlContext()
+            context.feed(prefix)
+            if context.protected or re.match(r"<(?:/?[A-Za-z]|[!?])", context.rawdata.lstrip()):
+                return None, None, None
+        # Backticks run before this processor. Preserve a backslash-escaped
+        # dollar too; math must run before Markdown consumes \( and \[.
+        if prefix.endswith("\\") and (len(prefix) - len(prefix.rstrip("\\"))) % 2:
+            return None, None, None
+        tex = next(group for group in match.groups() if group is not None)
+        display = match.group(1) is not None
+        return _placeholder(tex, display), match.start(), match.end()
+
+
+class _DisplayMath(BlockProcessor):
+    def test(self, parent, block):
+        return re.match(r"^ {0,3}(\$\$|\\\[)", block) is not None
+
+    def run(self, parent, blocks):
+        opening = re.match(r"^ {0,3}(\$\$|\\\[)", blocks[0])
+        closing = "$$" if opening.group(1) == "$$" else "\\]"
+        source = blocks[0][opening.end():]
+        consumed = 1
+        while closing not in source:
+            if consumed == len(blocks):
+                return False  # Leave incomplete input to ordinary Markdown.
+            source += "\n\n" + blocks[consumed]
+            consumed += 1
+        tex, _, tail = source.partition(closing)
+        del blocks[:consumed]
+        if tail.strip():
+            blocks.insert(0, tail)
+        parent.append(_placeholder(tex, True))
+
+
+class ReportMathExtension(Extension):
+    def extendMarkdown(self, md):
+        # Backticks (190) and indented code (80) retain precedence.
+        md.inlinePatterns.register(_InlineMath(_ESCAPED_MATH, md), "report_math_escaped", 185)
+        # Parse links before dollar math so URL destinations stay literal.
+        md.inlinePatterns.register(_InlineMath(_DOLLAR_MATH, md), "report_math_dollar", 155)
+        md.parser.blockprocessors.register(_DisplayMath(md.parser), "report_math", 79)

src/visual_report.py

@@ -23,6 +23,7 @@ from typing import Dict, List, Optional, Tuple
 from bs4 import BeautifulSoup
 
 from src.research_utils import strip_thinking
+from src.report_math import ReportMathExtension
 from urllib.parse import urlparse
 
 import markdown
@@ -47,6 +48,7 @@ for _t in ("td", "th"):
     _REPORT_ALLOWED_ATTRS.setdefault(_t, set()).add("align")
 _REPORT_ALLOWED_ATTRS.setdefault("a", set()).update({"href", "title", "target", "rel"})
 _REPORT_ALLOWED_ATTRS.setdefault("img", set()).update({"src", "alt", "title"})
+_REPORT_ALLOWED_ATTRS.setdefault("span", set()).add("data-display")
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -79,7 +81,7 @@ def _md_to_html(md_text: str) -> str:
     md_text = _autolink_urls(md_text)
     result = markdown.markdown(
         md_text,
-        extensions=["extra", "codehilite", "toc", "tables", "sane_lists", "md_in_html"],
+        extensions=["extra", "codehilite", "toc", "tables", "sane_lists", "md_in_html", ReportMathExtension()],
         extension_configs={
             "codehilite": {"css_class": "code", "guess_lang": False},
             "toc": {"marker": "", "toc_depth": "2-3"},
@@ -1021,6 +1023,8 @@ body[class^="standard-report-"] .report-section:nth-of-type(even) {{
 .content code {{ font-family: var(--font-mono); font-size: 0.86em; background: var(--bg-surface-alt); padding: 0.15em 0.4em; border-radius: 4px; }}
 .content pre {{ background: var(--bg-surface-alt); border: 1px solid var(--border); border-radius: var(--radius); padding: 1.25rem 1.5rem; overflow-x: auto; margin: 1.25rem 0; font-size: 0.86rem; line-height: 1.6; }}
 .content pre code {{ background: none; padding: 0; }}
+/* Match the chat renderer's handling of wide display equations. */
+.content .katex-display {{ overflow-x: auto; overflow-y: hidden; }}
 .content table {{ width: 100%; border-collapse: collapse; margin: 1.25rem 0; font-size: 0.9rem; border-radius: var(--radius); overflow: hidden; box-shadow: var(--shadow-sm); }}
 .content th {{ text-align: left; padding: 0.7rem 1rem; background: var(--accent-bg); font-weight: 600; border-bottom: 2px solid var(--border-strong); }}
 .content td {{ padding: 0.6rem 1rem; border-bottom: 1px solid var(--border); vertical-align: top; }}
@@ -1434,6 +1438,12 @@ body[class^="standard-report-"] .report-section:nth-of-type(even) {{
 
 <script>
 (function() {{
+  // Reuse the chat renderer and vendored assets only for reports with math.
+  if (document.querySelector('.ody-math-pending')) {{
+    import('/static/js/markdown.js').then(function(markdown) {{
+      return markdown.renderMath(document);
+    }}).catch(function(err) {{ console.warn('Report math render error:', err); }});
+  }}
   // ESC closes the report tab. window.close() works when the tab was
   // opened via window.open() (which is how the panel launches it). If the
   // browser blocks self-close (rare — e.g. report opened by direct URL),
@@ -2835,11 +2845,23 @@ def generate_visual_report(
     spare_images = section_pool[_consumed:]
 
     # Build TOC
+    math_labels = {}
+    if 'ody-math-pending' in report_html:
+        label_soup = BeautifulSoup(report_html, "html.parser")
+        for heading in label_soup.select('h2[id], h3[id]'):
+            if not heading.select_one('.ody-math-pending'):
+                continue
+            for child in list(heading.find_all(True)):
+                if 'ody-math-pending' in child.get('class', []):
+                    child['data-display'] = 'false'
+                else:
+                    child.unwrap()
+            math_labels[heading['id']] = heading.decode_contents()
     toc_lines = []
     for h in headings:
         depth_class = f"depth-{h['level']}"
         toc_lines.append(
-            f'<a href="#{h["slug"]}" class="{depth_class}">{html.escape(h["text"])}</a>'
+            f'<a href="#{h["slug"]}" class="{depth_class}">{math_labels.get(h["slug"], html.escape(h["text"]))}</a>'
         )
     toc_html = "\n      ".join(toc_lines) if toc_lines else ""
 
@@ -2848,8 +2870,11 @@ def generate_visual_report(
         options = []
         for h in headings:
             prefix = "\u2014 " if h["level"] == 3 else ""
+            text = h["text"]
+            if h["slug"] in math_labels:
+                text = BeautifulSoup(math_labels[h["slug"]], "html.parser").get_text()
             options.append(
-                f'<option value="{h["slug"]}">{prefix}{html.escape(h["text"])}</option>'
+                f'<option value="{h["slug"]}">{prefix}{html.escape(text)}</option>'
             )
         mobile_toc_html = (
             '<div class="mobile-section-nav">'

tests/test_visual_report_math.py

@@ -0,0 +1,76 @@
+"""Research reports preserve TeX for the shared, vendored math renderer."""
+from bs4 import BeautifulSoup
+import pytest
+
+from src.visual_report import _md_to_html, generate_visual_report
+
+
+@pytest.mark.parametrize("source,tex,display", [
+    (r"Value $x_i^2$.", "x_i^2", "false"),
+    (r"Value \(x_i^2\).", "x_i^2", "false"),
+    (r"Value $$\sum_i x_i$$.", r"\sum_i x_i", "true"),
+    (r"Value \[\sum_i x_i\].", r"\sum_i x_i", "true"),
+    ("$$a_i\n\nb_i$$", "a_i\n\nb_i", "true"),
+    ("\\[a_i\n\nb_i\\]", "a_i\n\nb_i", "true"),
+])
+def test_math_source_survives_markdown(source, tex, display):
+    soup = BeautifulSoup(_md_to_html(source), "html.parser")
+    pending = soup.select(".ody-math-pending")
+    assert len(pending) == 1
+    assert pending[0].get_text() == tex
+    assert pending[0]["data-display"] == display
+    assert pending[0].find("em") is None
+
+
+@pytest.mark.parametrize("source", [
+    "`$x_i$`", "```text\n$x_i$\n```", "~~~text\n$x_i$\n~~~",
+    "    $x_i$", "<code>$x_i$</code>", "<pre>$x_i$</pre>",
+    "Prices range from $5 to $10.", "An escaped \\$5 costs less than $10.",
+])
+def test_code_and_currency_are_not_math(source):
+    soup = BeautifulSoup(_md_to_html(source), "html.parser")
+    assert not soup.select(".ody-math-pending")
+    assert "$" in soup.get_text()
+
+
+def test_html_attributes_are_not_rewritten_as_math():
+    soup = BeautifulSoup(_md_to_html('<a title="$literal$">Value $x_i$</a>'), "html.parser")
+    assert soup.a["title"] == "$literal$"
+    assert soup.select_one("a .ody-math-pending").get_text() == "x_i"
+
+
+def test_link_destinations_stay_literal_while_link_labels_can_contain_math():
+    soup = BeautifulSoup(_md_to_html('[$x_i$](https://example.com/$literal$)'), "html.parser")
+    assert soup.a["href"] == "https://example.com/$literal$"
+    assert soup.select_one("a .ody-math-pending").get_text() == "x_i"
+
+
+def test_math_source_is_inert_and_report_sanitizer_stays_active():
+    soup = BeautifulSoup(_md_to_html(
+        r'$\text{<img src=x onerror="alert(1)">}$'
+        '\n\n<script>alert(2)</script><a href="javascript:alert(3)">link</a>'
+    ), "html.parser")
+    assert soup.select_one(".ody-math-pending").get_text() == r'\text{<img src=x onerror="alert(1)">}'
+    assert soup.find("img") is None
+    assert soup.find("script") is None
+    assert not soup.select('a[href^="javascript:"]')
+
+
+def test_heading_math_reaches_body_and_toc_without_changing_anchor_contract():
+    soup = BeautifulSoup(generate_visual_report(
+        "Formula report", "# Formula report\n\n## Formula $x_i$\n\nValue $x_i$.",
+        session_id="rp-math",
+    ), "html.parser")
+    link = soup.select_one(".toc-sidebar nav a")
+    heading = soup.find(id=link["href"][1:])
+    assert heading["id"] == "formula-xi"
+    for node in (link, heading):
+        assert node.select_one(".ody-math-pending").get_text() == "x_i"
+    assert soup.select_one("#mobile-section-select option").get_text() == "Formula x_i"
+
+
+def test_unclosed_display_math_stays_readable():
+    soup = BeautifulSoup(_md_to_html("$$unfinished\n\nA later paragraph."), "html.parser")
+    assert not soup.select(".ody-math-pending")
+    assert "unfinished" in soup.get_text()
+    assert "A later paragraph." in soup.get_text()

tests/test_visual_report_math_browser.py

@@ -0,0 +1,84 @@
+"""Exercise generated report HTML with the real vendored KaTeX in Chromium."""
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+
+import pytest
+
+from src.visual_report import generate_visual_report
+
+
+@pytest.mark.parametrize("width", [1440, 390])
+@pytest.mark.parametrize("mode", ["math", "plain", "blocked"])
+def test_report_math_in_browser(tmp_path, width, mode):
+    if not shutil.which("node"):
+        pytest.skip("node not installed")
+    markdown = (
+        "# Math report\n\n" + r"## Formula $x_i$" + "\n\n"
+        + r"Inline \(x_i^2\) and $a < b$." + "\n\n"
+        + r"\[\sum_{i=1}^{3} x_i\]" + "\n\n"
+        + r"Invalid: $\frac{1}$. Unsafe: $\href{javascript:alert(1)}{bad}$."
+        + "\n\n`$literal$` costs $5 to $10."
+        + "\n\n$$" + "+".join(["x_i"] * 30) + "$$"
+    ) if mode != "plain" else "## Findings\n\nNo math here."
+    document = tmp_path / "report.html"
+    document.write_text(generate_visual_report("Math report", markdown), encoding="utf-8")
+    script = r"""
+const fs = require('node:fs'), assert = require('node:assert/strict');
+let chromium;
+try { ({chromium} = require('playwright')); }
+catch (error) { if (error.code === 'MODULE_NOT_FOUND') process.exit(77); throw error; }
+(async () => {
+  const browser = await chromium.launch({headless:true});
+  try {
+    const page = await browser.newPage({viewport:{width:Number(process.argv[2]),height:900}});
+    const origin = process.env.ODYSSEUS_TEST_STATIC_ORIGIN;
+    const errors = [], requests = [], failures = [];
+    page.on('pageerror', e => errors.push(e.message));
+    page.on('request', r => requests.push(r.url()));
+    page.on('requestfailed', r => failures.push(r.url()));
+    page.on('dialog', async d => { errors.push('Unexpected dialog'); await d.dismiss(); });
+    const mode = process.argv[3];
+    if (mode === 'blocked') await page.route('**/static/lib/katex/katex.min.js', r => r.abort());
+    await page.route(origin + '/report-math-test', r => r.fulfill({
+      contentType:'text/html', body:fs.readFileSync(process.argv[1], 'utf8')
+    }));
+    await page.goto(origin + '/report-math-test');
+    if (mode === 'math') {
+      await page.locator('.content .katex-display').first().waitFor({state:'attached'});
+      await page.waitForFunction(() => !document.querySelector('.ody-math-pending'));
+      assert.equal(await page.locator('.toc-sidebar .katex').count(), 1);
+      assert.equal(await page.locator('.content .katex-error').count(), 1);
+      assert.equal(await page.locator('.content code').innerText(), '$literal$');
+      assert((await page.locator('.content').innerText()).includes('$5 to $10'));
+      assert.equal(await page.locator('a[href^="javascript:"]').count(), 0);
+      assert.equal(requests.filter(u => u.endsWith('/static/lib/katex/katex.min.js')).length, 1);
+      assert.equal(requests.filter(u => u.endsWith('/static/lib/katex/katex.min.css')).length, 1);
+      assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false);
+      assert.deepEqual(failures, []);
+    } else if (mode === 'blocked') {
+      await page.waitForFunction(() => performance.getEntriesByType('resource').some(e => e.name.includes('katex.min.js')));
+      assert((await page.locator('.content .ody-math-pending').first().innerText()).includes('x_i'));
+      assert.equal(await page.locator('.katex').count(), 0);
+    } else {
+      await page.waitForLoadState('networkidle');
+      assert.equal(requests.some(u => u.includes('/static/lib/katex/')), false);
+      assert.equal(requests.some(u => u.includes('/static/js/markdown.js')), false);
+    }
+    assert.deepEqual(errors, []);
+    assert(requests.every(u => u.startsWith(origin + '/')), JSON.stringify(requests));
+    console.log(JSON.stringify({mode, width:process.argv[2], requests:requests.length}));
+  } finally { await browser.close(); }
+})().catch(e => { console.error(e); process.exitCode=1; });
+"""
+    result = subprocess.run(
+        ["node", "-e", script, str(document), str(width), mode],
+        cwd=Path(__file__).resolve().parents[1], env=os.environ.copy(),
+        capture_output=True, text=True, timeout=60,
+    )
+    if result.returncode == 77:
+        pytest.skip("playwright not installed")
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert json.loads(result.stdout)["mode"] == mode