fix(llm): support scientific and decimal literals in degenerate guard

Preserve complete scientific notation and signed floating-point literals across
arbitrary streaming chunk boundaries in _DegenerateStreamGuard. Buffer pending
tokens until delimiter characters arrive to avoid splitting numbers into
incomplete or arbitrary alphanumeric tokens, while maintaining the 2048-number
runaway backstop and detection of repeated phrases and word loops.

src/llm_core.py

@@ -422,7 +422,18 @@ def _stream_delta_event(text: str, *, thinking: bool = False) -> str:
     return f"data: {json.dumps(payload)}\n\n"
 
 
-_DEGENERATE_WORD_RE = re.compile(r"[A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff]+")
+_WORD_CHARS = r"A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff"
+_DELIM_RE = re.compile(rf"[^{_WORD_CHARS}+\-.]")
+_NUMERIC_PATTERN = rf"(?<![{_WORD_CHARS}])[+-]?(?:[0-9]+\.[0-9]+|\.[0-9]+|[0-9]+)(?:[eE][+-]?[0-9]+)?(?![{_WORD_CHARS}])"
+_NUMERIC_FULLMATCH_RE = re.compile(rf"^[+-]?(?:[0-9]+\.[0-9]+|\.[0-9]+|[0-9]+)(?:[eE][+-]?[0-9]+)?$")
+_DEGENERATE_WORD_RE = re.compile(rf"[{_WORD_CHARS}]+")
+_TOKEN_RE = re.compile(rf"(?P<NUM>{_NUMERIC_PATTERN})|(?P<WORD>[{_WORD_CHARS}]+)")
+
+
+def _is_numeric_token(token: str) -> bool:
+    if token.isdigit():
+        return True
+    return _NUMERIC_FULLMATCH_RE.fullmatch(token) is not None
 
 
 class _DegenerateStreamGuard:
@@ -444,24 +455,38 @@ class _DegenerateStreamGuard:
         self.recent_tokens: List[str] = []
         self.periodic_tokens: List[str] = []
         self.total_chars = 0
+        self._buffer = ""
 
     def check(self, text: str) -> Optional[str]:
         if not text:
             return None
         self.total_chars += len(text)
-        # Keep one-character tokens. Structured output often varies through
+        self._buffer += text
-        # numeric selectors or coordinates; dropping those tokens can make
+
-        # valid CSS/SVG rows look like an exact repeated phrase.
+        m_delims = list(_DELIM_RE.finditer(self._buffer))
-        tokens = [t.lower() for t in _DEGENERATE_WORD_RE.findall(text)]
+        if m_delims:
+            split_idx = m_delims[-1].end()
+            head = self._buffer[:split_idx]
+            self._buffer = self._buffer[split_idx:]
+        elif len(self._buffer) >= 256:
+            head = self._buffer
+            self._buffer = ""
+        else:
+            return None
+
+        tokens = []
+        for m in _TOKEN_RE.finditer(head):
+            tokens.append((m.group(0).lower(), m.lastgroup == "NUM"))
         if not tokens:
             return None
-        for token in tokens:
+
+        for token, is_num in tokens:
             if token == self.last_token:
                 self.same_run += 1
             else:
                 self.last_token = token
                 self.same_run = 1
-            self.numeric_run = self.numeric_run + 1 if token.isdigit() else 0
+            self.numeric_run = self.numeric_run + 1 if is_num else 0
             self.recent_tokens.append(token)
             self.periodic_tokens.append(token)
         if len(self.recent_tokens) > 96:
@@ -470,19 +495,19 @@ class _DegenerateStreamGuard:
             self.periodic_tokens = self.periodic_tokens[-2048:]
 
         reason = None
-        if self.same_run >= 28 and self.total_chars >= 100 and not self.last_token.isdigit():
+        if self.same_run >= 28 and self.total_chars >= 100 and not _is_numeric_token(self.last_token):
             reason = f"repeated '{self.last_token}' {self.same_run} times"
         elif len(self.recent_tokens) >= 72:
             top = max(set(self.recent_tokens), key=self.recent_tokens.count)
             count = self.recent_tokens.count(top)
-            if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78 and not top.isdigit():
+            if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78 and not _is_numeric_token(top):
                 reason = f"repeated '{top}' {count}/{len(self.recent_tokens)} recent tokens"
         if (
             not reason
             and len(self.recent_tokens) >= 48
             # A window that is mostly numbers is matrix/table data; its
             # periodicity is real. Mixed loops still hit the exact-block check.
-            and sum(t.isdigit() for t in self.recent_tokens) * 2 < len(self.recent_tokens)
+            and sum(_is_numeric_token(t) for t in self.recent_tokens) * 2 < len(self.recent_tokens)
         ):
             # Detect a periodic suffix instead of counting repeated n-grams.
             # Reused property names are normal in CSS/SVG/JSON; a collapsed
@@ -518,9 +543,9 @@ class _DegenerateStreamGuard:
                 if history[previous:previous + anchor_size] != anchor:
                     continue
                 period = current_anchor - previous
-                if period < 24 or len(history) < period * 2:
+                if period < 24 or len(history) < period * 2 or self.numeric_run >= period:
                     continue
-                if history[-2 * period:-period] == history[-period:] and not all(t.isdigit() for t in history[-period:]):
+                if history[-2 * period:-period] == history[-period:] and not all(_is_numeric_token(t) for t in history[-period:]):
                     reason = f"repeated an exact {period}-token block twice"
                     break
 

tests/test_llm_core_fallback.py

@@ -1093,39 +1093,117 @@ def test_degenerate_stream_guard_allows_long_rows_with_changing_values():
 
 def _latex_matrix(cell):
     rows = (" & ".join(cell(i, j) for j in range(8)) for i in range(8))
-    return "$$\\begin{pmatrix}\n" + " \\\\\n".join(rows) + "\n\\end{pmatrix}$$"
+    return "$$\\begin{pmatrix}\n" + " \\\\\n".join(rows) + "\n\\end{pmatrix}$$\n"
 
 
 @pytest.mark.parametrize("text", [
     _latex_matrix(lambda i, j: "0"),
     _latex_matrix(lambda i, j: "1" if i == j else "0"),
-    "buf = [" + ", ".join("0" for _ in range(64)) + "]",
+    _latex_matrix(lambda i, j: "2e10"),
+    _latex_matrix(lambda i, j: "1e-5"),
+    _latex_matrix(lambda i, j: "1.25e-3"),
+    _latex_matrix(lambda i, j: "-0.125"),
+    _latex_matrix(lambda i, j: "0.0"),
+    _latex_matrix(lambda i, j: "3.14"),
+    "buf = [" + ", ".join("0" for _ in range(64)) + "]\n",
+    "buf = [" + ", ".join("1.25e-3" for _ in range(64)) + "]\n",
+    "buf = [" + ", ".join("-0.125" for _ in range(64)) + "]\n",
+    "t,v1,v2,v3\n" + "\n".join(f"{i},1.25e-3,-0.125,2e10" for i in range(25)) + "\n",
+    "\n".join(f"{i}. Solve: $\\frac{{{i}}}{{{i+1}}}x + \\frac{{1}}{{{i+2}}} = \\frac{{{i+3}}}{{{i+4}}}$" for i in range(1, 20)) + "\n",
+    "<svg>" + "".join(f'<rect x="{i*10}" y="{j*10}" width="10" height="10" fill="#abc"/>' for i in range(4) for j in range(4)) + "</svg>\n",
+    "{" + ", ".join(f'"key_{i}": "translation_{i}"' for i in range(40)) + "}\n",
 ])
-@pytest.mark.parametrize("chunk", [1, 6, 64])
+@pytest.mark.parametrize("chunk", [1, 3, 6, 8, 16, 32, 64, 128])
 def test_degenerate_stream_guard_allows_numeric_data(text, chunk):
-    """#5560: zero/identity matrices and padded buffers repeat numbers by
+    """#5560: legitimate matrices, scientific numbers, tables, and structured data
-    design; they are not token collapse."""
+    repeat values or formatting by design; they are not token collapse."""
     guard = llm_core._DegenerateStreamGuard("math-model")
 
     assert all(guard.check(text[i:i + chunk]) is None for i in range(0, len(text), chunk))
 
 
-def test_degenerate_stream_guard_still_stops_an_endless_number_run():
+@pytest.mark.parametrize("chunk", [1, 6, 64, 3000])
+def test_degenerate_stream_guard_allows_numeric_data_full_and_char_stream(chunk):
+    text = _latex_matrix(lambda i, j: "1.25e-3")
+    guard = llm_core._DegenerateStreamGuard("math-model")
+
+    assert all(guard.check(text[i:i + chunk]) is None for i in range(0, len(text), chunk))
+
+
+@pytest.mark.parametrize("loop_text", [
+    "0 " * 3000,
+    "1.25e-3 " * 3000,
+    "1 2 " * 1500,
+])
+@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 3000])
+def test_degenerate_stream_guard_still_stops_an_endless_number_run(loop_text, chunk):
     guard = llm_core._DegenerateStreamGuard("looping-model")
 
-    chunk = guard.check("0 " * 3000)
+    stopped = False
-
+    for i in range(0, len(loop_text), chunk):
-    assert chunk is not None
+        res = guard.check(loop_text[i:i + chunk])
-    assert "numbers in a row" in chunk
+        if res is not None:
+            assert "numbers in a row" in res
+            stopped = True
+            break
+    assert stopped
 
 
-def test_degenerate_stream_guard_still_stops_a_mixed_word_number_loop():
+@pytest.mark.parametrize("loop_text", [
+    "row 0 0 0 0 " * 300,
+    "Step 1 2 3 " * 200,
+])
+@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 1500])
+def test_degenerate_stream_guard_still_stops_a_mixed_word_number_loop(loop_text, chunk):
     guard = llm_core._DegenerateStreamGuard("looping-model")
 
-    chunk = guard.check("row 0 0 0 0 " * 300)
+    stopped = False
+    for i in range(0, len(loop_text), chunk):
+        res = guard.check(loop_text[i:i + chunk])
+        if res is not None:
+            assert "repeated" in res
+            stopped = True
+            break
+    assert stopped
 
-    assert chunk is not None
+
-    assert "repeated" in chunk
+@pytest.mark.parametrize("loop_text", [
+    "Summer " * 80,
+    "also be a software developer mode " * 30,
+])
+@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 500])
+def test_degenerate_stream_guard_still_stops_repeated_words_and_phrases(loop_text, chunk):
+    guard = llm_core._DegenerateStreamGuard("looping-model")
+
+    stopped = False
+    for i in range(0, len(loop_text), chunk):
+        res = guard.check(loop_text[i:i + chunk])
+        if res is not None:
+            assert "repeated" in res
+            stopped = True
+            break
+    assert stopped
+
+
+@pytest.mark.parametrize("bad_token", [
+    "1face ",
+    "1e ",
+    "25e ",
+    "0x12 ",
+])
+@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 250])
+def test_degenerate_stream_guard_stops_alphanumeric_repetition_resembling_numbers(bad_token, chunk):
+    guard = llm_core._DegenerateStreamGuard("looping-model")
+    text = bad_token * 50
+
+    stopped = False
+    for i in range(0, len(text), chunk):
+        res = guard.check(text[i:i + chunk])
+        if res is not None:
+            assert "repeated" in res
+            stopped = True
+            break
+    assert stopped
 
 
 def test_terminal_stream_retries_degenerate_generation_before_emitting(monkeypatch):