From 64b6a3d8260edc3c01e71fbc7e981a38f3842b98 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Fri, 11 Sep 2026 22:41:31 +0200 Subject: [PATCH] release v2.6.0 The cutover's first real lesson ships: tool_choice "required" was a workaround for a backend that ignored it, and became an unbreakable tool loop on a backend that obeys it every request (~80 s arithmetic turns, observed). The health check now learns which server answers behind OLLAMA_HOST (/props is llama-server's own surface) and only Ollama gets the advisory nudge. Probed unforced on llama-server: 3/3 tool calls via the --jinja template. Also carries BACKEND_SLOT_PINNING (default off) for the P3 pilot. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 10 ++++++ pyproject.toml | 2 +- src/anthropic/model_selector.py | 38 ++++++++++++++++++----- tests/anthropic/test_model_selector.py | 17 ++++++++++ tests/contracts/test_service_contracts.py | 10 +++--- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfe51c4..0623d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.6.0] - 2026-09-11 + ### Added - `BACKEND_SLOT_PINNING` (default off): pins each pipeline phase to a @@ -14,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 phase's stable prompt prefix stays in that slot's KV cache. No-op on Claude; ignored by Ollama. +### Fixed + +- Tool-calling no longer loops on llama-server backends: the health + check detects the server flavor, and `tool_choice: "required"` (an + advisory nudge on Ollama, an every-request mandate on llama-server) + is sent only to Ollama. On llama-server the model calls tools + unforced via its chat template. + ## [2.5.0] - 2026-09-11 ### Added diff --git a/pyproject.toml b/pyproject.toml index 8a19c10..347825b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "tatlock" -version = "2.5.0" +version = "2.6.0" description = "OpenAI-compatible API with Ollama backend" requires-python = ">=3.12" dependencies = [ diff --git a/src/anthropic/model_selector.py b/src/anthropic/model_selector.py index 8b1360a..a9d005f 100644 --- a/src/anthropic/model_selector.py +++ b/src/anthropic/model_selector.py @@ -28,6 +28,11 @@ logger = get_logger(__name__) # Cached health check results (set once at startup) _claude_available: bool | None = None _ollama_available: bool | None = None +# Which server answers behind OLLAMA_HOST: "llama-server" or "ollama". +# They disagree on tool_choice semantics (see get_tool_choice_settings), +# so the health check probes /props — served by llama-server only — and +# caches the answer here. +_local_flavor: str | None = None async def check_ollama_health() -> bool: @@ -40,7 +45,7 @@ async def check_ollama_health() -> bool: Returns: True if Ollama is reachable and OLLAMA_DEFAULT_MODEL is pulled. """ - global _ollama_available + global _ollama_available, _local_flavor host = str(config.OLLAMA_HOST).rstrip("/") model = config.OLLAMA_DEFAULT_MODEL @@ -53,12 +58,20 @@ async def check_ollama_health() -> bool: response.raise_for_status() names = [m.get("id", "") for m in response.json().get("data", [])] + # /props is llama-server's own surface; Ollama 404s it. + try: + props = await client.get(f"{host}/props") + _local_flavor = "llama-server" if props.status_code == 200 else "ollama" + except httpx.HTTPError: + _local_flavor = "ollama" + if model in names or f"{model}:latest" in names: _ollama_available = True logger.info( "ollama_health_check_passed", host=host, model=model, + flavor=_local_flavor, ) return True @@ -244,19 +257,30 @@ def get_model(prefer_cloud: bool | None = None) -> AnthropicModel | OpenAIChatMo def get_tool_choice_settings() -> ModelSettings: """ - Get model_settings for forcing tool calls on the first request. + Get model_settings for tool calling on the orchestration phase. - For Claude: PydanticAI handles tool_choice natively, so no extra_body needed. - For Ollama: Pass tool_choice="required" via extra_body to force tool calling. + Claude: PydanticAI handles tool_choice natively — no extra_body. + + Ollama: tool_choice="required" via extra_body. Advisory there (Ollama + ignores it), but it nudges gemma4 to actually call tools, which the + persona-suppression gotcha made necessary. + + llama-server: NO tool_choice. It enforces "required" on every request + in the run, so after a tool returns, the next generation is again + forced to call a tool — an unbreakable tool loop (~80 s turns, + observed at cutover). Its --jinja template renders tool definitions + the way gemma4 was trained, and the model calls tools reliably + unforced (probed 3/3). """ from pydantic_ai.settings import ModelSettings if resolve_backend() == "claude": # PydanticAI's Anthropic model handles tool_choice internally return ModelSettings() - else: - # Ollama needs explicit tool_choice via extra_body - return ModelSettings(extra_body={"tool_choice": "required"}) + if _local_flavor == "llama-server": + return ModelSettings() + # Ollama needs explicit tool_choice via extra_body + return ModelSettings(extra_body={"tool_choice": "required"}) def with_slot_pinning(settings: ModelSettings | None, slot: int) -> ModelSettings | None: diff --git a/tests/anthropic/test_model_selector.py b/tests/anthropic/test_model_selector.py index 9fc598e..ebcc60f 100644 --- a/tests/anthropic/test_model_selector.py +++ b/tests/anthropic/test_model_selector.py @@ -120,3 +120,20 @@ class TestWithSlotPinning: monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True) base = model_selector.get_tool_choice_settings() assert model_selector.with_slot_pinning(base, slot=1) is base + + +class TestLocalFlavorToolChoice: + def test_llama_server_flavor_sends_no_tool_choice(self, local_first, monkeypatch): + monkeypatch.setattr(model_selector, "_local_flavor", "llama-server") + settings = model_selector.get_tool_choice_settings() + assert not settings.get("extra_body") + + def test_ollama_flavor_keeps_advisory_required(self, local_first, monkeypatch): + monkeypatch.setattr(model_selector, "_local_flavor", "ollama") + settings = model_selector.get_tool_choice_settings() + assert settings["extra_body"] == {"tool_choice": "required"} + + def test_unknown_flavor_defaults_to_ollama_semantics(self, local_first, monkeypatch): + monkeypatch.setattr(model_selector, "_local_flavor", None) + settings = model_selector.get_tool_choice_settings() + assert settings["extra_body"] == {"tool_choice": "required"} diff --git a/tests/contracts/test_service_contracts.py b/tests/contracts/test_service_contracts.py index 614df06..0b6c72a 100644 --- a/tests/contracts/test_service_contracts.py +++ b/tests/contracts/test_service_contracts.py @@ -114,16 +114,18 @@ class TestLocalBackendContract: ) async def test_openai_compat_tool_calling(self): - # Mirrors the request PydanticAI's OpenAIChatModel sends for the - # orchestration phase, including the extra_body tool_choice. + # Mirrors the orchestration-phase request on llama-server: tools + # attached, NO tool_choice. The backend must call the tool unforced + # — "required" is deliberately absent because llama-server enforces + # it on every request in a run, which turns the tool loop + # unbreakable (observed at cutover: ~80 s turns). response = await _post_or_skip( f"{OLLAMA}/v1/chat/completions", - "ollama", + "local-backend", { "model": config.OLLAMA_DEFAULT_MODEL, "messages": [{"role": "user", "content": "What is 6 * 7? Use the calculator."}], "tools": [CALCULATOR_TOOL], - "tool_choice": "required", "stream": False, }, timeout=config.OLLAMA_TIMEOUT,