jpmschweitzer/tatlock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

fix: repair broken tests and ensure Claude backend is used in integration tests

Browse Source 
- Remove references to unimplemented get_benchmark_store from steward and
  tool tracking tests
- Fix steward test fixture calling async initialize_application synchronously
  by using sync register_household_members instead
- Rewrite tool tracking tests to assert actual logging behavior
- Change unit test fixture model from Tatlock to lorem-tester so unit tests
  don't require external services
- Add session-scoped _initialize_app fixture to run Claude health check,
  ensuring integration tests use Claude instead of falling back to Ollama
- Increase integration test timeouts from 30s to 120s to match OLLAMA_TIMEOUT
- Add Steward reasoning as ReasoningOutputItem in create_response_with_steward
  so <think> tags appear in chat completion responses
- Add test_tatlock_ollama_fallback to verify Ollama fallback path works

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jpmschweitzer
2026-02-05 20:15:59 +01:00
co-authored by Claude Opus 4.6
parent 8092740fa4
commit 334d313d17
5 changed files with 139 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/core/test_tool_tracking.py
+14 -28
View File
@@ -3,8 +3,6 @@ Tests for tool call tracking.
Tests capability extraction and recommendation matching.
"""
from unittest.mock import AsyncMock, patch
import pytest
from src.core.tool_tracking import ToolCallTracker
@@ -35,15 +33,11 @@ class TestToolCallTracker:
recommended_capabilities=["librarian", "biographer"]
)
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
mock_store.return_value.record = AsyncMock()
await tracker.track_call("delegate_to_librarian", 1.0)
await tracker.track_call("delegate_to_librarian", 1.0)
# Should NOT log warning since librarian was recommended
call_args = mock_store.return_value.record.call_args
benchmark = call_args[0][0]
assert benchmark.was_recommended is True
# Should record the call
assert "delegate_to_librarian" in tracker.actual_calls
assert tracker.actual_calls["delegate_to_librarian"] == [1.0]
@pytest.mark.asyncio
async def test_track_call_detects_not_recommended(self):
@@ -52,14 +46,12 @@ class TestToolCallTracker:
recommended_capabilities=["librarian"]
)
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
mock_store.return_value.record = AsyncMock()
await tracker.track_call("delegate_to_housekeeper", 1.0)
await tracker.track_call("delegate_to_housekeeper", 1.0)
call_args = mock_store.return_value.record.call_args
benchmark = call_args[0][0]
assert benchmark.was_recommended is False
# Should record the call even though not recommended
assert "delegate_to_housekeeper" in tracker.actual_calls
summary = tracker.get_summary()
assert summary["accuracy"]["not_recommended_but_used"] == 1
def test_get_summary_with_delegation_tools(self):
"""Test summary correctly maps delegation tools to capabilities."""
@@ -87,15 +79,9 @@ class TestToolCallTracker:
"delegate_to_librarian": [1.0],
}
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
mock_store.return_value.record = AsyncMock()
await tracker.finalize()
await tracker.finalize()
# Should record benchmark for unused biographer
assert mock_store.return_value.record.called
call_args = mock_store.return_value.record.call_args
benchmark = call_args[0][0]
assert benchmark.tool_name == "biographer"
assert benchmark.was_recommended is True
assert benchmark.was_actually_used is False
# Summary should show biographer as recommended but unused
summary = tracker.get_summary()
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer