fix: Redis bool storage, tool tracking matching, e2e fixture scope
Build and Push / build (release) Successful in 53s
Build and Push / build (release) Successful in 53s
- Convert booleans to strings for Redis hset (Redis doesn't accept bool) - Extract capability from delegate_to_X tool names for tracking - Use loop_scope="module" for pytest-asyncio module-scoped fixtures - Add note about using venv for tests in AGENTS.md 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
Tests for tool call tracking.
|
||||
|
||||
Tests capability extraction and recommendation matching.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
|
||||
|
||||
class TestToolCallTracker:
|
||||
"""Test ToolCallTracker functionality."""
|
||||
|
||||
def test_extract_capability_delegation_tool(self):
|
||||
"""Test extracting capability from delegation tool name."""
|
||||
tracker = ToolCallTracker(recommended_capabilities=["librarian"])
|
||||
|
||||
assert tracker._extract_capability("delegate_to_librarian") == "librarian"
|
||||
assert tracker._extract_capability("delegate_to_biographer") == "biographer"
|
||||
assert tracker._extract_capability("delegate_to_housekeeper") == "housekeeper"
|
||||
|
||||
def test_extract_capability_non_delegation_tool(self):
|
||||
"""Test that non-delegation tools return unchanged."""
|
||||
tracker = ToolCallTracker(recommended_capabilities=[])
|
||||
|
||||
assert tracker._extract_capability("calculate") == "calculate"
|
||||
assert tracker._extract_capability("search_web") == "search_web"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_call_recognizes_delegation_as_recommended(self):
|
||||
"""Test that delegate_to_X is recognized when X is recommended."""
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
# Should NOT log warning since librarian was recommended
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_call_detects_not_recommended(self):
|
||||
"""Test that unrecommended tools are flagged."""
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=["librarian"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is False
|
||||
|
||||
def test_get_summary_with_delegation_tools(self):
|
||||
"""Test summary correctly maps delegation tools to capabilities."""
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
tracker.actual_calls = {
|
||||
"delegate_to_librarian": [1.0, 2.0],
|
||||
"delegate_to_housekeeper": [0.5], # Not recommended
|
||||
}
|
||||
|
||||
summary = tracker.get_summary()
|
||||
|
||||
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
||||
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|
||||
assert summary["accuracy"]["not_recommended_but_used"] == 1 # housekeeper
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_with_delegation_tools(self):
|
||||
"""Test finalize correctly identifies unused recommendations."""
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
tracker.actual_calls = {
|
||||
"delegate_to_librarian": [1.0],
|
||||
}
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
await tracker.finalize()
|
||||
|
||||
# Should record benchmark for unused biographer
|
||||
assert mock_store.return_value.record.called
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.tool_name == "biographer"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is False
|
||||
Reference in New Issue
Block a user