jpmschweitzerandClaude
78066fab1b
Mechanical only, and separated from the judgment calls that follow so the reviewable changes are not buried in a 98-file whitespace diff. 227 automatic fixes: 60 blank lines carrying whitespace, 60 unsorted import blocks, 34 Optional[X] to X | None, 28 unused imports, 16 deprecated typing imports, 12 datetime.timezone.utc to datetime.UTC, and assorted smaller modernisations. Then `ruff format` over src and tests: 98 files reformatted, 35 already conforming. No file among the unused-import findings defines __all__ or is an __init__.py, so nothing here removes a re-export. `make test`: 658 passed, unchanged from HEAD. Two things observed while verifying, neither addressed here: `pytest tests/` cannot collect — tests/e2e/test_orchestration_e2e.py uses an `e2e` marker that is not registered, and the config is strict about markers. This fails identically at HEAD, so it predates this change; `make test` passes because it ignores tests/e2e, tests/integration and tests/contracts. test_tatlock_tool_call_logging_calculator is flaky. It failed once in a full run with these changes and passed on the next, passes in isolation with them, and fails in isolation at HEAD. It is order- or timing-dependent, not a regression from this commit — established by running the full suite both ways rather than by reasoning about which change could have caused it. Co-Authored-By: Claude <noreply@anthropic.com>
81 lines
3.2 KiB
Python
81 lines
3.2 KiB
Python
"""
|
|
Tests for tool call tracking.
|
|
|
|
Tests capability extraction and recommendation matching.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from src.core.tool_tracking import ToolCallTracker
|
|
|
|
|
|
class TestToolCallTracker:
|
|
"""Test ToolCallTracker functionality."""
|
|
|
|
def test_extract_capability_delegation_tool(self):
|
|
"""Test extracting capability from delegation tool name."""
|
|
tracker = ToolCallTracker(recommended_capabilities=["librarian"])
|
|
|
|
assert tracker._extract_capability("delegate_to_librarian") == "librarian"
|
|
assert tracker._extract_capability("delegate_to_biographer") == "biographer"
|
|
assert tracker._extract_capability("delegate_to_housekeeper") == "housekeeper"
|
|
|
|
def test_extract_capability_non_delegation_tool(self):
|
|
"""Test that non-delegation tools return unchanged."""
|
|
tracker = ToolCallTracker(recommended_capabilities=[])
|
|
|
|
assert tracker._extract_capability("calculate") == "calculate"
|
|
assert tracker._extract_capability("search_web") == "search_web"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_track_call_recognizes_delegation_as_recommended(self):
|
|
"""Test that delegate_to_X is recognized when X is recommended."""
|
|
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
|
|
|
|
await tracker.track_call("delegate_to_librarian", 1.0)
|
|
|
|
# Should record the call
|
|
assert "delegate_to_librarian" in tracker.actual_calls
|
|
assert tracker.actual_calls["delegate_to_librarian"] == [1.0]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_track_call_detects_not_recommended(self):
|
|
"""Test that unrecommended tools are flagged."""
|
|
tracker = ToolCallTracker(recommended_capabilities=["librarian"])
|
|
|
|
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
|
|
|
# Should record the call even though not recommended
|
|
assert "delegate_to_housekeeper" in tracker.actual_calls
|
|
summary = tracker.get_summary()
|
|
assert summary["accuracy"]["not_recommended_but_used"] == 1
|
|
|
|
def test_get_summary_with_delegation_tools(self):
|
|
"""Test summary correctly maps delegation tools to capabilities."""
|
|
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
|
|
tracker.actual_calls = {
|
|
"delegate_to_librarian": [1.0, 2.0],
|
|
"delegate_to_housekeeper": [0.5], # Not recommended
|
|
}
|
|
|
|
summary = tracker.get_summary()
|
|
|
|
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
|
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|
|
assert summary["accuracy"]["not_recommended_but_used"] == 1 # housekeeper
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_with_delegation_tools(self):
|
|
"""Test finalize correctly identifies unused recommendations."""
|
|
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
|
|
tracker.actual_calls = {
|
|
"delegate_to_librarian": [1.0],
|
|
}
|
|
|
|
await tracker.finalize()
|
|
|
|
# Summary should show biographer as recommended but unused
|
|
summary = tracker.get_summary()
|
|
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
|
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|