jpmschweitzerandClaude Sonnet 4.5
6eed5f4d13
Add comprehensive two-tier architecture where Steward analyzes requests and Tatlock executes with scoped tools. Includes full infrastructure for request preprocessing, tool tracking, benchmarking, and streaming. **Added:** - Steward agent for request analysis and capability recommendation - Household Registry for centralized capability management - Request preprocessing pipeline (Steward → Tatlock flow) - Tool usage tracking and benchmarking system - Streaming transparency (Steward reasoning visible in streams) - Structured logging with operation timing - Redis benchmark storage with 30-day expiry - Benchmark analysis CLI tools **Infrastructure:** - src/agents/steward/ - Steward agent implementation - src/agents/tatlock_core/ - Tatlock capability domain - src/core/preprocessing.py - Request preprocessing pipeline - src/core/tool_tracking.py - Tool call tracking - src/core/benchmarks.py - Benchmark recording system - src/core/household_registry.py - Capability registry - src/core/startup.py - Application startup coordination - src/core/logging_config.py - Structured logging setup **Integration:** - Responses API uses Steward for Tatlock requests - Chat Completions wraps Responses API for OpenAI compatibility - Streaming coordinator supports Steward + Tatlock flow - Tool scoping per request based on Steward recommendations **Testing:** - Integration tests for Steward-Tatlock flow - Benchmark and registry unit tests - Steward streaming tests See PHASE2_PLAN.md and PHASE2_COMPLETE.md for detailed documentation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
250 lines
11 KiB
Python
250 lines
11 KiB
Python
"""
|
|
Integration tests for Steward → Tatlock flow.
|
|
|
|
Tests the complete Phase 2 request pipeline:
|
|
1. Steward analyzes request and recommends capabilities
|
|
2. Tool tracker monitors tool usage
|
|
3. Tatlock runs with scoped tools
|
|
4. Response includes both Steward reasoning and Tatlock output
|
|
"""
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from src.responses.schemas import ResponseRequest
|
|
from src.responses.service import create_response_with_steward
|
|
from src.core.startup import initialize_application
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def setup_household_registry():
|
|
"""Initialize household registry before running tests."""
|
|
initialize_application()
|
|
|
|
|
|
class TestStewardTatlockIntegration:
|
|
"""Test full Steward → Tatlock integration flow."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_simple_math_request(self):
|
|
"""Test math request flows through Steward → Tatlock correctly."""
|
|
# Create a simple math request
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[{"role": "user", "content": "What's 2 + 2?"}],
|
|
)
|
|
|
|
# Mock the Steward analysis
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
# Mock Steward recommendation
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=["tatlock_core"],
|
|
reasoning="Math calculation requires tatlock_core",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(has_previous_context=False),
|
|
)
|
|
|
|
# Mock Tatlock response
|
|
mock_tatlock.return_value = "Certainly, sir. 2 + 2 equals 4."
|
|
|
|
# Execute the flow
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify Steward was called
|
|
assert mock_steward.called
|
|
assert mock_steward.call_args[0][0] == "What's 2 + 2?"
|
|
|
|
# Verify Tatlock was called with scoped tools
|
|
assert mock_tatlock.called
|
|
|
|
# Verify response structure
|
|
assert response.status == "completed"
|
|
assert len(response.output) == 2 # Reasoning + Message
|
|
|
|
# Check Steward reasoning output
|
|
reasoning_item = response.output[0]
|
|
assert reasoning_item.type == "reasoning"
|
|
assert "Math calculation" in reasoning_item.summary[1]
|
|
|
|
# Check Tatlock message output
|
|
message_item = response.output[1]
|
|
assert message_item.type == "message"
|
|
assert message_item.role == "assistant"
|
|
assert "4" in message_item.content[0].text
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_with_conversation_history(self):
|
|
"""Test that conversation history flows through to Steward."""
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[
|
|
{"role": "user", "content": "What's 5 times 3?"},
|
|
{"role": "assistant", "content": "That equals 15, sir."},
|
|
{"role": "user", "content": "And divided by 3?"},
|
|
],
|
|
)
|
|
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=["tatlock_core"],
|
|
reasoning="Follow-up calculation",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(
|
|
has_previous_context=True,
|
|
relevant_turns=[0],
|
|
context_summary="Previous calculation in turn 0"
|
|
),
|
|
)
|
|
|
|
mock_tatlock.return_value = "15 divided by 3 equals 5, sir."
|
|
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify Steward received conversation history
|
|
call_kwargs = mock_steward.call_args[1]
|
|
assert "conversation_history" in call_kwargs
|
|
assert len(call_kwargs["conversation_history"]) == 2 # First Q&A pair
|
|
|
|
# Verify Tatlock received history
|
|
tatlock_kwargs = mock_tatlock.call_args[1]
|
|
assert "message_history" in tatlock_kwargs
|
|
|
|
# Verify response completed
|
|
assert response.status == "completed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_capabilities_needed(self):
|
|
"""Test simple conversational request that needs no tools."""
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[{"role": "user", "content": "Hello!"}],
|
|
)
|
|
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=[], # No tools needed
|
|
reasoning="Simple greeting, no tools required",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(has_previous_context=False),
|
|
)
|
|
|
|
mock_tatlock.return_value = "Good day, sir. How may I assist you?"
|
|
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify empty scoped tools were passed
|
|
tatlock_kwargs = mock_tatlock.call_args[1]
|
|
assert "scoped_tools" in tatlock_kwargs
|
|
assert tatlock_kwargs["scoped_tools"] == [] # No tools
|
|
|
|
assert response.status == "completed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tool_tracker_integration(self):
|
|
"""Test that tool tracker is passed to Tatlock and finalized."""
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[{"role": "user", "content": "Calculate sqrt(16)"}],
|
|
)
|
|
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
with patch("src.core.tool_tracking.ToolCallTracker.finalize") as mock_finalize:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=["tatlock_core"],
|
|
reasoning="Calculator needed",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(has_previous_context=False),
|
|
)
|
|
|
|
mock_tatlock.return_value = "The square root of 16 is 4, sir."
|
|
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify tool tracker was finalized
|
|
assert mock_finalize.called
|
|
assert response.status == "completed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_capabilities_warning(self):
|
|
"""Test that missing capabilities are included in Steward's reasoning."""
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[{"role": "user", "content": "Generate an image of a sunset"}],
|
|
)
|
|
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=[],
|
|
reasoning="Image generation not available",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(has_previous_context=False),
|
|
missing_capabilities="Image generation capability would be needed",
|
|
)
|
|
|
|
mock_tatlock.return_value = "I'm afraid I don't have image generation capabilities, sir."
|
|
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify Steward's reasoning mentions missing capabilities
|
|
reasoning_item = response.output[0]
|
|
assert "not available" in reasoning_item.summary[1].lower()
|
|
|
|
assert response.status == "completed"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_conversation_id_propagation(self):
|
|
"""Test that conversation ID flows through entire pipeline."""
|
|
request = ResponseRequest(
|
|
model="tatlock",
|
|
input=[{"role": "user", "content": "Test request"}],
|
|
metadata={"conversation_id": "test_conv_123"},
|
|
)
|
|
|
|
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
|
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
|
with patch("src.responses.service.ToolCallTracker") as mock_tracker_class:
|
|
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
|
|
|
mock_steward.return_value = StewardRecommendation(
|
|
recommended_capabilities=["tatlock_core"],
|
|
reasoning="Test",
|
|
estimated_complexity="simple",
|
|
conversation_context=ConversationContext(has_previous_context=False),
|
|
)
|
|
|
|
mock_tatlock.return_value = "Test response"
|
|
|
|
mock_tracker = MagicMock()
|
|
mock_tracker.get_summary = MagicMock(return_value={})
|
|
mock_tracker.finalize = AsyncMock()
|
|
mock_tracker_class.return_value = mock_tracker
|
|
|
|
response = await create_response_with_steward(request)
|
|
|
|
# Verify conversation ID was passed to Steward
|
|
steward_kwargs = mock_steward.call_args[1]
|
|
assert steward_kwargs.get("conversation_id") == "test_conv_123"
|
|
|
|
# Verify conversation ID was passed to tracker
|
|
assert mock_tracker_class.called
|
|
tracker_call_args = mock_tracker_class.call_args
|
|
if tracker_call_args and len(tracker_call_args) > 1:
|
|
tracker_init_kwargs = tracker_call_args[1]
|
|
assert tracker_init_kwargs.get("conversation_id") == "test_conv_123"
|
|
|
|
assert response.status == "completed"
|