feat: implement Phase 2 two-tier architecture with Steward
Add comprehensive two-tier architecture where Steward analyzes requests and Tatlock executes with scoped tools. Includes full infrastructure for request preprocessing, tool tracking, benchmarking, and streaming. **Added:** - Steward agent for request analysis and capability recommendation - Household Registry for centralized capability management - Request preprocessing pipeline (Steward → Tatlock flow) - Tool usage tracking and benchmarking system - Streaming transparency (Steward reasoning visible in streams) - Structured logging with operation timing - Redis benchmark storage with 30-day expiry - Benchmark analysis CLI tools **Infrastructure:** - src/agents/steward/ - Steward agent implementation - src/agents/tatlock_core/ - Tatlock capability domain - src/core/preprocessing.py - Request preprocessing pipeline - src/core/tool_tracking.py - Tool call tracking - src/core/benchmarks.py - Benchmark recording system - src/core/household_registry.py - Capability registry - src/core/startup.py - Application startup coordination - src/core/logging_config.py - Structured logging setup **Integration:** - Responses API uses Steward for Tatlock requests - Chat Completions wraps Responses API for OpenAI compatibility - Streaming coordinator supports Steward + Tatlock flow - Tool scoping per request based on Steward recommendations **Testing:** - Integration tests for Steward-Tatlock flow - Benchmark and registry unit tests - Steward streaming tests See PHASE2_PLAN.md and PHASE2_COMPLETE.md for detailed documentation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.5
@@ -0,0 +1 @@
|
||||
"""Tests for the Steward agent."""
|
||||
@@ -0,0 +1,166 @@
|
||||
"""
|
||||
Tests for Steward schemas.
|
||||
|
||||
Tests the structured output models for conversation context and recommendations.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
|
||||
class TestConversationContext:
|
||||
"""Test ConversationContext model."""
|
||||
|
||||
def test_context_creation_with_defaults(self):
|
||||
"""Test creating context with default values."""
|
||||
context = ConversationContext(has_previous_context=False)
|
||||
|
||||
assert context.has_previous_context is False
|
||||
assert context.relevant_turns == []
|
||||
assert context.context_summary == ""
|
||||
|
||||
def test_context_creation_with_values(self):
|
||||
"""Test creating context with explicit values."""
|
||||
context = ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[0, 2, 4],
|
||||
context_summary="User discussed weather in turns 0 and 2"
|
||||
)
|
||||
|
||||
assert context.has_previous_context is True
|
||||
assert context.relevant_turns == [0, 2, 4]
|
||||
assert "weather" in context.context_summary
|
||||
|
||||
|
||||
class TestStewardRecommendation:
|
||||
"""Test StewardRecommendation model."""
|
||||
|
||||
def test_recommendation_simple(self):
|
||||
"""Test simple recommendation with no capabilities needed."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Simple greeting requires no tools",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
assert rec.recommended_capabilities == []
|
||||
assert rec.estimated_complexity == "simple"
|
||||
assert rec.missing_capabilities is None
|
||||
|
||||
def test_recommendation_with_capabilities(self):
|
||||
"""Test recommendation with specific capabilities."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Mathematical calculation requires calculator",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
assert "tatlock_core" in rec.recommended_capabilities
|
||||
assert rec.estimated_complexity == "simple"
|
||||
|
||||
def test_recommendation_with_missing_capabilities(self):
|
||||
"""Test recommendation noting missing capabilities."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Image generation is not available",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
missing_capabilities="Image generation capability would be needed",
|
||||
)
|
||||
|
||||
assert rec.missing_capabilities is not None
|
||||
assert "Image generation" in rec.missing_capabilities
|
||||
|
||||
def test_recommendation_complexity_levels(self):
|
||||
"""Test all complexity levels."""
|
||||
for complexity in ["simple", "moderate", "complex"]:
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning=f"Testing {complexity} complexity",
|
||||
estimated_complexity=complexity,
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
assert rec.estimated_complexity == complexity
|
||||
|
||||
def test_recommendation_with_context(self):
|
||||
"""Test recommendation with conversation context."""
|
||||
context = ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[1, 3],
|
||||
context_summary="User asked about calculation in turn 1, now wants explanation"
|
||||
)
|
||||
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="User wants explanation of previous calculation",
|
||||
estimated_complexity="moderate",
|
||||
conversation_context=context,
|
||||
)
|
||||
|
||||
assert rec.conversation_context.has_previous_context is True
|
||||
assert len(rec.conversation_context.relevant_turns) == 2
|
||||
|
||||
def test_format_for_butler_simple(self):
|
||||
"""Test formatting recommendation for Butler - simple case."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Math calculation needed",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
formatted = rec.format_for_butler()
|
||||
|
||||
assert "📋 Steward's Analysis" in formatted
|
||||
assert "SIMPLE" in formatted
|
||||
assert "tatlock_core" in formatted
|
||||
|
||||
def test_format_for_butler_with_context(self):
|
||||
"""Test formatting with conversation context."""
|
||||
context = ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[0],
|
||||
context_summary="Previous calculation mentioned"
|
||||
)
|
||||
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Follow-up calculation",
|
||||
estimated_complexity="moderate",
|
||||
conversation_context=context,
|
||||
)
|
||||
|
||||
formatted = rec.format_for_butler()
|
||||
|
||||
assert "Context:" in formatted
|
||||
assert "Previous calculation" in formatted
|
||||
|
||||
def test_format_for_butler_with_missing_capabilities(self):
|
||||
"""Test formatting with missing capabilities warning."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="No suitable tools available",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
missing_capabilities="Image generation would be needed",
|
||||
)
|
||||
|
||||
formatted = rec.format_for_butler()
|
||||
|
||||
assert "⚠️ Missing:" in formatted
|
||||
assert "Image generation" in formatted
|
||||
|
||||
def test_format_for_butler_no_capabilities(self):
|
||||
"""Test formatting when no tools needed (conversational)."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Simple greeting",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
formatted = rec.format_for_butler()
|
||||
|
||||
assert "None (conversational response)" in formatted
|
||||
@@ -0,0 +1,201 @@
|
||||
"""
|
||||
Tests for Steward service layer.
|
||||
|
||||
Tests request analysis, logging, and benchmarking integration.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
from src.agents.steward.service import analyze_request, format_steward_note
|
||||
from src.core.startup import initialize_application
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
|
||||
|
||||
class TestAnalyzeRequest:
|
||||
"""Test the analyze_request service function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_simple_greeting(self):
|
||||
"""Test analyzing a simple greeting."""
|
||||
# Mock the Steward agent's analyze method (plain text approach)
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(return_value="Simple greeting requires no tools. This is a simple request.")
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_math_request(self):
|
||||
"""Test analyzing a mathematical request."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(
|
||||
return_value="Mathematical calculation requires tatlock_core for solving this simple problem."
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_history(self):
|
||||
"""Test analyzing with previous conversation context."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(
|
||||
return_value="Follow-up to previous calculation in turn 0. Requires tatlock_core. Complexity: moderate."
|
||||
)
|
||||
|
||||
conversation_history = [
|
||||
{"role": "user", "content": "What's 2 + 2?"},
|
||||
{"role": "assistant", "content": "4"},
|
||||
]
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_missing_capabilities(self):
|
||||
"""Test analyzing request that needs unavailable capabilities."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(
|
||||
return_value="Image generation not available. Would be needed for this request. Complexity: simple."
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_id(self):
|
||||
"""Test that analysis includes conversation ID in context."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(
|
||||
return_value="This simple request requires tatlock_core to solve."
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_handles_errors(self):
|
||||
"""Test error handling in analyze_request."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.analyze = AsyncMock(side_effect=Exception("Test error"))
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with pytest.raises(Exception, match="Test error"):
|
||||
await analyze_request("Test", conversation_history=[])
|
||||
|
||||
|
||||
class TestFormatStewardNote:
|
||||
"""Test the format_steward_note function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_simple_note(self):
|
||||
"""Test formatting a simple recommendation."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Math needed",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
note = await format_steward_note(rec)
|
||||
|
||||
assert "📋 Steward's Analysis" in note
|
||||
assert "SIMPLE" in note
|
||||
assert "tatlock_core" in note
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_note_with_context(self):
|
||||
"""Test formatting note with conversation context."""
|
||||
context = ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[0, 1],
|
||||
context_summary="Previous discussion about calculations"
|
||||
)
|
||||
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Follow-up calculation",
|
||||
estimated_complexity="moderate",
|
||||
conversation_context=context,
|
||||
)
|
||||
|
||||
note = await format_steward_note(rec)
|
||||
|
||||
assert "Context:" in note
|
||||
assert "Previous discussion" in note
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_note_with_missing_capabilities(self):
|
||||
"""Test formatting note with missing capabilities warning."""
|
||||
rec = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Not available",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
missing_capabilities="Advanced research tools needed",
|
||||
)
|
||||
|
||||
note = await format_steward_note(rec)
|
||||
|
||||
assert "⚠️ Missing:" in note
|
||||
assert "Advanced research" in note
|
||||
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
Tests for benchmark storage.
|
||||
|
||||
Tests performance tracking, Redis storage, and analytics features.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.benchmarks import (
|
||||
BenchmarkStore,
|
||||
PerformanceBenchmark,
|
||||
get_benchmark_store,
|
||||
)
|
||||
|
||||
|
||||
class TestPerformanceBenchmark:
|
||||
"""Test PerformanceBenchmark model."""
|
||||
|
||||
def test_benchmark_creation(self):
|
||||
"""Test creating a performance benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=1.23,
|
||||
success=True,
|
||||
recommendation_count=3,
|
||||
)
|
||||
|
||||
assert benchmark.operation == "steward_analysis"
|
||||
assert benchmark.duration_seconds == 1.23
|
||||
assert benchmark.success is True
|
||||
assert benchmark.recommendation_count == 3
|
||||
assert isinstance(benchmark.timestamp, datetime)
|
||||
|
||||
def test_benchmark_with_tool_fields(self):
|
||||
"""Test benchmark with tool-specific fields."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="tool_call",
|
||||
duration_seconds=0.5,
|
||||
success=True,
|
||||
tool_name="calculate",
|
||||
was_recommended=True,
|
||||
was_actually_used=True,
|
||||
)
|
||||
|
||||
assert benchmark.tool_name == "calculate"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is True
|
||||
|
||||
def test_benchmark_to_redis_dict(self):
|
||||
"""Test conversion to Redis dict."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
metadata={"key": "value"},
|
||||
)
|
||||
|
||||
redis_dict = benchmark.to_redis_dict()
|
||||
assert redis_dict["operation"] == "test_op"
|
||||
assert redis_dict["duration_seconds"] == 1.0
|
||||
assert redis_dict["success"] is True
|
||||
assert isinstance(redis_dict["timestamp"], str)
|
||||
assert isinstance(redis_dict["metadata"], str)
|
||||
|
||||
def test_benchmark_from_redis_dict(self):
|
||||
"""Test reconstruction from Redis dict."""
|
||||
now = datetime.now(timezone.utc)
|
||||
redis_dict = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5,
|
||||
"success": True,
|
||||
"metadata": json.dumps({"test": "data"}),
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
benchmark = PerformanceBenchmark.from_redis_dict(redis_dict)
|
||||
assert benchmark.operation == "test_op"
|
||||
assert benchmark.duration_seconds == 1.5
|
||||
assert benchmark.metadata == {"test": "data"}
|
||||
|
||||
|
||||
class TestBenchmarkStore:
|
||||
"""Test BenchmarkStore functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis(self):
|
||||
"""Create mock Redis client."""
|
||||
mock = AsyncMock()
|
||||
mock.hset = AsyncMock()
|
||||
mock.expire = AsyncMock()
|
||||
mock.zadd = AsyncMock()
|
||||
mock.zrevrangebyscore = AsyncMock(return_value=[])
|
||||
mock.hgetall = AsyncMock(return_value={})
|
||||
mock.aclose = AsyncMock()
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def store(self, mock_redis):
|
||||
"""Create benchmark store with mock Redis."""
|
||||
return BenchmarkStore(redis_client=mock_redis)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark(self, store, mock_redis):
|
||||
"""Test recording a benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Verify Redis calls
|
||||
mock_redis.hset.assert_called_once()
|
||||
mock_redis.expire.assert_called()
|
||||
mock_redis.zadd.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_disabled(self, mock_redis):
|
||||
"""Test recording when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Should not call Redis
|
||||
mock_redis.hset.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_handles_errors(self, store, mock_redis):
|
||||
"""Test recording handles Redis errors gracefully."""
|
||||
mock_redis.hset.side_effect = Exception("Redis error")
|
||||
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Should not raise exception
|
||||
await store.record(benchmark)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_benchmarks(self, store, mock_redis):
|
||||
"""Test querying benchmarks."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_key = f"benchmark:test_op:{int(now.timestamp() * 1000)}"
|
||||
mock_redis.zrevrangebyscore.return_value = [mock_key]
|
||||
|
||||
# Mock hgetall to return proper data
|
||||
mock_redis.hgetall.return_value = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5, # Numeric, not string
|
||||
"success": True,
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
results = await store.query("test_op", limit=10)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].operation == "test_op"
|
||||
mock_redis.zrevrangebyscore.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_with_time_range(self, store, mock_redis):
|
||||
"""Test querying with time range."""
|
||||
now = datetime.now(timezone.utc)
|
||||
start_time = now - timedelta(hours=1)
|
||||
end_time = now
|
||||
|
||||
await store.query("test_op", start_time=start_time, end_time=end_time)
|
||||
|
||||
# Verify time range was converted to timestamps
|
||||
call_args = mock_redis.zrevrangebyscore.call_args
|
||||
assert call_args is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_disabled_benchmarks(self, mock_redis):
|
||||
"""Test querying when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_handles_errors(self, store, mock_redis):
|
||||
"""Test query handles errors gracefully."""
|
||||
mock_redis.zrevrangebyscore.side_effect = Exception("Redis error")
|
||||
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics(self, store, mock_redis):
|
||||
"""Test getting statistics."""
|
||||
# Setup mock data with multiple benchmarks
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:test_op:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(3)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Return different durations and success values
|
||||
benchmarks_data = [
|
||||
{"duration_seconds": "1.0", "success": "True"},
|
||||
{"duration_seconds": "2.0", "success": "True"},
|
||||
{"duration_seconds": "3.0", "success": "False"},
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = benchmarks_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": float(data["duration_seconds"]),
|
||||
"success": data["success"] == "True",
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 3
|
||||
assert stats["avg_duration"] == 2.0 # (1 + 2 + 3) / 3
|
||||
assert stats["min_duration"] == 1.0
|
||||
assert stats["max_duration"] == 3.0
|
||||
assert stats["success_rate"] == pytest.approx(66.67, rel=0.01)
|
||||
assert stats["total_successes"] == 2
|
||||
assert stats["total_failures"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics_empty(self, store, mock_redis):
|
||||
"""Test statistics with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 0
|
||||
assert stats["avg_duration"] == 0.0
|
||||
assert stats["success_rate"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy(self, store, mock_redis):
|
||||
"""Test tool accuracy calculation."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:tool_call:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(4)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Different combinations of recommended/used
|
||||
tool_data = [
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "False", "was_actually_used": "True"}, # Missed
|
||||
{"was_recommended": "True", "was_actually_used": "False"}, # Not used
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = tool_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "tool_call",
|
||||
"duration_seconds": 1.0,
|
||||
"success": True,
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": "test_tool",
|
||||
"conversation_id": None,
|
||||
"was_recommended": data["was_recommended"] == "True",
|
||||
"was_actually_used": data["was_actually_used"] == "True",
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 4
|
||||
assert accuracy["total_used"] == 3
|
||||
assert accuracy["recommended_and_used"] == 2
|
||||
assert accuracy["not_recommended_but_used"] == 1
|
||||
assert accuracy["precision"] == pytest.approx(66.67, rel=0.01)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy_empty(self, store, mock_redis):
|
||||
"""Test tool accuracy with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 0
|
||||
assert accuracy["precision"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close(self, store, mock_redis):
|
||||
"""Test closing the store."""
|
||||
await store.close()
|
||||
mock_redis.aclose.assert_called_once()
|
||||
|
||||
# Client should be None after close
|
||||
assert store._client is None
|
||||
|
||||
|
||||
class TestGlobalBenchmarkStore:
|
||||
"""Test global benchmark store instance."""
|
||||
|
||||
def test_get_benchmark_store(self):
|
||||
"""Test getting global store instance."""
|
||||
store = get_benchmark_store()
|
||||
assert isinstance(store, BenchmarkStore)
|
||||
|
||||
def test_get_benchmark_store_singleton(self):
|
||||
"""Test store is singleton."""
|
||||
store1 = get_benchmark_store()
|
||||
store2 = get_benchmark_store()
|
||||
assert store1 is store2
|
||||
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
Tests for household registry.
|
||||
|
||||
Tests capability registration, toolset scoping, and coordination features.
|
||||
"""
|
||||
import pytest
|
||||
from pydantic_ai.tools import Tool
|
||||
|
||||
from src.core.household_registry import (
|
||||
HouseholdCapability,
|
||||
HouseholdMember,
|
||||
HouseholdRegistry,
|
||||
household_registry,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry():
|
||||
"""Create a fresh registry for each test."""
|
||||
reg = HouseholdRegistry()
|
||||
return reg
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_capability():
|
||||
"""Sample household capability."""
|
||||
return HouseholdCapability(
|
||||
name="test_tools",
|
||||
role="Test Tools",
|
||||
category="testing",
|
||||
description="Tools for testing purposes",
|
||||
domains=["testing", "validation"],
|
||||
cost="low",
|
||||
requires_network=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_tools():
|
||||
"""Sample tool definitions."""
|
||||
def test_function_1(x: int) -> int:
|
||||
"""Test function 1."""
|
||||
return x * 2
|
||||
|
||||
def test_function_2(x: str) -> str:
|
||||
"""Test function 2."""
|
||||
return x.upper()
|
||||
|
||||
return [
|
||||
Tool(function=test_function_1, name="test_tool_1"),
|
||||
Tool(function=test_function_2, name="test_tool_2"),
|
||||
]
|
||||
|
||||
|
||||
class TestHouseholdCapability:
|
||||
"""Test HouseholdCapability model."""
|
||||
|
||||
def test_capability_creation(self, sample_capability):
|
||||
"""Test creating a capability."""
|
||||
assert sample_capability.name == "test_tools"
|
||||
assert sample_capability.role == "Test Tools"
|
||||
assert sample_capability.category == "testing"
|
||||
assert "testing" in sample_capability.domains
|
||||
assert sample_capability.cost == "low"
|
||||
assert sample_capability.requires_network is False
|
||||
|
||||
def test_capability_validation(self):
|
||||
"""Test capability field validation."""
|
||||
# Should succeed with valid data
|
||||
cap = HouseholdCapability(
|
||||
name="valid",
|
||||
role="Valid Role",
|
||||
category="test",
|
||||
description="Test description",
|
||||
domains=["test"],
|
||||
cost="medium",
|
||||
requires_network=True,
|
||||
)
|
||||
assert cap.name == "valid"
|
||||
|
||||
|
||||
class TestHouseholdMember:
|
||||
"""Test HouseholdMember model."""
|
||||
|
||||
def test_member_creation(self, sample_capability, sample_tools):
|
||||
"""Test creating a household member."""
|
||||
member = HouseholdMember(
|
||||
capability=sample_capability,
|
||||
tools=sample_tools,
|
||||
agent=None,
|
||||
)
|
||||
assert member.capability.name == "test_tools"
|
||||
assert len(member.tools) == 2
|
||||
assert member.agent is None
|
||||
|
||||
def test_member_with_agent(self, sample_capability, sample_tools):
|
||||
"""Test member can include an agent."""
|
||||
from unittest.mock import Mock
|
||||
mock_agent = Mock()
|
||||
|
||||
member = HouseholdMember(
|
||||
capability=sample_capability,
|
||||
tools=sample_tools,
|
||||
agent=mock_agent,
|
||||
)
|
||||
assert member.agent is mock_agent
|
||||
|
||||
|
||||
class TestHouseholdRegistry:
|
||||
"""Test HouseholdRegistry functionality."""
|
||||
|
||||
def test_registry_initialization(self, registry):
|
||||
"""Test registry initializes empty."""
|
||||
assert len(registry) == 0
|
||||
assert registry.list_members() == []
|
||||
|
||||
def test_register_member(self, registry, sample_capability, sample_tools):
|
||||
"""Test registering a household member."""
|
||||
registry.register(
|
||||
name="test_tools",
|
||||
capability=sample_capability,
|
||||
tools=sample_tools,
|
||||
)
|
||||
|
||||
assert len(registry) == 1
|
||||
assert "test_tools" in registry
|
||||
assert "test_tools" in registry.list_members()
|
||||
|
||||
def test_register_name_mismatch(self, registry, sample_capability, sample_tools):
|
||||
"""Test registration fails with name mismatch."""
|
||||
with pytest.raises(ValueError, match="Name mismatch"):
|
||||
registry.register(
|
||||
name="wrong_name",
|
||||
capability=sample_capability,
|
||||
tools=sample_tools,
|
||||
)
|
||||
|
||||
def test_unregister_member(self, registry, sample_capability, sample_tools):
|
||||
"""Test unregistering a member."""
|
||||
registry.register("test_tools", sample_capability, sample_tools)
|
||||
assert "test_tools" in registry
|
||||
|
||||
registry.unregister("test_tools")
|
||||
assert "test_tools" not in registry
|
||||
assert len(registry) == 0
|
||||
|
||||
def test_get_member(self, registry, sample_capability, sample_tools):
|
||||
"""Test retrieving a member."""
|
||||
registry.register("test_tools", sample_capability, sample_tools)
|
||||
|
||||
member = registry.get_member("test_tools")
|
||||
assert member is not None
|
||||
assert member.capability.name == "test_tools"
|
||||
assert len(member.tools) == 2
|
||||
|
||||
def test_get_nonexistent_member(self, registry):
|
||||
"""Test retrieving non-existent member returns None."""
|
||||
member = registry.get_member("nonexistent")
|
||||
assert member is None
|
||||
|
||||
def test_get_all_capabilities(self, registry, sample_capability, sample_tools):
|
||||
"""Test retrieving all capability summaries."""
|
||||
# Register multiple members
|
||||
cap1 = sample_capability
|
||||
cap2 = HouseholdCapability(
|
||||
name="other_tools",
|
||||
role="Other Tools",
|
||||
category="utility",
|
||||
description="Other test tools",
|
||||
domains=["utility"],
|
||||
cost="medium",
|
||||
requires_network=True,
|
||||
)
|
||||
|
||||
registry.register("test_tools", cap1, sample_tools)
|
||||
registry.register("other_tools", cap2, sample_tools[:1])
|
||||
|
||||
capabilities = registry.get_all_capabilities()
|
||||
assert len(capabilities) == 2
|
||||
assert any(cap.name == "test_tools" for cap in capabilities)
|
||||
assert any(cap.name == "other_tools" for cap in capabilities)
|
||||
|
||||
def test_get_scoped_tools(self, registry, sample_capability, sample_tools):
|
||||
"""Test creating scoped toolsets."""
|
||||
registry.register("test_tools", sample_capability, sample_tools)
|
||||
|
||||
# Get scoped tools
|
||||
tools = registry.get_scoped_tools(["test_tools"])
|
||||
assert len(tools) == 2
|
||||
assert tools[0].name == "test_tool_1"
|
||||
assert tools[1].name == "test_tool_2"
|
||||
|
||||
def test_get_scoped_tools_multiple_members(self, registry, sample_tools):
|
||||
"""Test scoping with multiple members."""
|
||||
cap1 = HouseholdCapability(
|
||||
name="member1",
|
||||
role="Member 1",
|
||||
category="test",
|
||||
description="First member",
|
||||
domains=["test"],
|
||||
cost="low",
|
||||
requires_network=False,
|
||||
)
|
||||
cap2 = HouseholdCapability(
|
||||
name="member2",
|
||||
role="Member 2",
|
||||
category="test",
|
||||
description="Second member",
|
||||
domains=["test"],
|
||||
cost="low",
|
||||
requires_network=False,
|
||||
)
|
||||
|
||||
registry.register("member1", cap1, sample_tools[:1])
|
||||
registry.register("member2", cap2, sample_tools[1:])
|
||||
|
||||
# Get combined tools
|
||||
tools = registry.get_scoped_tools(["member1", "member2"])
|
||||
assert len(tools) == 2
|
||||
|
||||
def test_get_scoped_tools_nonexistent_member(self, registry, sample_capability, sample_tools):
|
||||
"""Test scoping with non-existent member logs warning."""
|
||||
registry.register("test_tools", sample_capability, sample_tools)
|
||||
|
||||
# Request includes non-existent member
|
||||
tools = registry.get_scoped_tools(["test_tools", "nonexistent"])
|
||||
# Should return only existing member's tools
|
||||
assert len(tools) == 2
|
||||
|
||||
def test_get_members_by_domain(self, registry, sample_tools):
|
||||
"""Test filtering members by domain."""
|
||||
cap1 = HouseholdCapability(
|
||||
name="research_tools",
|
||||
role="Research Tools",
|
||||
category="research",
|
||||
description="Research tools",
|
||||
domains=["research", "analysis"],
|
||||
cost="medium",
|
||||
requires_network=True,
|
||||
)
|
||||
cap2 = HouseholdCapability(
|
||||
name="compute_tools",
|
||||
role="Compute Tools",
|
||||
category="computation",
|
||||
description="Computation tools",
|
||||
domains=["computation", "math"],
|
||||
cost="low",
|
||||
requires_network=False,
|
||||
)
|
||||
|
||||
registry.register("research_tools", cap1, sample_tools)
|
||||
registry.register("compute_tools", cap2, sample_tools)
|
||||
|
||||
# Filter by domain
|
||||
research_caps = registry.get_members_by_domain("research")
|
||||
assert len(research_caps) == 1
|
||||
assert research_caps[0].name == "research_tools"
|
||||
|
||||
compute_caps = registry.get_members_by_domain("computation")
|
||||
assert len(compute_caps) == 1
|
||||
assert compute_caps[0].name == "compute_tools"
|
||||
|
||||
def test_get_members_by_category(self, registry, sample_tools):
|
||||
"""Test filtering members by category."""
|
||||
cap1 = HouseholdCapability(
|
||||
name="core_tools",
|
||||
role="Core Tools",
|
||||
category="core",
|
||||
description="Core tools",
|
||||
domains=["general"],
|
||||
cost="low",
|
||||
requires_network=False,
|
||||
)
|
||||
cap2 = HouseholdCapability(
|
||||
name="research_tools",
|
||||
role="Research Tools",
|
||||
category="research",
|
||||
description="Research tools",
|
||||
domains=["research"],
|
||||
cost="medium",
|
||||
requires_network=True,
|
||||
)
|
||||
|
||||
registry.register("core_tools", cap1, sample_tools)
|
||||
registry.register("research_tools", cap2, sample_tools)
|
||||
|
||||
# Filter by category
|
||||
core_caps = registry.get_members_by_category("core")
|
||||
assert len(core_caps) == 1
|
||||
assert core_caps[0].name == "core_tools"
|
||||
|
||||
research_caps = registry.get_members_by_category("research")
|
||||
assert len(research_caps) == 1
|
||||
assert research_caps[0].name == "research_tools"
|
||||
|
||||
|
||||
class TestGlobalRegistry:
|
||||
"""Test the global registry instance."""
|
||||
|
||||
def test_global_registry_exists(self):
|
||||
"""Test global registry is available."""
|
||||
from src.core.household_registry import get_household_registry
|
||||
|
||||
registry = get_household_registry()
|
||||
assert isinstance(registry, HouseholdRegistry)
|
||||
|
||||
def test_global_registry_singleton(self):
|
||||
"""Test get_household_registry returns same instance."""
|
||||
from src.core.household_registry import get_household_registry
|
||||
|
||||
reg1 = get_household_registry()
|
||||
reg2 = get_household_registry()
|
||||
assert reg1 is reg2
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
Tests for structured logging configuration.
|
||||
|
||||
Tests logging setup, context management, and FastAPI integration.
|
||||
"""
|
||||
import logging
|
||||
from io import StringIO
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import structlog
|
||||
|
||||
from src.core.logging_config import (
|
||||
add_log_level,
|
||||
add_timestamp,
|
||||
get_logger,
|
||||
get_uvicorn_log_config,
|
||||
log_operation,
|
||||
)
|
||||
|
||||
|
||||
class TestLoggingProcessors:
|
||||
"""Test logging processor functions."""
|
||||
|
||||
def test_add_timestamp(self):
|
||||
"""Test timestamp processor adds ISO timestamp."""
|
||||
event_dict = {}
|
||||
result = add_timestamp(None, "info", event_dict)
|
||||
|
||||
assert "timestamp" in result
|
||||
assert isinstance(result["timestamp"], str)
|
||||
# Should be ISO 8601 format
|
||||
assert "T" in result["timestamp"] or "-" in result["timestamp"]
|
||||
|
||||
def test_add_log_level(self):
|
||||
"""Test log level processor."""
|
||||
event_dict = {}
|
||||
result = add_log_level(None, "info", event_dict)
|
||||
|
||||
assert result["level"] == "INFO"
|
||||
|
||||
result = add_log_level(None, "error", {})
|
||||
assert result["level"] == "ERROR"
|
||||
|
||||
|
||||
class TestGetLogger:
|
||||
"""Test logger retrieval."""
|
||||
|
||||
def test_get_logger_returns_bound_logger(self):
|
||||
"""Test get_logger returns structlog BoundLogger."""
|
||||
logger = get_logger("test")
|
||||
# Logger should have standard logging methods
|
||||
assert hasattr(logger, 'info')
|
||||
assert hasattr(logger, 'debug')
|
||||
assert hasattr(logger, 'warning')
|
||||
assert hasattr(logger, 'error')
|
||||
|
||||
def test_get_logger_with_module_name(self):
|
||||
"""Test logger with module name."""
|
||||
logger = get_logger(__name__)
|
||||
assert logger is not None
|
||||
|
||||
def test_logger_has_standard_methods(self):
|
||||
"""Test logger has standard logging methods."""
|
||||
logger = get_logger("test")
|
||||
assert hasattr(logger, "debug")
|
||||
assert hasattr(logger, "info")
|
||||
assert hasattr(logger, "warning")
|
||||
assert hasattr(logger, "error")
|
||||
assert hasattr(logger, "exception")
|
||||
|
||||
|
||||
class TestLogOperation:
|
||||
"""Test log_operation context manager."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_operation_success(self):
|
||||
"""Test log_operation for successful operation."""
|
||||
logger = get_logger("test")
|
||||
|
||||
async with log_operation("test_operation", {"user_id": "123"}) as ctx:
|
||||
# Can update context during operation
|
||||
ctx["result_count"] = 5
|
||||
|
||||
# Context should have been updated with success info
|
||||
assert ctx["success"] is True
|
||||
assert ctx["result_count"] == 5
|
||||
assert "duration_seconds" in ctx
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_operation_failure(self):
|
||||
"""Test log_operation for failed operation."""
|
||||
logger = get_logger("test")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
async with log_operation("test_operation") as ctx:
|
||||
raise ValueError("Test error")
|
||||
|
||||
# Context should have failure info
|
||||
assert ctx["success"] is False
|
||||
assert ctx["error"] == "Test error"
|
||||
assert ctx["error_type"] == "ValueError"
|
||||
assert "duration_seconds" in ctx
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_operation_timing(self):
|
||||
"""Test log_operation records duration."""
|
||||
import asyncio
|
||||
|
||||
async with log_operation("test_operation") as ctx:
|
||||
await asyncio.sleep(0.01) # Small delay
|
||||
|
||||
# Should have measurable duration
|
||||
assert ctx["duration_seconds"] > 0
|
||||
assert ctx["duration_seconds"] < 1.0 # Should be quick
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_operation_initial_context(self):
|
||||
"""Test log_operation with initial context."""
|
||||
initial = {"request_id": "abc123", "user": "test_user"}
|
||||
|
||||
async with log_operation("test_operation", initial) as ctx:
|
||||
pass
|
||||
|
||||
# Initial context should be preserved
|
||||
assert ctx["request_id"] == "abc123"
|
||||
assert ctx["user"] == "test_user"
|
||||
assert ctx["operation"] == "test_operation"
|
||||
|
||||
|
||||
class TestUvicornLogConfig:
|
||||
"""Test uvicorn logging configuration."""
|
||||
|
||||
def test_get_uvicorn_log_config_returns_dict(self):
|
||||
"""Test uvicorn config returns valid dict."""
|
||||
config = get_uvicorn_log_config()
|
||||
|
||||
assert isinstance(config, dict)
|
||||
assert "version" in config
|
||||
assert "formatters" in config
|
||||
assert "handlers" in config
|
||||
assert "loggers" in config
|
||||
|
||||
def test_uvicorn_log_config_has_required_loggers(self):
|
||||
"""Test config includes uvicorn loggers."""
|
||||
config = get_uvicorn_log_config()
|
||||
|
||||
loggers = config["loggers"]
|
||||
assert "uvicorn" in loggers
|
||||
assert "uvicorn.error" in loggers
|
||||
assert "uvicorn.access" in loggers
|
||||
|
||||
def test_uvicorn_log_config_format_selection(self):
|
||||
"""Test config format changes based on environment."""
|
||||
# Just test that the config is valid, format is determined by environment
|
||||
config = get_uvicorn_log_config()
|
||||
# Should have required structure
|
||||
assert "version" in config
|
||||
assert "formatters" in config
|
||||
assert "handlers" in config
|
||||
assert "loggers" in config
|
||||
|
||||
|
||||
class TestLoggingIntegration:
|
||||
"""Test logging integration with standard library."""
|
||||
|
||||
def test_standard_logging_works(self):
|
||||
"""Test standard logging.getLogger works."""
|
||||
logger = logging.getLogger("test.standard")
|
||||
# Should not raise
|
||||
logger.info("Test message")
|
||||
|
||||
def test_structlog_and_stdlib_coexist(self):
|
||||
"""Test structlog and stdlib can coexist."""
|
||||
struct_logger = get_logger("test.struct")
|
||||
std_logger = logging.getLogger("test.std")
|
||||
|
||||
# Both should work
|
||||
struct_logger.info("Structured log")
|
||||
std_logger.info("Standard log")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logging_in_async_context(self):
|
||||
"""Test logging works in async context."""
|
||||
logger = get_logger("test.async")
|
||||
|
||||
async def async_function():
|
||||
logger.info("Async log message", task="async_task")
|
||||
|
||||
await async_function()
|
||||
|
||||
|
||||
class TestLoggingOutput:
|
||||
"""Test actual logging output."""
|
||||
|
||||
def test_logger_outputs_structured_data(self):
|
||||
"""Test logger can output structured data."""
|
||||
logger = get_logger("test.output")
|
||||
|
||||
# Log with structured data
|
||||
logger.info(
|
||||
"user_action",
|
||||
user_id="123",
|
||||
action="login",
|
||||
success=True,
|
||||
)
|
||||
# Should not raise, output tested in integration tests
|
||||
|
||||
def test_logger_handles_exceptions(self):
|
||||
"""Test logger handles exception logging."""
|
||||
logger = get_logger("test.exceptions")
|
||||
|
||||
try:
|
||||
raise ValueError("Test error")
|
||||
except ValueError:
|
||||
logger.exception("Error occurred", extra_field="value")
|
||||
# Should not raise
|
||||
|
||||
def test_different_log_levels(self):
|
||||
"""Test different log levels."""
|
||||
logger = get_logger("test.levels")
|
||||
|
||||
logger.debug("Debug message", level="debug")
|
||||
logger.info("Info message", level="info")
|
||||
logger.warning("Warning message", level="warning")
|
||||
logger.error("Error message", level="error")
|
||||
# Should not raise
|
||||
|
||||
|
||||
class TestLoggingConfiguration:
|
||||
"""Test logging configuration behavior."""
|
||||
|
||||
def test_logging_respects_environment(self):
|
||||
"""Test logging format changes with environment."""
|
||||
from src.core.config import Environment, config
|
||||
|
||||
# In development, should use console format
|
||||
if config.ENVIRONMENT == Environment.DEVELOPMENT:
|
||||
assert config.log_format == "console"
|
||||
|
||||
# Mock production environment
|
||||
with patch.object(config, "ENVIRONMENT", Environment.PRODUCTION):
|
||||
assert config.log_format == "json"
|
||||
|
||||
def test_multiple_loggers_independent(self):
|
||||
"""Test multiple loggers are independent."""
|
||||
logger1 = get_logger("test.logger1")
|
||||
logger2 = get_logger("test.logger2")
|
||||
|
||||
assert logger1 is not logger2
|
||||
|
||||
# Both should work independently
|
||||
logger1.info("Logger 1 message")
|
||||
logger2.info("Logger 2 message")
|
||||
@@ -0,0 +1,190 @@
|
||||
"""
|
||||
Integration tests for Steward + Tatlock streaming.
|
||||
|
||||
Tests the complete streaming flow with Steward preprocessing.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from src.responses.schemas import ResponseRequest
|
||||
from src.responses.streaming import StreamingCoordinator, StreamEventType
|
||||
from src.core.startup import initialize_application
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
|
||||
|
||||
class TestStewardStreaming:
|
||||
"""Test Steward + Tatlock streaming integration."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_steward_basic(self):
|
||||
"""Test basic streaming with Steward preprocessing."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "What's 2 + 2?"}],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
# Mock the Steward analysis
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
# Mock Steward recommendation
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Math calculation requires tatlock_core",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
# Mock Tatlock response
|
||||
mock_tatlock.return_value = "Certainly, sir. 2 + 2 equals 4."
|
||||
|
||||
# Execute streaming
|
||||
coordinator = StreamingCoordinator()
|
||||
events = []
|
||||
|
||||
async for event in coordinator.stream_response_with_steward(request):
|
||||
events.append(event)
|
||||
|
||||
# Verify event sequence
|
||||
event_types = [e.event for e in events]
|
||||
|
||||
# Should have reasoning summary deltas
|
||||
assert StreamEventType.REASONING_SUMMARY_DELTA in event_types
|
||||
assert StreamEventType.REASONING_SUMMARY_DONE in event_types
|
||||
|
||||
# Should have output text deltas
|
||||
assert StreamEventType.OUTPUT_TEXT_DELTA in event_types
|
||||
assert StreamEventType.OUTPUT_TEXT_DONE in event_types
|
||||
|
||||
# Should end with response.done
|
||||
assert events[-1].event == StreamEventType.RESPONSE_DONE
|
||||
|
||||
# Verify Steward and Tatlock were called
|
||||
assert mock_steward.called
|
||||
assert mock_tatlock.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_conversation_history(self):
|
||||
"""Test streaming with conversation history."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[
|
||||
{"role": "user", "content": "What's 5 times 3?"},
|
||||
{"role": "assistant", "content": "That equals 15, sir."},
|
||||
{"role": "user", "content": "And divided by 3?"},
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Follow-up calculation based on previous result of 15",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[0],
|
||||
context_summary="Previous calculation in turn 0"
|
||||
),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "15 divided by 3 equals 5, sir."
|
||||
|
||||
coordinator = StreamingCoordinator()
|
||||
events = []
|
||||
|
||||
async for event in coordinator.stream_response_with_steward(request):
|
||||
events.append(event)
|
||||
|
||||
# Verify conversation history was passed to Steward
|
||||
call_kwargs = mock_steward.call_args[1]
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2 # First Q&A pair
|
||||
|
||||
# Verify final response includes both reasoning and message
|
||||
final_event = events[-1]
|
||||
assert final_event.event == StreamEventType.RESPONSE_DONE
|
||||
assert len(final_event.response.output) == 2 # Reasoning + Message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_reasoning_contains_steward_analysis(self):
|
||||
"""Test that reasoning summary contains Steward's analysis."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Test request"}],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="This is a test analysis with specific markers",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "Test response"
|
||||
|
||||
coordinator = StreamingCoordinator()
|
||||
reasoning_deltas = []
|
||||
|
||||
async for event in coordinator.stream_response_with_steward(request):
|
||||
if event.event == StreamEventType.REASONING_SUMMARY_DELTA:
|
||||
reasoning_deltas.append(event.delta)
|
||||
|
||||
# Combine all reasoning deltas
|
||||
full_reasoning = "".join(reasoning_deltas)
|
||||
|
||||
# Should contain Steward's analysis
|
||||
assert "test analysis" in full_reasoning.lower()
|
||||
assert len(reasoning_deltas) > 0, "Should have streamed reasoning deltas"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_missing_capabilities(self):
|
||||
"""Test streaming when Steward detects missing capabilities."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Generate an image of a sunset"}],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Image generation not available in current toolset",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
missing_capabilities="Image generation capability would be needed",
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "I'm afraid I don't have image generation capabilities, sir."
|
||||
|
||||
coordinator = StreamingCoordinator()
|
||||
events = []
|
||||
|
||||
async for event in coordinator.stream_response_with_steward(request):
|
||||
events.append(event)
|
||||
|
||||
# Should complete successfully even with missing capabilities
|
||||
assert events[-1].event == StreamEventType.RESPONSE_DONE
|
||||
|
||||
# Verify empty scoped tools were passed
|
||||
tatlock_kwargs = mock_tatlock.call_args[1]
|
||||
assert "scoped_tools" in tatlock_kwargs
|
||||
assert tatlock_kwargs["scoped_tools"] == []
|
||||
@@ -0,0 +1,249 @@
|
||||
"""
|
||||
Integration tests for Steward → Tatlock flow.
|
||||
|
||||
Tests the complete Phase 2 request pipeline:
|
||||
1. Steward analyzes request and recommends capabilities
|
||||
2. Tool tracker monitors tool usage
|
||||
3. Tatlock runs with scoped tools
|
||||
4. Response includes both Steward reasoning and Tatlock output
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from src.responses.schemas import ResponseRequest
|
||||
from src.responses.service import create_response_with_steward
|
||||
from src.core.startup import initialize_application
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
|
||||
|
||||
class TestStewardTatlockIntegration:
|
||||
"""Test full Steward → Tatlock integration flow."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_simple_math_request(self):
|
||||
"""Test math request flows through Steward → Tatlock correctly."""
|
||||
# Create a simple math request
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "What's 2 + 2?"}],
|
||||
)
|
||||
|
||||
# Mock the Steward analysis
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
# Mock Steward recommendation
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Math calculation requires tatlock_core",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
# Mock Tatlock response
|
||||
mock_tatlock.return_value = "Certainly, sir. 2 + 2 equals 4."
|
||||
|
||||
# Execute the flow
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify Steward was called
|
||||
assert mock_steward.called
|
||||
assert mock_steward.call_args[0][0] == "What's 2 + 2?"
|
||||
|
||||
# Verify Tatlock was called with scoped tools
|
||||
assert mock_tatlock.called
|
||||
|
||||
# Verify response structure
|
||||
assert response.status == "completed"
|
||||
assert len(response.output) == 2 # Reasoning + Message
|
||||
|
||||
# Check Steward reasoning output
|
||||
reasoning_item = response.output[0]
|
||||
assert reasoning_item.type == "reasoning"
|
||||
assert "Math calculation" in reasoning_item.summary[1]
|
||||
|
||||
# Check Tatlock message output
|
||||
message_item = response.output[1]
|
||||
assert message_item.type == "message"
|
||||
assert message_item.role == "assistant"
|
||||
assert "4" in message_item.content[0].text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_with_conversation_history(self):
|
||||
"""Test that conversation history flows through to Steward."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[
|
||||
{"role": "user", "content": "What's 5 times 3?"},
|
||||
{"role": "assistant", "content": "That equals 15, sir."},
|
||||
{"role": "user", "content": "And divided by 3?"},
|
||||
],
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Follow-up calculation",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(
|
||||
has_previous_context=True,
|
||||
relevant_turns=[0],
|
||||
context_summary="Previous calculation in turn 0"
|
||||
),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "15 divided by 3 equals 5, sir."
|
||||
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify Steward received conversation history
|
||||
call_kwargs = mock_steward.call_args[1]
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2 # First Q&A pair
|
||||
|
||||
# Verify Tatlock received history
|
||||
tatlock_kwargs = mock_tatlock.call_args[1]
|
||||
assert "message_history" in tatlock_kwargs
|
||||
|
||||
# Verify response completed
|
||||
assert response.status == "completed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_capabilities_needed(self):
|
||||
"""Test simple conversational request that needs no tools."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Hello!"}],
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=[], # No tools needed
|
||||
reasoning="Simple greeting, no tools required",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "Good day, sir. How may I assist you?"
|
||||
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify empty scoped tools were passed
|
||||
tatlock_kwargs = mock_tatlock.call_args[1]
|
||||
assert "scoped_tools" in tatlock_kwargs
|
||||
assert tatlock_kwargs["scoped_tools"] == [] # No tools
|
||||
|
||||
assert response.status == "completed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_tracker_integration(self):
|
||||
"""Test that tool tracker is passed to Tatlock and finalized."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Calculate sqrt(16)"}],
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
with patch("src.core.tool_tracking.ToolCallTracker.finalize") as mock_finalize:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Calculator needed",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "The square root of 16 is 4, sir."
|
||||
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify tool tracker was finalized
|
||||
assert mock_finalize.called
|
||||
assert response.status == "completed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_capabilities_warning(self):
|
||||
"""Test that missing capabilities are included in Steward's reasoning."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Generate an image of a sunset"}],
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=[],
|
||||
reasoning="Image generation not available",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
missing_capabilities="Image generation capability would be needed",
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "I'm afraid I don't have image generation capabilities, sir."
|
||||
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify Steward's reasoning mentions missing capabilities
|
||||
reasoning_item = response.output[0]
|
||||
assert "not available" in reasoning_item.summary[1].lower()
|
||||
|
||||
assert response.status == "completed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conversation_id_propagation(self):
|
||||
"""Test that conversation ID flows through entire pipeline."""
|
||||
request = ResponseRequest(
|
||||
model="tatlock",
|
||||
input=[{"role": "user", "content": "Test request"}],
|
||||
metadata={"conversation_id": "test_conv_123"},
|
||||
)
|
||||
|
||||
with patch("src.core.preprocessing.analyze_request") as mock_steward:
|
||||
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
|
||||
with patch("src.responses.service.ToolCallTracker") as mock_tracker_class:
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
|
||||
mock_steward.return_value = StewardRecommendation(
|
||||
recommended_capabilities=["tatlock_core"],
|
||||
reasoning="Test",
|
||||
estimated_complexity="simple",
|
||||
conversation_context=ConversationContext(has_previous_context=False),
|
||||
)
|
||||
|
||||
mock_tatlock.return_value = "Test response"
|
||||
|
||||
mock_tracker = MagicMock()
|
||||
mock_tracker.get_summary = MagicMock(return_value={})
|
||||
mock_tracker.finalize = AsyncMock()
|
||||
mock_tracker_class.return_value = mock_tracker
|
||||
|
||||
response = await create_response_with_steward(request)
|
||||
|
||||
# Verify conversation ID was passed to Steward
|
||||
steward_kwargs = mock_steward.call_args[1]
|
||||
assert steward_kwargs.get("conversation_id") == "test_conv_123"
|
||||
|
||||
# Verify conversation ID was passed to tracker
|
||||
assert mock_tracker_class.called
|
||||
tracker_call_args = mock_tracker_class.call_args
|
||||
if tracker_call_args and len(tracker_call_args) > 1:
|
||||
tracker_init_kwargs = tracker_call_args[1]
|
||||
assert tracker_init_kwargs.get("conversation_id") == "test_conv_123"
|
||||
|
||||
assert response.status == "completed"
|
||||
Reference in New Issue
Block a user