jpmschweitzer/tatlock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

style: apply ruff's automatic fixes and formatter

Browse Source 
Mechanical only, and separated from the judgment calls that follow so the
reviewable changes are not buried in a 98-file whitespace diff.

227 automatic fixes: 60 blank lines carrying whitespace, 60 unsorted import
blocks, 34 Optional[X] to X | None, 28 unused imports, 16 deprecated typing
imports, 12 datetime.timezone.utc to datetime.UTC, and assorted smaller
modernisations. Then `ruff format` over src and tests: 98 files reformatted,
35 already conforming.

No file among the unused-import findings defines __all__ or is an __init__.py,
so nothing here removes a re-export.

`make test`: 658 passed, unchanged from HEAD.

Two things observed while verifying, neither addressed here:

`pytest tests/` cannot collect — tests/e2e/test_orchestration_e2e.py uses an
`e2e` marker that is not registered, and the config is strict about markers.
This fails identically at HEAD, so it predates this change; `make test` passes
because it ignores tests/e2e, tests/integration and tests/contracts.

test_tatlock_tool_call_logging_calculator is flaky. It failed once in a full run
with these changes and passed on the next, passes in isolation with them, and
fails in isolation at HEAD. It is order- or timing-dependent, not a regression
from this commit — established by running the full suite both ways rather than
by reasoning about which change could have caused it.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jpmschweitzer
2026-08-11 17:25:18 +02:00
co-authored by Claude
parent 57fa6c13fc
commit 78066fab1b
103 changed files with 1601 additions and 1749 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/agents/biographer/test_capability.py
+3 -4
View File
@@ -2,9 +2,10 @@
Tests for Biographer capability registration.
"""
import pytest
from unittest.mock import MagicMock, patch
import pytest
from src.agents.biographer.capability import (
BIOGRAPHER_CAPABILITY,
get_biographer_capability,
@@ -73,9 +74,7 @@ class TestBiographerRegistration:
"src.agents.biographer.capability.get_household_registry",
return_value=mock_registry,
):
with patch(
"src.agents.biographer.capability.get_biographer_agent"
) as mock_get_agent:
with patch("src.agents.biographer.capability.get_biographer_agent") as mock_get_agent:
mock_agent = MagicMock()
mock_get_agent.return_value = mock_agent
tests/agents/housekeeper/test_capability.py
+3 -4
View File
@@ -2,9 +2,10 @@
Tests for Housekeeper capability registration.
"""
import pytest
from unittest.mock import MagicMock, patch
import pytest
from src.agents.housekeeper.capability import (
HOUSEKEEPER_CAPABILITY,
get_housekeeper_capability,
@@ -74,9 +75,7 @@ class TestHousekeeperRegistration:
"src.agents.housekeeper.capability.get_household_registry",
return_value=mock_registry,
):
with patch(
"src.agents.housekeeper.capability.get_housekeeper_agent"
) as mock_get_agent:
with patch("src.agents.housekeeper.capability.get_housekeeper_agent") as mock_get_agent:
mock_agent = MagicMock()
mock_get_agent.return_value = mock_agent
tests/agents/housekeeper/test_client.py
+2 -1
View File
@@ -2,9 +2,10 @@
Tests for the Core-API HTTP client.
"""
import pytest
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from src.agents.housekeeper.client import (
Area,
tests/agents/librarian/test_capability.py
+3 -4
View File
@@ -2,9 +2,10 @@
Tests for Librarian capability registration.
"""
import pytest
from unittest.mock import MagicMock, patch
import pytest
from src.agents.librarian.capability import (
LIBRARIAN_CAPABILITY,
get_librarian_capability,
@@ -67,9 +68,7 @@ class TestLibrarianRegistration:
"src.agents.librarian.capability.get_household_registry",
return_value=mock_registry,
):
with patch(
"src.agents.librarian.capability.get_librarian_agent"
) as mock_get_agent:
with patch("src.agents.librarian.capability.get_librarian_agent") as mock_get_agent:
mock_agent = MagicMock()
mock_get_agent.return_value = mock_agent
tests/agents/librarian/test_client.py
+8 -24
View File
@@ -137,9 +137,7 @@ class TestHybridSearch:
assert "docker" in result.keywords
@pytest.mark.asyncio
async def test_hybrid_search_empty_results(
self, client_with_mock, mock_httpx_client
):
async def test_hybrid_search_empty_results(self, client_with_mock, mock_httpx_client):
"""Test hybrid search with no results."""
mock_response = MagicMock()
mock_response.json.return_value = {
@@ -445,9 +443,7 @@ class TestUpdateWikiPage:
assert page.tags == ["projects", "devops"]
@pytest.mark.asyncio
async def test_update_wiki_page_multiple_fields(
self, client_with_mock, mock_httpx_client
):
async def test_update_wiki_page_multiple_fields(self, client_with_mock, mock_httpx_client):
"""Test updating multiple fields at once."""
mock_response = MagicMock()
mock_response.json.return_value = {
@@ -622,15 +618,11 @@ class TestExplicitUserContract:
@pytest.mark.asyncio
@pytest.mark.parametrize("method_name,kwargs", TENANT_SCOPED_METHODS)
async def test_user_from_context_is_sent_on_the_wire(
self, method_name, kwargs
):
async def test_user_from_context_is_sent_on_the_wire(self, method_name, kwargs):
"""With no explicit user, the context user is resolved and sent."""
client, mock_httpx = self._wire_client()
with patch(
"src.agents.librarian.client.get_user", return_value="llm_tester"
):
with patch("src.agents.librarian.client.get_user", return_value="llm_tester"):
await getattr(client, method_name)(**kwargs)
assert self._sent_user(mock_httpx) == "llm_tester"
@@ -647,9 +639,7 @@ class TestExplicitUserContract:
@pytest.mark.asyncio
@pytest.mark.parametrize("method_name,kwargs", TENANT_SCOPED_METHODS)
async def test_empty_context_user_fails_before_any_request(
self, method_name, kwargs
):
async def test_empty_context_user_fails_before_any_request(self, method_name, kwargs):
"""An empty resolved user raises before any bytes hit the wire."""
client, mock_httpx = self._wire_client()
@@ -698,9 +688,7 @@ class TestExplicitUserContract:
from src.core import config as config_module
from src.core.config import Environment
monkeypatch.setattr(
config_module.config, "ENVIRONMENT", Environment.DEVELOPMENT
)
monkeypatch.setattr(config_module.config, "ENVIRONMENT", Environment.DEVELOPMENT)
client, mock_httpx = self._wire_client()
await getattr(client, method_name)(user=explicit_user, **kwargs)
@@ -708,16 +696,12 @@ class TestExplicitUserContract:
assert self._sent_user(mock_httpx) == "llm_tester"
@pytest.mark.asyncio
async def test_explicit_production_tenant_passes_through_in_prod(
self, monkeypatch
):
async def test_explicit_production_tenant_passes_through_in_prod(self, monkeypatch):
"""In production the production tenant is sent unchanged."""
from src.core import config as config_module
from src.core.config import Environment
monkeypatch.setattr(
config_module.config, "ENVIRONMENT", Environment.PRODUCTION
)
monkeypatch.setattr(config_module.config, "ENVIRONMENT", Environment.PRODUCTION)
client, mock_httpx = self._wire_client()
await client.hybrid_search("q", user="jpmschweitzer")
tests/agents/librarian/test_client_retry.py
+5 -15
View File
@@ -91,9 +91,7 @@ class TestBoundedRetries:
mock_httpx.post.side_effect = httpx.ConnectError("Connection refused")
with pytest.raises(httpx.ConnectError):
await client_with_mock.create_wiki_page(
title="T", path="/t", content="c", user="u"
)
await client_with_mock.create_wiki_page(title="T", path="/t", content="c", user="u")
assert mock_httpx.post.call_count == 1
@@ -103,9 +101,7 @@ class TestBoundedRetries:
mock_httpx.post.side_effect = httpx.ConnectError("Connection refused")
with pytest.raises(httpx.ConnectError):
await client_with_mock.smart_create_wiki_page(
topic="T", tags=["x"], user="u"
)
await client_with_mock.smart_create_wiki_page(topic="T", tags=["x"], user="u")
assert mock_httpx.post.call_count == 1
@@ -184,9 +180,7 @@ class TestModelRetryEscalation:
@pytest.mark.asyncio
async def test_read_tool_raises_model_retry_on_transport_error(self):
mock_client = AsyncMock()
mock_client.hybrid_search.side_effect = httpx.ConnectError(
"Connection refused"
)
mock_client.hybrid_search.side_effect = httpx.ConnectError("Connection refused")
with self._patched_client(mock_client):
with pytest.raises(ModelRetry):
@@ -219,14 +213,10 @@ class TestModelRetryEscalation:
@pytest.mark.asyncio
async def test_write_tool_never_raises_model_retry(self):
mock_client = AsyncMock()
mock_client.create_wiki_page.side_effect = httpx.ConnectError(
"Connection refused"
)
mock_client.create_wiki_page.side_effect = httpx.ConnectError("Connection refused")
with self._patched_client(mock_client):
result = await create_wiki_page(
title="T", path="/t", content="c", tags=["x"]
)
result = await create_wiki_page(title="T", path="/t", content="c", tags=["x"])
assert "unable" in result
assert "Connection refused" not in result
tests/agents/librarian/test_hybrid_contract.py
+12 -28
View File
@@ -83,9 +83,7 @@ class TestHybridRAGContract:
assert source in SOURCE_ICONS, f"no icon for source '{source}'"
@pytest.mark.asyncio
async def test_context_maps_to_formatted_context(
self, client_with_recorded_response
):
async def test_context_maps_to_formatted_context(self, client_with_recorded_response):
"""Top-level 'context' field maps to formatted_context."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
@@ -94,9 +92,7 @@ class TestHybridRAGContract:
assert response.formatted_context != ""
@pytest.mark.asyncio
async def test_keywords_and_synonyms_from_dict(
self, client_with_recorded_response
):
async def test_keywords_and_synonyms_from_dict(self, client_with_recorded_response):
"""keywords is a dict: core_keywords + nested synonyms map."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
@@ -124,9 +120,7 @@ class TestHybridRAGContract:
assert len(response.related_dossiers) == len(set(response.related_dossiers))
@pytest.mark.asyncio
async def test_payload_never_sends_zero_limits(
self, client_with_recorded_response
):
async def test_payload_never_sends_zero_limits(self, client_with_recorded_response):
"""The live service 422s on limits < 1; disabled legs use enable_* flags."""
await client_with_recorded_response.hybrid_search(
"home server infrastructure",
@@ -151,24 +145,18 @@ class TestHybridRAGContract:
assert config["enable_volatile"] is False
@pytest.mark.asyncio
async def test_user_always_sent_as_query_param(
self, client_with_recorded_response
):
async def test_user_always_sent_as_query_param(self, client_with_recorded_response):
"""The tenant is always sent explicitly - library-desk is removing
its server-side default, so a missing user would 422."""
await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
params = client_with_recorded_response._client.post.call_args.kwargs[
"params"
]
params = client_with_recorded_response._client.post.call_args.kwargs["params"]
assert params["user"] == "testuser"
@pytest.mark.asyncio
async def test_source_counts_and_timing_parsed(
self, client_with_recorded_response
):
async def test_source_counts_and_timing_parsed(self, client_with_recorded_response):
"""source_counts and timing map into the response model."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
@@ -178,9 +166,7 @@ class TestHybridRAGContract:
assert response.timing.get("total_ms", 0) > 0
@pytest.mark.asyncio
async def test_source_status_absent_is_tolerated(
self, client_with_recorded_response
):
async def test_source_status_absent_is_tolerated(self, client_with_recorded_response):
"""Recorded response predates source_status/degraded - defaults apply."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
@@ -218,7 +204,9 @@ class TestHybridRAGContract:
assert response.source_status["volatile"] == "disabled"
@pytest.mark.asyncio
async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch):
async def test_tool_renders_no_unknown_results(
self, client_with_recorded_response, monkeypatch
):
"""The hybrid_search tool renders real sources and non-zero scores."""
class _Factory:
@@ -231,9 +219,7 @@ class TestHybridRAGContract:
async def __aexit__(self, *args):
return None
monkeypatch.setattr(
"src.agents.librarian.tools.LibraryDeskClient", _Factory()
)
monkeypatch.setattr("src.agents.librarian.tools.LibraryDeskClient", _Factory())
output = await hybrid_search("home server infrastructure")
@@ -278,9 +264,7 @@ class TestCoverageNote:
def test_wiki_leg_absence_is_not_degradation(self):
"""vector/graph missing from top-N counts is healthy ranking, not outage."""
response = self._response(
source_counts={"web": 2, "documents": 1, "volatile": 1}
)
response = self._response(source_counts={"web": 2, "documents": 1, "volatile": 1})
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
tests/agents/librarian/test_tool_schemas.py
+1 -3
View File
@@ -35,9 +35,7 @@ def _nullable_anyof_paths(schema: object, path: str = "") -> list[str]:
class TestLibrarianToolSchemas:
"""All registered librarian tools emit Ollama-safe parameter schemas."""
@pytest.mark.parametrize(
"tool_func", LIBRARIAN_TOOLS, ids=lambda f: f.__name__
)
@pytest.mark.parametrize("tool_func", LIBRARIAN_TOOLS, ids=lambda f: f.__name__)
def test_no_nullable_anyof_in_schema(self, tool_func):
schema = Tool(tool_func).function_schema.json_schema
tests/agents/librarian/test_tools.py
+26 -35
View File
@@ -36,6 +36,7 @@ def mock_client():
# Web Search Tests
# ============================================================================
@pytest.mark.unit
class TestSearchWeb:
"""Tests for search_web tool."""
@@ -68,9 +69,7 @@ class TestSearchWeb:
)
mock_client.search_web.return_value = mock_response
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -94,9 +93,7 @@ class TestSearchWeb:
)
mock_client.search_web.return_value = mock_response
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -111,9 +108,7 @@ class TestSearchWeb:
"Connection failed to http://internal-host:8089"
)
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -144,9 +139,7 @@ class TestSearchWeb:
)
mock_client.search_web.return_value = mock_response
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -164,6 +157,7 @@ class TestSearchWeb:
# Read URL Tests
# ============================================================================
@pytest.mark.unit
class TestReadUrl:
"""Tests for read_url tool."""
@@ -182,9 +176,7 @@ class TestReadUrl:
)
mock_client.extract_content.return_value = mock_result
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -205,9 +197,7 @@ class TestReadUrl:
)
mock_client.extract_content.return_value = mock_result
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -227,9 +217,7 @@ class TestReadUrl:
)
mock_client.extract_content.return_value = mock_result
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
@@ -246,6 +234,7 @@ class TestReadUrl:
# Batch URL Tests
# ============================================================================
@pytest.mark.unit
class TestReadUrlsBatch:
"""Tests for read_urls_batch tool."""
@@ -275,16 +264,16 @@ class TestReadUrlsBatch:
)
mock_client.extract_content_batch.return_value = mock_response
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
result = await read_urls_batch([
"https://example.com/1",
"https://example.com/2",
])
result = await read_urls_batch(
[
"https://example.com/1",
"https://example.com/2",
]
)
assert "Article 1" in result
assert "Article 2" in result
@@ -314,16 +303,16 @@ class TestReadUrlsBatch:
)
mock_client.extract_content_batch.return_value = mock_response
with patch(
"src.agents.librarian.tools.LibraryDeskClient"
) as mock_client_class:
with patch("src.agents.librarian.tools.LibraryDeskClient") as mock_client_class:
mock_client_class.return_value.__aenter__.return_value = mock_client
mock_client_class.return_value.__aexit__.return_value = None
result = await read_urls_batch([
"https://example.com/good",
"https://example.com/bad",
])
result = await read_urls_batch(
[
"https://example.com/good",
"https://example.com/bad",
]
)
# Should contain successful result
assert "Good Article" in result
@@ -336,6 +325,7 @@ class TestReadUrlsBatch:
# Response Model Tests
# ============================================================================
@pytest.mark.unit
class TestWebSearchModels:
"""Tests for web search response models."""
@@ -436,6 +426,7 @@ class TestWebSearchModels:
# Wiki Update Tests (tag sentinel behavior)
# ============================================================================
@pytest.mark.unit
class TestUpdateWikiPageTagSentinels:
"""Empty list leaves tags unchanged; the clear sentinel empties them."""
tests/agents/steward/test_steward_schemas.py
+3 -4
View File
@@ -3,7 +3,6 @@ Tests for Steward schemas.
Tests the structured output models for conversation context and recommendations.
"""
import pytest
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
@@ -24,7 +23,7 @@ class TestConversationContext:
context = ConversationContext(
has_previous_context=True,
relevant_turns=[0, 2, 4],
context_summary="User discussed weather in turns 0 and 2"
context_summary="User discussed weather in turns 0 and 2",
)
assert context.has_previous_context is True
@@ -89,7 +88,7 @@ class TestStewardRecommendation:
context = ConversationContext(
has_previous_context=True,
relevant_turns=[1, 3],
context_summary="User asked about calculation in turn 1, now wants explanation"
context_summary="User asked about calculation in turn 1, now wants explanation",
)
rec = StewardRecommendation(
@@ -122,7 +121,7 @@ class TestStewardRecommendation:
context = ConversationContext(
has_previous_context=True,
relevant_turns=[0],
context_summary="Previous calculation mentioned"
context_summary="Previous calculation mentioned",
)
rec = StewardRecommendation(
tests/agents/steward/test_steward_service.py
+17 -24
View File
@@ -3,6 +3,7 @@ Tests for Steward service layer.
Tests request analysis, logging, and benchmarking integration.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -31,7 +32,9 @@ class TestAnalyzeRequest:
"""Test analyzing a simple greeting."""
# Mock the Steward agent's analyze method (plain text approach)
mock_agent = MagicMock()
mock_agent.analyze = AsyncMock(return_value="Simple greeting requires no tools. This is a simple request.")
mock_agent.analyze = AsyncMock(
return_value="Simple greeting requires no tools. This is a simple request."
)
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
result = await analyze_request(
@@ -159,7 +162,7 @@ class TestFormatStewardNote:
context = ConversationContext(
has_previous_context=True,
relevant_turns=[0, 1],
context_summary="Previous discussion about calculations"
context_summary="Previous discussion about calculations",
)
rec = StewardRecommendation(
@@ -205,9 +208,7 @@ class TestBuildEnrichedQuery:
def test_enrichment_adds_location(self):
"""Test location is appended for weather queries."""
query = "What's the weather?"
memory_context = {
"profile": {"location": "Amsterdam", "timezone": "Europe/Amsterdam"}
}
memory_context = {"profile": {"location": "Amsterdam", "timezone": "Europe/Amsterdam"}}
result = _build_enriched_query(query, memory_context)
@@ -218,9 +219,7 @@ class TestBuildEnrichedQuery:
def test_no_location_when_specified(self):
"""Test location is not appended when already specified."""
query = "What's the weather in London?"
memory_context = {
"profile": {"location": "Amsterdam"}
}
memory_context = {"profile": {"location": "Amsterdam"}}
result = _build_enriched_query(query, memory_context)
@@ -230,9 +229,7 @@ class TestBuildEnrichedQuery:
def test_enrichment_adds_timezone(self):
"""Test timezone is appended for time queries."""
query = "What time is it?"
memory_context = {
"profile": {"timezone": "Europe/Amsterdam"}
}
memory_context = {"profile": {"timezone": "Europe/Amsterdam"}}
result = _build_enriched_query(query, memory_context)
@@ -241,9 +238,7 @@ class TestBuildEnrichedQuery:
def test_no_timezone_when_specified(self):
"""Test timezone is not appended when already specified."""
query = "What time is it in UTC?"
memory_context = {
"profile": {"timezone": "Europe/Amsterdam"}
}
memory_context = {"profile": {"timezone": "Europe/Amsterdam"}}
result = _build_enriched_query(query, memory_context)
@@ -254,7 +249,7 @@ class TestBuildEnrichedQuery:
query = "What's the weather?"
memory_context = {
"profile": {"location": "Amsterdam"},
"preferences": {"temperature_unit": "celsius"}
"preferences": {"temperature_unit": "celsius"},
}
result = _build_enriched_query(query, memory_context)
@@ -265,11 +260,8 @@ class TestBuildEnrichedQuery:
"""Test multiple context fields are appended."""
query = "What time and weather today?"
memory_context = {
"profile": {
"location": "Amsterdam",
"timezone": "Europe/Amsterdam"
},
"preferences": {"temperature_unit": "celsius"}
"profile": {"location": "Amsterdam", "timezone": "Europe/Amsterdam"},
"preferences": {"temperature_unit": "celsius"},
}
result = _build_enriched_query(query, memory_context)
@@ -281,9 +273,7 @@ class TestBuildEnrichedQuery:
def test_no_enrichment_for_unrelated_query(self):
"""Test no enrichment for queries that don't need context."""
query = "Tell me a joke"
memory_context = {
"profile": {"location": "Amsterdam", "timezone": "Europe/Amsterdam"}
}
memory_context = {"profile": {"location": "Amsterdam", "timezone": "Europe/Amsterdam"}}
result = _build_enriched_query(query, memory_context)
@@ -306,7 +296,10 @@ class TestExtractCapabilities:
("The user wants a description of the algorithm.", "script -> housekeeper"),
("I should discover what the answer is.", "cover -> housekeeper"),
("That sounds fantastic, let me compute it.", "fan -> housekeeper"),
("I acknowledge the request to add two numbers.", "knowledge/know -> librarian, biographer"),
(
"I acknowledge the request to add two numbers.",
"knowledge/know -> librarian, biographer",
),
("The user asks about the economy myth.", "my -> biographer"),
("Convert 98.6 Fahrenheit to Celsius.", "temperature is a housekeeper domain"),
]
tests/agents/test_delegation.py
+27 -13
View File
@@ -4,6 +4,7 @@ Tests for delegation infrastructure.
Tests the DelegationTask dataclass and delegation wrapper functions
that implement the agent-as-tool pattern.
"""
from unittest.mock import AsyncMock, patch
import pytest
@@ -41,9 +42,7 @@ class TestBuildDelegationContext:
assert "assistant: Docker is a container runtime." in context
def test_only_last_max_turns_kept(self):
history = [
{"role": "user", "content": f"message {i}"} for i in range(10)
]
history = [{"role": "user", "content": f"message {i}"} for i in range(10)]
context = build_delegation_context(history, max_turns=6)
@@ -60,9 +59,7 @@ class TestBuildDelegationContext:
assert "x" * 501 not in context
def test_structured_content_parts_tolerated(self):
history = [
{"role": "user", "content": [{"type": "text", "text": "hello there"}]}
]
history = [{"role": "user", "content": [{"type": "text", "text": "hello there"}]}]
context = build_delegation_context(history)
@@ -343,8 +340,12 @@ class TestHouseholdThinkMessages:
for action_type, messages in action_types.items():
for phase, msg in messages.items():
# Messages should NOT have <think> wrappers - they go to reasoning_content field
assert "<think>" not in msg, f"{expert}/{action_type}/{phase} should not have <think> wrapper"
assert "</think>" not in msg, f"{expert}/{action_type}/{phase} should not have </think> wrapper"
assert (
"<think>" not in msg
), f"{expert}/{action_type}/{phase} should not have <think> wrapper"
assert (
"</think>" not in msg
), f"{expert}/{action_type}/{phase} should not have </think> wrapper"
# Messages should be non-empty strings
assert isinstance(msg, str) and len(msg) > 0, f"{expert}/{action_type}/{phase}"
@@ -356,7 +357,9 @@ class TestDetectActionType:
def test_librarian_search_is_retrieve(self):
"""Test librarian search tasks are RETRIEVE."""
assert _detect_action_type("librarian", "search for Docker info") == ActionType.RETRIEVE
assert _detect_action_type("librarian", "find information about CI/CD") == ActionType.RETRIEVE
assert (
_detect_action_type("librarian", "find information about CI/CD") == ActionType.RETRIEVE
)
assert _detect_action_type("librarian", "look up Kubernetes docs") == ActionType.RETRIEVE
def test_librarian_web_search_is_research(self):
@@ -378,14 +381,25 @@ class TestDetectActionType:
def test_biographer_record_is_record(self):
"""Test biographer record tasks are RECORD."""
assert _detect_action_type("biographer", "remember that I work at Acme") == ActionType.RECORD
assert (
_detect_action_type("biographer", "remember that I work at Acme") == ActionType.RECORD
)
assert _detect_action_type("biographer", "note that my car is a Tesla") == ActionType.RECORD
assert _detect_action_type("biographer", "save my preference for dark mode") == ActionType.RECORD
assert (
_detect_action_type("biographer", "save my preference for dark mode")
== ActionType.RECORD
)
def test_housekeeper_status_is_retrieve(self):
"""Test housekeeper status tasks are RETRIEVE."""
assert _detect_action_type("housekeeper", "what devices are in the bedroom?") == ActionType.RETRIEVE
assert _detect_action_type("housekeeper", "is the living room light on?") == ActionType.RETRIEVE
assert (
_detect_action_type("housekeeper", "what devices are in the bedroom?")
== ActionType.RETRIEVE
)
assert (
_detect_action_type("housekeeper", "is the living room light on?")
== ActionType.RETRIEVE
)
def test_housekeeper_control_is_control(self):
"""Test housekeeper control tasks are CONTROL."""
tests/agents/test_lorem_tester.py
+2 -2
View File
@@ -6,9 +6,9 @@ import pytest
from src.agents.lorem_tester import LoremTesterAgent
from src.core.exceptions import (
RateLimitError,
ContextLengthError,
APIError,
ContextLengthError,
RateLimitError,
)
tests/agents/test_orchestration.py
+67 -42
View File
@@ -5,23 +5,25 @@ Tests the multi-expert coordination infrastructure including
delegation parsing, think updates, result handling, and
multi-expert sequential/parallel execution.
"""
import pytest
from unittest.mock import AsyncMock, patch
import pytest
from src.agents.delegation import DelegationResult, DelegationTask
from src.agents.orchestration import (
OrchestrationContext,
parse_delegation_from_steward_note,
execute_delegation,
orchestrate_with_think_updates,
extract_delegation_context,
ExecutionMode,
MultiExpertResult,
execute_sequential,
execute_parallel,
orchestrate_multi_expert,
OrchestrationContext,
_get_display_name,
execute_delegation,
execute_parallel,
execute_sequential,
extract_delegation_context,
orchestrate_multi_expert,
orchestrate_with_think_updates,
parse_delegation_from_steward_note,
)
from src.agents.delegation import DelegationTask, DelegationResult
@pytest.mark.unit
@@ -364,6 +366,7 @@ class TestOrchestrationContext:
# Multi-Expert Coordination Tests
# ============================================================================
@pytest.mark.unit
class TestMultiExpertResult:
"""Tests for MultiExpertResult aggregation."""
@@ -414,18 +417,22 @@ class TestMultiExpertResult:
"""Test aggregating outputs from multiple experts."""
result = MultiExpertResult()
result.add_result(DelegationResult(
expert_name="librarian",
task="search docs",
success=True,
output="Found Docker docs",
))
result.add_result(DelegationResult(
expert_name="memory",
task="get preferences",
success=True,
output="User prefers dark mode",
))
result.add_result(
DelegationResult(
expert_name="librarian",
task="search docs",
success=True,
output="Found Docker docs",
)
)
result.add_result(
DelegationResult(
expert_name="memory",
task="get preferences",
success=True,
output="User prefers dark mode",
)
)
combined = result.aggregate_outputs()
@@ -438,19 +445,23 @@ class TestMultiExpertResult:
"""Test that failed results are excluded from aggregate."""
result = MultiExpertResult()
result.add_result(DelegationResult(
expert_name="librarian",
task="search",
success=True,
output="Success output",
))
result.add_result(DelegationResult(
expert_name="memory",
task="get",
success=False,
output="",
error="Failed",
))
result.add_result(
DelegationResult(
expert_name="librarian",
task="search",
success=True,
output="Success output",
)
)
result.add_result(
DelegationResult(
expert_name="memory",
task="get",
success=False,
output="",
error="Failed",
)
)
combined = result.aggregate_outputs()
@@ -471,7 +482,9 @@ class TestExecuteSequential:
]
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="Result 1"),
DelegationResult(
expert_name="librarian", task="task 1", success=True, output="Result 1"
),
DelegationResult(expert_name="memory", task="task 2", success=True, output="Result 2"),
]
@@ -496,7 +509,9 @@ class TestExecuteSequential:
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="OK"),
DelegationResult(expert_name="memory", task="task 2", success=False, output="", error="Failed"),
DelegationResult(
expert_name="memory", task="task 2", success=False, output="", error="Failed"
),
]
with patch(
@@ -520,7 +535,9 @@ class TestExecuteSequential:
]
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=False, output="", error="Error"),
DelegationResult(
expert_name="librarian", task="task 1", success=False, output="", error="Error"
),
]
with patch(
@@ -548,7 +565,9 @@ class TestExecuteParallel:
]
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="Result 1"),
DelegationResult(
expert_name="librarian", task="task 1", success=True, output="Result 1"
),
DelegationResult(expert_name="memory", task="task 2", success=True, output="Result 2"),
]
@@ -572,7 +591,9 @@ class TestExecuteParallel:
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="OK"),
DelegationResult(expert_name="memory", task="task 2", success=False, output="", error="Timeout"),
DelegationResult(
expert_name="memory", task="task 2", success=False, output="", error="Timeout"
),
]
with patch(
@@ -629,7 +650,9 @@ class TestOrchestrateMultiExpert:
]
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="Result 1"),
DelegationResult(
expert_name="librarian", task="task 1", success=True, output="Result 1"
),
DelegationResult(expert_name="memory", task="task 2", success=True, output="Result 2"),
]
@@ -658,7 +681,9 @@ class TestOrchestrateMultiExpert:
]
mock_results = [
DelegationResult(expert_name="librarian", task="task 1", success=True, output="Result 1"),
DelegationResult(
expert_name="librarian", task="task 1", success=True, output="Result 1"
),
DelegationResult(expert_name="memory", task="task 2", success=True, output="Result 2"),
]
tests/agents/test_protocol.py
+1
View File
@@ -1,6 +1,7 @@
"""
Tests for the agent error protocol.
"""
import pytest
from src.agents.protocol import AgentError
tests/agents/test_registry.py
+5 -5
View File
@@ -4,8 +4,8 @@ Tests for model registry.
import pytest
from src.agents.registry import ModelRegistry
from src.agents.lorem_tester import LoremTesterAgent
from src.agents.registry import ModelRegistry
from src.agents.tatlock import TatlockAgent
from src.core.exceptions import ModelNotFoundError
@@ -63,10 +63,10 @@ async def test_tatlock_capabilities():
# Tatlock Phase 1 - basic streaming, reasoning, and permanent tools
assert capabilities["streaming"] is True
assert capabilities["reasoning"] is True # Basic reasoning summaries
assert capabilities["tools"] is True # Permanent tools: calculator, date/time, search
assert capabilities["vision"] is False # Future
assert capabilities["audio"] is False # Future
assert capabilities["reasoning"] is True # Basic reasoning summaries
assert capabilities["tools"] is True # Permanent tools: calculator, date/time, search
assert capabilities["vision"] is False # Future
assert capabilities["audio"] is False # Future
@pytest.mark.unit
tests/agents/test_tatlock_agent.py
+94 -132
View File
@@ -6,9 +6,7 @@ These tests verify:
2. Tool calls are logged to reasoning output (users see what tools are doing)
"""
import json
import pytest
from unittest.mock import patch, AsyncMock
from httpx import AsyncClient
@@ -28,14 +26,10 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
"messages": [
{"role": "user", "content": "My name is Alice and I love Python programming."}
],
"stream": False
"stream": False,
}
response_1 = await async_client.post(
"/v1/chat/completions",
json=request_data_1,
timeout=120.0
)
response_1 = await async_client.post("/v1/chat/completions", json=request_data_1, timeout=120.0)
assert response_1.status_code == 200
data_1 = response_1.json()
@@ -48,16 +42,15 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
"messages": [
{"role": "user", "content": "My name is Alice and I love Python programming."},
{"role": "assistant", "content": first_response},
{"role": "user", "content": "What did I say my name was? And what programming language did I mention?"}
{
"role": "user",
"content": "What did I say my name was? And what programming language did I mention?",
},
],
"stream": False
"stream": False,
}
response_2 = await async_client.post(
"/v1/chat/completions",
json=request_data_2,
timeout=120.0
)
response_2 = await async_client.post("/v1/chat/completions", json=request_data_2, timeout=120.0)
assert response_2.status_code == 200
data_2 = response_2.json()
@@ -68,7 +61,9 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
has_python = "python" in second_response
if not has_alice or not has_python:
pytest.xfail(f"LLM did not remember context (non-deterministic): alice={has_alice}, python={has_python}, response: {second_response[:200]}")
pytest.xfail(
f"LLM did not remember context (non-deterministic): alice={has_alice}, python={has_python}, response: {second_response[:200]}"
)
@pytest.mark.integration
@@ -86,48 +81,37 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
# Turn 1: Set up a topic
conversation.append({"role": "user", "content": "Let's talk about the number 42."})
request_1 = {
"model": "Tatlock",
"messages": conversation.copy(),
"stream": False
}
request_1 = {"model": "Tatlock", "messages": conversation.copy(), "stream": False}
response_1 = await async_client.post(
"/v1/chat/completions",
json=request_1,
timeout=120.0
)
response_1 = await async_client.post("/v1/chat/completions", json=request_1, timeout=120.0)
assert response_1.status_code == 200
data_1 = response_1.json()
conversation.append({
"role": "assistant",
"content": data_1["choices"][0]["message"]["content"]
})
conversation.append(
{"role": "assistant", "content": data_1["choices"][0]["message"]["content"]}
)
# Turn 2: Reference "it" (should refer to 42)
conversation.append({"role": "user", "content": "What number did I just mention?"})
request_2 = {
"model": "Tatlock",
"messages": conversation.copy(),
"stream": False
}
request_2 = {"model": "Tatlock", "messages": conversation.copy(), "stream": False}
response_2 = await async_client.post(
"/v1/chat/completions",
json=request_2,
timeout=120.0
)
response_2 = await async_client.post("/v1/chat/completions", json=request_2, timeout=120.0)
assert response_2.status_code == 200
data_2 = response_2.json()
final_response = data_2["choices"][0]["message"]["content"]
# Should reference 42 (check both as digit and word)
has_42 = "42" in final_response or "forty-two" in final_response.lower() or "forty two" in final_response.lower()
has_42 = (
"42" in final_response
or "forty-two" in final_response.lower()
or "forty two" in final_response.lower()
)
if not has_42:
pytest.xfail(f"LLM did not mention 42 in response (non-deterministic): {final_response[:200]}")
pytest.xfail(
f"LLM did not mention 42 in response (non-deterministic): {final_response[:200]}"
)
@pytest.mark.integration
@@ -142,16 +126,15 @@ async def test_tatlock_tool_call_logging_search(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Search for current information about Python 3.13 release date"}
{
"role": "user",
"content": "Search for current information about Python 3.13 release date",
}
],
"stream": False
"stream": False,
}
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=120.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=120.0)
assert response.status_code == 200
data = response.json()
@@ -167,8 +150,9 @@ async def test_tatlock_tool_call_logging_search(async_client: AsyncClient):
# If search was used, should show the 🔍 emoji
if "🔍" in full_response:
assert "search" in full_response.lower() or "python" in full_response.lower(), \
"Search query should be visible in the response"
assert (
"search" in full_response.lower() or "python" in full_response.lower()
), "Search query should be visible in the response"
@pytest.mark.integration
@@ -183,45 +167,38 @@ async def test_tatlock_tool_call_logging_calculator(async_client: AsyncClient):
"""
request_data = {
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is the square root of 144 plus 25?"}
],
"stream": False
"messages": [{"role": "user", "content": "What is the square root of 144 plus 25?"}],
"stream": False,
}
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=120.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=120.0)
assert response.status_code == 200
data = response.json()
full_response = data["choices"][0]["message"]["content"]
# Should have reasoning in <think> tags (from Steward analysis)
assert "<think>" in full_response, \
f"Should have reasoning output in <think> tags. Got: {full_response}"
assert (
"<think>" in full_response
), f"Should have reasoning output in <think> tags. Got: {full_response}"
# Should reference the calculation in some form
has_calculation_reference = (
"144" in full_response or
"sqrt" in full_response.lower() or
"square root" in full_response.lower()
"144" in full_response
or "sqrt" in full_response.lower()
or "square root" in full_response.lower()
)
assert has_calculation_reference, \
f"Should reference the calculation. Got: {full_response}"
assert has_calculation_reference, f"Should reference the calculation. Got: {full_response}"
# Should have the correct answer (37)
assert "37" in full_response, \
f"Should contain the answer 37. Got: {full_response}"
assert "37" in full_response, f"Should contain the answer 37. Got: {full_response}"
# Tool emoji is optional - depends on whether tool was used directly
# or computation was delegated to capability
if "🧮" in full_response:
print(f"\nCalculator tool was used directly")
print("\nCalculator tool was used directly")
else:
print(f"\nCalculation handled via tatlock_core capability")
print("\nCalculation handled via tatlock_core capability")
print(f"\nCalculator response: {full_response}")
@@ -234,17 +211,11 @@ async def test_tatlock_tool_call_logging_datetime(async_client: AsyncClient):
"""
request_data = {
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What was the date exactly 2 weeks ago?"}
],
"stream": False
"messages": [{"role": "user", "content": "What was the date exactly 2 weeks ago?"}],
"stream": False,
}
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=120.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=120.0)
assert response.status_code == 200
data = response.json()
@@ -257,19 +228,36 @@ async def test_tatlock_tool_call_logging_datetime(async_client: AsyncClient):
used_date_tool = "🕐" in full_response
# Should mention the calculation or the timeframe
assert "2 weeks ago" in full_response.lower() or "weeks" in full_response.lower(), \
f"Should reference the requested timeframe. Got: {full_response}"
assert (
"2 weeks ago" in full_response.lower() or "weeks" in full_response.lower()
), f"Should reference the requested timeframe. Got: {full_response}"
# Should provide a specific date (either YYYY-MM-DD format or natural language like "November 23")
import re
has_iso_date = bool(re.search(r'\d{4}-\d{2}-\d{2}', full_response))
has_month_mention = any(month in full_response.lower() for month in
['january', 'february', 'march', 'april', 'may', 'june',
'july', 'august', 'september', 'october', 'november', 'december'])
has_date_number = bool(re.search(r'\b\d{1,2}(st|nd|rd|th)?\b', full_response.lower()))
assert has_iso_date or has_month_mention or has_date_number, \
f"Should contain a specific date. Got: {full_response}"
has_iso_date = bool(re.search(r"\d{4}-\d{2}-\d{2}", full_response))
has_month_mention = any(
month in full_response.lower()
for month in [
"january",
"february",
"march",
"april",
"may",
"june",
"july",
"august",
"september",
"october",
"november",
"december",
]
)
has_date_number = bool(re.search(r"\b\d{1,2}(st|nd|rd|th)?\b", full_response.lower()))
assert (
has_iso_date or has_month_mention or has_date_number
), f"Should contain a specific date. Got: {full_response}"
print(f"\nDate/time response (tool used: {used_date_tool}): {full_response}")
@@ -284,17 +272,11 @@ async def test_tatlock_no_tool_calls_no_logging(async_client: AsyncClient):
"""
request_data = {
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Just say hello to me."}
],
"stream": False
"messages": [{"role": "user", "content": "Just say hello to me."}],
"stream": False,
}
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=120.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=120.0)
assert response.status_code == 200
data = response.json()
@@ -327,17 +309,9 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
# Turn 1: Do a calculation
conversation.append({"role": "user", "content": "Calculate 15 times 7 for me."})
request_1 = {
"model": "Tatlock",
"messages": conversation.copy(),
"stream": False
}
request_1 = {"model": "Tatlock", "messages": conversation.copy(), "stream": False}
response_1 = await async_client.post(
"/v1/chat/completions",
json=request_1,
timeout=120.0
)
response_1 = await async_client.post("/v1/chat/completions", json=request_1, timeout=120.0)
assert response_1.status_code == 200
data_1 = response_1.json()
@@ -353,17 +327,9 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
# Turn 2: Ask about previous calculation
conversation.append({"role": "user", "content": "What calculation did I just ask you to do?"})
request_2 = {
"model": "Tatlock",
"messages": conversation.copy(),
"stream": False
}
request_2 = {"model": "Tatlock", "messages": conversation.copy(), "stream": False}
response_2 = await async_client.post(
"/v1/chat/completions",
json=request_2,
timeout=120.0
)
response_2 = await async_client.post("/v1/chat/completions", json=request_2, timeout=120.0)
assert response_2.status_code == 200
data_2 = response_2.json()
@@ -371,13 +337,15 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
# Should remember the calculation (either as digits or words)
has_calculation = (
("15" in second_response and "7" in second_response) or # As digits
("fifteen" in second_response and "seven" in second_response) or # As words
"105" in second_response or # As answer
"multipl" in second_response # Mentions multiplication
("15" in second_response and "7" in second_response) # As digits
or ("fifteen" in second_response and "seven" in second_response) # As words
or "105" in second_response # As answer
or "multipl" in second_response # Mentions multiplication
)
if not has_calculation:
pytest.xfail(f"LLM did not remember calculation (non-deterministic): {second_response[:200]}")
pytest.xfail(
f"LLM did not remember calculation (non-deterministic): {second_response[:200]}"
)
@pytest.mark.integration
@@ -404,19 +372,13 @@ async def test_tatlock_ollama_fallback(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Say hello to me."}
],
"stream": False
"messages": [{"role": "user", "content": "Say hello to me."}],
"stream": False,
}
# 300s: this test forbids the Claude rescue, and the full local
# Steward -> orchestrate -> synthesize flow on gemma4 exceeds 120s
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=300.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=300.0)
assert response.status_code == 200
data = response.json()
tests/agents/test_tools.py
+3 -3
View File
@@ -5,21 +5,20 @@ Note: Web search has been moved to The Librarian agent.
See tests/agents/librarian/test_tools.py for search tests.
"""
import pytest
from datetime import datetime
from src.agents.tools import (
calculate,
get_current_datetime,
calculate_time_offset,
get_current_datetime,
time_difference,
)
# ============================================================================
# Calculator Tests
# ============================================================================
class TestCalculator:
"""Tests for the calculator tool."""
@@ -81,6 +80,7 @@ class TestCalculator:
# Date/Time Tests
# ============================================================================
class TestDateTime:
"""Tests for date/time toolkit."""
tests/anthropic/test_model_selector.py
+1
View File
@@ -4,6 +4,7 @@ Unit tests for backend selection (Ollama primary, Claude fallback).
These tests set the cached health-check globals directly so they are
deterministic regardless of which services are reachable.
"""
import pytest
from src.anthropic import model_selector
tests/chat/test_router.py
+11 -10
View File
@@ -1,6 +1,7 @@
"""
Tests for chat completions router.
"""
import json
import pytest
@@ -17,24 +18,24 @@ def test_chat_completion_non_streaming(
) -> None:
"""Test non-streaming chat completion."""
response = client.post("/v1/chat/completions", json=mock_chat_request)
assert response.status_code == 200
data = response.json()
# Verify response structure
assert "id" in data
assert data["object"] == constants.CHAT_COMPLETION_OBJECT
assert "created" in data
assert data["model"] == mock_chat_request["model"]
assert len(data["choices"]) == 1
# Verify choice structure
choice = data["choices"][0]
assert choice["index"] == 0
assert choice["message"]["role"] == constants.ROLE_ASSISTANT
assert choice["message"]["content"] # Should have content
assert choice["finish_reason"] == constants.FINISH_REASON_STOP
# Verify usage
assert "usage" in data
assert data["usage"]["prompt_tokens"] > 0
@@ -47,9 +48,9 @@ def test_chat_completion_validation_error(client: TestClient) -> None:
"""Test chat completion with invalid request."""
# Missing required field 'messages'
invalid_request = {"model": "Tatlock"}
response = client.post("/v1/chat/completions", json=invalid_request)
assert response.status_code == 422
data = response.json()
assert "error" in data
@@ -108,7 +109,7 @@ async def test_chat_completion_streaming(
# Execute with overall timeout
try:
chunks = await asyncio.wait_for(read_stream_with_timeout(), timeout=20.0)
except asyncio.TimeoutError:
except TimeoutError:
pytest.fail("Streaming test timed out after 20 seconds")
# Verify we got chunks
@@ -133,7 +134,7 @@ def test_chat_completion_temperature_validation(
invalid_request = {**mock_chat_request, "temperature": 3.0}
response = client.post("/v1/chat/completions", json=invalid_request)
assert response.status_code == 422
# Valid temperature
valid_request = {**mock_chat_request, "temperature": 0.5}
response = client.post("/v1/chat/completions", json=valid_request)
@@ -153,7 +154,7 @@ def test_chat_completion_message_roles(
{"role": "user", "content": "Hello!"},
],
}
response = client.post("/v1/chat/completions", json=request_with_system)
assert response.status_code == 200
@@ -168,6 +169,6 @@ def test_chat_completion_invalid_role(
**mock_chat_request,
"messages": [{"role": "invalid_role", "content": "test"}],
}
response = client.post("/v1/chat/completions", json=invalid_request)
assert response.status_code == 422
tests/chat/test_streaming_wrapper.py
+15 -27
View File
@@ -7,7 +7,9 @@ Tests that the wrapper correctly:
- Streams reasoning via reasoning_content field (DeepSeek R1 format)
- Streams both reasoning and content
"""
import json
import pytest
from httpx import AsyncClient
@@ -20,10 +22,8 @@ async def test_streaming_wrapper_enables_reasoning(async_client: AsyncClient):
"""Test that streaming wrapper automatically enables reasoning via reasoning_content."""
request_data = {
"model": "lorem-tester",
"messages": [
{"role": "user", "content": "Test message"}
],
"stream": True
"messages": [{"role": "user", "content": "Test message"}],
"stream": True,
}
chunks_received = []
@@ -74,10 +74,8 @@ async def test_streaming_wrapper_reasoning_before_content(async_client: AsyncCli
"""Test that reasoning_content comes before regular content."""
request_data = {
"model": "lorem-tester",
"messages": [
{"role": "user", "content": "Explain something"}
],
"stream": True
"messages": [{"role": "user", "content": "Explain something"}],
"stream": True,
}
chunk_types = [] # Track order: 'reasoning' or 'content'
@@ -127,11 +125,9 @@ async def test_streaming_wrapper_proper_chunk_structure(async_client: AsyncClien
"""Test that streaming chunks have proper structure."""
request_data = {
"model": "lorem-tester",
"messages": [
{"role": "user", "content": "Hello"}
],
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 0.8,
"stream": True
"stream": True,
}
first_chunk = None
@@ -201,9 +197,9 @@ async def test_streaming_wrapper_with_system_message(async_client: AsyncClient):
"model": "lorem-tester",
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello"}
{"role": "user", "content": "Hello"},
],
"stream": True
"stream": True,
}
chunks_received = []
@@ -243,10 +239,8 @@ async def test_streaming_wrapper_pipeline_prefix(async_client: AsyncClient):
"""Test streaming with pipeline prefix in model name."""
request_data = {
"model": "some_pipeline.lorem-tester",
"messages": [
{"role": "user", "content": "Test"}
],
"stream": True
"messages": [{"role": "user", "content": "Test"}],
"stream": True,
}
chunks_received = []
@@ -285,17 +279,11 @@ async def test_streaming_wrapper_non_streaming_fallback(async_client: AsyncClien
"""Test that non-streaming request works through wrapper."""
request_data = {
"model": "lorem-tester",
"messages": [
{"role": "user", "content": "Hello"}
],
"stream": False # Non-streaming
"messages": [{"role": "user", "content": "Hello"}],
"stream": False, # Non-streaming
}
response = await async_client.post(
"/v1/chat/completions",
json=request_data,
timeout=20.0
)
response = await async_client.post("/v1/chat/completions", json=request_data, timeout=20.0)
assert response.status_code == 200
data = response.json()
tests/conftest.py
+6 -12
View File
@@ -2,11 +2,12 @@
Shared test fixtures for all tests.
Following FastAPI testing best practices.
"""
import asyncio
import pytest
from fastapi.testclient import TestClient
from httpx import AsyncClient, ASGITransport
from httpx import ASGITransport, AsyncClient
from src.main import app
@@ -28,10 +29,7 @@ def _tenant_guard():
from src.core.multi_tenancy import get_memory_collection_name
effective = get_default_user()
if (
effective == PRODUCTION_TENANT
or config.effective_default_user == PRODUCTION_TENANT
):
if effective == PRODUCTION_TENANT or config.effective_default_user == PRODUCTION_TENANT:
pytest.exit(
f"TENANT GUARD: refusing to run the test suite - the effective "
f"tenant resolves to the production tenant '{PRODUCTION_TENANT}' "
@@ -59,6 +57,7 @@ def _initialize_app(_tenant_guard):
production tenant before any initialization happens.
"""
from src.core.startup import initialize_application
asyncio.run(initialize_application())
@@ -79,10 +78,7 @@ async def async_client() -> AsyncClient:
Use for testing async endpoints and streaming.
"""
async with AsyncClient(
transport=ASGITransport(app=app),
base_url="http://test"
) as client:
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
yield client
@@ -91,9 +87,7 @@ def mock_chat_request() -> dict:
"""Standard chat completion request fixture."""
return {
"model": "lorem-tester",
"messages": [
{"role": "user", "content": "Hello, world!"}
],
"messages": [{"role": "user", "content": "Hello, world!"}],
"temperature": 0.7,
"stream": False,
}
tests/contracts/test_service_contracts.py
+5 -6
View File
@@ -11,6 +11,7 @@ Semantics:
Run with: make test-contracts
"""
import json
import httpx
@@ -66,9 +67,9 @@ class TestOllamaContract:
assert response.status_code == 200
names = [m["name"] for m in response.json()["models"]]
model = config.OLLAMA_DEFAULT_MODEL
assert model in names or f"{model}:latest" in names, (
f"{model} not pulled; available: {names}"
)
assert (
model in names or f"{model}:latest" in names
), f"{model} not pulled; available: {names}"
async def test_generate_returns_plain_text(self):
# Mirrors StewardAgent._call_ollama()
@@ -94,9 +95,7 @@ class TestOllamaContract:
"ollama",
{
"model": config.OLLAMA_DEFAULT_MODEL,
"messages": [
{"role": "user", "content": "What is 6 * 7? Use the calculator."}
],
"messages": [{"role": "user", "content": "What is 6 * 7? Use the calculator."}],
"tools": [CALCULATOR_TOOL],
"tool_choice": "required",
"stream": False,
tests/core/test_household_registry.py
+8 -4
View File
@@ -3,6 +3,7 @@ Tests for household registry.
Tests capability registration, toolset scoping, and coordination features.
"""
import pytest
from pydantic_ai.tools import Tool
@@ -10,7 +11,6 @@ from src.core.household_registry import (
HouseholdCapability,
HouseholdMember,
HouseholdRegistry,
household_registry,
)
@@ -38,6 +38,7 @@ def sample_capability():
@pytest.fixture
def sample_tools():
"""Sample tool definitions."""
def test_function_1(x: int) -> int:
"""Test function 1."""
return x * 2
@@ -96,6 +97,7 @@ class TestHouseholdMember:
def test_member_with_agent(self, sample_capability, sample_tools):
"""Test member can include an agent."""
from unittest.mock import Mock
mock_agent = Mock()
member = HouseholdMember(
@@ -322,7 +324,9 @@ class TestGetDelegationTools:
assert callable(tools[0])
assert tools[0].__name__ == "delegate_to_librarian"
def test_delegation_tools_returns_raw_tools_for_member_without_agent(self, registry, sample_capability, sample_tools):
def test_delegation_tools_returns_raw_tools_for_member_without_agent(
self, registry, sample_capability, sample_tools
):
"""Test delegation tools returns raw tools when member has no agent."""
registry.register("test_tools", sample_capability, sample_tools)
@@ -373,8 +377,8 @@ class TestGetDelegationTools:
assert tools[0].__name__ == "delegate_to_librarian"
# Rest should be raw tools
assert hasattr(tools[1], 'name')
assert hasattr(tools[2], 'name')
assert hasattr(tools[1], "name")
assert hasattr(tools[2], "name")
def test_delegation_tools_nonexistent_member(self, registry):
"""Test delegation tools handles non-existent member gracefully."""
tests/core/test_logging_config.py
+5 -6
View File
@@ -3,12 +3,11 @@ Tests for structured logging configuration.
Tests logging setup, context management, and FastAPI integration.
"""
import logging
from io import StringIO
from unittest.mock import patch
import pytest
import structlog
from src.core.logging_config import (
add_log_level,
@@ -50,10 +49,10 @@ class TestGetLogger:
"""Test get_logger returns structlog BoundLogger."""
logger = get_logger("test")
# Logger should have standard logging methods
assert hasattr(logger, 'info')
assert hasattr(logger, 'debug')
assert hasattr(logger, 'warning')
assert hasattr(logger, 'error')
assert hasattr(logger, "info")
assert hasattr(logger, "debug")
assert hasattr(logger, "warning")
assert hasattr(logger, "error")
def test_get_logger_with_module_name(self):
"""Test logger with module name."""
tests/core/test_memory_service.py
+3 -2
View File
@@ -2,13 +2,14 @@
Tests for the memory service (direct access layer).
"""
from unittest.mock import AsyncMock, patch
import pytest
from unittest.mock import MagicMock, patch, AsyncMock
from src.core.memory_service import (
MemoryRecord,
MemoryService,
MemoryType,
MemoryRecord,
memory_service,
)
tests/core/test_router.py
+4 -3
View File
@@ -1,6 +1,7 @@
"""
Tests for core router (health check and root endpoints).
"""
import pytest
from fastapi.testclient import TestClient
@@ -11,7 +12,7 @@ from src.core.config import config
def test_health_check(client: TestClient) -> None:
"""Test health check endpoint returns healthy status."""
response = client.get("/health")
assert response.status_code == 200
data = response.json()
assert data["status"] == "healthy"
@@ -22,7 +23,7 @@ def test_health_check(client: TestClient) -> None:
def test_root_endpoint(client: TestClient) -> None:
"""Test root endpoint returns API information."""
response = client.get("/")
assert response.status_code == 200
data = response.json()
assert data["name"] == config.APP_NAME
@@ -42,7 +43,7 @@ def test_openapi_schema(client: TestClient) -> None:
"""Test that OpenAPI schema is accessible."""
response = client.get("/openapi.json")
assert response.status_code == 200
schema = response.json()
assert schema["info"]["title"] == config.APP_NAME
assert schema["info"]["version"] == config.APP_VERSION
tests/core/test_tenant_guard.py
+4 -10
View File
@@ -157,9 +157,7 @@ class TestRequestContextGuard:
"jpmschweitzer!",
],
)
@pytest.mark.parametrize(
"environment", [Environment.DEVELOPMENT, Environment.TESTING]
)
@pytest.mark.parametrize("environment", [Environment.DEVELOPMENT, Environment.TESTING])
def test_production_tenant_sanitization_variants_are_forced(
self, monkeypatch, environment, variant
):
@@ -175,9 +173,8 @@ class TestRequestContextGuard:
with RequestContext(user=variant):
effective = get_user()
assert effective == TEST_TENANT
assert (
get_memory_collection_name(effective)
!= get_memory_collection_name(PRODUCTION_TENANT)
assert get_memory_collection_name(effective) != get_memory_collection_name(
PRODUCTION_TENANT
)
def test_dev_non_colliding_user_is_not_forced(self, monkeypatch):
@@ -228,10 +225,7 @@ class TestSuiteRunsUnderTestTenant:
from src.core.context import get_default_user
from src.core.multi_tenancy import get_memory_collection_name
assert (
get_memory_collection_name(get_default_user())
== f"memories_{TEST_TENANT}"
)
assert get_memory_collection_name(get_default_user()) == f"memories_{TEST_TENANT}"
def test_redis_session_namespace_is_test_tenant(self):
from src.core.context import get_default_user
tests/core/test_tool_tracking.py
+5 -12
View File
@@ -3,6 +3,7 @@ Tests for tool call tracking.
Tests capability extraction and recommendation matching.
"""
import pytest
from src.core.tool_tracking import ToolCallTracker
@@ -29,9 +30,7 @@ class TestToolCallTracker:
@pytest.mark.asyncio
async def test_track_call_recognizes_delegation_as_recommended(self):
"""Test that delegate_to_X is recognized when X is recommended."""
tracker = ToolCallTracker(
recommended_capabilities=["librarian", "biographer"]
)
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
await tracker.track_call("delegate_to_librarian", 1.0)
@@ -42,9 +41,7 @@ class TestToolCallTracker:
@pytest.mark.asyncio
async def test_track_call_detects_not_recommended(self):
"""Test that unrecommended tools are flagged."""
tracker = ToolCallTracker(
recommended_capabilities=["librarian"]
)
tracker = ToolCallTracker(recommended_capabilities=["librarian"])
await tracker.track_call("delegate_to_housekeeper", 1.0)
@@ -55,9 +52,7 @@ class TestToolCallTracker:
def test_get_summary_with_delegation_tools(self):
"""Test summary correctly maps delegation tools to capabilities."""
tracker = ToolCallTracker(
recommended_capabilities=["librarian", "biographer"]
)
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
tracker.actual_calls = {
"delegate_to_librarian": [1.0, 2.0],
"delegate_to_housekeeper": [0.5], # Not recommended
@@ -72,9 +67,7 @@ class TestToolCallTracker:
@pytest.mark.asyncio
async def test_finalize_with_delegation_tools(self):
"""Test finalize correctly identifies unused recommendations."""
tracker = ToolCallTracker(
recommended_capabilities=["librarian", "biographer"]
)
tracker = ToolCallTracker(recommended_capabilities=["librarian", "biographer"])
tracker.actual_calls = {
"delegate_to_librarian": [1.0],
}
tests/e2e/test_api_endpoints.py
+66 -86
View File
@@ -7,10 +7,12 @@ These tests hit the actual running server and test the full stack:
- Tool execution
- Response formatting
"""
from collections.abc import AsyncGenerator
import httpx
import pytest
import pytest_asyncio
import httpx
from typing import AsyncGenerator
# Test server base URL (assumes server is running on localhost:8777 via ./wakeup.sh)
BASE_URL = "http://localhost:8777"
@@ -34,10 +36,8 @@ class TestChatCompletionsE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is 144 divided by 12?"}
],
}
"messages": [{"role": "user", "content": "What is 144 divided by 12?"}],
},
)
assert response.status_code == 200
@@ -68,7 +68,9 @@ class TestChatCompletionsE2E:
assert "usage" in data
assert data["usage"]["total_tokens"] > 0
print(f"✓ Calculator test passed. Found '12' in response. Tool indicator: {has_calculator_indicator}")
print(
f"✓ Calculator test passed. Found '12' in response. Tool indicator: {has_calculator_indicator}"
)
@pytest.mark.asyncio
async def test_web_search(self, client: httpx.AsyncClient):
@@ -80,7 +82,7 @@ class TestChatCompletionsE2E:
"messages": [
{"role": "user", "content": "Search for the current population of Tokyo"}
],
}
},
)
assert response.status_code == 200
@@ -114,10 +116,8 @@ class TestChatCompletionsE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is 15 times 4?"}
],
}
"messages": [{"role": "user", "content": "What is 15 times 4?"}],
},
)
assert response1.status_code == 200
@@ -135,9 +135,9 @@ class TestChatCompletionsE2E:
"messages": [
{"role": "user", "content": "What is 15 times 4?"},
{"role": "assistant", "content": message1},
{"role": "user", "content": "Now add 20 to that result."}
{"role": "user", "content": "Now add 20 to that result."},
],
}
},
)
assert response2.status_code == 200
@@ -152,7 +152,9 @@ class TestChatCompletionsE2E:
has_calculation = "60" in message2 and "20" in message2
assert has_answer or has_calculation, f"Expected '80' or calculation in: {message2}"
print(f"✓ Multi-turn test passed. Answer found: {has_answer}, Calculation shown: {has_calculation}")
print(
f"✓ Multi-turn test passed. Answer found: {has_answer}, Calculation shown: {has_calculation}"
)
@pytest.mark.asyncio
async def test_calculation_and_search(self, client: httpx.AsyncClient):
@@ -161,13 +163,8 @@ class TestChatCompletionsE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{
"role": "user",
"content": "Calculate the square root of 256"
}
],
}
"messages": [{"role": "user", "content": "Calculate the square root of 256"}],
},
)
assert response.status_code == 200
@@ -194,10 +191,8 @@ class TestChatCompletionsE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Hello, how are you?"}
],
}
"messages": [{"role": "user", "content": "Hello, how are you?"}],
},
)
assert response.status_code == 200
@@ -223,10 +218,8 @@ class TestChatCompletionsE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is today's date?"}
],
}
"messages": [{"role": "user", "content": "What is today's date?"}],
},
)
assert response.status_code == 200
@@ -242,11 +235,16 @@ class TestChatCompletionsE2E:
# Should contain some date/time information (flexible - varies in format)
import re
has_date = (
re.search(r'\d{4}', message) or # Year
re.search(r'\d{1,2}', message) or # Day/month number
re.search(r'(January|February|March|April|May|June|July|August|September|October|November|December)', message, re.IGNORECASE) or
"today" in message.lower()
re.search(r"\d{4}", message) # Year
or re.search(r"\d{1,2}", message) # Day/month number
or re.search(
r"(January|February|March|April|May|June|July|August|September|October|November|December)",
message,
re.IGNORECASE,
)
or "today" in message.lower()
)
assert has_date, f"Expected date/time information in: {message}"
@@ -263,11 +261,9 @@ class TestResponsesAPIE2E:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Calculate 25 times 16"}
],
"reasoning": {"effort": "medium", "summary": "auto"}
}
"input": [{"role": "user", "content": "Calculate 25 times 16"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
assert response.status_code == 200
@@ -300,7 +296,7 @@ class TestResponsesAPIE2E:
assert "usage" in data
assert data["usage"]["total_tokens"] > 0
print(f"✓ Responses API test passed. Found '400' with Steward reasoning.")
print("✓ Responses API test passed. Found '400' with Steward reasoning.")
@pytest.mark.asyncio
async def test_response_multi_turn(self, client: httpx.AsyncClient):
@@ -312,10 +308,10 @@ class TestResponsesAPIE2E:
"input": [
{"role": "user", "content": "What is 7 times 8?"},
{"role": "assistant", "content": "Certainly, sir. 7 times 8 equals 56."},
{"role": "user", "content": "Double that number."}
{"role": "user", "content": "Double that number."},
],
"reasoning": {"effort": "medium", "summary": "auto"}
}
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
assert response.status_code == 200
@@ -346,11 +342,9 @@ class TestStreamingE2E:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is 9 times 7?"}
],
"stream": True
}
"messages": [{"role": "user", "content": "What is 9 times 7?"}],
"stream": True,
},
) as response:
assert response.status_code == 200
@@ -362,6 +356,7 @@ class TestStreamingE2E:
break
import json
chunk = json.loads(data_str)
chunks.append(chunk)
@@ -373,8 +368,7 @@ class TestStreamingE2E:
# Should have received Steward's reasoning (in <think> tags)
full_content = "".join(
chunk["choices"][0]["delta"].get("content", "") or ""
for chunk in chunks
chunk["choices"][0]["delta"].get("content", "") or "" for chunk in chunks
)
assert "<think>" in full_content
assert "</think>" in full_content
@@ -393,10 +387,8 @@ class TestErrorHandling:
"/v1/chat/completions",
json={
"model": "nonexistent-model",
"messages": [
{"role": "user", "content": "Hello"}
],
}
"messages": [{"role": "user", "content": "Hello"}],
},
)
assert response.status_code == 404
@@ -411,7 +403,7 @@ class TestErrorHandling:
json={
"model": "Tatlock",
# Missing "messages" field
}
},
)
assert response.status_code == 422
@@ -425,11 +417,9 @@ class TestErrorHandling:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Hello"}
],
"temperature": 5.0 # Max is 2.0
}
"messages": [{"role": "user", "content": "Hello"}],
"temperature": 5.0, # Max is 2.0
},
)
assert response.status_code == 422
@@ -447,11 +437,9 @@ class TestChatResponsesWrapper:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Calculate 13 times 9"}
],
"reasoning": {"effort": "medium", "summary": "auto"}
}
"input": [{"role": "user", "content": "Calculate 13 times 9"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
assert response.status_code == 200
@@ -500,10 +488,8 @@ class TestChatResponsesWrapper:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "What is 5 plus 3?"}
],
}
"messages": [{"role": "user", "content": "What is 5 plus 3?"}],
},
)
assert response.status_code == 200
@@ -539,11 +525,9 @@ class TestChatResponsesWrapper:
"/v1/chat/completions",
json={
"model": "Tatlock",
"messages": [
{"role": "user", "content": "Count to 3"}
],
"stream": True
}
"messages": [{"role": "user", "content": "Count to 3"}],
"stream": True,
},
) as response:
assert response.status_code == 200
@@ -555,6 +539,7 @@ class TestChatResponsesWrapper:
break
import json
chunk = json.loads(data_str)
chunks.append(chunk)
@@ -574,10 +559,7 @@ class TestChatResponsesWrapper:
assert chunks[0]["choices"][0]["delta"]["role"] == "assistant"
# Should have content chunks
has_content = any(
"content" in chunk["choices"][0]["delta"]
for chunk in chunks
)
has_content = any("content" in chunk["choices"][0]["delta"] for chunk in chunks)
assert has_content
print(f"✓ Streaming format matches OpenAI spec ({len(chunks)} chunks)")
@@ -593,11 +575,9 @@ class TestStewardIntegration:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Calculate 123 times 456"}
],
"reasoning": {"effort": "medium", "summary": "auto"}
}
"input": [{"role": "user", "content": "Calculate 123 times 456"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
assert response.status_code == 200
@@ -620,10 +600,10 @@ class TestStewardIntegration:
"input": [
{"role": "user", "content": "My favorite number is 42"},
{"role": "assistant", "content": "Noted, sir. 42 is an excellent choice."},
{"role": "user", "content": "What was that number again?"}
{"role": "user", "content": "What was that number again?"},
],
"reasoning": {"effort": "medium", "summary": "auto"}
}
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
assert response.status_code == 200
tests/e2e/test_orchestration_e2e.py
+88 -68
View File
@@ -18,13 +18,14 @@ Requirements:
Note: LLM outputs are non-deterministic. Tests use flexible assertions
that check for behavioral patterns rather than exact text matches.
"""
import pytest
import httpx
import asyncio
import re
from typing import AsyncGenerator
from collections.abc import AsyncGenerator
from dataclasses import dataclass
import httpx
import pytest
from src.core.config import PRODUCTION_TENANT, TEST_TENANT
@@ -43,6 +44,7 @@ assert TEST_USER != PRODUCTION_TENANT
@dataclass
class LLMAssertionResult:
"""Result of an LLM output assertion check."""
passed: bool
evidence: str
confidence: str # "high", "medium", "low"
@@ -183,6 +185,7 @@ def assert_llm_behavior(
# Fixtures
# ============================================================================
@pytest.fixture
async def client() -> AsyncGenerator[httpx.AsyncClient, None]:
"""HTTP client for API requests."""
@@ -223,6 +226,7 @@ async def clean_test_memories(qdrant: QdrantVerifier):
# Memory System Tests
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestMemoryStorage:
@@ -245,9 +249,7 @@ class TestMemoryStorage:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Remember that my test color is purple"}
],
"input": [{"role": "user", "content": "Remember that my test color is purple"}],
},
)
@@ -281,7 +283,8 @@ class TestMemoryStorage:
# Verify data in Qdrant - look for any color-related or test-related memory
points = await qdrant.scroll_points(TEST_COLLECTION)
relevant_memories = [
p for p in points
p
for p in points
if "color" in p.get("payload", {}).get("key", "").lower()
or "purple" in str(p.get("payload", {}).get("value", "")).lower()
or "test" in p.get("payload", {}).get("key", "").lower()
@@ -341,9 +344,7 @@ class TestMemoryStorage:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What is my favorite food?"}
],
"input": [{"role": "user", "content": "What is my favorite food?"}],
},
)
@@ -431,6 +432,7 @@ class TestMemoryRecall:
# Steward Delegation Tests
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestStewardDelegation:
@@ -445,9 +447,7 @@ class TestStewardDelegation:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What do you know about me?"}
],
"input": [{"role": "user", "content": "What do you know about me?"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
@@ -479,9 +479,7 @@ class TestStewardDelegation:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Calculate 127 times 83"}
],
"input": [{"role": "user", "content": "Calculate 127 times 83"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
@@ -511,15 +509,16 @@ class TestStewardDelegation:
# Remove commas for number comparison
message_normalized = message_text.replace(",", "")
assert "10541" in message_normalized, (
f"Expected calculation result 10541. Got: {message_text[:200]}"
)
assert (
"10541" in message_normalized
), f"Expected calculation result 10541. Got: {message_text[:200]}"
# ============================================================================
# Direct Delegation Tests
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestDirectDelegation:
@@ -569,6 +568,7 @@ class TestDirectDelegation:
# Data Verification Tests
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestDataVerification:
@@ -610,8 +610,7 @@ class TestDataVerification:
payload = point.get("payload", {})
for field in required_fields:
assert field in payload, (
f"Point {point['id']} missing required field '{field}'. "
f"Payload: {payload}"
f"Point {point['id']} missing required field '{field}'. " f"Payload: {payload}"
)
async def test_qdrant_point_types_are_valid(
@@ -643,6 +642,7 @@ class TestDataVerification:
# Integration Health Tests
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestIntegrationHealth:
@@ -678,6 +678,7 @@ class TestIntegrationHealth:
# Orchestration Scenario Tests (from ORCHESTRATION_SCENARIOS.md)
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestScenario1WeatherWithMemory:
@@ -697,9 +698,7 @@ class TestScenario1WeatherWithMemory:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What's the weather like?"}
],
"input": [{"role": "user", "content": "What's the weather like?"}],
"reasoning": {"effort": "medium", "summary": "auto"},
},
)
@@ -717,13 +716,17 @@ class TestScenario1WeatherWithMemory:
print(f"Weather query - Steward: {reasoning_text[:200]}...")
# Should mention location/memory and search capabilities
has_memory_mention = "biographer" in reasoning_text.lower() or "memory" in reasoning_text.lower()
has_search_mention = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
has_memory_mention = (
"biographer" in reasoning_text.lower() or "memory" in reasoning_text.lower()
)
has_search_mention = (
"tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
)
# Weather query should trigger at least web search
assert has_search_mention, (
f"Weather query should recommend search capability. Got: {reasoning_text[:200]}"
)
assert (
has_search_mention
), f"Weather query should recommend search capability. Got: {reasoning_text[:200]}"
@pytest.mark.e2e
@@ -744,9 +747,7 @@ class TestScenario4SimpleExpertDelegation:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What is 847 times 293?"}
],
"input": [{"role": "user", "content": "What is 847 times 293?"}],
},
)
@@ -761,9 +762,9 @@ class TestScenario4SimpleExpertDelegation:
# Expected result: 248171 (may be formatted as 248,171)
message_normalized = message_text.replace(",", "")
assert "248171" in message_normalized, (
f"Calculator should compute 847 * 293 = 248171. Got: {message_text[:200]}"
)
assert (
"248171" in message_normalized
), f"Calculator should compute 847 * 293 = 248171. Got: {message_text[:200]}"
async def test_datetime_query_uses_datetime_tool(
self,
@@ -774,9 +775,7 @@ class TestScenario4SimpleExpertDelegation:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What day of the week is it?"}
],
"input": [{"role": "user", "content": "What day of the week is it?"}],
},
)
@@ -791,8 +790,20 @@ class TestScenario4SimpleExpertDelegation:
# Should mention a day of the week (full or abbreviated)
days = [
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
"mon", "tue", "wed", "thu", "fri", "sat", "sun"
"monday",
"tuesday",
"wednesday",
"thursday",
"friday",
"saturday",
"sunday",
"mon",
"tue",
"wed",
"thu",
"fri",
"sat",
"sun",
]
has_day = any(day in message_text.lower() for day in days)
@@ -838,9 +849,9 @@ class TestScenario6WikiCreation:
print(f"Wiki search - Steward: {reasoning_text[:200]}...")
# Should mention librarian
assert "librarian" in reasoning_text.lower(), (
f"Wiki search should recommend librarian. Got: {reasoning_text[:200]}"
)
assert (
"librarian" in reasoning_text.lower()
), f"Wiki search should recommend librarian. Got: {reasoning_text[:200]}"
@pytest.mark.e2e
@@ -862,7 +873,10 @@ class TestScenario8MultiExpertCoordination:
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What do you know about me? And also search for Python tutorials."}
{
"role": "user",
"content": "What do you know about me? And also search for Python tutorials.",
}
],
"reasoning": {"effort": "medium", "summary": "auto"},
},
@@ -885,15 +899,16 @@ class TestScenario8MultiExpertCoordination:
has_librarian = "librarian" in reasoning_text.lower()
has_search = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
assert has_biographer or has_librarian or has_search, (
f"Complex query should identify multiple capabilities. Got: {reasoning_text[:200]}"
)
assert (
has_biographer or has_librarian or has_search
), f"Complex query should identify multiple capabilities. Got: {reasoning_text[:200]}"
# ============================================================================
# New Scenarios from Today's Session
# ============================================================================
@pytest.mark.e2e
@pytest.mark.asyncio
class TestDirectDelegationBypass:
@@ -917,9 +932,7 @@ class TestDirectDelegationBypass:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Remember that my test value is alpha123"}
],
"input": [{"role": "user", "content": "Remember that my test value is alpha123"}],
},
)
@@ -979,7 +992,10 @@ class TestUserContextIsolation:
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": f"Remember that my isolation marker is {unique_value}"}
{
"role": "user",
"content": f"Remember that my isolation marker is {unique_value}",
}
],
},
)
@@ -1000,13 +1016,15 @@ class TestUserContextIsolation:
else:
in_prod_collection = False
print(f"Isolation test - In test collection: {in_test_collection}, In prod: {in_prod_collection}")
print(
f"Isolation test - In test collection: {in_test_collection}, In prod: {in_prod_collection}"
)
# Should be in test collection OR response acknowledged
# Should NOT be in production collection
assert not in_prod_collection, (
f"Test data leaked to production collection! Value: {unique_value}"
)
assert (
not in_prod_collection
), f"Test data leaked to production collection! Value: {unique_value}"
@pytest.mark.e2e
@@ -1025,16 +1043,16 @@ class TestErrorHandling:
"/v1/responses",
json={
"model": "nonexistent-model-xyz",
"input": [
{"role": "user", "content": "Hello"}
],
"input": [{"role": "user", "content": "Hello"}],
},
)
# Should return error status (404 or 400)
assert response.status_code in [400, 404, 422], (
f"Expected error status for invalid model. Got: {response.status_code}"
)
assert response.status_code in [
400,
404,
422,
], f"Expected error status for invalid model. Got: {response.status_code}"
data = response.json()
# Error could be in "error" or "detail" key
assert "error" in data or "detail" in data
@@ -1084,7 +1102,10 @@ class TestEvaluationReport:
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "Remember that my test pet is a hamster named Fluffy"}
{
"role": "user",
"content": "Remember that my test pet is a hamster named Fluffy",
}
],
},
)
@@ -1096,7 +1117,7 @@ class TestEvaluationReport:
break
report_lines.append("\n[STORE TEST]")
report_lines.append(f"Input: 'Remember that my test pet is a hamster named Fluffy'")
report_lines.append("Input: 'Remember that my test pet is a hamster named Fluffy'")
report_lines.append(f"Response: {store_text[:200]}...")
await asyncio.sleep(1.0)
@@ -1106,9 +1127,7 @@ class TestEvaluationReport:
"/v1/responses",
json={
"model": "Tatlock",
"input": [
{"role": "user", "content": "What pet do I have?"}
],
"input": [{"role": "user", "content": "What pet do I have?"}],
},
)
@@ -1119,7 +1138,7 @@ class TestEvaluationReport:
break
report_lines.append("\n[RECALL TEST]")
report_lines.append(f"Input: 'What pet do I have?'")
report_lines.append("Input: 'What pet do I have?'")
report_lines.append(f"Response: {recall_text[:200]}...")
# Check Qdrant state
@@ -1128,7 +1147,8 @@ class TestEvaluationReport:
report_lines.append(f"Total points in {TEST_COLLECTION}: {len(points)}")
pet_memories = [
p for p in points
p
for p in points
if "pet" in str(p.get("payload", {})).lower()
or "fluffy" in str(p.get("payload", {})).lower()
or "hamster" in str(p.get("payload", {})).lower()
tests/integration/test_steward_streaming.py
+19 -9
View File
@@ -3,12 +3,14 @@ Integration tests for Steward + Tatlock streaming.
Tests the complete streaming flow with Steward preprocessing.
"""
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from src.responses.schemas import ResponseRequest
from src.responses.streaming import StreamingCoordinator, StreamEventType
from unittest.mock import patch
import pytest
from src.core.startup import initialize_application
from src.responses.schemas import ResponseRequest
from src.responses.streaming import StreamEventType, StreamingCoordinator
@pytest.fixture(scope="module", autouse=True)
@@ -32,7 +34,9 @@ class TestStewardStreaming:
# Mock the Steward analysis
with patch("src.core.preprocessing.analyze_request") as mock_steward:
# Mock the streaming method (async generator)
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream") as mock_tatlock_stream:
with patch(
"src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream"
) as mock_tatlock_stream:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
# Mock Steward recommendation
@@ -89,7 +93,9 @@ class TestStewardStreaming:
)
with patch("src.core.preprocessing.analyze_request") as mock_steward:
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream") as mock_tatlock_stream:
with patch(
"src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream"
) as mock_tatlock_stream:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
mock_steward.return_value = StewardRecommendation(
@@ -99,7 +105,7 @@ class TestStewardStreaming:
conversation_context=ConversationContext(
has_previous_context=True,
relevant_turns=[0],
context_summary="Previous calculation in turn 0"
context_summary="Previous calculation in turn 0",
),
)
@@ -134,7 +140,9 @@ class TestStewardStreaming:
)
with patch("src.core.preprocessing.analyze_request") as mock_steward:
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream") as mock_tatlock_stream:
with patch(
"src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream"
) as mock_tatlock_stream:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
mock_steward.return_value = StewardRecommendation(
@@ -173,7 +181,9 @@ class TestStewardStreaming:
)
with patch("src.core.preprocessing.analyze_request") as mock_steward:
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream") as mock_tatlock_stream:
with patch(
"src.agents.tatlock.TatlockAgent.run_with_scoped_tools_stream"
) as mock_tatlock_stream:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
mock_steward.return_value = StewardRecommendation(
tests/integration/test_steward_tatlock_integration.py
+19 -8
View File
@@ -7,12 +7,14 @@ Tests the complete Phase 2 request pipeline:
3. Tatlock runs with scoped tools
4. Response includes both Steward reasoning and Tatlock output
"""
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.core.startup import initialize_application
from src.responses.schemas import ResponseRequest
from src.responses.service import create_response_with_steward
from src.core.startup import initialize_application
@pytest.fixture(scope="module", autouse=True)
@@ -56,8 +58,9 @@ class TestStewardTatlockIntegration:
assert mock_steward.called
# Note: preprocess_request injects temporal context
steward_call_arg = mock_steward.call_args[0][0]
assert steward_call_arg.startswith("What's 2 + 2?"), \
f"Expected request to start with original message, got: {steward_call_arg}"
assert steward_call_arg.startswith(
"What's 2 + 2?"
), f"Expected request to start with original message, got: {steward_call_arg}"
# Verify Tatlock was called with scoped tools
assert mock_tatlock.called
@@ -100,7 +103,7 @@ class TestStewardTatlockIntegration:
conversation_context=ConversationContext(
has_previous_context=True,
relevant_turns=[0],
context_summary="Previous calculation in turn 0"
context_summary="Previous calculation in turn 0",
),
)
@@ -161,7 +164,10 @@ class TestStewardTatlockIntegration:
with patch("src.core.preprocessing.analyze_request") as mock_steward:
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
with patch("src.core.tool_tracking.ToolCallTracker.finalize") as mock_finalize:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
from src.agents.steward.schemas import (
ConversationContext,
StewardRecommendation,
)
mock_steward.return_value = StewardRecommendation(
recommended_capabilities=["tatlock_core"],
@@ -198,7 +204,9 @@ class TestStewardTatlockIntegration:
missing_capabilities="Image generation capability would be needed",
)
mock_tatlock.return_value = "I'm afraid I don't have image generation capabilities, sir."
mock_tatlock.return_value = (
"I'm afraid I don't have image generation capabilities, sir."
)
response = await create_response_with_steward(request)
@@ -220,7 +228,10 @@ class TestStewardTatlockIntegration:
with patch("src.core.preprocessing.analyze_request") as mock_steward:
with patch("src.agents.tatlock.TatlockAgent.run_with_scoped_tools") as mock_tatlock:
with patch("src.responses.service.ToolCallTracker") as mock_tracker_class:
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
from src.agents.steward.schemas import (
ConversationContext,
StewardRecommendation,
)
mock_steward.return_value = StewardRecommendation(
recommended_capabilities=["tatlock_core"],
tests/integration/test_tatlock_streaming.py
+28 -25
View File
@@ -5,10 +5,12 @@ These tests verify the complete streaming flow from API endpoint through
StreamingCoordinator to TatlockAgent, ensuring no text duplication and
proper delta calculation.
"""
import json
import pytest
from httpx import AsyncClient
from fastapi.testclient import TestClient
from httpx import AsyncClient
@pytest.mark.integration
@@ -24,7 +26,7 @@ async def test_tatlock_streaming_no_duplication(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"input": [{"role": "user", "content": "Say hello"}],
"stream": True
"stream": True,
}
collected_deltas = []
@@ -74,12 +76,12 @@ async def test_tatlock_streaming_no_duplication(async_client: AsyncClient):
if len(words) > 0:
# Check for consecutive duplicate words (sign of duplication bug)
consecutive_dupes = sum(
1 for i in range(len(words) - 1)
if words[i] == words[i + 1] and len(words[i]) > 3
1 for i in range(len(words) - 1) if words[i] == words[i + 1] and len(words[i]) > 3
)
# Allow a few duplicates (natural language), but not excessive
assert consecutive_dupes < len(words) * 0.1, \
f"Too many consecutive duplicate words: {consecutive_dupes}/{len(words)}"
assert (
consecutive_dupes < len(words) * 0.1
), f"Too many consecutive duplicate words: {consecutive_dupes}/{len(words)}"
@pytest.mark.integration
@@ -94,7 +96,7 @@ async def test_tatlock_chat_streaming_no_duplication(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"messages": [{"role": "user", "content": "Hello"}],
"stream": True
"stream": True,
}
collected_content = []
@@ -138,11 +140,11 @@ async def test_tatlock_chat_streaming_no_duplication(async_client: AsyncClient):
words = full_response.lower().split()
if len(words) > 0:
consecutive_dupes = sum(
1 for i in range(len(words) - 1)
if words[i] == words[i + 1] and len(words[i]) > 3
1 for i in range(len(words) - 1) if words[i] == words[i + 1] and len(words[i]) > 3
)
assert consecutive_dupes < len(words) * 0.1, \
f"Too many consecutive duplicate words in chat response: {consecutive_dupes}/{len(words)}"
assert (
consecutive_dupes < len(words) * 0.1
), f"Too many consecutive duplicate words in chat response: {consecutive_dupes}/{len(words)}"
@pytest.mark.integration
@@ -153,7 +155,7 @@ def test_tatlock_non_streaming_responses_api(client: TestClient):
request_data = {
"model": "Tatlock",
"input": [{"role": "user", "content": "Say hello"}],
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data, timeout=30.0)
@@ -183,7 +185,7 @@ def test_tatlock_non_streaming_chat_api(client: TestClient):
request_data = {
"model": "Tatlock",
"messages": [{"role": "user", "content": "Hello"}],
"stream": False
"stream": False,
}
response = client.post("/v1/chat/completions", json=request_data, timeout=30.0)
@@ -216,7 +218,7 @@ async def test_tatlock_streaming_delta_accumulation(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"input": [{"role": "user", "content": "Count to three"}],
"stream": True
"stream": True,
}
collected_deltas = []
@@ -248,8 +250,9 @@ async def test_tatlock_streaming_delta_accumulation(async_client: AsyncClient):
# Verify each delta is new content
current_full = "".join(collected_deltas)
assert current_full.startswith(previous_full_text), \
"Deltas should accumulate progressively"
assert current_full.startswith(
previous_full_text
), "Deltas should accumulate progressively"
previous_full_text = current_full
except json.JSONDecodeError:
@@ -273,7 +276,7 @@ async def test_tatlock_with_reasoning(async_client: AsyncClient):
"model": "Tatlock",
"input": [{"role": "user", "content": "Hello"}],
"reasoning": {"effort": "medium", "summary": "auto"},
"stream": True
"stream": True,
}
has_reasoning = False
@@ -328,7 +331,7 @@ async def test_tatlock_markdown_formatting_preserved(async_client: AsyncClient):
request_data = {
"model": "Tatlock",
"input": [{"role": "user", "content": "Can you give me an HTML5 boilerplate template?"}],
"stream": True
"stream": True,
}
collected_deltas = []
@@ -365,15 +368,15 @@ async def test_tatlock_markdown_formatting_preserved(async_client: AsyncClient):
full_response = "".join(collected_deltas)
# Always print the response for debugging
print("\n" + "="*80)
print("\n" + "=" * 80)
print("FULL RESPONSE (repr):")
print("="*80)
print("=" * 80)
print(repr(full_response))
print("\n" + "="*80)
print("\n" + "=" * 80)
print("FULL RESPONSE (formatted):")
print("="*80)
print("=" * 80)
print(full_response)
print("="*80 + "\n")
print("=" * 80 + "\n")
# Verify we got a response (xfail if LLM didn't produce output)
if len(full_response) < 100:
@@ -384,7 +387,7 @@ async def test_tatlock_markdown_formatting_preserved(async_client: AsyncClient):
pytest.xfail("No markdown code blocks in response (LLM response varied)")
# Verify newlines are preserved (not all collapsed to spaces)
newline_count = full_response.count('\n')
newline_count = full_response.count("\n")
if newline_count < 5:
pytest.xfail(f"Only {newline_count} newlines, formatting may have been lost")
@@ -412,7 +415,7 @@ def test_tatlock_markdown_non_streaming(client: TestClient):
request_data = {
"model": "Tatlock",
"input": [{"role": "user", "content": "Give me a simple Python hello world code"}],
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data, timeout=30.0)
tests/models/test_router.py
+3 -2
View File
@@ -1,6 +1,7 @@
"""
Tests for models listing router.
"""
import pytest
from fastapi.testclient import TestClient
@@ -39,10 +40,10 @@ def test_list_models(client: TestClient) -> None:
def test_models_endpoint_returns_json(client: TestClient) -> None:
"""Test that models endpoint returns valid JSON."""
response = client.get("/v1/models")
assert response.status_code == 200
assert response.headers["content-type"] == "application/json"
# Should be able to parse as JSON
data = response.json()
assert isinstance(data, dict)
tests/responses/test_advanced_features.py
+53 -66
View File
@@ -6,7 +6,9 @@ Tests:
- Stop sequence detection and enforcement
- Max tokens enforcement
"""
import json
import pytest
from fastapi.testclient import TestClient
from httpx import AsyncClient
@@ -15,11 +17,11 @@ from pydantic import ValidationError
from src.responses.schemas import ResponseRequest
from src.responses.streaming import StreamingCoordinator
# ============================================================================
# Parameter Validation Tests
# ============================================================================
@pytest.mark.unit
def test_temperature_validation():
"""Test temperature parameter validation."""
@@ -27,9 +29,7 @@ def test_temperature_validation():
valid_temps = [0.0, 0.5, 1.0, 1.5, 2.0]
for temp in valid_temps:
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
temperature=temp
model="lorem-tester", input=[{"role": "user", "content": "Hello"}], temperature=temp
)
assert request.temperature == temp
@@ -38,7 +38,7 @@ def test_temperature_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
temperature=-0.1 # Too low
temperature=-0.1, # Too low
)
assert "temperature" in str(exc_info.value).lower()
@@ -46,7 +46,7 @@ def test_temperature_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
temperature=2.1 # Too high
temperature=2.1, # Too high
)
assert "temperature" in str(exc_info.value).lower()
@@ -55,12 +55,12 @@ def test_temperature_validation():
def test_reasoning_effort_validation():
"""Test reasoning.effort parameter validation."""
# Valid effort levels
valid_efforts = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']
valid_efforts = ["none", "minimal", "low", "medium", "high", "xhigh"]
for effort in valid_efforts:
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
reasoning={"effort": effort, "summary": "auto"}
reasoning={"effort": effort, "summary": "auto"},
)
assert request.reasoning["effort"] == effort
@@ -69,7 +69,7 @@ def test_reasoning_effort_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
reasoning={"effort": "invalid", "summary": "auto"}
reasoning={"effort": "invalid", "summary": "auto"},
)
assert "reasoning.effort" in str(exc_info.value)
@@ -78,12 +78,12 @@ def test_reasoning_effort_validation():
def test_reasoning_summary_validation():
"""Test reasoning.summary parameter validation."""
# Valid summary values
valid_summaries = ['auto', 'off']
valid_summaries = ["auto", "off"]
for summary in valid_summaries:
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
reasoning={"effort": "medium", "summary": summary}
reasoning={"effort": "medium", "summary": summary},
)
assert request.reasoning["summary"] == summary
@@ -92,7 +92,7 @@ def test_reasoning_summary_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
reasoning={"effort": "medium", "summary": "invalid"}
reasoning={"effort": "medium", "summary": "invalid"},
)
assert "reasoning.summary" in str(exc_info.value)
@@ -102,26 +102,20 @@ def test_max_output_tokens_validation():
"""Test max_output_tokens parameter validation."""
# Valid values
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
max_output_tokens=100
model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=100
)
assert request.max_output_tokens == 100
# None is valid (unlimited)
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
max_output_tokens=None
model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=None
)
assert request.max_output_tokens is None
# Invalid: zero or negative
with pytest.raises(ValidationError) as exc_info:
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
max_output_tokens=0
model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=0
)
assert "max_output_tokens" in str(exc_info.value)
@@ -129,7 +123,7 @@ def test_max_output_tokens_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
max_output_tokens=-10
max_output_tokens=-10,
)
assert "max_output_tokens" in str(exc_info.value)
@@ -141,9 +135,7 @@ def test_stop_sequences_validation():
for num_seqs in range(1, 5):
stop_seqs = [f"stop{i}" for i in range(num_seqs)]
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
stop=stop_seqs
model="lorem-tester", input=[{"role": "user", "content": "Hello"}], stop=stop_seqs
)
assert request.stop == stop_seqs
@@ -152,7 +144,7 @@ def test_stop_sequences_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
stop=["stop1", "stop2", "stop3", "stop4", "stop5"] # 5 sequences
stop=["stop1", "stop2", "stop3", "stop4", "stop5"], # 5 sequences
)
assert "4 stop sequences" in str(exc_info.value)
@@ -161,7 +153,7 @@ def test_stop_sequences_validation():
ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
stop=["stop1", ""] # Empty string
stop=["stop1", ""], # Empty string
)
assert "non-empty" in str(exc_info.value).lower()
@@ -170,6 +162,7 @@ def test_stop_sequences_validation():
# Stop Sequence Enforcement Tests
# ============================================================================
@pytest.mark.unit
def test_stop_sequence_detection_helper():
"""Test stop sequence detection helper method."""
@@ -181,26 +174,17 @@ def test_stop_sequence_detection_helper():
assert text == "Hello world"
# Stop sequence not present
found, text = coordinator._check_stop_sequence(
"Hello world",
["STOP", "END"]
)
found, text = coordinator._check_stop_sequence("Hello world", ["STOP", "END"])
assert found is False
assert text == "Hello world"
# Stop sequence found
found, text = coordinator._check_stop_sequence(
"Hello STOP this should not appear",
["STOP"]
)
found, text = coordinator._check_stop_sequence("Hello STOP this should not appear", ["STOP"])
assert found is True
assert text == "Hello "
# Multiple stop sequences, first one wins
found, text = coordinator._check_stop_sequence(
"Hello STOP this END that",
["STOP", "END"]
)
found, text = coordinator._check_stop_sequence("Hello STOP this END that", ["STOP", "END"])
assert found is True
assert text == "Hello "
@@ -216,7 +200,7 @@ async def test_stop_sequence_in_streaming(async_client: AsyncClient):
"model": "lorem-tester",
"input": [{"role": "user", "content": "Generate long text"}],
"stop": ["dolor"], # Common word in lorem ipsum
"stream": True
"stream": True,
}
chunks_received = []
@@ -251,6 +235,7 @@ async def test_stop_sequence_in_streaming(async_client: AsyncClient):
# Max Tokens Enforcement Tests
# ============================================================================
@pytest.mark.unit
def test_max_tokens_check_helper():
"""Test max tokens check helper method."""
@@ -296,7 +281,7 @@ async def test_max_tokens_in_streaming(async_client: AsyncClient):
"model": "lorem-tester",
"input": [{"role": "user", "content": "Generate text"}],
"max_output_tokens": 5, # Very low limit
"stream": True
"stream": True,
}
chunks_received = []
@@ -340,6 +325,7 @@ async def test_max_tokens_in_streaming(async_client: AsyncClient):
# Combined Features Test
# ============================================================================
@pytest.mark.unit
def test_combined_validation(client: TestClient):
"""Test combined parameter validation in actual request."""
@@ -351,7 +337,7 @@ def test_combined_validation(client: TestClient):
"max_output_tokens": 100,
"stop": ["STOP", "END"],
"reasoning": {"effort": "high", "summary": "auto"},
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -368,7 +354,7 @@ def test_invalid_combined_parameters(client: TestClient):
"model": "lorem-tester",
"input": [{"role": "user", "content": "Hello"}],
"temperature": 3.0, # Too high
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -382,6 +368,7 @@ def test_invalid_combined_parameters(client: TestClient):
# Streaming Delta Calculation Tests (No Duplication)
# ============================================================================
@pytest.mark.unit
@pytest.mark.asyncio
async def test_streaming_delta_calculation_no_duplication():
@@ -392,8 +379,10 @@ async def test_streaming_delta_calculation_no_duplication():
This test prevents the duplication bug where the same text was
streamed multiple times because we weren't computing deltas correctly.
"""
from collections.abc import AsyncGenerator
from typing import Any
from src.agents.base import AgentInterface, OutputItem
from typing import AsyncGenerator, Any
# Create a mock agent that simulates PydanticAI's behavior
# (yielding accumulated text, not deltas)
@@ -406,7 +395,7 @@ async def test_streaming_delta_calculation_no_duplication():
temperature: float = 1.0,
max_tokens: int | None = None,
stop: list[str] | None = None,
**kwargs: Any
**kwargs: Any,
) -> AsyncGenerator[OutputItem, None]:
"""
Simulate PydanticAI streaming behavior:
@@ -430,12 +419,8 @@ async def test_streaming_delta_calculation_no_duplication():
type="message",
id=msg_id,
role="assistant",
content=[{
"type": "output_text",
"text": text,
"annotations": []
}],
status="in_progress"
content=[{"type": "output_text", "text": text, "annotations": []}],
status="in_progress",
)
# Final message
@@ -443,12 +428,10 @@ async def test_streaming_delta_calculation_no_duplication():
type="message",
id=msg_id,
role="assistant",
content=[{
"type": "output_text",
"text": "Hello world how are you",
"annotations": []
}],
status="completed"
content=[
{"type": "output_text", "text": "Hello world how are you", "annotations": []}
],
status="completed",
)
async def supports_tools(self) -> bool:
@@ -462,7 +445,9 @@ async def test_streaming_delta_calculation_no_duplication():
# Register the mock agent
import time
from src.agents.registry import ModelRegistry
ModelRegistry.MODELS["mock-streaming"] = {
"agent_class": MockStreamingAgent,
"description": "Mock streaming agent for testing",
@@ -473,9 +458,7 @@ async def test_streaming_delta_calculation_no_duplication():
try:
# Create a test request
request = ResponseRequest(
model="mock-streaming",
input=[{"role": "user", "content": "Test"}],
stream=True
model="mock-streaming", input=[{"role": "user", "content": "Test"}], stream=True
)
# Stream the response
@@ -512,8 +495,10 @@ async def test_streaming_with_multiple_message_items():
Test that coordinator handles multiple message OutputItems correctly,
only streaming the delta between each one.
"""
from collections.abc import AsyncGenerator
from typing import Any
from src.agents.base import AgentInterface, OutputItem
from typing import AsyncGenerator, Any
class MockMultiMessageAgent(AgentInterface):
async def generate_response(
@@ -524,7 +509,7 @@ async def test_streaming_with_multiple_message_items():
temperature: float = 1.0,
max_tokens: int | None = None,
stop: list[str] | None = None,
**kwargs: Any
**kwargs: Any,
) -> AsyncGenerator[OutputItem, None]:
"""Yield multiple in_progress messages with accumulated text."""
# First chunk
@@ -533,7 +518,7 @@ async def test_streaming_with_multiple_message_items():
id="msg_1",
role="assistant",
content=[{"type": "output_text", "text": "The answer is", "annotations": []}],
status="in_progress"
status="in_progress",
)
# Second chunk (more text accumulated)
@@ -542,7 +527,7 @@ async def test_streaming_with_multiple_message_items():
id="msg_1",
role="assistant",
content=[{"type": "output_text", "text": "The answer is 42", "annotations": []}],
status="in_progress"
status="in_progress",
)
# Final chunk
@@ -551,7 +536,7 @@ async def test_streaming_with_multiple_message_items():
id="msg_1",
role="assistant",
content=[{"type": "output_text", "text": "The answer is 42", "annotations": []}],
status="completed"
status="completed",
)
async def supports_tools(self) -> bool:
@@ -565,7 +550,9 @@ async def test_streaming_with_multiple_message_items():
# Register mock agent
import time
from src.agents.registry import ModelRegistry
ModelRegistry.MODELS["mock-multi"] = {
"agent_class": MockMultiMessageAgent,
"description": "Mock multi-message agent for testing",
@@ -577,7 +564,7 @@ async def test_streaming_with_multiple_message_items():
request = ResponseRequest(
model="mock-multi",
input=[{"role": "user", "content": "What is the answer?"}],
stream=True
stream=True,
)
coordinator = StreamingCoordinator()
tests/responses/test_error_handling.py
+24 -52
View File
@@ -1,7 +1,9 @@
"""
Tests for error handling in Responses API.
"""
import json
import pytest
from fastapi.testclient import TestClient
from httpx import AsyncClient
@@ -12,10 +14,8 @@ def test_model_not_found_error(client: TestClient) -> None:
"""Test response when model doesn't exist."""
request_data = {
"model": "nonexistent-model-12345",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": False
"input": [{"role": "user", "content": "Hello"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -29,11 +29,7 @@ def test_model_not_found_error(client: TestClient) -> None:
@pytest.mark.unit
def test_validation_error_missing_model(client: TestClient) -> None:
"""Test validation error when model field is missing."""
request_data = {
"input": [
{"role": "user", "content": "Hello"}
]
}
request_data = {"input": [{"role": "user", "content": "Hello"}]}
response = client.post("/v1/responses", json=request_data)
@@ -45,10 +41,8 @@ def test_validation_error_invalid_temperature(client: TestClient) -> None:
"""Test validation error for out-of-range temperature."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Hello"}
],
"temperature": 3.0 # Max is 2.0
"input": [{"role": "user", "content": "Hello"}],
"temperature": 3.0, # Max is 2.0
}
response = client.post("/v1/responses", json=request_data)
@@ -61,10 +55,8 @@ def test_rate_limit_error(client: TestClient) -> None:
"""Test rate limit error trigger."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "trigger_rate_limit"}
],
"stream": False
"input": [{"role": "user", "content": "trigger_rate_limit"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -78,10 +70,8 @@ def test_context_overflow_error(client: TestClient) -> None:
"""Test context length overflow error trigger."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "trigger_context_overflow"}
],
"stream": False
"input": [{"role": "user", "content": "trigger_context_overflow"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -96,10 +86,8 @@ async def test_streaming_model_not_found(async_client: AsyncClient) -> None:
"""Test streaming response with nonexistent model."""
request_data = {
"model": "nonexistent-streaming-model",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": True
"input": [{"role": "user", "content": "Hello"}],
"stream": True,
}
async with async_client.stream(
@@ -133,10 +121,8 @@ async def test_streaming_rate_limit_error(async_client: AsyncClient) -> None:
"""Test streaming with rate limit error."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "trigger_rate_limit"}
],
"stream": True
"input": [{"role": "user", "content": "trigger_rate_limit"}],
"stream": True,
}
async with async_client.stream(
@@ -177,10 +163,8 @@ async def test_streaming_context_overflow_error(async_client: AsyncClient) -> No
"""Test streaming with context overflow error."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "trigger_context_overflow"}
],
"stream": True
"input": [{"role": "user", "content": "trigger_context_overflow"}],
"stream": True,
}
async with async_client.stream(
@@ -215,17 +199,9 @@ def test_function_call_output_item(client: TestClient) -> None:
"""Test response with function call items."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Use the search tool"}
],
"tools": [
{
"name": "search",
"description": "Search for information",
"parameters": {}
}
],
"stream": False
"input": [{"role": "user", "content": "Use the search tool"}],
"tools": [{"name": "search", "description": "Search for information", "parameters": {}}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -248,10 +224,8 @@ def test_pipeline_prefix_stripping(client: TestClient) -> None:
"""Test that pipeline prefixes are stripped from model names."""
request_data = {
"model": "some_pipeline.lorem-tester",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": False
"input": [{"role": "user", "content": "Hello"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -267,10 +241,8 @@ def test_multiple_pipeline_prefixes(client: TestClient) -> None:
"""Test multiple dots in model name (only first is prefix)."""
request_data = {
"model": "pipeline.sub.lorem-tester",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": False
"input": [{"role": "user", "content": "Hello"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
tests/responses/test_history.py
+29 -66
View File
@@ -1,13 +1,14 @@
"""
Tests for conversation history management.
"""
import pytest
from fastapi.testclient import TestClient
from src.responses.history import ConversationHistory
from src.responses.context import ContextWindow
from src.responses.schemas import ResponseRequest
from src.responses import service
from src.responses.context import ContextWindow
from src.responses.history import ConversationHistory
from src.responses.schemas import ResponseRequest
@pytest.mark.unit
@@ -19,7 +20,7 @@ async def test_conversation_id_from_metadata():
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}],
metadata={"conversation_id": "conv_123"}
metadata={"conversation_id": "conv_123"},
)
conv_id = await history.get_conversation_id(request)
@@ -34,7 +35,7 @@ async def test_conversation_id_generation():
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}]
input=[{"role": "user", "content": "Hello"}],
# No metadata provided
)
@@ -43,10 +44,7 @@ async def test_conversation_id_generation():
assert len(conv_id) == 16 # 16 character hex
# Same first message should generate same ID
request2 = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Hello"}]
)
request2 = ResponseRequest(model="lorem-tester", input=[{"role": "user", "content": "Hello"}])
conv_id2 = await history.get_conversation_id(request2)
assert conv_id == conv_id2
@@ -59,7 +57,7 @@ def test_conversation_history_tracking(client: TestClient):
"model": "lorem-tester",
"input": [{"role": "user", "content": "Hello"}],
"metadata": {"conversation_id": "test_conv_001"},
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -71,10 +69,10 @@ def test_conversation_history_tracking(client: TestClient):
"input": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
{"role": "user", "content": "How are you?"}
{"role": "user", "content": "How are you?"},
],
"metadata": {"conversation_id": "test_conv_001"},
"stream": False
"stream": False,
}
response2 = client.post("/v1/responses", json=request_data2)
@@ -94,7 +92,7 @@ async def test_conversation_history_retrieval():
request = ResponseRequest(
model="lorem-tester",
input=[{"role": "user", "content": "Test"}],
metadata={"conversation_id": "test_retrieve"}
metadata={"conversation_id": "test_retrieve"},
)
conv_id = await history.get_conversation_id(request)
@@ -104,24 +102,15 @@ async def test_conversation_history_retrieval():
assert len(retrieved) == 0
# Add mock response
from src.responses.schemas import Response, ResponseUsage, MessageOutputItem, OutputTextContent
from src.responses.schemas import MessageOutputItem, OutputTextContent, Response, ResponseUsage
mock_response = Response(
id="resp_123",
created_at=1234567890,
model="lorem-tester",
status="completed",
output=[
MessageOutputItem(
id="msg_1",
content=[OutputTextContent(text="Test response")]
)
],
usage=ResponseUsage(
input_tokens=10,
output_tokens=5,
reasoning_tokens=0,
total_tokens=15
)
output=[MessageOutputItem(id="msg_1", content=[OutputTextContent(text="Test response")])],
usage=ResponseUsage(input_tokens=10, output_tokens=5, reasoning_tokens=0, total_tokens=15),
)
await history.add_response(conv_id, mock_response)
@@ -138,7 +127,7 @@ async def test_conversation_history_trimming():
"""Test that history is trimmed to max_turns."""
history = ConversationHistory(max_turns=3)
from src.responses.schemas import Response, ResponseUsage, MessageOutputItem, OutputTextContent
from src.responses.schemas import MessageOutputItem, OutputTextContent, Response, ResponseUsage
conv_id = "test_trim"
@@ -150,17 +139,11 @@ async def test_conversation_history_trimming():
model="lorem-tester",
status="completed",
output=[
MessageOutputItem(
id=f"msg_{i}",
content=[OutputTextContent(text=f"Response {i}")]
)
MessageOutputItem(id=f"msg_{i}", content=[OutputTextContent(text=f"Response {i}")])
],
usage=ResponseUsage(
input_tokens=10,
output_tokens=5,
reasoning_tokens=0,
total_tokens=15
)
input_tokens=10, output_tokens=5, reasoning_tokens=0, total_tokens=15
),
)
await history.add_response(conv_id, response)
@@ -178,7 +161,7 @@ async def test_clear_conversation():
"""Test clearing conversation history."""
history = ConversationHistory()
from src.responses.schemas import Response, ResponseUsage, MessageOutputItem, OutputTextContent
from src.responses.schemas import MessageOutputItem, OutputTextContent, Response, ResponseUsage
conv_id = "test_clear"
@@ -188,18 +171,8 @@ async def test_clear_conversation():
created_at=1234567890,
model="lorem-tester",
status="completed",
output=[
MessageOutputItem(
id="msg_1",
content=[OutputTextContent(text="Test")]
)
],
usage=ResponseUsage(
input_tokens=10,
output_tokens=5,
reasoning_tokens=0,
total_tokens=15
)
output=[MessageOutputItem(id="msg_1", content=[OutputTextContent(text="Test")])],
usage=ResponseUsage(input_tokens=10, output_tokens=5, reasoning_tokens=0, total_tokens=15),
)
await history.add_response(conv_id, response)
@@ -227,7 +200,7 @@ async def test_conversation_count():
initial_count = await history.get_conversation_count()
# Add responses to 3 different conversations
from src.responses.schemas import Response, ResponseUsage, MessageOutputItem, OutputTextContent
from src.responses.schemas import MessageOutputItem, OutputTextContent, Response, ResponseUsage
for i in range(3):
response = Response(
@@ -235,18 +208,10 @@ async def test_conversation_count():
created_at=1234567890,
model="lorem-tester",
status="completed",
output=[
MessageOutputItem(
id=f"msg_{i}",
content=[OutputTextContent(text="Test")]
)
],
output=[MessageOutputItem(id=f"msg_{i}", content=[OutputTextContent(text="Test")])],
usage=ResponseUsage(
input_tokens=10,
output_tokens=5,
reasoning_tokens=0,
total_tokens=15
)
input_tokens=10, output_tokens=5, reasoning_tokens=0, total_tokens=15
),
)
await history.add_response(f"conv_{i}", response)
@@ -281,8 +246,8 @@ async def test_context_window_trimming():
# Create items that exceed limit
items = [
"This is a long message " * 20, # ~480 chars = ~120 tokens
"Another message " * 10, # ~160 chars = ~40 tokens
"Short message" # ~13 chars = ~3 tokens
"Another message " * 10, # ~160 chars = ~40 tokens
"Short message", # ~13 chars = ~3 tokens
]
# Trim with 10 token reserve
@@ -355,14 +320,12 @@ async def test_service_conversation_helpers():
@pytest.mark.asyncio
async def test_streaming_tracks_history(async_client):
"""Test that streaming responses also track conversation history."""
from httpx import AsyncClient
import json
request_data = {
"model": "lorem-tester",
"input": [{"role": "user", "content": "Hello"}],
"metadata": {"conversation_id": "stream_test_001"},
"stream": True
"stream": True,
}
async with async_client.stream(
tests/responses/test_router.py
+21 -50
View File
@@ -3,6 +3,7 @@ Tests for Responses API router.
"""
import json
import pytest
from fastapi.testclient import TestClient
from httpx import AsyncClient
@@ -13,10 +14,8 @@ def test_create_response_non_streaming(client: TestClient) -> None:
"""Test non-streaming response creation."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Hello, how are you?"}
],
"stream": False
"input": [{"role": "user", "content": "Hello, how are you?"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -48,14 +47,9 @@ def test_create_response_with_reasoning(client: TestClient) -> None:
"""Test response with reasoning enabled."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Explain something"}
],
"reasoning": {
"effort": "medium",
"summary": "auto"
},
"stream": False
"input": [{"role": "user", "content": "Explain something"}],
"reasoning": {"effort": "medium", "summary": "auto"},
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -88,17 +82,11 @@ def test_create_response_with_tools(client: TestClient) -> None:
"""Test response with tools available."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Use a tool to help"}
],
"input": [{"role": "user", "content": "Use a tool to help"}],
"tools": [
{
"name": "search_knowledge",
"description": "Search knowledge base",
"parameters": {}
}
{"name": "search_knowledge", "description": "Search knowledge base", "parameters": {}}
],
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -117,10 +105,8 @@ def test_create_response_invalid_model(client: TestClient) -> None:
"""Test response with non-existent model."""
request_data = {
"model": "nonexistent-model",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": False
"input": [{"role": "user", "content": "Hello"}],
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -134,11 +120,7 @@ def test_create_response_invalid_model(client: TestClient) -> None:
def test_create_response_validation_error(client: TestClient) -> None:
"""Test response with invalid request data."""
# Missing required 'model' field
request_data = {
"input": [
{"role": "user", "content": "Hello"}
]
}
request_data = {"input": [{"role": "user", "content": "Hello"}]}
response = client.post("/v1/responses", json=request_data)
@@ -151,10 +133,8 @@ async def test_create_response_streaming(async_client: AsyncClient) -> None:
"""Test streaming response creation."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Hello"}
],
"stream": True
"input": [{"role": "user", "content": "Hello"}],
"stream": True,
}
async with async_client.stream(
@@ -200,14 +180,9 @@ async def test_create_response_streaming_with_reasoning(async_client: AsyncClien
"""Test streaming response with reasoning."""
request_data = {
"model": "lorem-tester",
"input": [
{"role": "user", "content": "Explain"}
],
"reasoning": {
"effort": "medium",
"summary": "auto"
},
"stream": True
"input": [{"role": "user", "content": "Explain"}],
"reasoning": {"effort": "medium", "summary": "auto"},
"stream": True,
}
async with async_client.stream(
@@ -238,16 +213,12 @@ async def test_create_response_streaming_with_reasoning(async_client: AsyncClien
# Should have reasoning events
reasoning_events = [
e for e in events
if e["event"] == "response.reasoning_summary_text.delta"
e for e in events if e["event"] == "response.reasoning_summary_text.delta"
]
assert len(reasoning_events) > 0
# Should have output text events
output_events = [
e for e in events
if e["event"] == "response.output_text.delta"
]
output_events = [e for e in events if e["event"] == "response.output_text.delta"]
assert len(output_events) > 0
@@ -258,7 +229,7 @@ def test_create_response_temperature_parameter(client: TestClient) -> None:
"model": "lorem-tester",
"input": [{"role": "user", "content": "Test"}],
"temperature": 0.5,
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
@@ -273,7 +244,7 @@ def test_create_response_max_tokens_parameter(client: TestClient) -> None:
"model": "lorem-tester",
"input": [{"role": "user", "content": "Test"}],
"max_output_tokens": 100,
"stream": False
"stream": False,
}
response = client.post("/v1/responses", json=request_data)
tests/responses/test_streaming_delegation.py
+7 -15
View File
@@ -53,9 +53,7 @@ class TestStreamDirectDelegation:
) as mock_delegate:
gen = coordinator._stream_direct_delegation(
user_message="Search for Docker info",
recommendation=SimpleNamespace(
recommended_capabilities=["librarian"]
),
recommendation=SimpleNamespace(recommended_capabilities=["librarian"]),
tracker=tracker,
conversation_id="conv_1",
conversation_history=HISTORY,
@@ -66,9 +64,9 @@ class TestStreamDirectDelegation:
first = await gen.__anext__()
assert isinstance(first, ReasoningSummaryDelta)
assert first.delta.strip() != ""
assert mock_delegate.await_count == 0, (
"start think message must stream before the expert runs"
)
assert (
mock_delegate.await_count == 0
), "start think message must stream before the expert runs"
second = await gen.__anext__()
assert isinstance(second, ReasoningSummaryDone)
@@ -101,9 +99,7 @@ class TestStreamDirectDelegation:
event
async for event in coordinator._stream_direct_delegation(
user_message="And what services does it list?",
recommendation=SimpleNamespace(
recommended_capabilities=["librarian"]
),
recommendation=SimpleNamespace(recommended_capabilities=["librarian"]),
tracker=tracker,
conversation_id="conv_1",
conversation_history=HISTORY,
@@ -139,9 +135,7 @@ class TestStreamDirectDelegation:
event
async for event in coordinator._stream_direct_delegation(
user_message="Search for Docker info",
recommendation=SimpleNamespace(
recommended_capabilities=["librarian"]
),
recommendation=SimpleNamespace(recommended_capabilities=["librarian"]),
tracker=tracker,
conversation_id="conv_1",
results=results,
@@ -172,9 +166,7 @@ class TestServiceDelegationContext:
) as mock_delegate:
results = await _direct_delegation_with_results(
user_message="And what services does it list?",
recommendation=SimpleNamespace(
recommended_capabilities=["librarian"]
),
recommendation=SimpleNamespace(recommended_capabilities=["librarian"]),
tracker=tracker,
conversation_id="conv_1",
conversation_history=HISTORY,
tests/test_main.py
+15 -18
View File
@@ -8,6 +8,7 @@ Tests:
- Router registration
- Exception handlers
"""
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
@@ -56,8 +57,9 @@ def test_cors_middleware():
# Better check: verify CORS functionality by checking middleware stack
# The middleware should be present (wrapped as Middleware)
middleware_types = [type(m.cls).__name__ if hasattr(m, 'cls') else type(m).__name__
for m in app.user_middleware]
middleware_types = [
type(m.cls).__name__ if hasattr(m, "cls") else type(m).__name__ for m in app.user_middleware
]
assert "CORSMiddleware" in middleware_types or len(app.user_middleware) > 0
@@ -67,10 +69,7 @@ def test_cors_allows_all_origins(client: TestClient):
# Make OPTIONS request with Origin header
response = client.options(
"/v1/models",
headers={
"Origin": "http://example.com",
"Access-Control-Request-Method": "GET"
}
headers={"Origin": "http://example.com", "Access-Control-Request-Method": "GET"},
)
# Should allow CORS
@@ -81,10 +80,7 @@ def test_cors_allows_all_origins(client: TestClient):
@pytest.mark.unit
def test_cors_allows_credentials(client: TestClient):
"""Test that CORS allows credentials."""
response = client.get(
"/health",
headers={"Origin": "http://example.com"}
)
response = client.get("/health", headers={"Origin": "http://example.com"})
# Should include CORS headers
assert response.status_code == 200
@@ -96,10 +92,11 @@ def test_cors_allows_credentials(client: TestClient):
@pytest.mark.unit
def test_exception_handlers_registered():
"""Test that custom exception handlers are registered."""
from src.main import app
from src.core.exceptions import AppException
from fastapi.exceptions import RequestValidationError
from src.core.exceptions import AppException
from src.main import app
# App should have exception handlers
assert len(app.exception_handlers) > 0
@@ -119,8 +116,8 @@ def test_model_not_found_exception_handler(client: TestClient):
json={
"model": "non-existent-model-xyz",
"input": [{"role": "user", "content": "test"}],
"stream": False
}
"stream": False,
},
)
assert response.status_code == 404
@@ -137,8 +134,8 @@ def test_app_exception_handler(client: TestClient):
json={
"model": "lorem-tester",
"input": [], # Empty input might cause issues
"temperature": 3.0 # Invalid temperature
}
"temperature": 3.0, # Invalid temperature
},
)
# Should return validation error
@@ -149,8 +146,8 @@ def test_app_exception_handler(client: TestClient):
@pytest.mark.asyncio
async def test_lifespan_startup():
"""Test lifespan startup event."""
from src.main import app
from contextlib import asynccontextmanager
# The lifespan context manager should be defined
assert hasattr(app.router, "lifespan_context")
@@ -238,7 +235,7 @@ def test_validation_error_handler(client: TestClient):
json={
# Missing "model" field
"messages": [{"role": "user", "content": "test"}]
}
},
)
assert response.status_code == 422