style: apply ruff's automatic fixes and formatter
Mechanical only, and separated from the judgment calls that follow so the reviewable changes are not buried in a 98-file whitespace diff. 227 automatic fixes: 60 blank lines carrying whitespace, 60 unsorted import blocks, 34 Optional[X] to X | None, 28 unused imports, 16 deprecated typing imports, 12 datetime.timezone.utc to datetime.UTC, and assorted smaller modernisations. Then `ruff format` over src and tests: 98 files reformatted, 35 already conforming. No file among the unused-import findings defines __all__ or is an __init__.py, so nothing here removes a re-export. `make test`: 658 passed, unchanged from HEAD. Two things observed while verifying, neither addressed here: `pytest tests/` cannot collect — tests/e2e/test_orchestration_e2e.py uses an `e2e` marker that is not registered, and the config is strict about markers. This fails identically at HEAD, so it predates this change; `make test` passes because it ignores tests/e2e, tests/integration and tests/contracts. test_tatlock_tool_call_logging_calculator is flaky. It failed once in a full run with these changes and passed on the next, passes in isolation with them, and fails in isolation at HEAD. It is order- or timing-dependent, not a regression from this commit — established by running the full suite both ways rather than by reasoning about which change could have caused it. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -7,10 +7,12 @@ These tests hit the actual running server and test the full stack:
|
||||
- Tool execution
|
||||
- Response formatting
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import httpx
|
||||
from typing import AsyncGenerator
|
||||
|
||||
# Test server base URL (assumes server is running on localhost:8777 via ./wakeup.sh)
|
||||
BASE_URL = "http://localhost:8777"
|
||||
@@ -34,10 +36,8 @@ class TestChatCompletionsE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 144 divided by 12?"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "What is 144 divided by 12?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -68,7 +68,9 @@ class TestChatCompletionsE2E:
|
||||
assert "usage" in data
|
||||
assert data["usage"]["total_tokens"] > 0
|
||||
|
||||
print(f"✓ Calculator test passed. Found '12' in response. Tool indicator: {has_calculator_indicator}")
|
||||
print(
|
||||
f"✓ Calculator test passed. Found '12' in response. Tool indicator: {has_calculator_indicator}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_search(self, client: httpx.AsyncClient):
|
||||
@@ -80,7 +82,7 @@ class TestChatCompletionsE2E:
|
||||
"messages": [
|
||||
{"role": "user", "content": "Search for the current population of Tokyo"}
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -114,10 +116,8 @@ class TestChatCompletionsE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 15 times 4?"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "What is 15 times 4?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response1.status_code == 200
|
||||
@@ -135,9 +135,9 @@ class TestChatCompletionsE2E:
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 15 times 4?"},
|
||||
{"role": "assistant", "content": message1},
|
||||
{"role": "user", "content": "Now add 20 to that result."}
|
||||
{"role": "user", "content": "Now add 20 to that result."},
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert response2.status_code == 200
|
||||
@@ -152,7 +152,9 @@ class TestChatCompletionsE2E:
|
||||
has_calculation = "60" in message2 and "20" in message2
|
||||
assert has_answer or has_calculation, f"Expected '80' or calculation in: {message2}"
|
||||
|
||||
print(f"✓ Multi-turn test passed. Answer found: {has_answer}, Calculation shown: {has_calculation}")
|
||||
print(
|
||||
f"✓ Multi-turn test passed. Answer found: {has_answer}, Calculation shown: {has_calculation}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_calculation_and_search(self, client: httpx.AsyncClient):
|
||||
@@ -161,13 +163,8 @@ class TestChatCompletionsE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Calculate the square root of 256"
|
||||
}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "Calculate the square root of 256"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -194,10 +191,8 @@ class TestChatCompletionsE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -223,10 +218,8 @@ class TestChatCompletionsE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is today's date?"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "What is today's date?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -242,11 +235,16 @@ class TestChatCompletionsE2E:
|
||||
|
||||
# Should contain some date/time information (flexible - varies in format)
|
||||
import re
|
||||
|
||||
has_date = (
|
||||
re.search(r'\d{4}', message) or # Year
|
||||
re.search(r'\d{1,2}', message) or # Day/month number
|
||||
re.search(r'(January|February|March|April|May|June|July|August|September|October|November|December)', message, re.IGNORECASE) or
|
||||
"today" in message.lower()
|
||||
re.search(r"\d{4}", message) # Year
|
||||
or re.search(r"\d{1,2}", message) # Day/month number
|
||||
or re.search(
|
||||
r"(January|February|March|April|May|June|July|August|September|October|November|December)",
|
||||
message,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
or "today" in message.lower()
|
||||
)
|
||||
|
||||
assert has_date, f"Expected date/time information in: {message}"
|
||||
@@ -263,11 +261,9 @@ class TestResponsesAPIE2E:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Calculate 25 times 16"}
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"}
|
||||
}
|
||||
"input": [{"role": "user", "content": "Calculate 25 times 16"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -300,7 +296,7 @@ class TestResponsesAPIE2E:
|
||||
assert "usage" in data
|
||||
assert data["usage"]["total_tokens"] > 0
|
||||
|
||||
print(f"✓ Responses API test passed. Found '400' with Steward reasoning.")
|
||||
print("✓ Responses API test passed. Found '400' with Steward reasoning.")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_multi_turn(self, client: httpx.AsyncClient):
|
||||
@@ -312,10 +308,10 @@ class TestResponsesAPIE2E:
|
||||
"input": [
|
||||
{"role": "user", "content": "What is 7 times 8?"},
|
||||
{"role": "assistant", "content": "Certainly, sir. 7 times 8 equals 56."},
|
||||
{"role": "user", "content": "Double that number."}
|
||||
{"role": "user", "content": "Double that number."},
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"}
|
||||
}
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -346,11 +342,9 @@ class TestStreamingE2E:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 9 times 7?"}
|
||||
],
|
||||
"stream": True
|
||||
}
|
||||
"messages": [{"role": "user", "content": "What is 9 times 7?"}],
|
||||
"stream": True,
|
||||
},
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -362,6 +356,7 @@ class TestStreamingE2E:
|
||||
break
|
||||
|
||||
import json
|
||||
|
||||
chunk = json.loads(data_str)
|
||||
chunks.append(chunk)
|
||||
|
||||
@@ -373,8 +368,7 @@ class TestStreamingE2E:
|
||||
|
||||
# Should have received Steward's reasoning (in <think> tags)
|
||||
full_content = "".join(
|
||||
chunk["choices"][0]["delta"].get("content", "") or ""
|
||||
for chunk in chunks
|
||||
chunk["choices"][0]["delta"].get("content", "") or "" for chunk in chunks
|
||||
)
|
||||
assert "<think>" in full_content
|
||||
assert "</think>" in full_content
|
||||
@@ -393,10 +387,8 @@ class TestErrorHandling:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "nonexistent-model",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
@@ -411,7 +403,7 @@ class TestErrorHandling:
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
# Missing "messages" field
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
@@ -425,11 +417,9 @@ class TestErrorHandling:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
"temperature": 5.0 # Max is 2.0
|
||||
}
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"temperature": 5.0, # Max is 2.0
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
@@ -447,11 +437,9 @@ class TestChatResponsesWrapper:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Calculate 13 times 9"}
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"}
|
||||
}
|
||||
"input": [{"role": "user", "content": "Calculate 13 times 9"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -500,10 +488,8 @@ class TestChatResponsesWrapper:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 5 plus 3?"}
|
||||
],
|
||||
}
|
||||
"messages": [{"role": "user", "content": "What is 5 plus 3?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -539,11 +525,9 @@ class TestChatResponsesWrapper:
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Count to 3"}
|
||||
],
|
||||
"stream": True
|
||||
}
|
||||
"messages": [{"role": "user", "content": "Count to 3"}],
|
||||
"stream": True,
|
||||
},
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -555,6 +539,7 @@ class TestChatResponsesWrapper:
|
||||
break
|
||||
|
||||
import json
|
||||
|
||||
chunk = json.loads(data_str)
|
||||
chunks.append(chunk)
|
||||
|
||||
@@ -574,10 +559,7 @@ class TestChatResponsesWrapper:
|
||||
assert chunks[0]["choices"][0]["delta"]["role"] == "assistant"
|
||||
|
||||
# Should have content chunks
|
||||
has_content = any(
|
||||
"content" in chunk["choices"][0]["delta"]
|
||||
for chunk in chunks
|
||||
)
|
||||
has_content = any("content" in chunk["choices"][0]["delta"] for chunk in chunks)
|
||||
assert has_content
|
||||
|
||||
print(f"✓ Streaming format matches OpenAI spec ({len(chunks)} chunks)")
|
||||
@@ -593,11 +575,9 @@ class TestStewardIntegration:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Calculate 123 times 456"}
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"}
|
||||
}
|
||||
"input": [{"role": "user", "content": "Calculate 123 times 456"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -620,10 +600,10 @@ class TestStewardIntegration:
|
||||
"input": [
|
||||
{"role": "user", "content": "My favorite number is 42"},
|
||||
{"role": "assistant", "content": "Noted, sir. 42 is an excellent choice."},
|
||||
{"role": "user", "content": "What was that number again?"}
|
||||
{"role": "user", "content": "What was that number again?"},
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"}
|
||||
}
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -18,13 +18,14 @@ Requirements:
|
||||
Note: LLM outputs are non-deterministic. Tests use flexible assertions
|
||||
that check for behavioral patterns rather than exact text matches.
|
||||
"""
|
||||
import pytest
|
||||
import httpx
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from typing import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src.core.config import PRODUCTION_TENANT, TEST_TENANT
|
||||
|
||||
@@ -43,6 +44,7 @@ assert TEST_USER != PRODUCTION_TENANT
|
||||
@dataclass
|
||||
class LLMAssertionResult:
|
||||
"""Result of an LLM output assertion check."""
|
||||
|
||||
passed: bool
|
||||
evidence: str
|
||||
confidence: str # "high", "medium", "low"
|
||||
@@ -183,6 +185,7 @@ def assert_llm_behavior(
|
||||
# Fixtures
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client() -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""HTTP client for API requests."""
|
||||
@@ -223,6 +226,7 @@ async def clean_test_memories(qdrant: QdrantVerifier):
|
||||
# Memory System Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestMemoryStorage:
|
||||
@@ -245,9 +249,7 @@ class TestMemoryStorage:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Remember that my test color is purple"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "Remember that my test color is purple"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -281,7 +283,8 @@ class TestMemoryStorage:
|
||||
# Verify data in Qdrant - look for any color-related or test-related memory
|
||||
points = await qdrant.scroll_points(TEST_COLLECTION)
|
||||
relevant_memories = [
|
||||
p for p in points
|
||||
p
|
||||
for p in points
|
||||
if "color" in p.get("payload", {}).get("key", "").lower()
|
||||
or "purple" in str(p.get("payload", {}).get("value", "")).lower()
|
||||
or "test" in p.get("payload", {}).get("key", "").lower()
|
||||
@@ -341,9 +344,7 @@ class TestMemoryStorage:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What is my favorite food?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What is my favorite food?"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -431,6 +432,7 @@ class TestMemoryRecall:
|
||||
# Steward Delegation Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestStewardDelegation:
|
||||
@@ -445,9 +447,7 @@ class TestStewardDelegation:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What do you know about me?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What do you know about me?"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
@@ -479,9 +479,7 @@ class TestStewardDelegation:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Calculate 127 times 83"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "Calculate 127 times 83"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
@@ -511,15 +509,16 @@ class TestStewardDelegation:
|
||||
|
||||
# Remove commas for number comparison
|
||||
message_normalized = message_text.replace(",", "")
|
||||
assert "10541" in message_normalized, (
|
||||
f"Expected calculation result 10541. Got: {message_text[:200]}"
|
||||
)
|
||||
assert (
|
||||
"10541" in message_normalized
|
||||
), f"Expected calculation result 10541. Got: {message_text[:200]}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Direct Delegation Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestDirectDelegation:
|
||||
@@ -569,6 +568,7 @@ class TestDirectDelegation:
|
||||
# Data Verification Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestDataVerification:
|
||||
@@ -610,8 +610,7 @@ class TestDataVerification:
|
||||
payload = point.get("payload", {})
|
||||
for field in required_fields:
|
||||
assert field in payload, (
|
||||
f"Point {point['id']} missing required field '{field}'. "
|
||||
f"Payload: {payload}"
|
||||
f"Point {point['id']} missing required field '{field}'. " f"Payload: {payload}"
|
||||
)
|
||||
|
||||
async def test_qdrant_point_types_are_valid(
|
||||
@@ -643,6 +642,7 @@ class TestDataVerification:
|
||||
# Integration Health Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestIntegrationHealth:
|
||||
@@ -678,6 +678,7 @@ class TestIntegrationHealth:
|
||||
# Orchestration Scenario Tests (from ORCHESTRATION_SCENARIOS.md)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestScenario1WeatherWithMemory:
|
||||
@@ -697,9 +698,7 @@ class TestScenario1WeatherWithMemory:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What's the weather like?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What's the weather like?"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
)
|
||||
@@ -717,13 +716,17 @@ class TestScenario1WeatherWithMemory:
|
||||
print(f"Weather query - Steward: {reasoning_text[:200]}...")
|
||||
|
||||
# Should mention location/memory and search capabilities
|
||||
has_memory_mention = "biographer" in reasoning_text.lower() or "memory" in reasoning_text.lower()
|
||||
has_search_mention = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
|
||||
has_memory_mention = (
|
||||
"biographer" in reasoning_text.lower() or "memory" in reasoning_text.lower()
|
||||
)
|
||||
has_search_mention = (
|
||||
"tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
|
||||
)
|
||||
|
||||
# Weather query should trigger at least web search
|
||||
assert has_search_mention, (
|
||||
f"Weather query should recommend search capability. Got: {reasoning_text[:200]}"
|
||||
)
|
||||
assert (
|
||||
has_search_mention
|
||||
), f"Weather query should recommend search capability. Got: {reasoning_text[:200]}"
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@@ -744,9 +747,7 @@ class TestScenario4SimpleExpertDelegation:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What is 847 times 293?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What is 847 times 293?"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -761,9 +762,9 @@ class TestScenario4SimpleExpertDelegation:
|
||||
|
||||
# Expected result: 248171 (may be formatted as 248,171)
|
||||
message_normalized = message_text.replace(",", "")
|
||||
assert "248171" in message_normalized, (
|
||||
f"Calculator should compute 847 * 293 = 248171. Got: {message_text[:200]}"
|
||||
)
|
||||
assert (
|
||||
"248171" in message_normalized
|
||||
), f"Calculator should compute 847 * 293 = 248171. Got: {message_text[:200]}"
|
||||
|
||||
async def test_datetime_query_uses_datetime_tool(
|
||||
self,
|
||||
@@ -774,9 +775,7 @@ class TestScenario4SimpleExpertDelegation:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What day of the week is it?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What day of the week is it?"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -791,8 +790,20 @@ class TestScenario4SimpleExpertDelegation:
|
||||
|
||||
# Should mention a day of the week (full or abbreviated)
|
||||
days = [
|
||||
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
|
||||
"mon", "tue", "wed", "thu", "fri", "sat", "sun"
|
||||
"monday",
|
||||
"tuesday",
|
||||
"wednesday",
|
||||
"thursday",
|
||||
"friday",
|
||||
"saturday",
|
||||
"sunday",
|
||||
"mon",
|
||||
"tue",
|
||||
"wed",
|
||||
"thu",
|
||||
"fri",
|
||||
"sat",
|
||||
"sun",
|
||||
]
|
||||
has_day = any(day in message_text.lower() for day in days)
|
||||
|
||||
@@ -838,9 +849,9 @@ class TestScenario6WikiCreation:
|
||||
print(f"Wiki search - Steward: {reasoning_text[:200]}...")
|
||||
|
||||
# Should mention librarian
|
||||
assert "librarian" in reasoning_text.lower(), (
|
||||
f"Wiki search should recommend librarian. Got: {reasoning_text[:200]}"
|
||||
)
|
||||
assert (
|
||||
"librarian" in reasoning_text.lower()
|
||||
), f"Wiki search should recommend librarian. Got: {reasoning_text[:200]}"
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@@ -862,7 +873,10 @@ class TestScenario8MultiExpertCoordination:
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What do you know about me? And also search for Python tutorials."}
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What do you know about me? And also search for Python tutorials.",
|
||||
}
|
||||
],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
},
|
||||
@@ -885,15 +899,16 @@ class TestScenario8MultiExpertCoordination:
|
||||
has_librarian = "librarian" in reasoning_text.lower()
|
||||
has_search = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
|
||||
|
||||
assert has_biographer or has_librarian or has_search, (
|
||||
f"Complex query should identify multiple capabilities. Got: {reasoning_text[:200]}"
|
||||
)
|
||||
assert (
|
||||
has_biographer or has_librarian or has_search
|
||||
), f"Complex query should identify multiple capabilities. Got: {reasoning_text[:200]}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# New Scenarios from Today's Session
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.asyncio
|
||||
class TestDirectDelegationBypass:
|
||||
@@ -917,9 +932,7 @@ class TestDirectDelegationBypass:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Remember that my test value is alpha123"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "Remember that my test value is alpha123"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -979,7 +992,10 @@ class TestUserContextIsolation:
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": f"Remember that my isolation marker is {unique_value}"}
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Remember that my isolation marker is {unique_value}",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1000,13 +1016,15 @@ class TestUserContextIsolation:
|
||||
else:
|
||||
in_prod_collection = False
|
||||
|
||||
print(f"Isolation test - In test collection: {in_test_collection}, In prod: {in_prod_collection}")
|
||||
print(
|
||||
f"Isolation test - In test collection: {in_test_collection}, In prod: {in_prod_collection}"
|
||||
)
|
||||
|
||||
# Should be in test collection OR response acknowledged
|
||||
# Should NOT be in production collection
|
||||
assert not in_prod_collection, (
|
||||
f"Test data leaked to production collection! Value: {unique_value}"
|
||||
)
|
||||
assert (
|
||||
not in_prod_collection
|
||||
), f"Test data leaked to production collection! Value: {unique_value}"
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@@ -1025,16 +1043,16 @@ class TestErrorHandling:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "nonexistent-model-xyz",
|
||||
"input": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
# Should return error status (404 or 400)
|
||||
assert response.status_code in [400, 404, 422], (
|
||||
f"Expected error status for invalid model. Got: {response.status_code}"
|
||||
)
|
||||
assert response.status_code in [
|
||||
400,
|
||||
404,
|
||||
422,
|
||||
], f"Expected error status for invalid model. Got: {response.status_code}"
|
||||
data = response.json()
|
||||
# Error could be in "error" or "detail" key
|
||||
assert "error" in data or "detail" in data
|
||||
@@ -1084,7 +1102,10 @@ class TestEvaluationReport:
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "Remember that my test pet is a hamster named Fluffy"}
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Remember that my test pet is a hamster named Fluffy",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
@@ -1096,7 +1117,7 @@ class TestEvaluationReport:
|
||||
break
|
||||
|
||||
report_lines.append("\n[STORE TEST]")
|
||||
report_lines.append(f"Input: 'Remember that my test pet is a hamster named Fluffy'")
|
||||
report_lines.append("Input: 'Remember that my test pet is a hamster named Fluffy'")
|
||||
report_lines.append(f"Response: {store_text[:200]}...")
|
||||
|
||||
await asyncio.sleep(1.0)
|
||||
@@ -1106,9 +1127,7 @@ class TestEvaluationReport:
|
||||
"/v1/responses",
|
||||
json={
|
||||
"model": "Tatlock",
|
||||
"input": [
|
||||
{"role": "user", "content": "What pet do I have?"}
|
||||
],
|
||||
"input": [{"role": "user", "content": "What pet do I have?"}],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1119,7 +1138,7 @@ class TestEvaluationReport:
|
||||
break
|
||||
|
||||
report_lines.append("\n[RECALL TEST]")
|
||||
report_lines.append(f"Input: 'What pet do I have?'")
|
||||
report_lines.append("Input: 'What pet do I have?'")
|
||||
report_lines.append(f"Response: {recall_text[:200]}...")
|
||||
|
||||
# Check Qdrant state
|
||||
@@ -1128,7 +1147,8 @@ class TestEvaluationReport:
|
||||
report_lines.append(f"Total points in {TEST_COLLECTION}: {len(points)}")
|
||||
|
||||
pet_memories = [
|
||||
p for p in points
|
||||
p
|
||||
for p in points
|
||||
if "pet" in str(p.get("payload", {})).lower()
|
||||
or "fluffy" in str(p.get("payload", {})).lower()
|
||||
or "hamster" in str(p.get("payload", {})).lower()
|
||||
|
||||
Reference in New Issue
Block a user