jpmschweitzerandClaude Fable 5
b47c5b9281
Session-scoped autouse guard in tests/conftest.py refuses to run any test (pytest.exit, returncode 1) when the effective tenant resolves to the production tenant jpmschweitzer - the same guard library-desk applies on its side. _initialize_app now depends on the guard so the refusal happens before any initialization. Suite-level assertions pin that the live session runs under the llm_tester namespaces: Qdrant memories_llm_tester collection and Redis session:llm_tester:* keys. The biographer/memory unit tests already run fully mocked (no shared-service writes); the e2e isolation tests already used llm_tester - their constants now derive from the shared TEST_TENANT/PRODUCTION_TENANT config constants so a drift fails loudly instead of silently splitting. Verified: ENVIRONMENT=production pytest run exits 1 with the TENANT GUARD message and zero tests executed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1164 lines
38 KiB
Python
1164 lines
38 KiB
Python
"""
|
|
End-to-end tests for orchestration scenarios.
|
|
|
|
Tests the full stack including:
|
|
- Memory storage and recall via REST API
|
|
- Qdrant data verification
|
|
- Multi-expert coordination
|
|
- Direct delegation bypass
|
|
|
|
These tests hit the actual running server and verify data persistence.
|
|
They use the `llm_tester` user for isolation from production data.
|
|
|
|
Requirements:
|
|
- Server running on localhost:8777 (use ./wakeup.sh)
|
|
- Qdrant running on localhost:6333
|
|
- Ollama running with mistral-nemo model
|
|
|
|
Note: LLM outputs are non-deterministic. Tests use flexible assertions
|
|
that check for behavioral patterns rather than exact text matches.
|
|
"""
|
|
import pytest
|
|
import httpx
|
|
import asyncio
|
|
import re
|
|
from typing import AsyncGenerator
|
|
from dataclasses import dataclass
|
|
|
|
|
|
from src.core.config import PRODUCTION_TENANT, TEST_TENANT
|
|
|
|
# Test configuration
|
|
BASE_URL = "http://localhost:8777"
|
|
QDRANT_URL = "http://localhost:6333"
|
|
API_TIMEOUT = 120.0 # LLM calls can be slow
|
|
# All e2e writes to shared services go to the reserved test tenant's
|
|
# namespaces (Qdrant memories_llm_tester, wiki llm_tester scope) - never
|
|
# the production tenant's.
|
|
TEST_USER = TEST_TENANT
|
|
TEST_COLLECTION = f"memories_{TEST_USER}"
|
|
assert TEST_USER != PRODUCTION_TENANT
|
|
|
|
|
|
@dataclass
|
|
class LLMAssertionResult:
|
|
"""Result of an LLM output assertion check."""
|
|
passed: bool
|
|
evidence: str
|
|
confidence: str # "high", "medium", "low"
|
|
|
|
|
|
class QdrantVerifier:
|
|
"""Helper for verifying data in Qdrant."""
|
|
|
|
def __init__(self, base_url: str = QDRANT_URL):
|
|
self.base_url = base_url
|
|
|
|
async def collection_exists(self, collection_name: str) -> bool:
|
|
"""Check if a collection exists."""
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.get(f"{self.base_url}/collections/{collection_name}")
|
|
return response.status_code == 200
|
|
|
|
async def get_points_count(self, collection_name: str) -> int:
|
|
"""Get number of points in a collection."""
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.get(f"{self.base_url}/collections/{collection_name}")
|
|
if response.status_code != 200:
|
|
return 0
|
|
data = response.json()
|
|
return data.get("result", {}).get("points_count", 0)
|
|
|
|
async def scroll_points(
|
|
self,
|
|
collection_name: str,
|
|
limit: int = 100,
|
|
with_payload: bool = True,
|
|
) -> list[dict]:
|
|
"""Get all points from a collection."""
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.post(
|
|
f"{self.base_url}/collections/{collection_name}/points/scroll",
|
|
json={
|
|
"limit": limit,
|
|
"with_payload": with_payload,
|
|
"with_vector": False,
|
|
},
|
|
)
|
|
if response.status_code != 200:
|
|
return []
|
|
data = response.json()
|
|
return data.get("result", {}).get("points", [])
|
|
|
|
async def find_memory_by_key(
|
|
self,
|
|
collection_name: str,
|
|
key: str,
|
|
memory_type: str | None = None,
|
|
) -> dict | None:
|
|
"""Find a specific memory by key."""
|
|
points = await self.scroll_points(collection_name)
|
|
for point in points:
|
|
payload = point.get("payload", {})
|
|
if payload.get("key") == key:
|
|
if memory_type is None or payload.get("type") == memory_type:
|
|
return point
|
|
return None
|
|
|
|
async def delete_points_by_key(
|
|
self,
|
|
collection_name: str,
|
|
keys: list[str],
|
|
) -> bool:
|
|
"""Delete points by key (for cleanup)."""
|
|
points = await self.scroll_points(collection_name)
|
|
point_ids = []
|
|
for point in points:
|
|
if point.get("payload", {}).get("key") in keys:
|
|
point_ids.append(point["id"])
|
|
|
|
if not point_ids:
|
|
return True
|
|
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.post(
|
|
f"{self.base_url}/collections/{collection_name}/points/delete",
|
|
json={"points": point_ids},
|
|
)
|
|
return response.status_code == 200
|
|
|
|
|
|
def assert_llm_behavior(
|
|
response_text: str,
|
|
expected_patterns: list[str],
|
|
unexpected_patterns: list[str] | None = None,
|
|
min_matches: int = 1,
|
|
) -> LLMAssertionResult:
|
|
"""
|
|
Assert LLM behavior using flexible pattern matching.
|
|
|
|
This handles the non-deterministic nature of LLM outputs by checking
|
|
for behavioral patterns rather than exact text.
|
|
|
|
Args:
|
|
response_text: The LLM's response text
|
|
expected_patterns: Regex patterns that should match (at least min_matches)
|
|
unexpected_patterns: Patterns that should NOT be present
|
|
min_matches: Minimum number of expected patterns that must match
|
|
|
|
Returns:
|
|
LLMAssertionResult with pass/fail and evidence
|
|
"""
|
|
response_lower = response_text.lower()
|
|
matches = []
|
|
unexpected_matches = []
|
|
|
|
for pattern in expected_patterns:
|
|
if re.search(pattern, response_text, re.IGNORECASE):
|
|
matches.append(pattern)
|
|
|
|
if unexpected_patterns:
|
|
for pattern in unexpected_patterns:
|
|
if re.search(pattern, response_text, re.IGNORECASE):
|
|
unexpected_matches.append(pattern)
|
|
|
|
passed = len(matches) >= min_matches and len(unexpected_matches) == 0
|
|
|
|
# Determine confidence
|
|
if len(matches) >= len(expected_patterns):
|
|
confidence = "high"
|
|
elif len(matches) >= min_matches:
|
|
confidence = "medium"
|
|
else:
|
|
confidence = "low"
|
|
|
|
evidence = f"Matched {len(matches)}/{len(expected_patterns)} patterns: {matches}"
|
|
if unexpected_matches:
|
|
evidence += f"; Unexpected: {unexpected_matches}"
|
|
|
|
return LLMAssertionResult(passed=passed, evidence=evidence, confidence=confidence)
|
|
|
|
|
|
# ============================================================================
|
|
# Fixtures
|
|
# ============================================================================
|
|
|
|
@pytest.fixture
|
|
async def client() -> AsyncGenerator[httpx.AsyncClient, None]:
|
|
"""HTTP client for API requests."""
|
|
async with httpx.AsyncClient(base_url=BASE_URL, timeout=API_TIMEOUT) as client:
|
|
yield client
|
|
|
|
|
|
@pytest.fixture
|
|
def qdrant() -> QdrantVerifier:
|
|
"""Qdrant verification helper."""
|
|
return QdrantVerifier()
|
|
|
|
|
|
@pytest.fixture
|
|
async def clean_test_memories(qdrant: QdrantVerifier):
|
|
"""Clean up test memories before and after each test."""
|
|
# Test-specific keys we might create
|
|
test_keys = [
|
|
"test_color",
|
|
"test_pet",
|
|
"test_location",
|
|
"test_food",
|
|
"favorite_color",
|
|
"favorite_food",
|
|
"pet_name",
|
|
]
|
|
|
|
# Clean before test
|
|
await qdrant.delete_points_by_key(TEST_COLLECTION, test_keys)
|
|
|
|
yield
|
|
|
|
# Clean after test
|
|
await qdrant.delete_points_by_key(TEST_COLLECTION, test_keys)
|
|
|
|
|
|
# ============================================================================
|
|
# Memory System Tests
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestMemoryStorage:
|
|
"""Tests for memory storage via REST API."""
|
|
|
|
async def test_store_memory_creates_qdrant_point(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
qdrant: QdrantVerifier,
|
|
clean_test_memories,
|
|
):
|
|
"""
|
|
Test that asking to remember something creates a Qdrant point.
|
|
|
|
This tests the direct delegation bypass path:
|
|
Steward -> recommends biographer -> direct delegation -> biographer stores
|
|
"""
|
|
# Send remember request
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Remember that my test color is purple"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["status"] == "completed"
|
|
|
|
# Check response indicates memory was handled
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
# LLM response should indicate acknowledgment (flexible check)
|
|
result = assert_llm_behavior(
|
|
message_text,
|
|
expected_patterns=[
|
|
r"(remember|noted|recorded|stored|got it|understood|certainly|sure|of course)",
|
|
r"(purple|color|test)",
|
|
],
|
|
min_matches=1,
|
|
)
|
|
|
|
print(f"Store response: {message_text[:200]}...")
|
|
print(f"Assertion: {result.evidence}")
|
|
|
|
# Give Qdrant a moment to sync
|
|
await asyncio.sleep(1.0)
|
|
|
|
# Verify data in Qdrant - look for any color-related or test-related memory
|
|
points = await qdrant.scroll_points(TEST_COLLECTION)
|
|
relevant_memories = [
|
|
p for p in points
|
|
if "color" in p.get("payload", {}).get("key", "").lower()
|
|
or "purple" in str(p.get("payload", {}).get("value", "")).lower()
|
|
or "test" in p.get("payload", {}).get("key", "").lower()
|
|
]
|
|
|
|
print(f"Points found in {TEST_COLLECTION}: {len(points)}")
|
|
print(f"Relevant memories found: {len(relevant_memories)}")
|
|
|
|
# The test passes if either:
|
|
# 1. Memory was stored in Qdrant
|
|
# 2. LLM acknowledged the request (even if storage failed)
|
|
# 3. The API responded successfully (delegation happened)
|
|
api_success = response.status_code == 200
|
|
|
|
if len(relevant_memories) == 0 and not result.passed and not api_success:
|
|
pytest.xfail(
|
|
f"Memory may not have stored. "
|
|
f"Points found: {len(relevant_memories)}, Response check: {result.evidence}"
|
|
)
|
|
|
|
# At minimum, the API should have succeeded
|
|
assert api_success, f"API call failed: {response.status_code}"
|
|
|
|
async def test_store_and_recall_memory(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
qdrant: QdrantVerifier,
|
|
clean_test_memories,
|
|
):
|
|
"""
|
|
Test storing a memory and then recalling it.
|
|
|
|
This is the critical end-to-end flow:
|
|
1. Store: User request -> Steward -> Biographer -> Qdrant
|
|
2. Recall: User request -> Steward -> Biographer -> Qdrant search -> response
|
|
"""
|
|
# Step 1: Store a unique memory
|
|
unique_value = "chartreuse" # Distinctive value
|
|
store_response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": f"Remember that my favorite food is {unique_value}"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert store_response.status_code == 200
|
|
print(f"Store response status: {store_response.json()['status']}")
|
|
|
|
# Give system time to process
|
|
await asyncio.sleep(1.0)
|
|
|
|
# Step 2: Recall the memory
|
|
recall_response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What is my favorite food?"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert recall_response.status_code == 200
|
|
recall_data = recall_response.json()
|
|
|
|
# Extract response text
|
|
recall_text = ""
|
|
for output in recall_data["output"]:
|
|
if output["type"] == "message":
|
|
recall_text = output["content"][0]["text"]
|
|
break
|
|
|
|
print(f"Recall response: {recall_text[:200]}...")
|
|
|
|
# Check if the unique value was recalled
|
|
result = assert_llm_behavior(
|
|
recall_text,
|
|
expected_patterns=[
|
|
rf"{unique_value}", # The specific value
|
|
r"(food|favorite)", # Context about food
|
|
],
|
|
min_matches=1,
|
|
)
|
|
|
|
print(f"Recall assertion: {result.evidence}, confidence: {result.confidence}")
|
|
|
|
# This is a soft assertion - we note if it failed but don't fail the test
|
|
# because LLM behavior is non-deterministic
|
|
if not result.passed:
|
|
pytest.xfail(
|
|
f"Memory recall did not return expected value. "
|
|
f"This may be due to LLM non-determinism. Evidence: {result.evidence}"
|
|
)
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestMemoryRecall:
|
|
"""Tests for memory recall scenarios."""
|
|
|
|
async def test_recall_nonexistent_memory(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test asking about something not in memory."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What is my favorite quantum physics theorem?"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
# Should indicate no memory found (various phrasings acceptable)
|
|
result = assert_llm_behavior(
|
|
message_text,
|
|
expected_patterns=[
|
|
r"(don't have|no record|not sure|haven't|can't recall|don't know)",
|
|
r"(would you like|shall I|tell me)",
|
|
],
|
|
min_matches=1,
|
|
)
|
|
|
|
print(f"No-memory response: {message_text[:150]}...")
|
|
print(f"Assertion: {result.evidence}")
|
|
|
|
# Soft assertion
|
|
if not result.passed:
|
|
pytest.xfail(f"Response unclear about missing memory. Evidence: {result.evidence}")
|
|
|
|
|
|
# ============================================================================
|
|
# Steward Delegation Tests
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestStewardDelegation:
|
|
"""Tests for Steward's delegation decisions."""
|
|
|
|
async def test_steward_recommends_biographer_for_memory(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that Steward recommends biographer for memory requests."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What do you know about me?"}
|
|
],
|
|
"reasoning": {"effort": "medium", "summary": "auto"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check Steward's reasoning
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
print(f"Steward reasoning: {reasoning_text[:200]}...")
|
|
|
|
# Steward should mention biographer
|
|
assert "biographer" in reasoning_text.lower(), (
|
|
f"Steward should recommend biographer for 'what do you know about me'. "
|
|
f"Got: {reasoning_text[:200]}"
|
|
)
|
|
|
|
async def test_steward_recommends_calculator_for_math(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that Steward recommends tatlock_core for math."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Calculate 127 times 83"}
|
|
],
|
|
"reasoning": {"effort": "medium", "summary": "auto"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check Steward's reasoning
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
# Steward should recommend tatlock_core for calculation
|
|
assert "tatlock_core" in reasoning_text.lower() or "calculat" in reasoning_text.lower(), (
|
|
f"Steward should recommend tatlock_core for calculation. "
|
|
f"Got: {reasoning_text[:200]}"
|
|
)
|
|
|
|
# Should also get the correct answer (10541 or 10,541)
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
# Remove commas for number comparison
|
|
message_normalized = message_text.replace(",", "")
|
|
assert "10541" in message_normalized, (
|
|
f"Expected calculation result 10541. Got: {message_text[:200]}"
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# Direct Delegation Tests
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestDirectDelegation:
|
|
"""Tests for direct delegation bypass (when only biographer/librarian needed)."""
|
|
|
|
async def test_direct_delegation_for_pure_memory_request(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""
|
|
Test that pure memory requests bypass Tatlock and go directly to biographer.
|
|
|
|
When Steward recommends ONLY biographer, we should skip Tatlock's LLM
|
|
call and delegate directly.
|
|
"""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Remember that I am testing direct delegation"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check that response was generated
|
|
assert data["status"] == "completed"
|
|
assert len(data["output"]) >= 2 # reasoning + message
|
|
|
|
# Steward should recommend biographer
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
print(f"Direct delegation test - Steward: {reasoning_text[:200]}...")
|
|
|
|
# Should mention biographer
|
|
assert "biographer" in reasoning_text.lower()
|
|
|
|
|
|
# ============================================================================
|
|
# Data Verification Tests
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestDataVerification:
|
|
"""Tests that verify data presence and structure in storage."""
|
|
|
|
async def test_qdrant_collection_exists_for_test_user(
|
|
self,
|
|
qdrant: QdrantVerifier,
|
|
):
|
|
"""Verify the test user's Qdrant collection exists."""
|
|
exists = await qdrant.collection_exists(TEST_COLLECTION)
|
|
|
|
# Collection might not exist if no memories stored yet
|
|
if not exists:
|
|
pytest.skip(
|
|
f"Collection {TEST_COLLECTION} does not exist. "
|
|
"Run memory tests first to create it."
|
|
)
|
|
|
|
assert exists, f"Collection {TEST_COLLECTION} should exist"
|
|
|
|
async def test_qdrant_points_have_required_fields(
|
|
self,
|
|
qdrant: QdrantVerifier,
|
|
):
|
|
"""Verify Qdrant points have the required payload structure."""
|
|
exists = await qdrant.collection_exists(TEST_COLLECTION)
|
|
if not exists:
|
|
pytest.skip(f"Collection {TEST_COLLECTION} does not exist")
|
|
|
|
points = await qdrant.scroll_points(TEST_COLLECTION, limit=10)
|
|
|
|
if not points:
|
|
pytest.skip("No points in collection to verify")
|
|
|
|
required_fields = ["type", "key", "value"]
|
|
|
|
for point in points:
|
|
payload = point.get("payload", {})
|
|
for field in required_fields:
|
|
assert field in payload, (
|
|
f"Point {point['id']} missing required field '{field}'. "
|
|
f"Payload: {payload}"
|
|
)
|
|
|
|
async def test_qdrant_point_types_are_valid(
|
|
self,
|
|
qdrant: QdrantVerifier,
|
|
):
|
|
"""Verify Qdrant points have valid memory types."""
|
|
exists = await qdrant.collection_exists(TEST_COLLECTION)
|
|
if not exists:
|
|
pytest.skip(f"Collection {TEST_COLLECTION} does not exist")
|
|
|
|
points = await qdrant.scroll_points(TEST_COLLECTION)
|
|
|
|
if not points:
|
|
pytest.skip("No points in collection to verify")
|
|
|
|
valid_types = ["user_profile", "preference", "learned_fact"]
|
|
|
|
for point in points:
|
|
payload = point.get("payload", {})
|
|
memory_type = payload.get("type")
|
|
assert memory_type in valid_types, (
|
|
f"Invalid memory type '{memory_type}' in point {point['id']}. "
|
|
f"Valid types: {valid_types}"
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# Integration Health Tests
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestIntegrationHealth:
|
|
"""Tests for verifying integration health."""
|
|
|
|
async def test_api_is_reachable(self, client: httpx.AsyncClient):
|
|
"""Test that the API server is running and reachable."""
|
|
response = await client.get("/health")
|
|
assert response.status_code == 200
|
|
|
|
async def test_models_endpoint_returns_tatlock(self, client: httpx.AsyncClient):
|
|
"""Test that Tatlock is listed in available models."""
|
|
response = await client.get("/v1/models")
|
|
assert response.status_code == 200
|
|
|
|
data = response.json()
|
|
model_ids = [m["id"] for m in data.get("data", [])]
|
|
|
|
assert "Tatlock" in model_ids, f"Tatlock not in models: {model_ids}"
|
|
|
|
async def test_qdrant_is_reachable(self, qdrant: QdrantVerifier):
|
|
"""Test that Qdrant is running and reachable."""
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.get(f"{QDRANT_URL}/collections")
|
|
assert response.status_code == 200
|
|
|
|
|
|
# ============================================================================
|
|
# Test Evaluation Helpers
|
|
# ============================================================================
|
|
|
|
# ============================================================================
|
|
# Orchestration Scenario Tests (from ORCHESTRATION_SCENARIOS.md)
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestScenario1WeatherWithMemory:
|
|
"""
|
|
Scenario 1: Weather Check (Multi-Step with Memory Lookup)
|
|
|
|
Tests: Location determination from memory + weather lookup
|
|
Flow: Steward -> Memory (location) -> Web search (weather) -> Response
|
|
"""
|
|
|
|
async def test_weather_query_triggers_memory_lookup(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that weather query attempts to get user's location."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What's the weather like?"}
|
|
],
|
|
"reasoning": {"effort": "medium", "summary": "auto"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check Steward reasoning
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
print(f"Weather query - Steward: {reasoning_text[:200]}...")
|
|
|
|
# Should mention location/memory and search capabilities
|
|
has_memory_mention = "biographer" in reasoning_text.lower() or "memory" in reasoning_text.lower()
|
|
has_search_mention = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
|
|
|
|
# Weather query should trigger at least web search
|
|
assert has_search_mention, (
|
|
f"Weather query should recommend search capability. Got: {reasoning_text[:200]}"
|
|
)
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestScenario4SimpleExpertDelegation:
|
|
"""
|
|
Scenario 4: Simple Expert Delegation (Calculator)
|
|
|
|
Tests: Direct tool use for simple requests
|
|
"""
|
|
|
|
async def test_calculation_uses_calculator_tool(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that math requests use the calculator."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What is 847 times 293?"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
# Expected result: 248171 (may be formatted as 248,171)
|
|
message_normalized = message_text.replace(",", "")
|
|
assert "248171" in message_normalized, (
|
|
f"Calculator should compute 847 * 293 = 248171. Got: {message_text[:200]}"
|
|
)
|
|
|
|
async def test_datetime_query_uses_datetime_tool(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that date/time requests use datetime tools."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What day of the week is it?"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
# Should mention a day of the week (full or abbreviated)
|
|
days = [
|
|
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
|
|
"mon", "tue", "wed", "thu", "fri", "sat", "sun"
|
|
]
|
|
has_day = any(day in message_text.lower() for day in days)
|
|
|
|
assert has_day, f"Response should mention day of week. Got: {message_text[:200]}"
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestScenario6WikiCreation:
|
|
"""
|
|
Scenario 6: Create Wiki Page (Expert with Research)
|
|
|
|
Tests: Librarian delegation for wiki operations
|
|
Note: Requires library-desk to be running
|
|
"""
|
|
|
|
async def test_wiki_creation_delegates_to_librarian(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that wiki creation requests delegate to librarian."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Search the wiki for information about Docker"}
|
|
],
|
|
"reasoning": {"effort": "medium", "summary": "auto"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check Steward reasoning
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
print(f"Wiki search - Steward: {reasoning_text[:200]}...")
|
|
|
|
# Should mention librarian
|
|
assert "librarian" in reasoning_text.lower(), (
|
|
f"Wiki search should recommend librarian. Got: {reasoning_text[:200]}"
|
|
)
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestScenario8MultiExpertCoordination:
|
|
"""
|
|
Scenario 8: Complex Multi-Expert Coordination
|
|
|
|
Tests: Multiple experts working together
|
|
"""
|
|
|
|
async def test_complex_query_identifies_multiple_capabilities(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that complex queries recommend multiple capabilities."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What do you know about me? And also search for Python tutorials."}
|
|
],
|
|
"reasoning": {"effort": "medium", "summary": "auto"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Check Steward reasoning
|
|
reasoning_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "reasoning":
|
|
reasoning_text = " ".join(output.get("summary", []))
|
|
break
|
|
|
|
print(f"Multi-expert query - Steward: {reasoning_text[:300]}...")
|
|
|
|
# Should mention multiple capabilities
|
|
has_biographer = "biographer" in reasoning_text.lower()
|
|
has_librarian = "librarian" in reasoning_text.lower()
|
|
has_search = "tatlock_core" in reasoning_text.lower() or "search" in reasoning_text.lower()
|
|
|
|
assert has_biographer or has_librarian or has_search, (
|
|
f"Complex query should identify multiple capabilities. Got: {reasoning_text[:200]}"
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# New Scenarios from Today's Session
|
|
# ============================================================================
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestDirectDelegationBypass:
|
|
"""
|
|
Tests for direct delegation bypass (new feature from today).
|
|
|
|
When Steward recommends ONLY biographer or librarian, we skip
|
|
Tatlock's LLM call and delegate directly to work around
|
|
mistral-nemo's unreliable tool calling.
|
|
"""
|
|
|
|
async def test_pure_memory_request_uses_direct_delegation(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
qdrant: QdrantVerifier,
|
|
clean_test_memories,
|
|
):
|
|
"""Test that pure memory requests bypass Tatlock LLM."""
|
|
# Make a pure memory request
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Remember that my test value is alpha123"}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# Should complete successfully
|
|
assert data["status"] == "completed"
|
|
|
|
# Get response text
|
|
message_text = ""
|
|
for output in data["output"]:
|
|
if output["type"] == "message":
|
|
message_text = output["content"][0]["text"]
|
|
break
|
|
|
|
print(f"Direct delegation test response: {message_text[:200]}...")
|
|
|
|
# The response should come from biographer, not Tatlock
|
|
# Check for biographer-style language or memory acknowledgment
|
|
result = assert_llm_behavior(
|
|
message_text,
|
|
expected_patterns=[
|
|
r"(remember|noted|recorded|stored|acknowledged|got it)",
|
|
r"(alpha123|test|value)",
|
|
],
|
|
min_matches=1,
|
|
)
|
|
|
|
print(f"Direct delegation assertion: {result.evidence}")
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestUserContextIsolation:
|
|
"""
|
|
Tests for user context isolation.
|
|
|
|
Verifies that:
|
|
- Development uses llm_tester user
|
|
- Data is stored in user-specific collections
|
|
- Production user data is not affected
|
|
"""
|
|
|
|
async def test_memories_go_to_test_user_collection(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
qdrant: QdrantVerifier,
|
|
clean_test_memories,
|
|
):
|
|
"""Test that memories are stored in llm_tester collection."""
|
|
unique_value = f"isolation_test_{asyncio.get_event_loop().time()}"
|
|
|
|
# Store a memory
|
|
await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": f"Remember that my isolation marker is {unique_value}"}
|
|
],
|
|
},
|
|
)
|
|
|
|
await asyncio.sleep(1.0)
|
|
|
|
# Check llm_tester collection
|
|
test_points = await qdrant.scroll_points(TEST_COLLECTION)
|
|
test_values = [str(p.get("payload", {})) for p in test_points]
|
|
in_test_collection = any(unique_value in v for v in test_values)
|
|
|
|
# Check production collection (should NOT be there)
|
|
prod_collection = f"memories_{PRODUCTION_TENANT}"
|
|
if await qdrant.collection_exists(prod_collection):
|
|
prod_points = await qdrant.scroll_points(prod_collection)
|
|
prod_values = [str(p.get("payload", {})) for p in prod_points]
|
|
in_prod_collection = any(unique_value in v for v in prod_values)
|
|
else:
|
|
in_prod_collection = False
|
|
|
|
print(f"Isolation test - In test collection: {in_test_collection}, In prod: {in_prod_collection}")
|
|
|
|
# Should be in test collection OR response acknowledged
|
|
# Should NOT be in production collection
|
|
assert not in_prod_collection, (
|
|
f"Test data leaked to production collection! Value: {unique_value}"
|
|
)
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestErrorHandling:
|
|
"""
|
|
Error handling scenarios from ORCHESTRATION_SCENARIOS.md
|
|
"""
|
|
|
|
async def test_invalid_model_returns_error(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that invalid model returns proper error."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "nonexistent-model-xyz",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
},
|
|
)
|
|
|
|
# Should return error status (404 or 400)
|
|
assert response.status_code in [400, 404, 422], (
|
|
f"Expected error status for invalid model. Got: {response.status_code}"
|
|
)
|
|
data = response.json()
|
|
# Error could be in "error" or "detail" key
|
|
assert "error" in data or "detail" in data
|
|
|
|
async def test_empty_input_returns_error(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
):
|
|
"""Test that empty input returns validation error."""
|
|
response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [],
|
|
},
|
|
)
|
|
|
|
# Should either reject or handle gracefully
|
|
assert response.status_code in [200, 400, 422]
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.asyncio
|
|
class TestEvaluationReport:
|
|
"""
|
|
Meta-tests that generate evaluation reports.
|
|
|
|
These aren't pass/fail tests - they generate reports about
|
|
system behavior for human review.
|
|
"""
|
|
|
|
async def test_generate_memory_behavior_report(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
qdrant: QdrantVerifier,
|
|
):
|
|
"""
|
|
Generate a report on memory system behavior.
|
|
|
|
This test always passes but outputs diagnostic information.
|
|
"""
|
|
report_lines = ["=" * 60, "MEMORY SYSTEM BEHAVIOR REPORT", "=" * 60]
|
|
|
|
# Test 1: Store command
|
|
store_response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "Remember that my test pet is a hamster named Fluffy"}
|
|
],
|
|
},
|
|
)
|
|
|
|
store_text = ""
|
|
for output in store_response.json()["output"]:
|
|
if output["type"] == "message":
|
|
store_text = output["content"][0]["text"]
|
|
break
|
|
|
|
report_lines.append("\n[STORE TEST]")
|
|
report_lines.append(f"Input: 'Remember that my test pet is a hamster named Fluffy'")
|
|
report_lines.append(f"Response: {store_text[:200]}...")
|
|
|
|
await asyncio.sleep(1.0)
|
|
|
|
# Test 2: Recall command
|
|
recall_response = await client.post(
|
|
"/v1/responses",
|
|
json={
|
|
"model": "Tatlock",
|
|
"input": [
|
|
{"role": "user", "content": "What pet do I have?"}
|
|
],
|
|
},
|
|
)
|
|
|
|
recall_text = ""
|
|
for output in recall_response.json()["output"]:
|
|
if output["type"] == "message":
|
|
recall_text = output["content"][0]["text"]
|
|
break
|
|
|
|
report_lines.append("\n[RECALL TEST]")
|
|
report_lines.append(f"Input: 'What pet do I have?'")
|
|
report_lines.append(f"Response: {recall_text[:200]}...")
|
|
|
|
# Check Qdrant state
|
|
points = await qdrant.scroll_points(TEST_COLLECTION)
|
|
report_lines.append("\n[QDRANT STATE]")
|
|
report_lines.append(f"Total points in {TEST_COLLECTION}: {len(points)}")
|
|
|
|
pet_memories = [
|
|
p for p in points
|
|
if "pet" in str(p.get("payload", {})).lower()
|
|
or "fluffy" in str(p.get("payload", {})).lower()
|
|
or "hamster" in str(p.get("payload", {})).lower()
|
|
]
|
|
report_lines.append(f"Pet-related memories found: {len(pet_memories)}")
|
|
|
|
for mem in pet_memories:
|
|
payload = mem.get("payload", {})
|
|
report_lines.append(
|
|
f" - {payload.get('type')}: {payload.get('key')} = {payload.get('value')}"
|
|
)
|
|
|
|
# Assessment
|
|
report_lines.append("\n[ASSESSMENT]")
|
|
store_acknowledged = any(
|
|
kw in store_text.lower()
|
|
for kw in ["remember", "noted", "recorded", "got it", "understood", "fluffy", "hamster"]
|
|
)
|
|
recall_correct = "fluffy" in recall_text.lower() or "hamster" in recall_text.lower()
|
|
data_persisted = len(pet_memories) > 0
|
|
|
|
report_lines.append(f"Store acknowledged: {'✓' if store_acknowledged else '✗'}")
|
|
report_lines.append(f"Recall correct: {'✓' if recall_correct else '✗'}")
|
|
report_lines.append(f"Data persisted in Qdrant: {'✓' if data_persisted else '✗'}")
|
|
|
|
report_lines.append("\n" + "=" * 60)
|
|
|
|
# Print report
|
|
print("\n".join(report_lines))
|
|
|
|
# This test always passes - it's for generating reports
|
|
assert True
|