Add Responses API with streaming, history, and advanced features
Implements Phases 2, 3, and 6: Complete Responses API implementation Core API (Phase 2): - OpenAI Responses API format with structured output items - Streaming and non-streaming support via SSE-Starlette - Reasoning items (thinking summaries) - Function call items (tool execution) - Message items (assistant responses) - Router, schemas, service, and streaming coordinator Conversation History (Phase 3): - Hybrid client/server approach - Auto-generated deterministic conversation IDs - Configurable max turns with automatic trimming - Context window management with token counting - Token usage statistics - Placeholder for future vector memory integration Advanced Features (Phase 6): - Parameter validation with Pydantic field validators: - Temperature: 0.0-2.0 range enforcement - Reasoning effort: 6 levels (none to xhigh) - Max output tokens: positive integer enforcement - Stop sequences: up to 4, non-empty strings - Real-time stop sequence detection during streaming - Real-time max tokens enforcement with token counting - Graceful error handling and OpenAI-compatible error format Testing: - 9 unit tests for API endpoints and streaming - 11 unit tests for error handling - 13 unit tests for conversation history and context - 12 unit tests for advanced features and validation - Total: 45 tests with comprehensive coverage 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,378 @@
|
||||
"""
|
||||
Tests for Phase 6 advanced features.
|
||||
|
||||
Tests:
|
||||
- Parameter validation (reasoning effort, max_output_tokens, stop sequences)
|
||||
- Stop sequence detection and enforcement
|
||||
- Max tokens enforcement
|
||||
"""
|
||||
import json
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import AsyncClient
|
||||
from pydantic import ValidationError
|
||||
|
||||
from src.responses.schemas import ResponseRequest
|
||||
from src.responses.streaming import StreamingCoordinator
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Parameter Validation Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_temperature_validation():
|
||||
"""Test temperature parameter validation."""
|
||||
# Valid temperatures
|
||||
valid_temps = [0.0, 0.5, 1.0, 1.5, 2.0]
|
||||
for temp in valid_temps:
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
temperature=temp
|
||||
)
|
||||
assert request.temperature == temp
|
||||
|
||||
# Invalid temperatures
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
temperature=-0.1 # Too low
|
||||
)
|
||||
assert "temperature" in str(exc_info.value).lower()
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
temperature=2.1 # Too high
|
||||
)
|
||||
assert "temperature" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reasoning_effort_validation():
|
||||
"""Test reasoning.effort parameter validation."""
|
||||
# Valid effort levels
|
||||
valid_efforts = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh']
|
||||
for effort in valid_efforts:
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
reasoning={"effort": effort, "summary": "auto"}
|
||||
)
|
||||
assert request.reasoning["effort"] == effort
|
||||
|
||||
# Invalid effort level
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
reasoning={"effort": "invalid", "summary": "auto"}
|
||||
)
|
||||
assert "reasoning.effort" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reasoning_summary_validation():
|
||||
"""Test reasoning.summary parameter validation."""
|
||||
# Valid summary values
|
||||
valid_summaries = ['auto', 'off']
|
||||
for summary in valid_summaries:
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
reasoning={"effort": "medium", "summary": summary}
|
||||
)
|
||||
assert request.reasoning["summary"] == summary
|
||||
|
||||
# Invalid summary value
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
reasoning={"effort": "medium", "summary": "invalid"}
|
||||
)
|
||||
assert "reasoning.summary" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_max_output_tokens_validation():
|
||||
"""Test max_output_tokens parameter validation."""
|
||||
# Valid values
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
max_output_tokens=100
|
||||
)
|
||||
assert request.max_output_tokens == 100
|
||||
|
||||
# None is valid (unlimited)
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
max_output_tokens=None
|
||||
)
|
||||
assert request.max_output_tokens is None
|
||||
|
||||
# Invalid: zero or negative
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
max_output_tokens=0
|
||||
)
|
||||
assert "max_output_tokens" in str(exc_info.value)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
max_output_tokens=-10
|
||||
)
|
||||
assert "max_output_tokens" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stop_sequences_validation():
|
||||
"""Test stop sequences parameter validation."""
|
||||
# Valid: up to 4 stop sequences
|
||||
for num_seqs in range(1, 5):
|
||||
stop_seqs = [f"stop{i}" for i in range(num_seqs)]
|
||||
request = ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
stop=stop_seqs
|
||||
)
|
||||
assert request.stop == stop_seqs
|
||||
|
||||
# Invalid: more than 4 stop sequences
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
stop=["stop1", "stop2", "stop3", "stop4", "stop5"] # 5 sequences
|
||||
)
|
||||
assert "4 stop sequences" in str(exc_info.value)
|
||||
|
||||
# Invalid: empty string in stop sequences
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
ResponseRequest(
|
||||
model="lorem-tester",
|
||||
input=[{"role": "user", "content": "Hello"}],
|
||||
stop=["stop1", ""] # Empty string
|
||||
)
|
||||
assert "non-empty" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Stop Sequence Enforcement Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stop_sequence_detection_helper():
|
||||
"""Test stop sequence detection helper method."""
|
||||
coordinator = StreamingCoordinator()
|
||||
|
||||
# No stop sequences
|
||||
found, text = coordinator._check_stop_sequence("Hello world", None)
|
||||
assert found is False
|
||||
assert text == "Hello world"
|
||||
|
||||
# Stop sequence not present
|
||||
found, text = coordinator._check_stop_sequence(
|
||||
"Hello world",
|
||||
["STOP", "END"]
|
||||
)
|
||||
assert found is False
|
||||
assert text == "Hello world"
|
||||
|
||||
# Stop sequence found
|
||||
found, text = coordinator._check_stop_sequence(
|
||||
"Hello STOP this should not appear",
|
||||
["STOP"]
|
||||
)
|
||||
assert found is True
|
||||
assert text == "Hello "
|
||||
|
||||
# Multiple stop sequences, first one wins
|
||||
found, text = coordinator._check_stop_sequence(
|
||||
"Hello STOP this END that",
|
||||
["STOP", "END"]
|
||||
)
|
||||
assert found is True
|
||||
assert text == "Hello "
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_sequence_in_streaming(async_client: AsyncClient):
|
||||
"""Test stop sequence enforcement during streaming."""
|
||||
# We'll use lorem-tester which generates predictable text
|
||||
# The agent generates lorem ipsum text, so we use a stop sequence
|
||||
# that's likely to appear
|
||||
request_data = {
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Generate long text"}],
|
||||
"stop": ["dolor"], # Common word in lorem ipsum
|
||||
"stream": True
|
||||
}
|
||||
|
||||
chunks_received = []
|
||||
async with async_client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
json=request_data,
|
||||
timeout=20.0,
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:].strip()
|
||||
if data_str != "[DONE]":
|
||||
try:
|
||||
chunk = json.loads(data_str)
|
||||
chunks_received.append(chunk)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Should have received chunks and stopped early
|
||||
# (Can't verify exact stop behavior with random lorem ipsum,
|
||||
# but test ensures no errors occur)
|
||||
assert len(chunks_received) > 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Max Tokens Enforcement Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_max_tokens_check_helper():
|
||||
"""Test max tokens check helper method."""
|
||||
coordinator = StreamingCoordinator()
|
||||
|
||||
# No limit
|
||||
assert coordinator._check_max_tokens(100, None) is False
|
||||
|
||||
# Under limit
|
||||
assert coordinator._check_max_tokens(50, 100) is False
|
||||
|
||||
# At limit
|
||||
assert coordinator._check_max_tokens(100, 100) is True
|
||||
|
||||
# Over limit
|
||||
assert coordinator._check_max_tokens(150, 100) is True
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_token_counting_helper():
|
||||
"""Test approximate token counting."""
|
||||
coordinator = StreamingCoordinator()
|
||||
|
||||
# Empty string
|
||||
assert coordinator._count_tokens_approx("") == 0
|
||||
|
||||
# ~4 chars per token
|
||||
text = "Hello world" # 11 chars
|
||||
tokens = coordinator._count_tokens_approx(text)
|
||||
assert tokens == 2 # 11 // 4 = 2
|
||||
|
||||
text = "A" * 100 # 100 chars
|
||||
tokens = coordinator._count_tokens_approx(text)
|
||||
assert tokens == 25 # 100 // 4 = 25
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_tokens_in_streaming(async_client: AsyncClient):
|
||||
"""Test max tokens enforcement during streaming."""
|
||||
# Set very low max_output_tokens to force early stop
|
||||
request_data = {
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Generate text"}],
|
||||
"max_output_tokens": 5, # Very low limit
|
||||
"stream": True
|
||||
}
|
||||
|
||||
chunks_received = []
|
||||
text_chunks = []
|
||||
|
||||
async with async_client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
json=request_data,
|
||||
timeout=20.0,
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
if line.startswith("event: "):
|
||||
event_type = line[7:].strip()
|
||||
elif line.startswith("data: "):
|
||||
data_str = line[6:].strip()
|
||||
if data_str != "[DONE]":
|
||||
try:
|
||||
chunk = json.loads(data_str)
|
||||
chunks_received.append(chunk)
|
||||
|
||||
# Collect text deltas
|
||||
if "delta" in chunk:
|
||||
text_chunks.append(chunk["delta"])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Should have stopped early due to token limit
|
||||
total_text = "".join(text_chunks)
|
||||
# With max_output_tokens=5, total text should be small
|
||||
# (Approximate: 5 tokens * 4 chars ≈ 20 chars)
|
||||
assert len(total_text) < 100 # Reasonable upper bound
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Combined Features Test
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_combined_validation(client: TestClient):
|
||||
"""Test combined parameter validation in actual request."""
|
||||
# Valid request with all advanced features
|
||||
request_data = {
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"temperature": 0.7,
|
||||
"max_output_tokens": 100,
|
||||
"stop": ["STOP", "END"],
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"stream": False
|
||||
}
|
||||
|
||||
response = client.post("/v1/responses", json=request_data)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "completed"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_invalid_combined_parameters(client: TestClient):
|
||||
"""Test that invalid parameters are rejected."""
|
||||
# Invalid temperature
|
||||
request_data = {
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"temperature": 3.0, # Too high
|
||||
"stream": False
|
||||
}
|
||||
|
||||
response = client.post("/v1/responses", json=request_data)
|
||||
assert response.status_code == 422
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
assert data["error"]["type"] == "invalid_request_error"
|
||||
Reference in New Issue
Block a user