jpmschweitzerandClaude
ff6c3cf1b5
Implements Phases 2, 3, and 6: Complete Responses API implementation Core API (Phase 2): - OpenAI Responses API format with structured output items - Streaming and non-streaming support via SSE-Starlette - Reasoning items (thinking summaries) - Function call items (tool execution) - Message items (assistant responses) - Router, schemas, service, and streaming coordinator Conversation History (Phase 3): - Hybrid client/server approach - Auto-generated deterministic conversation IDs - Configurable max turns with automatic trimming - Context window management with token counting - Token usage statistics - Placeholder for future vector memory integration Advanced Features (Phase 6): - Parameter validation with Pydantic field validators: - Temperature: 0.0-2.0 range enforcement - Reasoning effort: 6 levels (none to xhigh) - Max output tokens: positive integer enforcement - Stop sequences: up to 4, non-empty strings - Real-time stop sequence detection during streaming - Real-time max tokens enforcement with token counting - Graceful error handling and OpenAI-compatible error format Testing: - 9 unit tests for API endpoints and streaming - 11 unit tests for error handling - 13 unit tests for conversation history and context - 12 unit tests for advanced features and validation - Total: 45 tests with comprehensive coverage 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
282 lines
8.1 KiB
Python
282 lines
8.1 KiB
Python
"""
|
|
Tests for error handling in Responses API.
|
|
"""
|
|
import json
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from httpx import AsyncClient
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_model_not_found_error(client: TestClient) -> None:
|
|
"""Test response when model doesn't exist."""
|
|
request_data = {
|
|
"model": "nonexistent-model-12345",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 404
|
|
data = response.json()
|
|
assert "detail" in data
|
|
assert "not found" in data["detail"].lower()
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_validation_error_missing_model(client: TestClient) -> None:
|
|
"""Test validation error when model field is missing."""
|
|
request_data = {
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
]
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_validation_error_invalid_temperature(client: TestClient) -> None:
|
|
"""Test validation error for out-of-range temperature."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
"temperature": 3.0 # Max is 2.0
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_rate_limit_error(client: TestClient) -> None:
|
|
"""Test rate limit error trigger."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "trigger_rate_limit"}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should get an error response
|
|
assert response.status_code in [429, 500] # Rate limit or internal error
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_context_overflow_error(client: TestClient) -> None:
|
|
"""Test context length overflow error trigger."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "trigger_context_overflow"}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should get an error response
|
|
assert response.status_code in [400, 500] # Bad request or internal error
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_model_not_found(async_client: AsyncClient) -> None:
|
|
"""Test streaming response with nonexistent model."""
|
|
request_data = {
|
|
"model": "nonexistent-streaming-model",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
"stream": True
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# Streaming always returns 200 OK, errors are sent as events
|
|
assert response.status_code == 200
|
|
|
|
# Collect events and look for error event
|
|
error_found = False
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
if event_type == "error":
|
|
error_found = True
|
|
break
|
|
|
|
# Should have received an error event
|
|
assert error_found, "Expected error event in stream"
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_rate_limit_error(async_client: AsyncClient) -> None:
|
|
"""Test streaming with rate limit error."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "trigger_rate_limit"}
|
|
],
|
|
"stream": True
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# May get error status or error event
|
|
# Collect all events
|
|
events = []
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
elif line.startswith("data: "):
|
|
data_str = line[6:].strip()
|
|
try:
|
|
data = json.loads(data_str)
|
|
if event_type:
|
|
events.append({"event": event_type, "data": data})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Should either have error status or error event
|
|
if response.status_code == 200:
|
|
# Check for error event
|
|
error_events = [e for e in events if e["event"] == "error"]
|
|
# May or may not have error event depending on where error occurs
|
|
# At minimum, should not crash
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_context_overflow_error(async_client: AsyncClient) -> None:
|
|
"""Test streaming with context overflow error."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "trigger_context_overflow"}
|
|
],
|
|
"stream": True
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# May get error status or error event
|
|
events = []
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
elif line.startswith("data: "):
|
|
data_str = line[6:].strip()
|
|
try:
|
|
data = json.loads(data_str)
|
|
if event_type:
|
|
events.append({"event": event_type, "data": data})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Should not crash
|
|
assert response.status_code in [200, 400, 500]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_function_call_output_item(client: TestClient) -> None:
|
|
"""Test response with function call items."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "Use the search tool"}
|
|
],
|
|
"tools": [
|
|
{
|
|
"name": "search",
|
|
"description": "Search for information",
|
|
"parameters": {}
|
|
}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# May or may not have function_call items (random in lorem-tester)
|
|
# But should successfully handle them if present
|
|
output_items = data["output"]
|
|
for item in output_items:
|
|
assert item["type"] in ["message", "reasoning", "function_call"]
|
|
if item["type"] == "function_call":
|
|
assert "name" in item
|
|
assert "arguments" in item
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_pipeline_prefix_stripping(client: TestClient) -> None:
|
|
"""Test that pipeline prefixes are stripped from model names."""
|
|
request_data = {
|
|
"model": "some_pipeline.lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should successfully strip prefix and find model
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["status"] == "completed"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_multiple_pipeline_prefixes(client: TestClient) -> None:
|
|
"""Test multiple dots in model name (only first is prefix)."""
|
|
request_data = {
|
|
"model": "pipeline.sub.lorem-tester",
|
|
"input": [
|
|
{"role": "user", "content": "Hello"}
|
|
],
|
|
"stream": False
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should strip only first part
|
|
# "pipeline.sub.lorem-tester" -> "sub.lorem-tester"
|
|
# This should fail since "sub.lorem-tester" doesn't exist
|
|
assert response.status_code == 404
|