jpmschweitzerandClaude
78066fab1b
Mechanical only, and separated from the judgment calls that follow so the reviewable changes are not buried in a 98-file whitespace diff. 227 automatic fixes: 60 blank lines carrying whitespace, 60 unsorted import blocks, 34 Optional[X] to X | None, 28 unused imports, 16 deprecated typing imports, 12 datetime.timezone.utc to datetime.UTC, and assorted smaller modernisations. Then `ruff format` over src and tests: 98 files reformatted, 35 already conforming. No file among the unused-import findings defines __all__ or is an __init__.py, so nothing here removes a re-export. `make test`: 658 passed, unchanged from HEAD. Two things observed while verifying, neither addressed here: `pytest tests/` cannot collect — tests/e2e/test_orchestration_e2e.py uses an `e2e` marker that is not registered, and the config is strict about markers. This fails identically at HEAD, so it predates this change; `make test` passes because it ignores tests/e2e, tests/integration and tests/contracts. test_tatlock_tool_call_logging_calculator is flaky. It failed once in a full run with these changes and passed on the next, passes in isolation with them, and fails in isolation at HEAD. It is order- or timing-dependent, not a regression from this commit — established by running the full suite both ways rather than by reasoning about which change could have caused it. Co-Authored-By: Claude <noreply@anthropic.com>
254 lines
7.8 KiB
Python
254 lines
7.8 KiB
Python
"""
|
|
Tests for error handling in Responses API.
|
|
"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from httpx import AsyncClient
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_model_not_found_error(client: TestClient) -> None:
|
|
"""Test response when model doesn't exist."""
|
|
request_data = {
|
|
"model": "nonexistent-model-12345",
|
|
"input": [{"role": "user", "content": "Hello"}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 404
|
|
data = response.json()
|
|
assert "detail" in data
|
|
assert "not found" in data["detail"].lower()
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_validation_error_missing_model(client: TestClient) -> None:
|
|
"""Test validation error when model field is missing."""
|
|
request_data = {"input": [{"role": "user", "content": "Hello"}]}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_validation_error_invalid_temperature(client: TestClient) -> None:
|
|
"""Test validation error for out-of-range temperature."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "Hello"}],
|
|
"temperature": 3.0, # Max is 2.0
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 422
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_rate_limit_error(client: TestClient) -> None:
|
|
"""Test rate limit error trigger."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "trigger_rate_limit"}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should get an error response
|
|
assert response.status_code in [429, 500] # Rate limit or internal error
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_context_overflow_error(client: TestClient) -> None:
|
|
"""Test context length overflow error trigger."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "trigger_context_overflow"}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should get an error response
|
|
assert response.status_code in [400, 500] # Bad request or internal error
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_model_not_found(async_client: AsyncClient) -> None:
|
|
"""Test streaming response with nonexistent model."""
|
|
request_data = {
|
|
"model": "nonexistent-streaming-model",
|
|
"input": [{"role": "user", "content": "Hello"}],
|
|
"stream": True,
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# Streaming always returns 200 OK, errors are sent as events
|
|
assert response.status_code == 200
|
|
|
|
# Collect events and look for error event
|
|
error_found = False
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
if event_type == "error":
|
|
error_found = True
|
|
break
|
|
|
|
# Should have received an error event
|
|
assert error_found, "Expected error event in stream"
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_rate_limit_error(async_client: AsyncClient) -> None:
|
|
"""Test streaming with rate limit error."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "trigger_rate_limit"}],
|
|
"stream": True,
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# May get error status or error event
|
|
# Collect all events
|
|
events = []
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
elif line.startswith("data: "):
|
|
data_str = line[6:].strip()
|
|
try:
|
|
data = json.loads(data_str)
|
|
if event_type:
|
|
events.append({"event": event_type, "data": data})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Should either have error status or error event
|
|
if response.status_code == 200:
|
|
# Check for error event
|
|
error_events = [e for e in events if e["event"] == "error"]
|
|
# May or may not have error event depending on where error occurs
|
|
# At minimum, should not crash
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.asyncio
|
|
async def test_streaming_context_overflow_error(async_client: AsyncClient) -> None:
|
|
"""Test streaming with context overflow error."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "trigger_context_overflow"}],
|
|
"stream": True,
|
|
}
|
|
|
|
async with async_client.stream(
|
|
"POST",
|
|
"/v1/responses",
|
|
json=request_data,
|
|
timeout=20.0,
|
|
) as response:
|
|
# May get error status or error event
|
|
events = []
|
|
async for line in response.aiter_lines():
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.startswith("event: "):
|
|
event_type = line[7:].strip()
|
|
elif line.startswith("data: "):
|
|
data_str = line[6:].strip()
|
|
try:
|
|
data = json.loads(data_str)
|
|
if event_type:
|
|
events.append({"event": event_type, "data": data})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Should not crash
|
|
assert response.status_code in [200, 400, 500]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_function_call_output_item(client: TestClient) -> None:
|
|
"""Test response with function call items."""
|
|
request_data = {
|
|
"model": "lorem-tester",
|
|
"input": [{"role": "user", "content": "Use the search tool"}],
|
|
"tools": [{"name": "search", "description": "Search for information", "parameters": {}}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
|
|
# May or may not have function_call items (random in lorem-tester)
|
|
# But should successfully handle them if present
|
|
output_items = data["output"]
|
|
for item in output_items:
|
|
assert item["type"] in ["message", "reasoning", "function_call"]
|
|
if item["type"] == "function_call":
|
|
assert "name" in item
|
|
assert "arguments" in item
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_pipeline_prefix_stripping(client: TestClient) -> None:
|
|
"""Test that pipeline prefixes are stripped from model names."""
|
|
request_data = {
|
|
"model": "some_pipeline.lorem-tester",
|
|
"input": [{"role": "user", "content": "Hello"}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should successfully strip prefix and find model
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["status"] == "completed"
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_multiple_pipeline_prefixes(client: TestClient) -> None:
|
|
"""Test multiple dots in model name (only first is prefix)."""
|
|
request_data = {
|
|
"model": "pipeline.sub.lorem-tester",
|
|
"input": [{"role": "user", "content": "Hello"}],
|
|
"stream": False,
|
|
}
|
|
|
|
response = client.post("/v1/responses", json=request_data)
|
|
|
|
# Should strip only first part
|
|
# "pipeline.sub.lorem-tester" -> "sub.lorem-tester"
|
|
# This should fail since "sub.lorem-tester" doesn't exist
|
|
assert response.status_code == 404
|