""" Tests for Phase 6 advanced features. Tests: - Parameter validation (reasoning effort, max_output_tokens, stop sequences) - Stop sequence detection and enforcement - Max tokens enforcement """ import json import pytest from fastapi.testclient import TestClient from httpx import AsyncClient from pydantic import ValidationError from src.responses.schemas import ResponseRequest from src.responses.streaming import StreamingCoordinator # ============================================================================ # Parameter Validation Tests # ============================================================================ @pytest.mark.unit def test_temperature_validation(): """Test temperature parameter validation.""" # Valid temperatures valid_temps = [0.0, 0.5, 1.0, 1.5, 2.0] for temp in valid_temps: request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], temperature=temp ) assert request.temperature == temp # Invalid temperatures with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], temperature=-0.1 # Too low ) assert "temperature" in str(exc_info.value).lower() with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], temperature=2.1 # Too high ) assert "temperature" in str(exc_info.value).lower() @pytest.mark.unit def test_reasoning_effort_validation(): """Test reasoning.effort parameter validation.""" # Valid effort levels valid_efforts = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh'] for effort in valid_efforts: request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], reasoning={"effort": effort, "summary": "auto"} ) assert request.reasoning["effort"] == effort # Invalid effort level with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], reasoning={"effort": "invalid", "summary": "auto"} ) assert "reasoning.effort" in str(exc_info.value) @pytest.mark.unit def test_reasoning_summary_validation(): """Test reasoning.summary parameter validation.""" # Valid summary values valid_summaries = ['auto', 'off'] for summary in valid_summaries: request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], reasoning={"effort": "medium", "summary": summary} ) assert request.reasoning["summary"] == summary # Invalid summary value with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], reasoning={"effort": "medium", "summary": "invalid"} ) assert "reasoning.summary" in str(exc_info.value) @pytest.mark.unit def test_max_output_tokens_validation(): """Test max_output_tokens parameter validation.""" # Valid values request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=100 ) assert request.max_output_tokens == 100 # None is valid (unlimited) request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=None ) assert request.max_output_tokens is None # Invalid: zero or negative with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=0 ) assert "max_output_tokens" in str(exc_info.value) with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], max_output_tokens=-10 ) assert "max_output_tokens" in str(exc_info.value) @pytest.mark.unit def test_stop_sequences_validation(): """Test stop sequences parameter validation.""" # Valid: up to 4 stop sequences for num_seqs in range(1, 5): stop_seqs = [f"stop{i}" for i in range(num_seqs)] request = ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], stop=stop_seqs ) assert request.stop == stop_seqs # Invalid: more than 4 stop sequences with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], stop=["stop1", "stop2", "stop3", "stop4", "stop5"] # 5 sequences ) assert "4 stop sequences" in str(exc_info.value) # Invalid: empty string in stop sequences with pytest.raises(ValidationError) as exc_info: ResponseRequest( model="lorem-tester", input=[{"role": "user", "content": "Hello"}], stop=["stop1", ""] # Empty string ) assert "non-empty" in str(exc_info.value).lower() # ============================================================================ # Stop Sequence Enforcement Tests # ============================================================================ @pytest.mark.unit def test_stop_sequence_detection_helper(): """Test stop sequence detection helper method.""" coordinator = StreamingCoordinator() # No stop sequences found, text = coordinator._check_stop_sequence("Hello world", None) assert found is False assert text == "Hello world" # Stop sequence not present found, text = coordinator._check_stop_sequence( "Hello world", ["STOP", "END"] ) assert found is False assert text == "Hello world" # Stop sequence found found, text = coordinator._check_stop_sequence( "Hello STOP this should not appear", ["STOP"] ) assert found is True assert text == "Hello " # Multiple stop sequences, first one wins found, text = coordinator._check_stop_sequence( "Hello STOP this END that", ["STOP", "END"] ) assert found is True assert text == "Hello " @pytest.mark.unit @pytest.mark.asyncio async def test_stop_sequence_in_streaming(async_client: AsyncClient): """Test stop sequence enforcement during streaming.""" # We'll use lorem-tester which generates predictable text # The agent generates lorem ipsum text, so we use a stop sequence # that's likely to appear request_data = { "model": "lorem-tester", "input": [{"role": "user", "content": "Generate long text"}], "stop": ["dolor"], # Common word in lorem ipsum "stream": True } chunks_received = [] async with async_client.stream( "POST", "/v1/responses", json=request_data, timeout=20.0, ) as response: assert response.status_code == 200 async for line in response.aiter_lines(): if not line.strip(): continue if line.startswith("data: "): data_str = line[6:].strip() if data_str != "[DONE]": try: chunk = json.loads(data_str) chunks_received.append(chunk) except json.JSONDecodeError: pass # Should have received chunks and stopped early # (Can't verify exact stop behavior with random lorem ipsum, # but test ensures no errors occur) assert len(chunks_received) > 0 # ============================================================================ # Max Tokens Enforcement Tests # ============================================================================ @pytest.mark.unit def test_max_tokens_check_helper(): """Test max tokens check helper method.""" coordinator = StreamingCoordinator() # No limit assert coordinator._check_max_tokens(100, None) is False # Under limit assert coordinator._check_max_tokens(50, 100) is False # At limit assert coordinator._check_max_tokens(100, 100) is True # Over limit assert coordinator._check_max_tokens(150, 100) is True @pytest.mark.unit def test_token_counting_helper(): """Test approximate token counting.""" coordinator = StreamingCoordinator() # Empty string assert coordinator._count_tokens_approx("") == 0 # ~4 chars per token text = "Hello world" # 11 chars tokens = coordinator._count_tokens_approx(text) assert tokens == 2 # 11 // 4 = 2 text = "A" * 100 # 100 chars tokens = coordinator._count_tokens_approx(text) assert tokens == 25 # 100 // 4 = 25 @pytest.mark.unit @pytest.mark.asyncio async def test_max_tokens_in_streaming(async_client: AsyncClient): """Test max tokens enforcement during streaming.""" # Set very low max_output_tokens to force early stop request_data = { "model": "lorem-tester", "input": [{"role": "user", "content": "Generate text"}], "max_output_tokens": 5, # Very low limit "stream": True } chunks_received = [] text_chunks = [] async with async_client.stream( "POST", "/v1/responses", json=request_data, timeout=20.0, ) as response: assert response.status_code == 200 async for line in response.aiter_lines(): if not line.strip(): continue if line.startswith("event: "): event_type = line[7:].strip() elif line.startswith("data: "): data_str = line[6:].strip() if data_str != "[DONE]": try: chunk = json.loads(data_str) chunks_received.append(chunk) # Collect text deltas if "delta" in chunk: text_chunks.append(chunk["delta"]) except json.JSONDecodeError: pass # Should have stopped early due to token limit total_text = "".join(text_chunks) # With max_output_tokens=5, total text should be small # (Approximate: 5 tokens * 4 chars ≈ 20 chars) assert len(total_text) < 100 # Reasonable upper bound # ============================================================================ # Combined Features Test # ============================================================================ @pytest.mark.unit def test_combined_validation(client: TestClient): """Test combined parameter validation in actual request.""" # Valid request with all advanced features request_data = { "model": "lorem-tester", "input": [{"role": "user", "content": "Hello"}], "temperature": 0.7, "max_output_tokens": 100, "stop": ["STOP", "END"], "reasoning": {"effort": "high", "summary": "auto"}, "stream": False } response = client.post("/v1/responses", json=request_data) assert response.status_code == 200 data = response.json() assert data["status"] == "completed" @pytest.mark.unit def test_invalid_combined_parameters(client: TestClient): """Test that invalid parameters are rejected.""" # Invalid temperature request_data = { "model": "lorem-tester", "input": [{"role": "user", "content": "Hello"}], "temperature": 3.0, # Too high "stream": False } response = client.post("/v1/responses", json=request_data) assert response.status_code == 422 data = response.json() assert "error" in data assert data["error"]["type"] == "invalid_request_error"