feat: add retry logic for transient failures
- @with_retry decorator and retry_async() function - Exponential backoff with jitter - Retries on: timeout, connection errors, HTTP 429/5xx - Web search tool now retries on network failures - Configurable via RETRY_MAX_ATTEMPTS, RETRY_BASE_DELAY, RETRY_MAX_DELAY - 29 new tests (205 total passing) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -0,0 +1,244 @@
|
||||
"""
|
||||
Tests for retry utilities.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
|
||||
from src.shared.retry import (
|
||||
with_retry,
|
||||
retry_async,
|
||||
is_retryable_exception,
|
||||
is_retryable_http_status,
|
||||
calculate_backoff,
|
||||
)
|
||||
|
||||
|
||||
class TestIsRetryableHttpStatus:
|
||||
"""Tests for HTTP status code checking."""
|
||||
|
||||
def test_429_is_retryable(self):
|
||||
"""429 Too Many Requests should be retryable."""
|
||||
assert is_retryable_http_status(429) is True
|
||||
|
||||
def test_500_is_retryable(self):
|
||||
"""500 Internal Server Error should be retryable."""
|
||||
assert is_retryable_http_status(500) is True
|
||||
|
||||
def test_502_is_retryable(self):
|
||||
"""502 Bad Gateway should be retryable."""
|
||||
assert is_retryable_http_status(502) is True
|
||||
|
||||
def test_503_is_retryable(self):
|
||||
"""503 Service Unavailable should be retryable."""
|
||||
assert is_retryable_http_status(503) is True
|
||||
|
||||
def test_504_is_retryable(self):
|
||||
"""504 Gateway Timeout should be retryable."""
|
||||
assert is_retryable_http_status(504) is True
|
||||
|
||||
def test_400_not_retryable(self):
|
||||
"""400 Bad Request should not be retryable."""
|
||||
assert is_retryable_http_status(400) is False
|
||||
|
||||
def test_401_not_retryable(self):
|
||||
"""401 Unauthorized should not be retryable."""
|
||||
assert is_retryable_http_status(401) is False
|
||||
|
||||
def test_404_not_retryable(self):
|
||||
"""404 Not Found should not be retryable."""
|
||||
assert is_retryable_http_status(404) is False
|
||||
|
||||
def test_200_not_retryable(self):
|
||||
"""200 OK should not be retryable."""
|
||||
assert is_retryable_http_status(200) is False
|
||||
|
||||
|
||||
class TestIsRetryableException:
|
||||
"""Tests for exception checking."""
|
||||
|
||||
def test_timeout_exception_is_retryable(self):
|
||||
"""Timeout exceptions should be retryable."""
|
||||
exc = httpx.TimeoutException("timeout")
|
||||
assert is_retryable_exception(exc) is True
|
||||
|
||||
def test_connect_error_is_retryable(self):
|
||||
"""Connection errors should be retryable."""
|
||||
exc = httpx.ConnectError("connection failed")
|
||||
assert is_retryable_exception(exc) is True
|
||||
|
||||
def test_connection_error_is_retryable(self):
|
||||
"""Python ConnectionError should be retryable."""
|
||||
exc = ConnectionError("connection refused")
|
||||
assert is_retryable_exception(exc) is True
|
||||
|
||||
def test_timeout_error_is_retryable(self):
|
||||
"""Python TimeoutError should be retryable."""
|
||||
exc = TimeoutError("timed out")
|
||||
assert is_retryable_exception(exc) is True
|
||||
|
||||
def test_value_error_not_retryable(self):
|
||||
"""ValueError should not be retryable."""
|
||||
exc = ValueError("invalid value")
|
||||
assert is_retryable_exception(exc) is False
|
||||
|
||||
def test_key_error_not_retryable(self):
|
||||
"""KeyError should not be retryable."""
|
||||
exc = KeyError("missing key")
|
||||
assert is_retryable_exception(exc) is False
|
||||
|
||||
|
||||
class TestCalculateBackoff:
|
||||
"""Tests for backoff calculation."""
|
||||
|
||||
def test_first_attempt_base_delay(self):
|
||||
"""First attempt should use base delay."""
|
||||
delay = calculate_backoff(0, base_delay=1.0, jitter=False)
|
||||
assert delay == 1.0
|
||||
|
||||
def test_second_attempt_doubles(self):
|
||||
"""Second attempt should double the delay."""
|
||||
delay = calculate_backoff(1, base_delay=1.0, jitter=False)
|
||||
assert delay == 2.0
|
||||
|
||||
def test_third_attempt_quadruples(self):
|
||||
"""Third attempt should quadruple the delay."""
|
||||
delay = calculate_backoff(2, base_delay=1.0, jitter=False)
|
||||
assert delay == 4.0
|
||||
|
||||
def test_max_delay_respected(self):
|
||||
"""Delay should not exceed max_delay."""
|
||||
delay = calculate_backoff(10, base_delay=1.0, max_delay=30.0, jitter=False)
|
||||
assert delay == 30.0
|
||||
|
||||
def test_jitter_adds_randomness(self):
|
||||
"""Jitter should add randomness to delay."""
|
||||
delays = [calculate_backoff(1, base_delay=1.0, jitter=True) for _ in range(10)]
|
||||
# With jitter, delays should vary (not all identical)
|
||||
assert len(set(delays)) > 1
|
||||
|
||||
def test_jitter_within_bounds(self):
|
||||
"""Jitter should keep delay within reasonable bounds."""
|
||||
for _ in range(100):
|
||||
delay = calculate_backoff(0, base_delay=2.0, jitter=True)
|
||||
# Attempt 0 with base 2.0 = 2.0, with jitter should be 0.75-1.25x = 1.5-2.5
|
||||
assert 1.5 <= delay <= 2.5
|
||||
|
||||
|
||||
class TestWithRetryDecorator:
|
||||
"""Tests for the @with_retry decorator."""
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_success_on_first_attempt(self):
|
||||
"""Function should return on first successful attempt."""
|
||||
call_count = 0
|
||||
|
||||
@with_retry(max_attempts=3)
|
||||
async def successful_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return "success"
|
||||
|
||||
result = await successful_func()
|
||||
assert result == "success"
|
||||
assert call_count == 1
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_retry_on_timeout(self):
|
||||
"""Should retry on timeout exception."""
|
||||
call_count = 0
|
||||
|
||||
@with_retry(max_attempts=3, base_delay=0.01)
|
||||
async def flaky_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 3:
|
||||
raise httpx.TimeoutException("timeout")
|
||||
return "success"
|
||||
|
||||
result = await flaky_func()
|
||||
assert result == "success"
|
||||
assert call_count == 3
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_retry_on_value_error(self):
|
||||
"""Should not retry on non-retryable exceptions."""
|
||||
call_count = 0
|
||||
|
||||
@with_retry(max_attempts=3)
|
||||
async def bad_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
raise ValueError("bad value")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await bad_func()
|
||||
assert call_count == 1
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_exhausted_retries(self):
|
||||
"""Should raise last exception after all retries exhausted."""
|
||||
call_count = 0
|
||||
|
||||
@with_retry(max_attempts=3, base_delay=0.01)
|
||||
async def always_fails():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
raise httpx.TimeoutException("always times out")
|
||||
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
await always_fails()
|
||||
assert call_count == 3
|
||||
|
||||
|
||||
class TestRetryAsync:
|
||||
"""Tests for the retry_async function."""
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_success_on_first_attempt(self):
|
||||
"""Function should return on first successful attempt."""
|
||||
async def successful_func():
|
||||
return "success"
|
||||
|
||||
result = await retry_async(successful_func, max_attempts=3)
|
||||
assert result == "success"
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_retry_on_connect_error(self):
|
||||
"""Should retry on connection errors."""
|
||||
call_count = 0
|
||||
|
||||
async def flaky_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count < 2:
|
||||
raise httpx.ConnectError("connection failed")
|
||||
return "success"
|
||||
|
||||
result = await retry_async(flaky_func, max_attempts=3, base_delay=0.01)
|
||||
assert result == "success"
|
||||
assert call_count == 2
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_passes_args_and_kwargs(self):
|
||||
"""Should pass arguments to the function."""
|
||||
async def add(a, b, multiplier=1):
|
||||
return (a + b) * multiplier
|
||||
|
||||
result = await retry_async(add, 2, 3, max_attempts=1, multiplier=2)
|
||||
assert result == 10
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_retry_on_key_error(self):
|
||||
"""Should not retry on non-retryable exceptions."""
|
||||
call_count = 0
|
||||
|
||||
async def bad_func():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
raise KeyError("missing")
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
await retry_async(bad_func, max_attempts=3)
|
||||
assert call_count == 1
|
||||
Reference in New Issue
Block a user