diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ea088..807983f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.2] - 2026-01-11 + +### Added +- Retry logic for transient failures with exponential backoff + - `src/shared/retry.py` - `@with_retry` decorator and `retry_async()` function + - Retries on: timeout, connection errors, HTTP 429/5xx + - Configurable: `RETRY_MAX_ATTEMPTS`, `RETRY_BASE_DELAY`, `RETRY_MAX_DELAY` +- Web search tool now automatically retries on network failures +- 29 retry tests (205 total tests passing) + ## [0.4.1] - 2026-01-11 ### Fixed diff --git a/webber-api/docs/COVERAGE.md b/webber-api/docs/COVERAGE.md index 8a83826..c19873f 100644 --- a/webber-api/docs/COVERAGE.md +++ b/webber-api/docs/COVERAGE.md @@ -126,7 +126,7 @@ Last updated: 2026-01-11 | **Todo tracking** | CLI | Built-in task list (`/todo`) | Medium | | **Git integration** | CLI | Auto-commit, branch management | Medium | | **Agent handoff** | Orchestration | Explore → Plan → Task workflow | High | -| **Retry logic** | Infrastructure | Auto-retry on tool failures | Low | +| ~~**Retry logic**~~ | Infrastructure | ✅ Auto-retry with exponential backoff | Low | ### Low Priority @@ -151,11 +151,12 @@ Last updated: 2026-01-11 | Task agent tests | 15 | 15 | ✅ | | Conversation tests | 19 | 19 | ✅ | | Token tests | 6 | 6 | ✅ | +| Retry tests | 29 | 29 | ✅ | | Security tests | 14 | 14 | ✅ | | Integration tests | 10 | 10 | ✅ Agent + real LLM | | E2E tests | 12 | 12 | ✅ Full API workflow | -**Total: 176 tests passing** +**Total: 205 tests passing** **Test breakdown:** - Read/Glob/Grep tools: 17 tests @@ -168,6 +169,7 @@ Last updated: 2026-01-11 - Task agent: 15 tests - Conversations: 19 tests - Tokens: 6 tests +- Retry: 29 tests - Security: 14 tests - Health checks: 2 tests - Integration (LLM): 10 tests diff --git a/webber-api/docs/architecture.md b/webber-api/docs/architecture.md index f1a6d14..20eb662 100644 --- a/webber-api/docs/architecture.md +++ b/webber-api/docs/architecture.md @@ -217,6 +217,9 @@ All settings via environment variables or `.env`: | SUMMARIZATION_THRESHOLD | 0.8 | Summarize at N% of max tokens | | SUMMARIZATION_TARGET_TOKENS | 500 | Target summary size | | KEEP_RECENT_MESSAGES | 6 | Messages to keep unsummarized | +| RETRY_MAX_ATTEMPTS | 3 | Max retry attempts for transient failures | +| RETRY_BASE_DELAY | 1.0 | Base delay between retries (seconds) | +| RETRY_MAX_DELAY | 30.0 | Maximum delay between retries (seconds) | --- diff --git a/webber-api/pyproject.toml b/webber-api/pyproject.toml index df23fda..4d00598 100644 --- a/webber-api/pyproject.toml +++ b/webber-api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "webber-api" -version = "0.4.1" +version = "0.4.2" description = "Webber API - Multi-Agent AI Development Server" authors = [ {name = "jpmschweitzer"} diff --git a/webber-api/src/domains/tools/search/web.py b/webber-api/src/domains/tools/search/web.py index 85b5c4b..9274c09 100644 --- a/webber-api/src/domains/tools/search/web.py +++ b/webber-api/src/domains/tools/search/web.py @@ -9,6 +9,7 @@ import httpx from src.domains.tools.base import BaseTool, ToolResult from src.shared.config import get_settings from src.shared.logging import logged, get_logger +from src.shared.retry import retry_async logger = get_logger(__name__) @@ -73,6 +74,24 @@ IMPORTANT: self.searxng_url = (searxng_url or settings.searxng_url).rstrip("/") self.timeout = timeout or settings.searxng_timeout self.max_results = max_results + # Retry settings + self.retry_max_attempts = settings.retry_max_attempts + self.retry_base_delay = settings.retry_base_delay + self.retry_max_delay = settings.retry_max_delay + + async def _fetch_search_results(self, params: dict) -> dict: + """ + Fetch search results from SearXNG. + + This method is wrapped with retry logic for transient failures. + """ + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.searxng_url}/search", + params=params, + ) + response.raise_for_status() + return response.json() @logged() async def execute( @@ -111,16 +130,15 @@ IMPORTANT: params["categories"] = categories try: - async with httpx.AsyncClient(timeout=self.timeout) as client: - response = await client.get( - f"{self.searxng_url}/search", - params=params, - ) - response.raise_for_status() - data = response.json() - + data = await retry_async( + self._fetch_search_results, + params, + max_attempts=self.retry_max_attempts, + base_delay=self.retry_base_delay, + max_delay=self.retry_max_delay, + ) except httpx.TimeoutException: - return self._error(f"Search timed out after {self.timeout}s") + return self._error(f"Search timed out after {self.timeout}s (all retries exhausted)") except httpx.HTTPStatusError as e: return self._error(f"Search failed: HTTP {e.response.status_code}") except httpx.RequestError as e: diff --git a/webber-api/src/shared/config.py b/webber-api/src/shared/config.py index 9c08609..8f7ca1f 100644 --- a/webber-api/src/shared/config.py +++ b/webber-api/src/shared/config.py @@ -90,6 +90,11 @@ class Settings(BaseSettings): summarization_target_tokens: int = 500 # Target summary size keep_recent_messages: int = 6 # Messages to keep unsummarized (3 turns) + # Retry logic + retry_max_attempts: int = 3 # Max retry attempts for transient failures + retry_base_delay: float = 1.0 # Base delay in seconds + retry_max_delay: float = 30.0 # Maximum delay in seconds + model_config = SettingsConfigDict( env_file=".env", case_sensitive=False, diff --git a/webber-api/src/shared/retry.py b/webber-api/src/shared/retry.py new file mode 100644 index 0000000..481e633 --- /dev/null +++ b/webber-api/src/shared/retry.py @@ -0,0 +1,215 @@ +""" +Retry utilities for handling transient failures. + +Provides decorators and helpers for automatic retry with exponential backoff. +""" +import asyncio +import random +from collections.abc import Awaitable, Callable +from functools import wraps +from typing import Any, TypeVar + +import httpx + +from src.shared.logging import get_logger + +logger = get_logger(__name__) + +T = TypeVar("T") + + +# Exceptions that should trigger a retry +RETRYABLE_EXCEPTIONS = ( + httpx.TimeoutException, + httpx.ConnectError, + httpx.ReadError, + httpx.WriteError, + httpx.ConnectTimeout, + httpx.ReadTimeout, + httpx.WriteTimeout, + httpx.PoolTimeout, + ConnectionError, + TimeoutError, + OSError, # Covers many network-related errors +) + + +def is_retryable_http_status(status_code: int) -> bool: + """ + Check if an HTTP status code should trigger a retry. + + Retryable: + - 429 Too Many Requests (rate limited) + - 500 Internal Server Error + - 502 Bad Gateway + - 503 Service Unavailable + - 504 Gateway Timeout + """ + return status_code in (429, 500, 502, 503, 504) + + +def is_retryable_exception(exc: Exception) -> bool: + """Check if an exception should trigger a retry.""" + if isinstance(exc, RETRYABLE_EXCEPTIONS): + return True + + # Check for retryable HTTP status codes + if isinstance(exc, httpx.HTTPStatusError): + return is_retryable_http_status(exc.response.status_code) + + return False + + +def calculate_backoff( + attempt: int, + base_delay: float = 1.0, + max_delay: float = 60.0, + jitter: bool = True, +) -> float: + """ + Calculate exponential backoff delay with optional jitter. + + Args: + attempt: Current attempt number (0-indexed) + base_delay: Base delay in seconds + max_delay: Maximum delay in seconds + jitter: Add random jitter to prevent thundering herd + + Returns: + Delay in seconds + """ + # Exponential backoff: base_delay * 2^attempt + delay = min(base_delay * (2 ** attempt), max_delay) + + if jitter: + # Add up to 25% random jitter + delay = delay * (0.75 + random.random() * 0.5) + + return delay + + +def with_retry( + max_attempts: int = 3, + base_delay: float = 1.0, + max_delay: float = 60.0, + retryable_exceptions: tuple[type[Exception], ...] | None = None, +) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]: + """ + Decorator for async functions that should retry on transient failures. + + Args: + max_attempts: Maximum number of attempts (including initial) + base_delay: Base delay between retries in seconds + max_delay: Maximum delay between retries in seconds + retryable_exceptions: Additional exceptions to retry on + + Returns: + Decorated function with retry logic + + Example: + @with_retry(max_attempts=3, base_delay=1.0) + async def fetch_data(): + async with httpx.AsyncClient() as client: + response = await client.get(url) + return response.json() + """ + extra_exceptions = retryable_exceptions or () + + def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]: + @wraps(func) + async def wrapper(*args: Any, **kwargs: Any) -> T: + last_exception: Exception | None = None + + for attempt in range(max_attempts): + try: + return await func(*args, **kwargs) + + except (*RETRYABLE_EXCEPTIONS, *extra_exceptions) as e: + last_exception = e + should_retry = True + + except httpx.HTTPStatusError as e: + last_exception = e + should_retry = is_retryable_http_status(e.response.status_code) + + except Exception: + # Non-retryable exception, re-raise immediately + raise + + if should_retry and attempt < max_attempts - 1: + delay = calculate_backoff(attempt, base_delay, max_delay) + logger.warning( + f"Retry {attempt + 1}/{max_attempts - 1} for {func.__name__} " + f"after {delay:.2f}s due to: {last_exception}" + ) + await asyncio.sleep(delay) + elif not should_retry: + # Non-retryable HTTP error + raise last_exception # type: ignore + + # All retries exhausted + logger.error( + f"All {max_attempts} attempts failed for {func.__name__}: {last_exception}" + ) + raise last_exception # type: ignore + + return wrapper + return decorator + + +async def retry_async( + func: Callable[..., Awaitable[T]], + *args: Any, + max_attempts: int = 3, + base_delay: float = 1.0, + max_delay: float = 60.0, + **kwargs: Any, +) -> T: + """ + Retry an async function with exponential backoff. + + Alternative to decorator when you need per-call control. + + Args: + func: Async function to call + *args: Positional arguments for func + max_attempts: Maximum number of attempts + base_delay: Base delay between retries + max_delay: Maximum delay between retries + **kwargs: Keyword arguments for func + + Returns: + Result of func + + Raises: + Last exception if all retries fail + + Example: + result = await retry_async( + fetch_data, + url, + max_attempts=5, + timeout=30, + ) + """ + last_exception: Exception | None = None + + for attempt in range(max_attempts): + try: + return await func(*args, **kwargs) + + except Exception as e: + last_exception = e + + if not is_retryable_exception(e): + raise + + if attempt < max_attempts - 1: + delay = calculate_backoff(attempt, base_delay, max_delay) + logger.warning( + f"Retry {attempt + 1}/{max_attempts - 1} " + f"after {delay:.2f}s due to: {e}" + ) + await asyncio.sleep(delay) + + raise last_exception # type: ignore diff --git a/webber-api/tests/test_retry.py b/webber-api/tests/test_retry.py new file mode 100644 index 0000000..e73ec43 --- /dev/null +++ b/webber-api/tests/test_retry.py @@ -0,0 +1,244 @@ +""" +Tests for retry utilities. +""" +import pytest +from unittest.mock import AsyncMock, patch + +import httpx + +from src.shared.retry import ( + with_retry, + retry_async, + is_retryable_exception, + is_retryable_http_status, + calculate_backoff, +) + + +class TestIsRetryableHttpStatus: + """Tests for HTTP status code checking.""" + + def test_429_is_retryable(self): + """429 Too Many Requests should be retryable.""" + assert is_retryable_http_status(429) is True + + def test_500_is_retryable(self): + """500 Internal Server Error should be retryable.""" + assert is_retryable_http_status(500) is True + + def test_502_is_retryable(self): + """502 Bad Gateway should be retryable.""" + assert is_retryable_http_status(502) is True + + def test_503_is_retryable(self): + """503 Service Unavailable should be retryable.""" + assert is_retryable_http_status(503) is True + + def test_504_is_retryable(self): + """504 Gateway Timeout should be retryable.""" + assert is_retryable_http_status(504) is True + + def test_400_not_retryable(self): + """400 Bad Request should not be retryable.""" + assert is_retryable_http_status(400) is False + + def test_401_not_retryable(self): + """401 Unauthorized should not be retryable.""" + assert is_retryable_http_status(401) is False + + def test_404_not_retryable(self): + """404 Not Found should not be retryable.""" + assert is_retryable_http_status(404) is False + + def test_200_not_retryable(self): + """200 OK should not be retryable.""" + assert is_retryable_http_status(200) is False + + +class TestIsRetryableException: + """Tests for exception checking.""" + + def test_timeout_exception_is_retryable(self): + """Timeout exceptions should be retryable.""" + exc = httpx.TimeoutException("timeout") + assert is_retryable_exception(exc) is True + + def test_connect_error_is_retryable(self): + """Connection errors should be retryable.""" + exc = httpx.ConnectError("connection failed") + assert is_retryable_exception(exc) is True + + def test_connection_error_is_retryable(self): + """Python ConnectionError should be retryable.""" + exc = ConnectionError("connection refused") + assert is_retryable_exception(exc) is True + + def test_timeout_error_is_retryable(self): + """Python TimeoutError should be retryable.""" + exc = TimeoutError("timed out") + assert is_retryable_exception(exc) is True + + def test_value_error_not_retryable(self): + """ValueError should not be retryable.""" + exc = ValueError("invalid value") + assert is_retryable_exception(exc) is False + + def test_key_error_not_retryable(self): + """KeyError should not be retryable.""" + exc = KeyError("missing key") + assert is_retryable_exception(exc) is False + + +class TestCalculateBackoff: + """Tests for backoff calculation.""" + + def test_first_attempt_base_delay(self): + """First attempt should use base delay.""" + delay = calculate_backoff(0, base_delay=1.0, jitter=False) + assert delay == 1.0 + + def test_second_attempt_doubles(self): + """Second attempt should double the delay.""" + delay = calculate_backoff(1, base_delay=1.0, jitter=False) + assert delay == 2.0 + + def test_third_attempt_quadruples(self): + """Third attempt should quadruple the delay.""" + delay = calculate_backoff(2, base_delay=1.0, jitter=False) + assert delay == 4.0 + + def test_max_delay_respected(self): + """Delay should not exceed max_delay.""" + delay = calculate_backoff(10, base_delay=1.0, max_delay=30.0, jitter=False) + assert delay == 30.0 + + def test_jitter_adds_randomness(self): + """Jitter should add randomness to delay.""" + delays = [calculate_backoff(1, base_delay=1.0, jitter=True) for _ in range(10)] + # With jitter, delays should vary (not all identical) + assert len(set(delays)) > 1 + + def test_jitter_within_bounds(self): + """Jitter should keep delay within reasonable bounds.""" + for _ in range(100): + delay = calculate_backoff(0, base_delay=2.0, jitter=True) + # Attempt 0 with base 2.0 = 2.0, with jitter should be 0.75-1.25x = 1.5-2.5 + assert 1.5 <= delay <= 2.5 + + +class TestWithRetryDecorator: + """Tests for the @with_retry decorator.""" + + @pytest.mark.anyio + async def test_success_on_first_attempt(self): + """Function should return on first successful attempt.""" + call_count = 0 + + @with_retry(max_attempts=3) + async def successful_func(): + nonlocal call_count + call_count += 1 + return "success" + + result = await successful_func() + assert result == "success" + assert call_count == 1 + + @pytest.mark.anyio + async def test_retry_on_timeout(self): + """Should retry on timeout exception.""" + call_count = 0 + + @with_retry(max_attempts=3, base_delay=0.01) + async def flaky_func(): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise httpx.TimeoutException("timeout") + return "success" + + result = await flaky_func() + assert result == "success" + assert call_count == 3 + + @pytest.mark.anyio + async def test_no_retry_on_value_error(self): + """Should not retry on non-retryable exceptions.""" + call_count = 0 + + @with_retry(max_attempts=3) + async def bad_func(): + nonlocal call_count + call_count += 1 + raise ValueError("bad value") + + with pytest.raises(ValueError): + await bad_func() + assert call_count == 1 + + @pytest.mark.anyio + async def test_exhausted_retries(self): + """Should raise last exception after all retries exhausted.""" + call_count = 0 + + @with_retry(max_attempts=3, base_delay=0.01) + async def always_fails(): + nonlocal call_count + call_count += 1 + raise httpx.TimeoutException("always times out") + + with pytest.raises(httpx.TimeoutException): + await always_fails() + assert call_count == 3 + + +class TestRetryAsync: + """Tests for the retry_async function.""" + + @pytest.mark.anyio + async def test_success_on_first_attempt(self): + """Function should return on first successful attempt.""" + async def successful_func(): + return "success" + + result = await retry_async(successful_func, max_attempts=3) + assert result == "success" + + @pytest.mark.anyio + async def test_retry_on_connect_error(self): + """Should retry on connection errors.""" + call_count = 0 + + async def flaky_func(): + nonlocal call_count + call_count += 1 + if call_count < 2: + raise httpx.ConnectError("connection failed") + return "success" + + result = await retry_async(flaky_func, max_attempts=3, base_delay=0.01) + assert result == "success" + assert call_count == 2 + + @pytest.mark.anyio + async def test_passes_args_and_kwargs(self): + """Should pass arguments to the function.""" + async def add(a, b, multiplier=1): + return (a + b) * multiplier + + result = await retry_async(add, 2, 3, max_attempts=1, multiplier=2) + assert result == 10 + + @pytest.mark.anyio + async def test_no_retry_on_key_error(self): + """Should not retry on non-retryable exceptions.""" + call_count = 0 + + async def bad_func(): + nonlocal call_count + call_count += 1 + raise KeyError("missing") + + with pytest.raises(KeyError): + await retry_async(bad_func, max_attempts=3) + assert call_count == 1