feat: add retry logic for transient failures

- @with_retry decorator and retry_async() function
- Exponential backoff with jitter
- Retries on: timeout, connection errors, HTTP 429/5xx
- Web search tool now retries on network failures
- Configurable via RETRY_MAX_ATTEMPTS, RETRY_BASE_DELAY, RETRY_MAX_DELAY
- 29 new tests (205 total passing)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.2] - 2026-01-11
+
+### Added
+- Retry logic for transient failures with exponential backoff
+  - `src/shared/retry.py` - `@with_retry` decorator and `retry_async()` function
+  - Retries on: timeout, connection errors, HTTP 429/5xx
+  - Configurable: `RETRY_MAX_ATTEMPTS`, `RETRY_BASE_DELAY`, `RETRY_MAX_DELAY`
+- Web search tool now automatically retries on network failures
+- 29 retry tests (205 total tests passing)
+
 ## [0.4.1] - 2026-01-11
 
 ### Fixed

webber-api/docs/COVERAGE.md

@@ -126,7 +126,7 @@ Last updated: 2026-01-11
 | **Todo tracking** | CLI | Built-in task list (`/todo`) | Medium |
 | **Git integration** | CLI | Auto-commit, branch management | Medium |
 | **Agent handoff** | Orchestration | Explore → Plan → Task workflow | High |
-| **Retry logic** | Infrastructure | Auto-retry on tool failures | Low |
+| ~~**Retry logic**~~ | Infrastructure | ✅ Auto-retry with exponential backoff | Low |
 
 ### Low Priority
 
@@ -151,11 +151,12 @@ Last updated: 2026-01-11
 | Task agent tests | 15 | 15 | ✅ |
 | Conversation tests | 19 | 19 | ✅ |
 | Token tests | 6 | 6 | ✅ |
+| Retry tests | 29 | 29 | ✅ |
 | Security tests | 14 | 14 | ✅ |
 | Integration tests | 10 | 10 | ✅ Agent + real LLM |
 | E2E tests | 12 | 12 | ✅ Full API workflow |
 
-**Total: 176 tests passing**
+**Total: 205 tests passing**
 
 **Test breakdown:**
 - Read/Glob/Grep tools: 17 tests
@@ -168,6 +169,7 @@ Last updated: 2026-01-11
 - Task agent: 15 tests
 - Conversations: 19 tests
 - Tokens: 6 tests
+- Retry: 29 tests
 - Security: 14 tests
 - Health checks: 2 tests
 - Integration (LLM): 10 tests

webber-api/docs/architecture.md

@@ -217,6 +217,9 @@ All settings via environment variables or `.env`:
 | SUMMARIZATION_THRESHOLD | 0.8 | Summarize at N% of max tokens |
 | SUMMARIZATION_TARGET_TOKENS | 500 | Target summary size |
 | KEEP_RECENT_MESSAGES | 6 | Messages to keep unsummarized |
+| RETRY_MAX_ATTEMPTS | 3 | Max retry attempts for transient failures |
+| RETRY_BASE_DELAY | 1.0 | Base delay between retries (seconds) |
+| RETRY_MAX_DELAY | 30.0 | Maximum delay between retries (seconds) |
 
 ---
 

webber-api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "webber-api"
-version = "0.4.1"
+version = "0.4.2"
 description = "Webber API - Multi-Agent AI Development Server"
 authors = [
     {name = "jpmschweitzer"}

webber-api/src/domains/tools/search/web.py

@@ -9,6 +9,7 @@ import httpx
 from src.domains.tools.base import BaseTool, ToolResult
 from src.shared.config import get_settings
 from src.shared.logging import logged, get_logger
+from src.shared.retry import retry_async
 
 logger = get_logger(__name__)
 
@@ -73,6 +74,24 @@ IMPORTANT:
         self.searxng_url = (searxng_url or settings.searxng_url).rstrip("/")
         self.timeout = timeout or settings.searxng_timeout
         self.max_results = max_results
+        # Retry settings
+        self.retry_max_attempts = settings.retry_max_attempts
+        self.retry_base_delay = settings.retry_base_delay
+        self.retry_max_delay = settings.retry_max_delay
+
+    async def _fetch_search_results(self, params: dict) -> dict:
+        """
+        Fetch search results from SearXNG.
+
+        This method is wrapped with retry logic for transient failures.
+        """
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await client.get(
+                f"{self.searxng_url}/search",
+                params=params,
+            )
+            response.raise_for_status()
+            return response.json()
 
     @logged()
     async def execute(
@@ -111,16 +130,15 @@ IMPORTANT:
             params["categories"] = categories
 
         try:
-            async with httpx.AsyncClient(timeout=self.timeout) as client:
-                response = await client.get(
-                    f"{self.searxng_url}/search",
-                    params=params,
+            data = await retry_async(
+                self._fetch_search_results,
+                params,
+                max_attempts=self.retry_max_attempts,
+                base_delay=self.retry_base_delay,
+                max_delay=self.retry_max_delay,
             )
-                response.raise_for_status()
-                data = response.json()
-
         except httpx.TimeoutException:
-            return self._error(f"Search timed out after {self.timeout}s")
+            return self._error(f"Search timed out after {self.timeout}s (all retries exhausted)")
         except httpx.HTTPStatusError as e:
             return self._error(f"Search failed: HTTP {e.response.status_code}")
         except httpx.RequestError as e:

webber-api/src/shared/config.py

@@ -90,6 +90,11 @@ class Settings(BaseSettings):
     summarization_target_tokens: int = 500  # Target summary size
     keep_recent_messages: int = 6  # Messages to keep unsummarized (3 turns)
 
+    # Retry logic
+    retry_max_attempts: int = 3  # Max retry attempts for transient failures
+    retry_base_delay: float = 1.0  # Base delay in seconds
+    retry_max_delay: float = 30.0  # Maximum delay in seconds
+
     model_config = SettingsConfigDict(
         env_file=".env",
         case_sensitive=False,

webber-api/src/shared/retry.py

@@ -0,0 +1,215 @@
+"""
+Retry utilities for handling transient failures.
+
+Provides decorators and helpers for automatic retry with exponential backoff.
+"""
+import asyncio
+import random
+from collections.abc import Awaitable, Callable
+from functools import wraps
+from typing import Any, TypeVar
+
+import httpx
+
+from src.shared.logging import get_logger
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+
+
+# Exceptions that should trigger a retry
+RETRYABLE_EXCEPTIONS = (
+    httpx.TimeoutException,
+    httpx.ConnectError,
+    httpx.ReadError,
+    httpx.WriteError,
+    httpx.ConnectTimeout,
+    httpx.ReadTimeout,
+    httpx.WriteTimeout,
+    httpx.PoolTimeout,
+    ConnectionError,
+    TimeoutError,
+    OSError,  # Covers many network-related errors
+)
+
+
+def is_retryable_http_status(status_code: int) -> bool:
+    """
+    Check if an HTTP status code should trigger a retry.
+
+    Retryable:
+    - 429 Too Many Requests (rate limited)
+    - 500 Internal Server Error
+    - 502 Bad Gateway
+    - 503 Service Unavailable
+    - 504 Gateway Timeout
+    """
+    return status_code in (429, 500, 502, 503, 504)
+
+
+def is_retryable_exception(exc: Exception) -> bool:
+    """Check if an exception should trigger a retry."""
+    if isinstance(exc, RETRYABLE_EXCEPTIONS):
+        return True
+
+    # Check for retryable HTTP status codes
+    if isinstance(exc, httpx.HTTPStatusError):
+        return is_retryable_http_status(exc.response.status_code)
+
+    return False
+
+
+def calculate_backoff(
+    attempt: int,
+    base_delay: float = 1.0,
+    max_delay: float = 60.0,
+    jitter: bool = True,
+) -> float:
+    """
+    Calculate exponential backoff delay with optional jitter.
+
+    Args:
+        attempt: Current attempt number (0-indexed)
+        base_delay: Base delay in seconds
+        max_delay: Maximum delay in seconds
+        jitter: Add random jitter to prevent thundering herd
+
+    Returns:
+        Delay in seconds
+    """
+    # Exponential backoff: base_delay * 2^attempt
+    delay = min(base_delay * (2 ** attempt), max_delay)
+
+    if jitter:
+        # Add up to 25% random jitter
+        delay = delay * (0.75 + random.random() * 0.5)
+
+    return delay
+
+
+def with_retry(
+    max_attempts: int = 3,
+    base_delay: float = 1.0,
+    max_delay: float = 60.0,
+    retryable_exceptions: tuple[type[Exception], ...] | None = None,
+) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
+    """
+    Decorator for async functions that should retry on transient failures.
+
+    Args:
+        max_attempts: Maximum number of attempts (including initial)
+        base_delay: Base delay between retries in seconds
+        max_delay: Maximum delay between retries in seconds
+        retryable_exceptions: Additional exceptions to retry on
+
+    Returns:
+        Decorated function with retry logic
+
+    Example:
+        @with_retry(max_attempts=3, base_delay=1.0)
+        async def fetch_data():
+            async with httpx.AsyncClient() as client:
+                response = await client.get(url)
+                return response.json()
+    """
+    extra_exceptions = retryable_exceptions or ()
+
+    def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> T:
+            last_exception: Exception | None = None
+
+            for attempt in range(max_attempts):
+                try:
+                    return await func(*args, **kwargs)
+
+                except (*RETRYABLE_EXCEPTIONS, *extra_exceptions) as e:
+                    last_exception = e
+                    should_retry = True
+
+                except httpx.HTTPStatusError as e:
+                    last_exception = e
+                    should_retry = is_retryable_http_status(e.response.status_code)
+
+                except Exception:
+                    # Non-retryable exception, re-raise immediately
+                    raise
+
+                if should_retry and attempt < max_attempts - 1:
+                    delay = calculate_backoff(attempt, base_delay, max_delay)
+                    logger.warning(
+                        f"Retry {attempt + 1}/{max_attempts - 1} for {func.__name__} "
+                        f"after {delay:.2f}s due to: {last_exception}"
+                    )
+                    await asyncio.sleep(delay)
+                elif not should_retry:
+                    # Non-retryable HTTP error
+                    raise last_exception  # type: ignore
+
+            # All retries exhausted
+            logger.error(
+                f"All {max_attempts} attempts failed for {func.__name__}: {last_exception}"
+            )
+            raise last_exception  # type: ignore
+
+        return wrapper
+    return decorator
+
+
+async def retry_async(
+    func: Callable[..., Awaitable[T]],
+    *args: Any,
+    max_attempts: int = 3,
+    base_delay: float = 1.0,
+    max_delay: float = 60.0,
+    **kwargs: Any,
+) -> T:
+    """
+    Retry an async function with exponential backoff.
+
+    Alternative to decorator when you need per-call control.
+
+    Args:
+        func: Async function to call
+        *args: Positional arguments for func
+        max_attempts: Maximum number of attempts
+        base_delay: Base delay between retries
+        max_delay: Maximum delay between retries
+        **kwargs: Keyword arguments for func
+
+    Returns:
+        Result of func
+
+    Raises:
+        Last exception if all retries fail
+
+    Example:
+        result = await retry_async(
+            fetch_data,
+            url,
+            max_attempts=5,
+            timeout=30,
+        )
+    """
+    last_exception: Exception | None = None
+
+    for attempt in range(max_attempts):
+        try:
+            return await func(*args, **kwargs)
+
+        except Exception as e:
+            last_exception = e
+
+            if not is_retryable_exception(e):
+                raise
+
+            if attempt < max_attempts - 1:
+                delay = calculate_backoff(attempt, base_delay, max_delay)
+                logger.warning(
+                    f"Retry {attempt + 1}/{max_attempts - 1} "
+                    f"after {delay:.2f}s due to: {e}"
+                )
+                await asyncio.sleep(delay)
+
+    raise last_exception  # type: ignore

webber-api/tests/test_retry.py

@@ -0,0 +1,244 @@
+"""
+Tests for retry utilities.
+"""
+import pytest
+from unittest.mock import AsyncMock, patch
+
+import httpx
+
+from src.shared.retry import (
+    with_retry,
+    retry_async,
+    is_retryable_exception,
+    is_retryable_http_status,
+    calculate_backoff,
+)
+
+
+class TestIsRetryableHttpStatus:
+    """Tests for HTTP status code checking."""
+
+    def test_429_is_retryable(self):
+        """429 Too Many Requests should be retryable."""
+        assert is_retryable_http_status(429) is True
+
+    def test_500_is_retryable(self):
+        """500 Internal Server Error should be retryable."""
+        assert is_retryable_http_status(500) is True
+
+    def test_502_is_retryable(self):
+        """502 Bad Gateway should be retryable."""
+        assert is_retryable_http_status(502) is True
+
+    def test_503_is_retryable(self):
+        """503 Service Unavailable should be retryable."""
+        assert is_retryable_http_status(503) is True
+
+    def test_504_is_retryable(self):
+        """504 Gateway Timeout should be retryable."""
+        assert is_retryable_http_status(504) is True
+
+    def test_400_not_retryable(self):
+        """400 Bad Request should not be retryable."""
+        assert is_retryable_http_status(400) is False
+
+    def test_401_not_retryable(self):
+        """401 Unauthorized should not be retryable."""
+        assert is_retryable_http_status(401) is False
+
+    def test_404_not_retryable(self):
+        """404 Not Found should not be retryable."""
+        assert is_retryable_http_status(404) is False
+
+    def test_200_not_retryable(self):
+        """200 OK should not be retryable."""
+        assert is_retryable_http_status(200) is False
+
+
+class TestIsRetryableException:
+    """Tests for exception checking."""
+
+    def test_timeout_exception_is_retryable(self):
+        """Timeout exceptions should be retryable."""
+        exc = httpx.TimeoutException("timeout")
+        assert is_retryable_exception(exc) is True
+
+    def test_connect_error_is_retryable(self):
+        """Connection errors should be retryable."""
+        exc = httpx.ConnectError("connection failed")
+        assert is_retryable_exception(exc) is True
+
+    def test_connection_error_is_retryable(self):
+        """Python ConnectionError should be retryable."""
+        exc = ConnectionError("connection refused")
+        assert is_retryable_exception(exc) is True
+
+    def test_timeout_error_is_retryable(self):
+        """Python TimeoutError should be retryable."""
+        exc = TimeoutError("timed out")
+        assert is_retryable_exception(exc) is True
+
+    def test_value_error_not_retryable(self):
+        """ValueError should not be retryable."""
+        exc = ValueError("invalid value")
+        assert is_retryable_exception(exc) is False
+
+    def test_key_error_not_retryable(self):
+        """KeyError should not be retryable."""
+        exc = KeyError("missing key")
+        assert is_retryable_exception(exc) is False
+
+
+class TestCalculateBackoff:
+    """Tests for backoff calculation."""
+
+    def test_first_attempt_base_delay(self):
+        """First attempt should use base delay."""
+        delay = calculate_backoff(0, base_delay=1.0, jitter=False)
+        assert delay == 1.0
+
+    def test_second_attempt_doubles(self):
+        """Second attempt should double the delay."""
+        delay = calculate_backoff(1, base_delay=1.0, jitter=False)
+        assert delay == 2.0
+
+    def test_third_attempt_quadruples(self):
+        """Third attempt should quadruple the delay."""
+        delay = calculate_backoff(2, base_delay=1.0, jitter=False)
+        assert delay == 4.0
+
+    def test_max_delay_respected(self):
+        """Delay should not exceed max_delay."""
+        delay = calculate_backoff(10, base_delay=1.0, max_delay=30.0, jitter=False)
+        assert delay == 30.0
+
+    def test_jitter_adds_randomness(self):
+        """Jitter should add randomness to delay."""
+        delays = [calculate_backoff(1, base_delay=1.0, jitter=True) for _ in range(10)]
+        # With jitter, delays should vary (not all identical)
+        assert len(set(delays)) > 1
+
+    def test_jitter_within_bounds(self):
+        """Jitter should keep delay within reasonable bounds."""
+        for _ in range(100):
+            delay = calculate_backoff(0, base_delay=2.0, jitter=True)
+            # Attempt 0 with base 2.0 = 2.0, with jitter should be 0.75-1.25x = 1.5-2.5
+            assert 1.5 <= delay <= 2.5
+
+
+class TestWithRetryDecorator:
+    """Tests for the @with_retry decorator."""
+
+    @pytest.mark.anyio
+    async def test_success_on_first_attempt(self):
+        """Function should return on first successful attempt."""
+        call_count = 0
+
+        @with_retry(max_attempts=3)
+        async def successful_func():
+            nonlocal call_count
+            call_count += 1
+            return "success"
+
+        result = await successful_func()
+        assert result == "success"
+        assert call_count == 1
+
+    @pytest.mark.anyio
+    async def test_retry_on_timeout(self):
+        """Should retry on timeout exception."""
+        call_count = 0
+
+        @with_retry(max_attempts=3, base_delay=0.01)
+        async def flaky_func():
+            nonlocal call_count
+            call_count += 1
+            if call_count < 3:
+                raise httpx.TimeoutException("timeout")
+            return "success"
+
+        result = await flaky_func()
+        assert result == "success"
+        assert call_count == 3
+
+    @pytest.mark.anyio
+    async def test_no_retry_on_value_error(self):
+        """Should not retry on non-retryable exceptions."""
+        call_count = 0
+
+        @with_retry(max_attempts=3)
+        async def bad_func():
+            nonlocal call_count
+            call_count += 1
+            raise ValueError("bad value")
+
+        with pytest.raises(ValueError):
+            await bad_func()
+        assert call_count == 1
+
+    @pytest.mark.anyio
+    async def test_exhausted_retries(self):
+        """Should raise last exception after all retries exhausted."""
+        call_count = 0
+
+        @with_retry(max_attempts=3, base_delay=0.01)
+        async def always_fails():
+            nonlocal call_count
+            call_count += 1
+            raise httpx.TimeoutException("always times out")
+
+        with pytest.raises(httpx.TimeoutException):
+            await always_fails()
+        assert call_count == 3
+
+
+class TestRetryAsync:
+    """Tests for the retry_async function."""
+
+    @pytest.mark.anyio
+    async def test_success_on_first_attempt(self):
+        """Function should return on first successful attempt."""
+        async def successful_func():
+            return "success"
+
+        result = await retry_async(successful_func, max_attempts=3)
+        assert result == "success"
+
+    @pytest.mark.anyio
+    async def test_retry_on_connect_error(self):
+        """Should retry on connection errors."""
+        call_count = 0
+
+        async def flaky_func():
+            nonlocal call_count
+            call_count += 1
+            if call_count < 2:
+                raise httpx.ConnectError("connection failed")
+            return "success"
+
+        result = await retry_async(flaky_func, max_attempts=3, base_delay=0.01)
+        assert result == "success"
+        assert call_count == 2
+
+    @pytest.mark.anyio
+    async def test_passes_args_and_kwargs(self):
+        """Should pass arguments to the function."""
+        async def add(a, b, multiplier=1):
+            return (a + b) * multiplier
+
+        result = await retry_async(add, 2, 3, max_attempts=1, multiplier=2)
+        assert result == 10
+
+    @pytest.mark.anyio
+    async def test_no_retry_on_key_error(self):
+        """Should not retry on non-retryable exceptions."""
+        call_count = 0
+
+        async def bad_func():
+            nonlocal call_count
+            call_count += 1
+            raise KeyError("missing")
+
+        with pytest.raises(KeyError):
+            await retry_async(bad_func, max_attempts=3)
+        assert call_count == 1