test: add integration and E2E test infrastructure

- Add pytest markers (integration, e2e, slow) with skip logic - Add command line options (--run-integration, --run-e2e) - Create sample_project and sample_project_with_bug fixtures - Add test_integration.py with 10 LLM tests - Add test_e2e.py with 12 API server tests - Update COVERAGE.md to reflect ~65% complete Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-11 19:33:21 +01:00
parent ef69d9c945
commit c839e263f9
5 changed files with 748 additions and 8 deletions
@@ -1,12 +1,75 @@
 """
 Pytest configuration and fixtures.
+
+Test categories:
+- Unit tests: Run by default, no external dependencies
+- Integration tests: Require Ollama, run with --run-integration
+- E2E tests: Require running API server, run with --run-e2e
+
+Usage:
+    pytest tests/                           # Run unit tests only
+    pytest tests/ --run-integration         # Include integration tests
+    pytest tests/ --run-e2e                 # Include E2E tests
+    pytest tests/ --run-integration --run-e2e  # Run all tests
 """
+import os
+import tempfile
+from pathlib import Path
+
 import pytest
 from httpx import ASGITransport, AsyncClient

 from src.main import app


+# =============================================================================
+# Command Line Options
+# =============================================================================
+
+def pytest_addoption(parser):
+    """Add custom command line options."""
+    parser.addoption(
+        "--run-integration",
+        action="store_true",
+        default=False,
+        help="Run integration tests (require Ollama to be running)",
+    )
+    parser.addoption(
+        "--run-e2e",
+        action="store_true",
+        default=False,
+        help="Run E2E tests (require API server to be running)",
+    )
+    parser.addoption(
+        "--ollama-url",
+        action="store",
+        default="http://192.168.86.149:11434",
+        help="Ollama API URL for integration tests",
+    )
+    parser.addoption(
+        "--api-url",
+        action="store",
+        default="http://localhost:8095",
+        help="Webber API URL for E2E tests",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    """Skip integration/e2e tests unless explicitly requested."""
+    skip_integration = pytest.mark.skip(reason="need --run-integration option to run")
+    skip_e2e = pytest.mark.skip(reason="need --run-e2e option to run")
+
+    for item in items:
+        if "integration" in item.keywords and not config.getoption("--run-integration"):
+            item.add_marker(skip_integration)
+        if "e2e" in item.keywords and not config.getoption("--run-e2e"):
+            item.add_marker(skip_e2e)
+
+
+# =============================================================================
+# Basic Fixtures
+# =============================================================================
+
@pytest.fixture
 def anyio_backend():
    """Use asyncio for async tests."""
@@ -15,7 +78,7 @@ def anyio_backend():

@pytest.fixture
 async def client():
-    """Async HTTP client for testing."""
+    """Async HTTP client for testing (no auth)."""
    async with AsyncClient(
        transport=ASGITransport(app=app),
        base_url="http://test"
@@ -32,3 +95,249 @@ async def auth_client():
        headers={"X-API-Key": "test-api-key"}
    ) as ac:
        yield ac
+
+
+# =============================================================================
+# Integration Test Fixtures
+# =============================================================================
+
+@pytest.fixture
+def ollama_url(request):
+    """Get Ollama URL from command line or environment."""
+    return request.config.getoption("--ollama-url") or os.environ.get(
+        "OLLAMA_URL", "http://192.168.86.149:11434"
+    )
+
+
+@pytest.fixture
+def api_url(request):
+    """Get API URL from command line or environment."""
+    return request.config.getoption("--api-url") or os.environ.get(
+        "WEBBER_API_URL", "http://localhost:8095"
+    )
+
+
+@pytest.fixture
+def sample_project():
+    """Create a sample Python project for testing."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        project = Path(tmpdir)
+
+        # Create a realistic project structure
+        (project / "src").mkdir()
+        (project / "tests").mkdir()
+
+        # Main application file
+        (project / "src" / "__init__.py").write_text("")
+        (project / "src" / "main.py").write_text('''"""Main application module."""
+
+def greet(name: str) -> str:
+    """Greet someone by name.
+
+    Args:
+        name: The name to greet
+
+    Returns:
+        A greeting string
+    """
+    return f"Hello, {name}!"
+
+
+def add(a: int, b: int) -> int:
+    """Add two numbers.
+
+    Args:
+        a: First number
+        b: Second number
+
+    Returns:
+        Sum of a and b
+    """
+    return a + b
+
+
+def divide(a: float, b: float) -> float:
+    """Divide two numbers.
+
+    Args:
+        a: Dividend
+        b: Divisor
+
+    Returns:
+        Result of a / b
+
+    Raises:
+        ValueError: If b is zero
+    """
+    if b == 0:
+        raise ValueError("Cannot divide by zero")
+    return a / b
+
+
+if __name__ == "__main__":
+    print(greet("World"))
+''')
+
+        # Utility module
+        (project / "src" / "utils.py").write_text('''"""Utility functions."""
+
+def is_even(n: int) -> bool:
+    """Check if a number is even."""
+    return n % 2 == 0
+
+
+def is_prime(n: int) -> bool:
+    """Check if a number is prime."""
+    if n < 2:
+        return False
+    for i in range(2, int(n ** 0.5) + 1):
+        if n % i == 0:
+            return False
+    return True
+
+
+def factorial(n: int) -> int:
+    """Calculate factorial recursively."""
+    if n <= 1:
+        return 1
+    return n * factorial(n - 1)
+
+
+def fibonacci(n: int) -> list[int]:
+    """Generate Fibonacci sequence up to n terms."""
+    if n <= 0:
+        return []
+    if n == 1:
+        return [0]
+
+    fib = [0, 1]
+    for _ in range(2, n):
+        fib.append(fib[-1] + fib[-2])
+    return fib
+''')
+
+        # Test file
+        (project / "tests" / "__init__.py").write_text("")
+        (project / "tests" / "test_main.py").write_text('''"""Tests for main module."""
+import pytest
+from src.main import greet, add, divide
+
+
+def test_greet():
+    assert greet("World") == "Hello, World!"
+
+
+def test_add():
+    assert add(2, 3) == 5
+
+
+def test_divide():
+    assert divide(10, 2) == 5.0
+
+
+def test_divide_by_zero():
+    with pytest.raises(ValueError):
+        divide(1, 0)
+''')
+
+        # README
+        (project / "README.md").write_text('''# Sample Project
+
+A simple Python project for testing Webber's code exploration.
+
+## Features
+
+- Greeting functionality
+- Math utilities
+- Comprehensive test suite
+
+## Usage
+
+```python
+from src.main import greet, add
+print(greet("World"))
+print(add(2, 3))
+```
+
+## Testing
+
+```bash
+pytest tests/
+```
+''')
+
+        # Configuration files
+        (project / "pyproject.toml").write_text('''[project]
+name = "sample-project"
+version = "0.1.0"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+''')
+
+        yield project
+
+
+@pytest.fixture
+def sample_project_with_bug():
+    """Create a sample project with intentional bugs for testing."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        project = Path(tmpdir)
+
+        (project / "buggy.py").write_text('''"""Module with intentional bugs."""
+
+def divide_numbers(a, b):
+    """Divide two numbers - BUG: no zero check."""
+    return a / b  # BUG: ZeroDivisionError if b is 0
+
+
+def get_item(lst, index):
+    """Get item from list - BUG: no bounds check."""
+    return lst[index]  # BUG: IndexError if out of bounds
+
+
+def parse_int(s):
+    """Parse string to int - BUG: no error handling."""
+    return int(s)  # BUG: ValueError if not a valid int
+
+
+# TODO: Fix the division bug
+# FIXME: Add bounds checking to get_item
+''')
+
+        yield project
+
+
+# =============================================================================
+# E2E Test Fixtures
+# =============================================================================
+
+@pytest.fixture
+async def live_client(api_url):
+    """HTTP client for E2E tests against running server."""
+    async with AsyncClient(base_url=api_url, timeout=30.0) as client:
+        yield client
+
+
+# =============================================================================
+# Helper Functions
+# =============================================================================
+
+def assert_contains_any(text: str, substrings: list[str], case_sensitive: bool = False) -> bool:
+    """Assert that text contains at least one of the substrings."""
+    check_text = text if case_sensitive else text.lower()
+    check_subs = substrings if case_sensitive else [s.lower() for s in substrings]
+
+    found = [s for s in check_subs if s in check_text]
+    assert found, f"Expected text to contain one of {substrings}, but none found in: {text[:200]}..."
+    return True
+
+
+def assert_contains_all(text: str, substrings: list[str], case_sensitive: bool = False) -> bool:
+    """Assert that text contains all of the substrings."""
+    check_text = text if case_sensitive else text.lower()
+    check_subs = substrings if case_sensitive else [s.lower() for s in substrings]
+
+    missing = [s for s in check_subs if s not in check_text]
+    assert not missing, f"Expected text to contain all of {substrings}, missing: {missing}"
+    return True