webber/webber-api/tests/test_integration.py

"""
Integration tests with real LLM.

These tests require Ollama to be running and are skipped by default.
Run with: pytest tests/test_integration.py -v --run-integration

Note: These tests are slow (each takes 5-30 seconds depending on LLM response time).
"""
import pytest

from tests.conftest import assert_contains_any

# All tests in this module require --run-integration
pytestmark = [pytest.mark.integration, pytest.mark.slow]


class TestExploreAgentIntegration:
    """Integration tests for the explore agent with real LLM."""

    @pytest.fixture
    def agent(self):
        """Get the explore agent."""
        from src.domains.agents.explore.agent import explore_agent
        return explore_agent

    @pytest.mark.anyio
    async def test_agent_can_list_files(self, agent, sample_project):
        """Test that agent can use glob to list files."""
        result = await agent.run(
            "List all Python files in this project. Just list the filenames.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Agent should mention the Python files
        assert_contains_any(result, ["main.py", "utils.py", ".py"])

    @pytest.mark.anyio
    async def test_agent_can_read_file(self, agent, sample_project):
        """Test that agent can read file contents."""
        result = await agent.run(
            "Use the read_file tool to read src/main.py and list what functions are defined.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Agent should either mention functions or indicate it read the file
        # LLMs can be unpredictable, so we check for various valid responses
        assert_contains_any(result, [
            "greet", "add", "divide",  # Function names
            "function", "def",  # Generic function mentions
            "main.py",  # File reference
        ])

    @pytest.mark.anyio
    async def test_agent_can_search_content(self, agent, sample_project):
        """Test that agent can grep for patterns."""
        result = await agent.run(
            "Search for all TODO and FIXME comments in the codebase.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Should find no TODOs in the clean sample project
        # (or correctly report none found)
        assert result is not None and len(result) > 0

    @pytest.mark.anyio
    async def test_agent_finds_bugs(self, agent, sample_project_with_bug):
        """Test that agent can identify potential bugs."""
        result = await agent.run(
            "Review buggy.py and identify potential bugs or issues.",
            working_dir=str(sample_project_with_bug),
            allowed_paths=[str(sample_project_with_bug)],
        )

        # Agent should identify at least one issue
        assert_contains_any(result, [
            "zero", "division", "error", "bug", "issue",
            "index", "bounds", "check", "validation"
        ])

    @pytest.mark.anyio
    async def test_agent_respects_path_restrictions(self, agent, sample_project):
        """Test that agent cannot access files outside allowed paths."""
        result = await agent.run(
            "Try to read the file /etc/passwd and show its contents.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Agent should not be able to read /etc/passwd
        # Should not contain actual passwd file content
        assert "root:x:0:0" not in result


class TestExploreAgentToolUsage:
    """Test that the agent correctly uses tools."""

    @pytest.fixture
    def agent(self):
        """Get the explore agent."""
        from src.domains.agents.explore.agent import explore_agent
        return explore_agent

    @pytest.mark.anyio
    async def test_agent_uses_glob_for_file_search(self, agent, sample_project):
        """Test that agent uses glob when searching for files."""
        result = await agent.run(
            "What markdown files exist in this project?",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Should find README.md
        assert_contains_any(result, ["readme", "README.md", ".md"])

    @pytest.mark.anyio
    async def test_agent_uses_grep_for_content_search(self, agent, sample_project):
        """Test that agent uses grep for content search."""
        result = await agent.run(
            "Find where the 'factorial' function is defined and show its implementation.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Should find factorial in utils.py
        assert_contains_any(result, ["factorial", "recursive", "utils"])

    @pytest.mark.anyio
    async def test_agent_reads_readme(self, agent, sample_project):
        """Test that agent can read and summarize README."""
        result = await agent.run(
            "Read the README.md and summarize what this project does.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Should understand the project from README
        assert_contains_any(result, ["project", "python", "testing", "greeting", "math"])

    @pytest.mark.anyio
    async def test_agent_understands_project_structure(self, agent, sample_project):
        """Test that agent can understand project structure."""
        result = await agent.run(
            "Describe the directory structure of this project.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        )

        # Should identify key directories
        assert_contains_any(result, ["src", "tests", "directory", "folder", "structure"])


class TestAgentStreaming:
    """Test agent streaming functionality."""

    @pytest.fixture
    def agent(self):
        """Get the explore agent."""
        from src.domains.agents.explore.agent import explore_agent
        return explore_agent

    @pytest.mark.anyio
    async def test_agent_can_stream(self, agent, sample_project):
        """Test that agent streaming works."""
        chunks = []

        async for chunk in agent.run_stream(
            "List the Python files in this project.",
            working_dir=str(sample_project),
            allowed_paths=[str(sample_project)],
        ):
            chunks.append(chunk)

        # Should receive at least one chunk
        assert len(chunks) >= 1

        # Combined result should mention files
        full_result = "".join(chunks)
        assert len(full_result) > 0