portainer-core/services/library-desk/tests/test_integration.py

"""Integration tests for Library Desk service clients.

These tests require actual service connectivity:
- Neo4j running at bolt://neo4j:7687
- Qdrant running at http://qdrant:6333
- Wiki.js running at http://wiki:3000
- SearXNG running at http://searxng:8080
- Ollama running at http://ollama:11434

Run with: pytest tests/test_integration.py -v
"""

import pytest
import pytest_asyncio
from typing import AsyncGenerator

from src.clients.neo4j_client import Neo4jClient
from src.clients.qdrant_client import QdrantClientWrapper
from src.clients.wikijs_client import WikiJSClient
from src.clients.searxng_client import SearXNGClient
from src.clients.ollama_client import OllamaClient
from src.jobs.job_manager import JobManager, JobType, JobStatus
from src.config import get_settings


@pytest.fixture
def settings():
    """Get application settings."""
    return get_settings()


@pytest_asyncio.fixture
async def neo4j_client(settings) -> AsyncGenerator[Neo4jClient, None]:
    """Get connected Neo4j client."""
    client = Neo4jClient(
        uri=settings.neo4j_uri,
        user=settings.neo4j_user,
        password=settings.neo4j_password
    )
    await client.connect()
    yield client
    await client.close()


@pytest.fixture
def qdrant_client(settings) -> QdrantClientWrapper:
    """Get Qdrant client."""
    return QdrantClientWrapper(url=settings.qdrant_url)


@pytest_asyncio.fixture
async def wikijs_client(settings) -> AsyncGenerator[WikiJSClient, None]:
    """Get Wiki.js client."""
    client = WikiJSClient(
        base_url=settings.wikijs_url,
        api_key=settings.wikijs_api_key
    )
    yield client
    await client.close()


@pytest_asyncio.fixture
async def searxng_client(settings) -> AsyncGenerator[SearXNGClient, None]:
    """Get SearXNG client."""
    client = SearXNGClient(base_url=settings.searxng_url)
    yield client
    await client.close()


@pytest_asyncio.fixture
async def ollama_client(settings) -> AsyncGenerator[OllamaClient, None]:
    """Get Ollama client."""
    client = OllamaClient(
        base_url=settings.ollama_url,
        model=settings.ollama_model
    )
    yield client
    await client.close()


@pytest_asyncio.fixture
async def job_manager(settings) -> AsyncGenerator[JobManager, None]:
    """Get job manager."""
    manager = JobManager(redis_url=settings.redis_url)
    await manager.connect()
    yield manager
    await manager.close()


class TestNeo4jIntegration:
    """Test Neo4j connectivity and basic operations."""

    @pytest.mark.asyncio
    async def test_connection(self, neo4j_client):
        """Test Neo4j connection."""
        result = await neo4j_client.execute_query("RETURN 1 as test", {})
        assert result
        assert result[0]["test"] == 1

    @pytest.mark.asyncio
    async def test_create_and_get_document(self, neo4j_client, test_user):
        """Test creating and retrieving a document node."""
        doc_id = "test_doc_integration"

        # Create document
        doc = await neo4j_client.create_document_node(
            user=test_user,
            doc_id=doc_id,
            properties={
                "title": "Integration Test Document",
                "source": "test"
            }
        )
        assert doc is not None
        assert doc["id"] == doc_id

        # Retrieve document
        retrieved = await neo4j_client.get_document_node(test_user, doc_id)
        assert retrieved is not None
        assert retrieved["id"] == doc_id
        assert retrieved["title"] == "Integration Test Document"

        # Cleanup
        deleted = await neo4j_client.delete_document_node(test_user, doc_id)
        assert deleted is True


class TestQdrantIntegration:
    """Test Qdrant connectivity and basic operations."""

    @pytest.mark.asyncio
    async def test_collection_creation(self, qdrant_client, test_user):
        """Test creating a collection."""
        await qdrant_client.ensure_collection(test_user)

        collection_name = qdrant_client.get_collection_name(test_user)
        collections = qdrant_client.client.get_collections()
        collection_names = [c.name for c in collections.collections]

        assert collection_name in collection_names

    @pytest.mark.asyncio
    async def test_upsert_and_search(self, qdrant_client, test_user):
        """Test upserting and searching chunks."""
        await qdrant_client.ensure_collection(test_user)

        # Create test chunks with 768-dimensional embeddings
        chunks = [
            {"content": "Test chunk 1", "metadata": {}},
            {"content": "Test chunk 2", "metadata": {}}
        ]
        embeddings = [
            [0.1] * 768,
            [0.2] * 768
        ]

        # Upsert
        count = await qdrant_client.upsert_document_chunks(
            user=test_user,
            doc_id="test_doc_qdrant",
            chunks=chunks,
            embeddings=embeddings
        )
        assert count == 2

        # Search
        results = await qdrant_client.search(
            user=test_user,
            query_vector=[0.1] * 768,
            limit=5,
            score_threshold=0.0
        )
        assert len(results) > 0

        # Cleanup
        deleted = await qdrant_client.delete_document(test_user, "test_doc_qdrant")
        assert deleted is True


class TestWikiJSIntegration:
    """Test Wiki.js connectivity and basic operations."""

    @pytest.mark.asyncio
    async def test_list_pages(self, wikijs_client):
        """Test listing pages."""
        pages = await wikijs_client.list_pages(limit=10)
        assert isinstance(pages, list)
        # May be empty if wiki is new

    @pytest.mark.asyncio
    async def test_search_pages(self, wikijs_client):
        """Test searching pages."""
        results = await wikijs_client.search_pages("test")
        assert isinstance(results, list)


class TestSearXNGIntegration:
    """Test SearXNG connectivity and search."""

    @pytest.mark.asyncio
    async def test_general_search(self, searxng_client):
        """Test general web search."""
        results = await searxng_client.search_general("python programming", limit=5)
        assert isinstance(results, list)
        if results:
            assert "title" in results[0]
            assert "url" in results[0]

    @pytest.mark.asyncio
    async def test_search_with_suggestions(self, searxng_client):
        """Test getting search suggestions."""
        suggestions = await searxng_client.get_suggestions("pytho")
        assert isinstance(suggestions, list)


class TestOllamaIntegration:
    """Test Ollama connectivity and embeddings."""

    @pytest.mark.asyncio
    async def test_health_check(self, ollama_client):
        """Test Ollama health check."""
        is_healthy = await ollama_client.health_check()
        # May be False if model not pulled
        assert isinstance(is_healthy, bool)

    @pytest.mark.asyncio
    async def test_list_models(self, ollama_client):
        """Test listing available models."""
        models = await ollama_client.list_models()
        assert isinstance(models, list)

    @pytest.mark.asyncio
    @pytest.mark.skipif(
        True,  # Skip by default as embeddings can be slow
        reason="Embedding generation is slow - enable manually if needed"
    )
    async def test_generate_embedding(self, ollama_client):
        """Test generating a single embedding."""
        embedding = await ollama_client.embed("test text")
        if embedding:
            assert isinstance(embedding, list)
            assert len(embedding) > 0


class TestJobManagerIntegration:
    """Test Job Manager with Redis."""

    @pytest.mark.asyncio
    async def test_create_and_get_job(self, job_manager, test_user):
        """Test creating and retrieving a job."""
        # Create job
        job_id = await job_manager.create_job(
            job_type=JobType.DOCUMENT_INGESTION,
            user=test_user,
            parameters={"doc_url": "https://example.com/doc.pdf"}
        )
        assert job_id

        # Get job
        job = await job_manager.get_job(job_id)
        assert job is not None
        assert job["job_id"] == job_id
        assert job["user"] == test_user
        assert job["status"] == JobStatus.QUEUED.value

        # Update job
        await job_manager.update_job_status(
            job_id,
            JobStatus.PROCESSING,
            progress=50
        )

        updated_job = await job_manager.get_job(job_id)
        assert updated_job["status"] == JobStatus.PROCESSING.value
        assert updated_job["progress"] == 50

        # Cleanup
        deleted = await job_manager.delete_job(job_id, test_user)
        assert deleted is True


class TestDependencyInjection:
    """Test dependency injection and lifecycle management."""

    @pytest.mark.asyncio
    async def test_startup_clients(self):
        """Test client startup."""
        from src.core.dependencies import startup_clients

        # Should not raise exceptions
        await startup_clients()

    @pytest.mark.asyncio
    async def test_check_service_health(self):
        """Test health check for all services."""
        from src.core.dependencies import check_service_health

        health = await check_service_health()

        assert isinstance(health, dict)
        assert "neo4j" in health
        assert "qdrant" in health
        assert "wikijs" in health
        assert "searxng" in health
        assert "ollama" in health

    @pytest.mark.asyncio
    async def test_shutdown_clients(self):
        """Test client shutdown."""
        from src.core.dependencies import shutdown_clients

        # Should not raise exceptions
        await shutdown_clients()