Initial commit: library-desk service extraction from portainer-core

src/services/vector_service.py

@@ -0,0 +1,358 @@
+"""
+Vector service for Library Desk Qdrant operations.
+
+Handles semantic search, document chunking, and embeddings.
+"""
+
+import re
+import time
+import hashlib
+import uuid
+from typing import List, Dict, Any, Optional
+import logging
+
+from src.clients.qdrant_client import QdrantClientWrapper
+from src.clients.wikijs_client import WikiJSClient
+from src.clients.ollama_client import OllamaClient
+from src.core.multi_tenancy import get_qdrant_collection_name
+from src.models.vector import (
+    SearchResult, SearchResponse, VectorUpdateSummary,
+    DocumentChunk, CollectionInfo, CollectionListResponse
+)
+
+logger = logging.getLogger(__name__)
+
+
+class VectorService:
+    """
+    Service for Qdrant vector operations.
+
+    Responsibilities:
+    - Document chunking
+    - Embedding generation
+    - Semantic search
+    - Vector CRUD operations
+    """
+
+    def __init__(
+        self,
+        qdrant_client: QdrantClientWrapper,
+        wikijs_client: WikiJSClient,
+        ollama_client: OllamaClient,
+        chunk_size: int = 500,
+        chunk_overlap: int = 50
+    ):
+        """
+        Initialize vector service.
+
+        Args:
+            qdrant_client: Qdrant database client
+            wikijs_client: Wiki.js client for fetching pages
+            ollama_client: Ollama client for embeddings
+            chunk_size: Target chunk size in tokens (approximate)
+            chunk_overlap: Overlap between chunks in tokens
+        """
+        self.qdrant = qdrant_client
+        self.wiki = wikijs_client
+        self.ollama = ollama_client
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """
+        Chunk text into overlapping segments.
+
+        Simple word-based chunking for now.
+        TODO: Use tiktoken or similar for token-accurate chunking.
+
+        Args:
+            text: Text to chunk
+
+        Returns:
+            List of text chunks
+        """
+        # Remove extra whitespace
+        text = re.sub(r'\s+', ' ', text).strip()
+
+        # Split into words (approximates tokens)
+        words = text.split()
+
+        if len(words) <= self.chunk_size:
+            return [text]
+
+        chunks = []
+        start = 0
+
+        while start < len(words):
+            end = start + self.chunk_size
+            chunk_words = words[start:end]
+            chunks.append(' '.join(chunk_words))
+
+            # Move start forward with overlap
+            start = end - self.chunk_overlap
+
+        return chunks
+
+    async def update_from_page(
+        self,
+        page_id: int,
+        user: str,
+        force_refresh: bool = False
+    ) -> VectorUpdateSummary:
+        """
+        Update vector embeddings from a wiki page.
+
+        Chunks the page content, generates embeddings, and upserts to Qdrant.
+
+        Args:
+            page_id: Wiki page ID
+            user: User identifier
+            force_refresh: Force re-embedding even if unchanged
+
+        Returns:
+            Summary of update operation
+        """
+        start_time = time.time()
+
+        try:
+            # Fetch page from Wiki.js
+            page = await self.wiki.get_page(page_id)
+            if not page:
+                raise ValueError(f"Page {page_id} not found")
+
+            # Get collection name for user
+            collection_name = get_qdrant_collection_name(user)
+
+            # Ensure collection exists
+            await self.qdrant.ensure_collection(collection_name)
+
+            # Extract content
+            content = page.get("content", "")
+            title = page.get("title", "")
+            path = page.get("path", "")
+
+            if not content:
+                logger.warning(f"Page {page_id} has no content, skipping vector update")
+                return VectorUpdateSummary(
+                    page_id=page_id,
+                    page_title=title,
+                    processing_time_ms=(time.time() - start_time) * 1000,
+                    success=True
+                )
+
+            # Chunk the content
+            chunks = self._chunk_text(content)
+            logger.info(f"Split page {page_id} into {len(chunks)} chunks")
+
+            # Delete existing chunks for this page
+            deleted_count = await self.qdrant.delete_by_filter(
+                collection_name=collection_name,
+                filter_conditions={"page_id": page_id}
+            )
+
+            # Generate embeddings and upsert chunks
+            chunks_created = 0
+            for idx, chunk_text in enumerate(chunks):
+                # Generate deterministic UUID from page_id and chunk_index
+                chunk_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"page_{page_id}_chunk_{idx}"))
+
+                # Generate embedding
+                embedding = await self.ollama.embed(chunk_text)
+                if not embedding:
+                    logger.error(f"Failed to generate embedding for chunk {chunk_id}")
+                    continue
+
+                # Prepare metadata
+                metadata = {
+                    "page_id": page_id,
+                    "page_title": title,
+                    "page_path": path,
+                    "chunk_index": idx,
+                    "chunk_text": chunk_text,
+                    "user": user
+                }
+
+                # Upsert to Qdrant
+                success = await self.qdrant.upsert_vector(
+                    collection_name=collection_name,
+                    vector_id=chunk_id,
+                    vector=embedding,
+                    payload=metadata
+                )
+
+                if success:
+                    chunks_created += 1
+
+            processing_time_ms = (time.time() - start_time) * 1000
+
+            logger.info(
+                f"Updated vectors for page {page_id}: "
+                f"{chunks_created} chunks created, {deleted_count} old chunks deleted"
+            )
+
+            return VectorUpdateSummary(
+                page_id=page_id,
+                page_title=title,
+                chunks_created=chunks_created,
+                chunks_deleted=deleted_count,
+                total_chunks=chunks_created,
+                embedding_dim=len(embedding) if embedding else 768,
+                processing_time_ms=processing_time_ms,
+                success=True
+            )
+
+        except Exception as e:
+            processing_time_ms = (time.time() - start_time) * 1000
+            logger.error(f"Failed to update vectors for page {page_id}: {e}", exc_info=True)
+
+            return VectorUpdateSummary(
+                page_id=page_id,
+                page_title="Unknown",
+                processing_time_ms=processing_time_ms,
+                success=False,
+                error_message=str(e)
+            )
+
+    async def search(
+        self,
+        query: str,
+        user: str,
+        limit: int = 10,
+        score_threshold: float = 0.5
+    ) -> SearchResponse:
+        """
+        Perform semantic search across user's documents.
+
+        Args:
+            query: Search query text
+            user: User identifier
+            limit: Maximum results to return
+            score_threshold: Minimum similarity score (0-1)
+
+        Returns:
+            Search results with similarity scores
+        """
+        start_time = time.time()
+
+        try:
+            # Get collection name
+            collection_name = get_qdrant_collection_name(user)
+
+            # Check if collection exists
+            exists = await self.qdrant.collection_exists(collection_name)
+            if not exists:
+                logger.info(f"Collection {collection_name} doesn't exist, returning empty results")
+                return SearchResponse(
+                    query=query,
+                    results=[],
+                    total=0,
+                    user=user
+                )
+
+            # Generate query embedding
+            query_embedding = await self.ollama.embed(query)
+            if not query_embedding:
+                raise ValueError("Failed to generate query embedding")
+
+            # Search in Qdrant
+            search_results = await self.qdrant.search_vectors(
+                collection_name=collection_name,
+                query_vector=query_embedding,
+                limit=limit,
+                score_threshold=score_threshold
+            )
+
+            # Convert to SearchResult models
+            results = []
+            for result in search_results:
+                payload = result.get("payload", {})
+                results.append(SearchResult(
+                    chunk_id=result["id"],
+                    page_id=payload.get("page_id", 0),
+                    page_title=payload.get("page_title"),
+                    page_path=payload.get("page_path"),
+                    chunk_index=payload.get("chunk_index", 0),
+                    content=payload.get("chunk_text", ""),
+                    score=result["score"],
+                    metadata=payload
+                ))
+
+            query_time_ms = (time.time() - start_time) * 1000
+            logger.info(f"Semantic search completed in {query_time_ms:.2f}ms: {len(results)} results")
+
+            return SearchResponse(
+                query=query,
+                results=results,
+                total=len(results),
+                user=user
+            )
+
+        except Exception as e:
+            logger.error(f"Semantic search failed: {e}", exc_info=True)
+            return SearchResponse(
+                query=query,
+                results=[],
+                total=0,
+                user=user
+            )
+
+    async def delete_page_chunks(
+        self,
+        page_id: int,
+        user: str
+    ) -> int:
+        """
+        Delete all chunks for a wiki page.
+
+        Args:
+            page_id: Wiki page ID
+            user: User identifier
+
+        Returns:
+            Number of chunks deleted
+        """
+        collection_name = get_qdrant_collection_name(user)
+
+        try:
+            deleted_count = await self.qdrant.delete_by_filter(
+                collection_name=collection_name,
+                filter_conditions={"page_id": page_id}
+            )
+
+            logger.info(f"Deleted {deleted_count} chunks for page {page_id}")
+            return deleted_count
+
+        except Exception as e:
+            logger.error(f"Failed to delete chunks for page {page_id}: {e}", exc_info=True)
+            return 0
+
+    async def list_collections(self) -> CollectionListResponse:
+        """
+        List all Qdrant collections.
+
+        Returns:
+            List of collections with stats
+        """
+        try:
+            collections_data = await self.qdrant.list_collections()
+
+            collections = []
+            for coll in collections_data:
+                collections.append(CollectionInfo(
+                    name=coll["name"],
+                    vectors_count=coll.get("vectors_count", 0),
+                    points_count=coll.get("points_count", 0),
+                    segments_count=coll.get("segments_count", 0)
+                ))
+
+            return CollectionListResponse(
+                collections=collections,
+                total=len(collections)
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to list collections: {e}", exc_info=True)
+            return CollectionListResponse(
+                collections=[],
+                total=0
+            )