feat: refactor volatile cache to vector storage with HybridRAG integration

- Migrate volatile backend from Redis to Qdrant for semantic search - Add natural language conversion for structured data embedding - Simplify API: /volatile/search, /volatile/store, /{namespace}/{key} - Integrate volatile into HybridRAG with priority boost in RRF fusion - Add POST /maintenance/cleanup/volatile for expiry purging - Update tests for new Qdrant-based architecture (37/37 pass) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-24 20:03:37 +01:00
parent 1f848c4878
commit 37f8e1819e
10 changed files with 1098 additions and 391 deletions
@@ -2,6 +2,7 @@
 Volatile cache router for Library Desk API.

 Endpoints for ephemeral cached data with TTL - weather, news, financial, etc.
+Data is stored as vectors in Qdrant for semantic search retrieval.
 """

 from fastapi import APIRouter, HTTPException, Depends, Query
@@ -14,11 +15,11 @@ from src.models.volatile import (
    VolatileScheduledResponse,
    VolatileStatsResponse,
    VolatileDeleteResponse,
-    VolatileBulkDeleteResponse,
    VolatileNamespace,
+    NAMESPACE_DEFAULT_TTL,
 )
 from src.services.volatile_service import VolatileCacheService
-from src.core.dependencies import verify_api_key, RedisDep
+from src.core.dependencies import verify_api_key, QdrantDep, OllamaDep
 from src.core.multi_tenancy import DEFAULT_USER
 from src.config import get_settings

@@ -27,16 +28,21 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/volatile", tags=["Volatile Cache"])


-def get_volatile_service(redis: RedisDep) -> VolatileCacheService:
+def get_volatile_service(qdrant: QdrantDep, ollama: OllamaDep) -> VolatileCacheService:
    """Get volatile cache service instance."""
    settings = get_settings()
-    return VolatileCacheService(redis_client=redis, settings=settings)
+    return VolatileCacheService(
+        qdrant_client=qdrant,
+        ollama_client=ollama,
+        settings=settings
+    )


@router.get("/stats", response_model=VolatileStatsResponse)
 async def get_stats(
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
@@ -44,14 +50,14 @@ async def get_stats(

    Returns counts of records by namespace and scheduled refresh info.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    stats = await service.get_stats(user)

    return VolatileStatsResponse(
        total_records=stats["total_records"],
        by_namespace=stats["by_namespace"],
        scheduled_count=stats["scheduled_count"],
-        total_memory_bytes=stats.get("total_memory_bytes"),
+        total_memory_bytes=None,
        user=user,
    )

@@ -59,7 +65,8 @@ async def get_stats(
@router.get("/scheduled", response_model=VolatileScheduledResponse)
 async def get_scheduled(
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
@@ -68,7 +75,7 @@ async def get_scheduled(
    Used by scheduler to determine what volatile data needs refreshing.
    Returns all records that have a refresh_schedule cron expression set.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    records = await service.get_scheduled(user)

    return VolatileScheduledResponse(
@@ -86,10 +93,7 @@ async def list_namespaces(
    List available namespaces and their default TTLs.

    Returns predefined namespaces with their default TTL values.
-    Custom namespaces can also be used with the default TTL.
    """
-    from src.models.volatile import NAMESPACE_DEFAULT_TTL
-
    return {
        "namespaces": [
            {
@@ -120,103 +124,62 @@ def _get_namespace_description(ns: VolatileNamespace) -> str:
    return descriptions.get(ns, "Custom namespace")


-@router.get("/{namespace}", response_model=VolatileListResponse)
-async def list_keys(
-    namespace: str,
+@router.get("/search")
+async def search_volatile(
+    q: str = Query(..., min_length=1, description="Search query"),
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    limit: int = Query(default=5, ge=1, le=20, description="Maximum results"),
+    threshold: float = Query(default=0.75, ge=0.5, le=1.0, description="Minimum similarity score"),
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    List all keys in a namespace.
+    Semantic search across volatile data.

-    Returns the list of keys stored in the specified namespace.
-    """
-    service = get_volatile_service(redis)
-    keys = await service.list_namespace(user, namespace)
-
-    return VolatileListResponse(
-        namespace=namespace,
-        keys=keys,
-        count=len(keys),
-        user=user,
-    )
-
-
-@router.delete("/{namespace}", response_model=VolatileBulkDeleteResponse)
-async def delete_namespace(
-    namespace: str,
-    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
-    api_key: str = Depends(verify_api_key)
-):
-    """
-    Delete all records in a namespace.
-
-    Removes all volatile data for the specified namespace.
-    """
-    service = get_volatile_service(redis)
-    deleted = await service.delete_namespace(user, namespace)
-
-    return VolatileBulkDeleteResponse(
-        namespace=namespace,
-        deleted_count=deleted,
-        user=user,
-    )
-
-
-@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
-async def get_record(
-    namespace: str,
-    key: str,
-    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
-    api_key: str = Depends(verify_api_key)
-):
-    """
-    Get a volatile record.
-
-    Returns the record if it exists and has not expired.
+    Searches all volatile data for semantically similar content.
+    Higher threshold = stricter matching.

    **Example:**
    ```
-    GET /volatile/weather/rotterdam?user=jpmschweitzer
+    GET /volatile/search?q=weather%20rotterdam&user=jpmschweitzer
    ```
    """
-    service = get_volatile_service(redis)
-    record = await service.get(user, namespace, key)
+    service = get_volatile_service(qdrant, ollama)
+    results = await service.search(user, q, limit=limit, score_threshold=threshold)

-    if not record:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Record '{key}' not found in namespace '{namespace}'"
-        )
-
-    return record
+    return {
+        "query": q,
+        "results": results,
+        "count": len(results),
+        "user": user,
+    }


-@router.post("/{namespace}/{key}", response_model=VolatileRecordResponse)
-async def set_record(
-    namespace: str,
-    key: str,
-    request: VolatileRecordCreate,
+@router.post("/store", response_model=VolatileRecordResponse)
+async def store_volatile(
+    namespace: str = Query(..., description="Data namespace (weather, news, etc.)"),
+    key: str = Query(..., description="Record key (e.g., 'rotterdam', 'nos-headlines')"),
+    request: VolatileRecordCreate = None,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    Store or update a volatile record.
+    Store volatile data.

-    Creates or updates a record with the specified TTL.
-    If TTL is not provided, the namespace default is used.
+    Data is converted to natural language and embedded for semantic search.
+    If the same namespace+key already exists, it will be updated.

    **Example Request:**
    ```json
+    POST /volatile/store?namespace=weather&key=rotterdam
    {
        "data": {
-            "temperature": 18,
-            "conditions": "Partly cloudy",
-            "humidity": 65
+            "temperature": 8,
+            "conditions": "Cloudy",
+            "humidity": 85
        },
        "source": "openweathermap",
        "ttl": 1800,
@@ -228,10 +191,21 @@ async def set_record(
    Optional cron expression for automatic refresh. The scheduler
    will query `/volatile/scheduled` and trigger refreshes.
    """
-    service = get_volatile_service(redis)
+    # Validate namespace if not custom
+    if namespace != VolatileNamespace.CUSTOM:
+        try:
+            VolatileNamespace(namespace)
+        except ValueError:
+            valid = [ns.value for ns in VolatileNamespace]
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid namespace '{namespace}'. Valid: {valid}"
+            )
+
+    service = get_volatile_service(qdrant, ollama)

    try:
-        record = await service.set(
+        record = await service.store(
            user=user,
            namespace=namespace,
            key=key,
@@ -244,7 +218,36 @@ async def set_record(

    except Exception as e:
        logger.error(f"Failed to store volatile record: {e}")
-        raise HTTPException(status_code=500, detail="Failed to store record")
+        raise HTTPException(status_code=500, detail=f"Failed to store record: {str(e)}")
+
+
+@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
+async def get_record(
+    namespace: str,
+    key: str,
+    user: str = Query(default=DEFAULT_USER, description="User identifier"),
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Get a specific volatile record by namespace and key.
+
+    **Example:**
+    ```
+    GET /volatile/weather/rotterdam?user=jpmschweitzer
+    ```
+    """
+    service = get_volatile_service(qdrant, ollama)
+    record = await service.get(user, namespace, key)
+
+    if not record:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Record '{key}' not found in namespace '{namespace}'"
+        )
+
+    return record


@router.delete("/{namespace}/{key}", response_model=VolatileDeleteResponse)
@@ -252,15 +255,14 @@ async def delete_record(
    namespace: str,
    key: str,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    Delete a volatile record.
-
-    Removes the record from the cache.
+    Delete a specific volatile record.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    deleted = await service.delete(user, namespace, key)

    return VolatileDeleteResponse(