feat: refactor volatile cache to vector storage with HybridRAG integration

- Migrate volatile backend from Redis to Qdrant for semantic search
- Add natural language conversion for structured data embedding
- Simplify API: /volatile/search, /volatile/store, /{namespace}/{key}
- Integrate volatile into HybridRAG with priority boost in RRF fusion
- Add POST /maintenance/cleanup/volatile for expiry purging
- Update tests for new Qdrant-based architecture (37/37 pass)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

tests/test_volatile.py

@@ -1,11 +1,13 @@
 """
-Tests for volatile cache router and service.
+Tests for volatile cache router and service (Qdrant backend).
 
 Tests:
 - Volatile record CRUD operations
 - Namespace listing and management
 - Scheduled record retrieval
-- TTL behavior
+- TTL behavior and expiry filtering
+- Semantic search
+- Natural language conversion
 """
 
 import pytest
@@ -106,6 +108,10 @@ class TestVolatileNamespaces:
         """Test sports namespace default TTL (fast updates)."""
         assert NAMESPACE_DEFAULT_TTL[VolatileNamespace.SPORTS] == 60  # 1 min
 
+    def test_namespace_count(self):
+        """Test we have the expected number of namespaces."""
+        assert len(VolatileNamespace) == 11
+
 
 class TestVolatileListResponse:
     """Test list response models."""
@@ -210,18 +216,28 @@ class TestVolatileDeleteResponses:
 
 
 class TestVolatileService:
-    """Test VolatileCacheService functionality."""
+    """Test VolatileCacheService functionality (Qdrant backend)."""
 
     @pytest.fixture
-    def mock_redis(self):
-        """Create mock Redis client."""
-        redis = AsyncMock()
-        redis.get = AsyncMock(return_value=None)
-        redis.setex = AsyncMock()
-        redis.delete = AsyncMock(return_value=1)
-        redis.ttl = AsyncMock(return_value=1500)
-        redis.scan_iter = MagicMock(return_value=iter([]))
-        return redis
+    def mock_qdrant(self):
+        """Create mock Qdrant client."""
+        qdrant = AsyncMock()
+        qdrant.ensure_collection = AsyncMock()
+        qdrant.collection_exists = AsyncMock(return_value=True)
+        qdrant.upsert_vector = AsyncMock(return_value=True)
+        qdrant.delete_by_ids = AsyncMock(return_value=1)
+        qdrant.search_with_expiry_filter = AsyncMock(return_value=[])
+        qdrant.scroll_all_points = AsyncMock(return_value=[])
+        qdrant.delete_expired_vectors = AsyncMock(return_value=0)
+        qdrant.get_volatile_collections = AsyncMock(return_value=[])
+        return qdrant
+
+    @pytest.fixture
+    def mock_ollama(self):
+        """Create mock Ollama client."""
+        ollama = AsyncMock()
+        ollama.embed = AsyncMock(return_value=[0.1] * 768)  # Return 768-dim embedding
+        return ollama
 
     @pytest.fixture
     def mock_settings(self):
@@ -231,29 +247,29 @@ class TestVolatileService:
         return settings
 
     @pytest.fixture
-    def volatile_service(self, mock_redis, mock_settings):
+    def volatile_service(self, mock_qdrant, mock_ollama, mock_settings):
         """Create VolatileCacheService with mocks."""
         from src.services.volatile_service import VolatileCacheService
         return VolatileCacheService(
-            redis_client=mock_redis,
+            qdrant_client=mock_qdrant,
+            ollama_client=mock_ollama,
             settings=mock_settings
         )
 
-    def test_build_key(self, volatile_service):
-        """Test Redis key building."""
-        key = volatile_service._build_key("jpmschweitzer", "weather", "rotterdam")
-        assert key.startswith("jpmschweitzer:volatile:weather:")
-        assert len(key) > 30  # Has hash suffix
+    def test_collection_name(self, volatile_service):
+        """Test collection naming pattern."""
+        name = volatile_service._collection_name("jpmschweitzer")
+        assert name == "volatile_jpmschweitzer"
 
-    def test_build_pattern(self, volatile_service):
-        """Test pattern building."""
-        pattern = volatile_service._build_pattern("jpmschweitzer", "weather")
-        assert pattern == "jpmschweitzer:volatile:weather:*"
+    def test_make_vector_id(self, volatile_service):
+        """Test deterministic vector ID generation."""
+        id1 = volatile_service._make_vector_id("weather", "rotterdam")
+        id2 = volatile_service._make_vector_id("weather", "rotterdam")
+        id3 = volatile_service._make_vector_id("weather", "amsterdam")
 
-    def test_build_pattern_all(self, volatile_service):
-        """Test pattern building for all namespaces."""
-        pattern = volatile_service._build_pattern("jpmschweitzer")
-        assert pattern == "jpmschweitzer:volatile:*"
+        assert id1 == id2  # Same namespace+key = same ID
+        assert id1 != id3  # Different key = different ID
+        assert len(id1) == 32  # MD5 hex length
 
     def test_get_default_ttl_known_namespace(self, volatile_service):
         """Test default TTL for known namespace."""
@@ -265,23 +281,287 @@ class TestVolatileService:
         ttl = volatile_service._get_default_ttl("unknown_namespace")
         assert ttl == 3600  # Falls back to settings default
 
-    @pytest.mark.asyncio
-    async def test_get_not_found(self, volatile_service, mock_redis):
-        """Test get when record not found."""
-        mock_redis.get.return_value = None
-        result = await volatile_service.get("jpmschweitzer", "weather", "rotterdam")
-        assert result is None
+    def test_to_natural_language_weather(self, volatile_service):
+        """Test natural language conversion for weather data."""
+        text = volatile_service._to_natural_language(
+            namespace="weather",
+            key="rotterdam",
+            data={"temperature": 18, "conditions": "Cloudy", "humidity": 75}
+        )
+        assert "rotterdam" in text.lower()
+        assert "18" in text
+        assert "Cloudy" in text
+        assert "75" in text
+
+    def test_to_natural_language_news(self, volatile_service):
+        """Test natural language conversion for news data."""
+        text = volatile_service._to_natural_language(
+            namespace="news",
+            key="nos-headlines",
+            data={"title": "Breaking News", "summary": "Something happened", "source": "NOS"}
+        )
+        assert "Breaking News" in text
+        assert "Something happened" in text
+        assert "NOS" in text
+
+    def test_to_natural_language_financial(self, volatile_service):
+        """Test natural language conversion for financial data."""
+        text = volatile_service._to_natural_language(
+            namespace="financial",
+            key="AAPL",
+            data={"symbol": "AAPL", "price": 150.50, "change": 2.3}
+        )
+        assert "AAPL" in text
+        assert "price" in text.lower()
+        assert "change" in text.lower()
+
+    def test_to_natural_language_transit(self, volatile_service):
+        """Test natural language conversion for transit data."""
+        text = volatile_service._to_natural_language(
+            namespace="transit",
+            key="ns-intercity",
+            data={"route": "Amsterdam-Rotterdam", "status": "On time", "delay": 0}
+        )
+        assert "Amsterdam-Rotterdam" in text or "ns-intercity" in text.lower()
+        assert "On time" in text
+
+    def test_to_natural_language_fallback(self, volatile_service):
+        """Test natural language fallback for unknown namespace."""
+        text = volatile_service._to_natural_language(
+            namespace="custom",
+            key="test-key",
+            data={"foo": "bar", "count": 42}
+        )
+        assert "custom" in text.lower()
+        assert "foo" in text or "bar" in text
 
     @pytest.mark.asyncio
-    async def test_delete_success(self, volatile_service, mock_redis):
+    async def test_store_success(self, volatile_service, mock_qdrant, mock_ollama):
+        """Test successful store operation."""
+        result = await volatile_service.store(
+            user="jpmschweitzer",
+            namespace="weather",
+            key="rotterdam",
+            data={"temperature": 18, "conditions": "Sunny"},
+            source="openweathermap",
+            ttl=1800
+        )
+
+        assert result.key == "rotterdam"
+        assert result.namespace == "weather"
+        assert result.ttl == 1800
+        mock_qdrant.ensure_collection.assert_called_once()
+        mock_ollama.embed.assert_called_once()
+        mock_qdrant.upsert_vector.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_store_uses_namespace_default_ttl(self, volatile_service, mock_qdrant, mock_ollama):
+        """Test store uses namespace default TTL when not specified."""
+        result = await volatile_service.store(
+            user="jpmschweitzer",
+            namespace="weather",
+            key="amsterdam",
+            data={"temperature": 16},
+            source="openweathermap",
+            ttl=None  # Not specified
+        )
+
+        assert result.ttl == 1800  # Weather default
+
+    @pytest.mark.asyncio
+    async def test_search_empty_collection(self, volatile_service, mock_qdrant, mock_ollama):
+        """Test search when collection doesn't exist."""
+        mock_qdrant.collection_exists.return_value = False
+
+        results = await volatile_service.search(
+            user="jpmschweitzer",
+            query="weather rotterdam"
+        )
+
+        assert results == []
+        mock_ollama.embed.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_search_with_results(self, volatile_service, mock_qdrant, mock_ollama):
+        """Test search returns results."""
+        import time
+        now_ms = int(time.time() * 1000)
+
+        mock_qdrant.search_with_expiry_filter.return_value = [
+            {
+                "score": 0.95,
+                "payload": {
+                    "key": "rotterdam",
+                    "namespace": "weather",
+                    "raw_data": {"temperature": 18},
+                    "source": "openweathermap",
+                    "created_at": datetime.utcnow().isoformat(),
+                    "updated_at": datetime.utcnow().isoformat(),
+                    "ttl": 1800,
+                    "ttl_expiry": now_ms + 900000,  # 15 min remaining
+                    "refresh_schedule": None,
+                    "user": "jpmschweitzer"
+                }
+            }
+        ]
+
+        results = await volatile_service.search(
+            user="jpmschweitzer",
+            query="weather rotterdam"
+        )
+
+        assert len(results) == 1
+        assert results[0].key == "rotterdam"
+        assert results[0].namespace == "weather"
+
+    @pytest.mark.asyncio
+    async def test_delete_success(self, volatile_service, mock_qdrant):
         """Test successful delete."""
-        mock_redis.delete.return_value = 1
+        mock_qdrant.delete_by_ids.return_value = 1
+
         result = await volatile_service.delete("jpmschweitzer", "weather", "rotterdam")
+
         assert result is True
+        mock_qdrant.delete_by_ids.assert_called_once()
 
     @pytest.mark.asyncio
-    async def test_delete_not_found(self, volatile_service, mock_redis):
+    async def test_delete_not_found(self, volatile_service, mock_qdrant):
         """Test delete when record not found."""
-        mock_redis.delete.return_value = 0
+        mock_qdrant.delete_by_ids.return_value = 0
+
         result = await volatile_service.delete("jpmschweitzer", "weather", "nonexistent")
+
         assert result is False
+
+    @pytest.mark.asyncio
+    async def test_get_stats_empty(self, volatile_service, mock_qdrant):
+        """Test stats with no records."""
+        mock_qdrant.collection_exists.return_value = False
+
+        stats = await volatile_service.get_stats("jpmschweitzer")
+
+        assert stats["total_records"] == 0
+        assert stats["by_namespace"] == {}
+        assert stats["scheduled_count"] == 0
+
+    @pytest.mark.asyncio
+    async def test_get_stats_with_records(self, volatile_service, mock_qdrant):
+        """Test stats with records."""
+        import time
+        now_ms = int(time.time() * 1000)
+
+        mock_qdrant.scroll_all_points.return_value = [
+            {"payload": {"namespace": "weather", "ttl_expiry": now_ms + 100000}},
+            {"payload": {"namespace": "weather", "ttl_expiry": now_ms + 100000, "refresh_schedule": "0 * * * *"}},
+            {"payload": {"namespace": "news", "ttl_expiry": now_ms + 100000}},
+            {"payload": {"namespace": "weather", "ttl_expiry": now_ms - 100000}},  # Expired
+        ]
+
+        stats = await volatile_service.get_stats("jpmschweitzer")
+
+        assert stats["total_records"] == 3  # Excludes expired
+        assert stats["by_namespace"]["weather"] == 2
+        assert stats["by_namespace"]["news"] == 1
+        assert stats["scheduled_count"] == 1
+        assert stats["expired_count"] == 1
+
+    @pytest.mark.asyncio
+    async def test_purge_expired(self, volatile_service, mock_qdrant):
+        """Test purging expired records."""
+        mock_qdrant.delete_expired_vectors.return_value = 5
+
+        result = await volatile_service.purge_expired("jpmschweitzer")
+
+        assert result == 5
+        mock_qdrant.delete_expired_vectors.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_purge_all_expired(self, volatile_service, mock_qdrant):
+        """Test purging expired from all collections."""
+        mock_qdrant.get_volatile_collections.return_value = [
+            "volatile_user1",
+            "volatile_user2"
+        ]
+        mock_qdrant.delete_expired_vectors.side_effect = [3, 2]
+
+        results = await volatile_service.purge_all_expired()
+
+        assert results["volatile_user1"] == 3
+        assert results["volatile_user2"] == 2
+
+    @pytest.mark.asyncio
+    async def test_get_scheduled(self, volatile_service, mock_qdrant):
+        """Test getting scheduled records."""
+        import time
+        now_ms = int(time.time() * 1000)
+
+        mock_qdrant.scroll_all_points.return_value = [
+            {
+                "payload": {
+                    "key": "nos-headlines",
+                    "namespace": "news",
+                    "raw_data": {"headlines": []},
+                    "source": "nos.nl",
+                    "created_at": datetime.utcnow().isoformat(),
+                    "updated_at": datetime.utcnow().isoformat(),
+                    "ttl": 3600,
+                    "ttl_expiry": now_ms + 1800000,
+                    "refresh_schedule": "0 */6 * * *",
+                    "user": "jpmschweitzer"
+                }
+            },
+            {
+                "payload": {
+                    "key": "rotterdam",
+                    "namespace": "weather",
+                    "raw_data": {"temperature": 18},
+                    "source": "openweathermap",
+                    "created_at": datetime.utcnow().isoformat(),
+                    "updated_at": datetime.utcnow().isoformat(),
+                    "ttl": 1800,
+                    "ttl_expiry": now_ms + 900000,
+                    "refresh_schedule": None,  # Not scheduled
+                    "user": "jpmschweitzer"
+                }
+            }
+        ]
+
+        scheduled = await volatile_service.get_scheduled("jpmschweitzer")
+
+        assert len(scheduled) == 1
+        assert scheduled[0].key == "nos-headlines"
+        assert scheduled[0].refresh_schedule == "0 */6 * * *"
+
+
+class TestVolatileCleanupEndpoint:
+    """Test volatile cleanup in maintenance router."""
+
+    @pytest.mark.asyncio
+    async def test_cleanup_volatile(self):
+        """Test volatile cleanup endpoint."""
+        from src.routers.maintenance import cleanup_volatile, VolatileCleanupResponse
+
+        mock_qdrant = AsyncMock()
+        mock_qdrant.get_volatile_collections = AsyncMock(return_value=[
+            "volatile_user1",
+            "volatile_user2"
+        ])
+        mock_qdrant.delete_expired_vectors = AsyncMock(side_effect=[3, 2])
+
+        mock_ollama = AsyncMock()
+
+        mock_settings = MagicMock()
+        mock_settings.volatile_default_ttl = 3600
+
+        with patch('src.routers.maintenance.get_settings', return_value=mock_settings):
+            result = await cleanup_volatile(
+                qdrant=mock_qdrant,
+                ollama=mock_ollama,
+                api_key="test"
+            )
+
+        assert result.success is True
+        assert result.collections_processed == 2
+        assert result.total_expired_purged == 5
+        assert result.by_collection["volatile_user1"] == 3
+        assert result.by_collection["volatile_user2"] == 2