jpmschweitzerandClaude
a687b770ef
105 findings to zero. Most were mechanical — 67 unused imports, and assorted
f-strings without placeholders. Three groups needed a decision.
The 15 F821 "undefined name" were forward references, not runtime errors. Each
annotation is quoted — `-> "WikiService"`, `Optional["IngestionService"]` — with
the real import inside the function body to break an import cycle. A quoted
annotation is never evaluated, so the code ran; the names were simply
unresolvable to any checker. They now have a TYPE_CHECKING block, which costs
nothing at import time and keeps the cycle broken.
The 6 E402 split two ways. `import secrets`, `Security`, `Request` and
`HTTPBearer` in dependencies.py had drifted below several hundred lines of
factory functions for no reason — stdlib and fastapi, no cycle to avoid — and
moved up. The other three are deliberate and now say so: the VectorService and
GraphService aliases import back into dependencies.py, and main.py's routers
expect a configured app, so both must stay put.
Bare `except:` narrowed to `except Exception:` in three places, which stops them
swallowing KeyboardInterrupt and SystemExit.
The 5 unused locals were all genuinely dead. One is worth naming rather than
fixing: qdrant_client.delete()'s return value was bound and never read, so a
failed delete is indistinguishable from a successful one — the assignment is
gone, but nothing checks the status either way and that has not changed here.
`timing = {}` in _retrieve_parallel looked like it might mean the reported
per-leg timings were always zero; traced, and they come from output["timing"],
so the local was only vestigial.
426 passed, 29 skipped, unchanged. The app imports and the service aliases still
resolve, which is the check that mattered after moving imports in
dependencies.py.
The gate still prints "not gated here yet: test (T-56)" — lint is green, tests
remain unwired, and that is left visible rather than silently absent.
Co-Authored-By: Claude <noreply@anthropic.com>
856 lines
26 KiB
Python
856 lines
26 KiB
Python
"""
|
|
Qdrant client wrapper for Library Desk.
|
|
|
|
Provides async vector operations with:
|
|
- Collection-per-user multi-tenancy
|
|
- Document chunk storage with embeddings
|
|
- Semantic search
|
|
- Similarity queries
|
|
"""
|
|
|
|
from qdrant_client import AsyncQdrantClient
|
|
from qdrant_client.models import (
|
|
Distance, VectorParams, PointStruct,
|
|
Filter, FieldCondition, MatchValue, Range
|
|
)
|
|
from typing import List, Dict, Any, Optional
|
|
import uuid
|
|
import logging
|
|
|
|
from src.core.multi_tenancy import get_qdrant_collection_name
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class QdrantClientWrapper:
|
|
"""
|
|
Qdrant client wrapper with multi-tenancy support.
|
|
|
|
Pattern: Collection per user (from qdrant_memory.py)
|
|
Each user has isolated vector collection for their documents.
|
|
"""
|
|
|
|
def __init__(self, url: str, embedding_dim: int = 768, timeout: float = 30.0):
|
|
"""
|
|
Initialize Qdrant client.
|
|
|
|
Uses AsyncQdrantClient so vector calls never block the FastAPI
|
|
event loop, with an explicit timeout so a hung Qdrant cannot
|
|
stall requests indefinitely.
|
|
|
|
Args:
|
|
url: Qdrant server URL (e.g., "http://qdrant:6333")
|
|
embedding_dim: Vector embedding dimension (default 768 for nomic-embed-text)
|
|
timeout: Per-request timeout in seconds (default 30.0)
|
|
"""
|
|
self.client = AsyncQdrantClient(url=url, timeout=timeout)
|
|
self.embedding_dim = embedding_dim
|
|
logger.info(f"Initialized async Qdrant client: {url} (timeout={timeout}s)")
|
|
|
|
def get_collection_name(self, user: str) -> str:
|
|
"""
|
|
Get collection name for user.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Collection name (e.g., "library_desk_jpmschweitzer")
|
|
"""
|
|
return get_qdrant_collection_name(user)
|
|
|
|
async def ensure_collection(self, collection_name: str):
|
|
"""
|
|
Create collection if it doesn't exist.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
"""
|
|
try:
|
|
collections = await self.client.get_collections()
|
|
existing = [c.name for c in collections.collections]
|
|
|
|
if collection_name not in existing:
|
|
logger.info(f"Creating Qdrant collection: {collection_name}")
|
|
await self.client.create_collection(
|
|
collection_name=collection_name,
|
|
vectors_config=VectorParams(
|
|
size=self.embedding_dim,
|
|
distance=Distance.COSINE
|
|
)
|
|
)
|
|
logger.info(f"Created collection: {collection_name}")
|
|
except Exception as e:
|
|
logger.error(f"Error ensuring collection: {e}", exc_info=True)
|
|
raise
|
|
|
|
async def collection_exists(self, collection_name: str) -> bool:
|
|
"""
|
|
Check if collection exists.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
|
|
Returns:
|
|
True if collection exists
|
|
"""
|
|
try:
|
|
collections = await self.client.get_collections()
|
|
existing = [c.name for c in collections.collections]
|
|
return collection_name in existing
|
|
except Exception as e:
|
|
logger.error(f"Error checking collection: {e}", exc_info=True)
|
|
return False
|
|
|
|
async def upsert_document_chunks(
|
|
self,
|
|
user: str,
|
|
doc_id: str,
|
|
chunks: List[Dict[str, Any]],
|
|
embeddings: List[List[float]]
|
|
) -> int:
|
|
"""
|
|
Upsert document chunks with embeddings.
|
|
|
|
Point structure:
|
|
{
|
|
id: "doc_123_chunk_0",
|
|
vector: [...],
|
|
payload: {
|
|
doc_id: "doc_123",
|
|
chunk_index: 0,
|
|
content: "text content",
|
|
metadata: {...}
|
|
}
|
|
}
|
|
|
|
Args:
|
|
user: User identifier
|
|
doc_id: Document ID
|
|
chunks: List of chunk dictionaries with content
|
|
embeddings: List of embedding vectors
|
|
|
|
Returns:
|
|
Number of chunks upserted
|
|
|
|
Raises:
|
|
ValueError: If chunks and embeddings length mismatch
|
|
"""
|
|
if len(chunks) != len(embeddings):
|
|
raise ValueError(
|
|
f"Chunks ({len(chunks)}) and embeddings ({len(embeddings)}) length mismatch"
|
|
)
|
|
|
|
collection_name = self.get_collection_name(user)
|
|
# Ensure the tenant-scoped collection (passing the raw user here used
|
|
# to create a stray collection named after the bare user string).
|
|
await self.ensure_collection(collection_name)
|
|
|
|
points = []
|
|
for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
|
|
# Generate deterministic point ID
|
|
point_id = str(uuid.uuid5(
|
|
uuid.NAMESPACE_DNS,
|
|
f"{doc_id}_chunk_{i}"
|
|
))
|
|
|
|
points.append(PointStruct(
|
|
id=point_id,
|
|
vector=embedding,
|
|
payload={
|
|
"doc_id": doc_id,
|
|
"chunk_index": i,
|
|
"content": chunk.get("content", ""),
|
|
"metadata": chunk.get("metadata", {})
|
|
}
|
|
))
|
|
|
|
try:
|
|
await self.client.upsert(
|
|
collection_name=collection_name,
|
|
points=points
|
|
)
|
|
logger.info(f"Upserted {len(points)} chunks for {doc_id}")
|
|
return len(points)
|
|
except Exception as e:
|
|
logger.error(f"Failed to upsert chunks: {e}", exc_info=True)
|
|
raise
|
|
|
|
async def search(
|
|
self,
|
|
user: str,
|
|
query_vector: List[float],
|
|
limit: int = 10,
|
|
score_threshold: float = 0.7,
|
|
filter_dict: Optional[Dict[str, Any]] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Semantic search in user's collection.
|
|
|
|
Args:
|
|
user: User identifier
|
|
query_vector: Query embedding vector
|
|
limit: Maximum results to return
|
|
score_threshold: Minimum similarity score (0.0-1.0)
|
|
filter_dict: Optional payload filters
|
|
|
|
Returns:
|
|
List of matches with scores and payloads
|
|
|
|
Example:
|
|
results = await client.search(
|
|
user="jpmschweitzer",
|
|
query_vector=[0.1, 0.2, ...],
|
|
limit=5,
|
|
filter_dict={"doc_id": "doc_123"}
|
|
)
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
# Build filter if provided
|
|
query_filter = None
|
|
if filter_dict:
|
|
conditions = []
|
|
for key, value in filter_dict.items():
|
|
conditions.append(
|
|
FieldCondition(key=key, match=MatchValue(value=value))
|
|
)
|
|
query_filter = Filter(must=conditions)
|
|
|
|
try:
|
|
response = await self.client.query_points(
|
|
collection_name=collection_name,
|
|
query=query_vector,
|
|
limit=limit,
|
|
score_threshold=score_threshold,
|
|
query_filter=query_filter,
|
|
with_payload=True
|
|
)
|
|
|
|
return [
|
|
{
|
|
"id": str(point.id),
|
|
"score": point.score,
|
|
"doc_id": point.payload["doc_id"],
|
|
"chunk_index": point.payload["chunk_index"],
|
|
"content": point.payload["content"],
|
|
"metadata": point.payload.get("metadata", {})
|
|
}
|
|
for point in response.points
|
|
]
|
|
except Exception as e:
|
|
logger.error(f"Search failed: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def delete_document(self, user: str, doc_id: str) -> bool:
|
|
"""
|
|
Delete all chunks for a document.
|
|
|
|
Args:
|
|
user: User identifier
|
|
doc_id: Document ID
|
|
|
|
Returns:
|
|
True if deleted successfully
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
try:
|
|
await self.client.delete(
|
|
collection_name=collection_name,
|
|
points_selector=Filter(
|
|
must=[
|
|
FieldCondition(
|
|
key="doc_id",
|
|
match=MatchValue(value=doc_id)
|
|
)
|
|
]
|
|
)
|
|
)
|
|
logger.info(f"Deleted chunks for {doc_id}")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete document: {e}", exc_info=True)
|
|
return False
|
|
|
|
async def get_document_chunks(
|
|
self,
|
|
user: str,
|
|
doc_id: str
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get all chunks for a document.
|
|
|
|
Args:
|
|
user: User identifier
|
|
doc_id: Document ID
|
|
|
|
Returns:
|
|
List of chunks with content and metadata
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
try:
|
|
# Scroll through points with doc_id filter
|
|
points, _ = await self.client.scroll(
|
|
collection_name=collection_name,
|
|
scroll_filter=Filter(
|
|
must=[
|
|
FieldCondition(
|
|
key="doc_id",
|
|
match=MatchValue(value=doc_id)
|
|
)
|
|
]
|
|
),
|
|
limit=1000,
|
|
with_payload=True,
|
|
with_vectors=False
|
|
)
|
|
|
|
return [
|
|
{
|
|
"id": str(point.id),
|
|
"chunk_index": point.payload["chunk_index"],
|
|
"content": point.payload["content"],
|
|
"metadata": point.payload.get("metadata", {})
|
|
}
|
|
for point in points
|
|
]
|
|
except Exception as e:
|
|
logger.error(f"Failed to get document chunks: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def count_documents(self, user: str) -> int:
|
|
"""
|
|
Count total number of unique documents in user's collection.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of unique documents
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
try:
|
|
# Get collection info
|
|
collection_info = await self.client.get_collection(collection_name)
|
|
# This gives total points, not unique docs
|
|
# For unique docs, would need to aggregate by doc_id
|
|
return collection_info.points_count
|
|
except Exception as e:
|
|
logger.error(f"Failed to count documents: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def delete_collection(self, user: str) -> bool:
|
|
"""
|
|
Delete user's entire collection.
|
|
|
|
Warning: This removes all data for the user!
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
True if deleted successfully
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
try:
|
|
await self.client.delete_collection(collection_name)
|
|
logger.warning(f"Deleted collection: {collection_name}")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete collection: {e}", exc_info=True)
|
|
return False
|
|
|
|
async def find_similar_chunks(
|
|
self,
|
|
user: str,
|
|
doc_id: str,
|
|
limit: int = 10
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Find chunks similar to those in a given document.
|
|
|
|
Strategy: Get all chunks from doc, use their vectors to find similar chunks.
|
|
|
|
Args:
|
|
user: User identifier
|
|
doc_id: Source document ID
|
|
limit: Maximum results per chunk
|
|
|
|
Returns:
|
|
List of similar chunks from other documents
|
|
"""
|
|
collection_name = self.get_collection_name(user)
|
|
|
|
try:
|
|
# Get source document chunks with vectors
|
|
source_points, _ = await self.client.scroll(
|
|
collection_name=collection_name,
|
|
scroll_filter=Filter(
|
|
must=[
|
|
FieldCondition(
|
|
key="doc_id",
|
|
match=MatchValue(value=doc_id)
|
|
)
|
|
]
|
|
),
|
|
limit=10, # Sample first 10 chunks
|
|
with_payload=True,
|
|
with_vectors=True
|
|
)
|
|
|
|
if not source_points:
|
|
return []
|
|
|
|
# Search using first chunk's vector
|
|
# (could aggregate multiple chunks for better results)
|
|
first_vector = source_points[0].vector
|
|
|
|
response = await self.client.query_points(
|
|
collection_name=collection_name,
|
|
query=first_vector,
|
|
limit=limit * 2, # Get more to filter out same doc
|
|
with_payload=True
|
|
)
|
|
|
|
# Filter out chunks from same document
|
|
similar = [
|
|
{
|
|
"id": str(point.id),
|
|
"score": point.score,
|
|
"doc_id": point.payload["doc_id"],
|
|
"content": point.payload["content"]
|
|
}
|
|
for point in response.points
|
|
if point.payload["doc_id"] != doc_id
|
|
]
|
|
|
|
return similar[:limit]
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to find similar chunks: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def upsert_vector(
|
|
self,
|
|
collection_name: str,
|
|
vector_id: str,
|
|
vector: List[float],
|
|
payload: Dict[str, Any]
|
|
) -> bool:
|
|
"""
|
|
Upsert a single vector point.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
vector_id: Point ID
|
|
vector: Embedding vector
|
|
payload: Point payload/metadata
|
|
|
|
Returns:
|
|
True if successful
|
|
"""
|
|
try:
|
|
await self.client.upsert(
|
|
collection_name=collection_name,
|
|
points=[PointStruct(
|
|
id=vector_id,
|
|
vector=vector,
|
|
payload=payload
|
|
)]
|
|
)
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to upsert vector: {e}", exc_info=True)
|
|
return False
|
|
|
|
async def upsert_points(
|
|
self,
|
|
collection_name: str,
|
|
points: List[Dict[str, Any]]
|
|
) -> int:
|
|
"""
|
|
Upsert a batch of vector points in a single request.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
points: List of {"id": str, "vector": List[float], "payload": dict}
|
|
|
|
Returns:
|
|
Number of points upserted
|
|
|
|
Raises:
|
|
Exception: If the upsert fails (callers decide how to degrade)
|
|
"""
|
|
if not points:
|
|
return 0
|
|
|
|
structs = [
|
|
PointStruct(
|
|
id=p["id"],
|
|
vector=p["vector"],
|
|
payload=p.get("payload", {})
|
|
)
|
|
for p in points
|
|
]
|
|
|
|
await self.client.upsert(
|
|
collection_name=collection_name,
|
|
points=structs
|
|
)
|
|
logger.info(f"Upserted {len(structs)} points into {collection_name}")
|
|
return len(structs)
|
|
|
|
async def delete_by_filter(
|
|
self,
|
|
collection_name: str,
|
|
filter_conditions: Dict[str, Any]
|
|
) -> int:
|
|
"""
|
|
Delete points matching filter conditions.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
filter_conditions: Filter conditions (e.g., {"page_id": 5})
|
|
|
|
Returns:
|
|
Number of points deleted (approximation)
|
|
"""
|
|
try:
|
|
# Build filter
|
|
conditions = []
|
|
for key, value in filter_conditions.items():
|
|
conditions.append(
|
|
FieldCondition(key=key, match=MatchValue(value=value))
|
|
)
|
|
query_filter = Filter(must=conditions)
|
|
|
|
# Delete points
|
|
await self.client.delete(
|
|
collection_name=collection_name,
|
|
points_selector=query_filter
|
|
)
|
|
|
|
# Return operation status (Qdrant doesn't return count directly)
|
|
logger.info(f"Deleted points with filter {filter_conditions}")
|
|
return 1 # Placeholder, actual count not available from API
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete by filter: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def search_vectors(
|
|
self,
|
|
collection_name: str,
|
|
query_vector: List[float],
|
|
limit: int = 10,
|
|
score_threshold: float = 0.5,
|
|
filter_conditions: Optional[Dict[str, Any]] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Search vectors in collection.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
query_vector: Query embedding vector
|
|
limit: Maximum results
|
|
score_threshold: Minimum similarity score
|
|
filter_conditions: Optional filter conditions
|
|
|
|
Returns:
|
|
List of search results with scores and payloads
|
|
"""
|
|
# Build filter if provided
|
|
query_filter = None
|
|
if filter_conditions:
|
|
conditions = []
|
|
for key, value in filter_conditions.items():
|
|
conditions.append(
|
|
FieldCondition(key=key, match=MatchValue(value=value))
|
|
)
|
|
query_filter = Filter(must=conditions)
|
|
|
|
try:
|
|
response = await self.client.query_points(
|
|
collection_name=collection_name,
|
|
query=query_vector,
|
|
limit=limit,
|
|
score_threshold=score_threshold,
|
|
query_filter=query_filter,
|
|
with_payload=True
|
|
)
|
|
|
|
return [
|
|
{
|
|
"id": str(point.id),
|
|
"score": point.score,
|
|
"payload": dict(point.payload)
|
|
}
|
|
for point in response.points
|
|
]
|
|
except Exception as e:
|
|
logger.error(f"Search failed: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def scroll_all_points(
|
|
self,
|
|
collection_name: str,
|
|
batch_size: int = 100,
|
|
with_payload: bool = True,
|
|
with_vectors: bool = False,
|
|
filter_conditions: Optional[Dict[str, Any]] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Scroll through all points in a collection.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
batch_size: Number of points per batch
|
|
with_payload: Include payload in results
|
|
with_vectors: Include vectors in results
|
|
filter_conditions: Optional filter conditions
|
|
|
|
Returns:
|
|
List of all points with id and payload
|
|
"""
|
|
all_points = []
|
|
offset = None
|
|
|
|
# Build filter if provided
|
|
scroll_filter = None
|
|
if filter_conditions:
|
|
conditions = []
|
|
for key, value in filter_conditions.items():
|
|
conditions.append(
|
|
FieldCondition(key=key, match=MatchValue(value=value))
|
|
)
|
|
scroll_filter = Filter(must=conditions)
|
|
|
|
try:
|
|
while True:
|
|
points, next_offset = await self.client.scroll(
|
|
collection_name=collection_name,
|
|
scroll_filter=scroll_filter,
|
|
limit=batch_size,
|
|
offset=offset,
|
|
with_payload=with_payload,
|
|
with_vectors=with_vectors
|
|
)
|
|
|
|
for point in points:
|
|
entry = {
|
|
"id": str(point.id),
|
|
"payload": dict(point.payload) if point.payload else {}
|
|
}
|
|
if with_vectors:
|
|
entry["vector"] = point.vector
|
|
all_points.append(entry)
|
|
|
|
if next_offset is None:
|
|
break
|
|
offset = next_offset
|
|
|
|
return all_points
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to scroll collection {collection_name}: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def delete_by_ids(
|
|
self,
|
|
collection_name: str,
|
|
point_ids: List[str]
|
|
) -> int:
|
|
"""
|
|
Delete points by their IDs.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
point_ids: List of point IDs to delete
|
|
|
|
Returns:
|
|
Number of points deleted
|
|
"""
|
|
if not point_ids:
|
|
return 0
|
|
|
|
try:
|
|
await self.client.delete(
|
|
collection_name=collection_name,
|
|
points_selector=point_ids
|
|
)
|
|
logger.info(f"Deleted {len(point_ids)} points from {collection_name}")
|
|
return len(point_ids)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete points by IDs: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def list_collections(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
List all collections with stats.
|
|
|
|
Returns:
|
|
List of collection info dictionaries
|
|
"""
|
|
try:
|
|
collections = await self.client.get_collections()
|
|
result = []
|
|
|
|
for coll in collections.collections:
|
|
# Get detailed collection info
|
|
try:
|
|
info = await self.client.get_collection(coll.name)
|
|
result.append({
|
|
"name": coll.name,
|
|
"vectors_count": info.vectors_count or 0,
|
|
"points_count": info.points_count or 0,
|
|
"segments_count": info.segments_count or 0
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get info for collection {coll.name}: {e}")
|
|
result.append({
|
|
"name": coll.name,
|
|
"vectors_count": 0,
|
|
"points_count": 0,
|
|
"segments_count": 0
|
|
})
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to list collections: {e}", exc_info=True)
|
|
return []
|
|
|
|
# ========== Volatile Data Methods ==========
|
|
|
|
async def search_with_expiry_filter(
|
|
self,
|
|
collection_name: str,
|
|
query_vector: List[float],
|
|
current_timestamp: int,
|
|
limit: int = 10,
|
|
score_threshold: float = 0.7
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Search vectors filtering out expired records.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
query_vector: Query embedding vector
|
|
current_timestamp: Current time in milliseconds
|
|
limit: Maximum results
|
|
score_threshold: Minimum similarity score
|
|
|
|
Returns:
|
|
List of non-expired search results
|
|
"""
|
|
# Filter: ttl_expiry > current_timestamp (not expired)
|
|
expiry_filter = Filter(
|
|
must=[
|
|
FieldCondition(
|
|
key="ttl_expiry",
|
|
range=Range(gt=current_timestamp)
|
|
)
|
|
]
|
|
)
|
|
|
|
try:
|
|
response = await self.client.query_points(
|
|
collection_name=collection_name,
|
|
query=query_vector,
|
|
limit=limit,
|
|
score_threshold=score_threshold,
|
|
query_filter=expiry_filter,
|
|
with_payload=True
|
|
)
|
|
|
|
return [
|
|
{
|
|
"id": str(point.id),
|
|
"score": point.score,
|
|
"payload": dict(point.payload)
|
|
}
|
|
for point in response.points
|
|
]
|
|
except Exception as e:
|
|
logger.error(f"Volatile search failed: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def delete_expired_vectors(
|
|
self,
|
|
collection_name: str,
|
|
current_timestamp: int
|
|
) -> int:
|
|
"""
|
|
Delete all vectors where ttl_expiry < current_timestamp.
|
|
|
|
Args:
|
|
collection_name: Collection name
|
|
current_timestamp: Current time in milliseconds
|
|
|
|
Returns:
|
|
Number of points deleted (approximate)
|
|
"""
|
|
# Filter: ttl_expiry < current_timestamp (expired)
|
|
expiry_filter = Filter(
|
|
must=[
|
|
FieldCondition(
|
|
key="ttl_expiry",
|
|
range=Range(lt=current_timestamp)
|
|
)
|
|
]
|
|
)
|
|
|
|
try:
|
|
# First count how many will be deleted (scroll to count)
|
|
count = 0
|
|
offset = None
|
|
while True:
|
|
points, next_offset = await self.client.scroll(
|
|
collection_name=collection_name,
|
|
scroll_filter=expiry_filter,
|
|
limit=100,
|
|
offset=offset,
|
|
with_payload=False
|
|
)
|
|
count += len(points)
|
|
if next_offset is None:
|
|
break
|
|
offset = next_offset
|
|
|
|
if count == 0:
|
|
return 0
|
|
|
|
# Delete expired points
|
|
await self.client.delete(
|
|
collection_name=collection_name,
|
|
points_selector=expiry_filter
|
|
)
|
|
|
|
logger.info(f"Deleted {count} expired vectors from {collection_name}")
|
|
return count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete expired vectors: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def get_volatile_collections(self) -> List[str]:
|
|
"""
|
|
Get all volatile collections (prefixed with 'volatile_').
|
|
|
|
Returns:
|
|
List of volatile collection names
|
|
"""
|
|
try:
|
|
collections = await self.client.get_collections()
|
|
return [
|
|
c.name for c in collections.collections
|
|
if c.name.startswith("volatile_")
|
|
]
|
|
except Exception as e:
|
|
logger.error(f"Failed to list volatile collections: {e}", exc_info=True)
|
|
return [] |