Add POST /maintenance/cleanup/test-data endpoint to purge LLM test data from wiki, graph, and vectors. Security-restricted to test user namespace only (users/llm-tester/*, users/llm_tester/*). - Supports dry_run=true (default) to preview before deleting - Cleans vectors, graph nodes, and wiki pages - Scheduler task configured for weekly cleanup (Sunday 3:00 AM) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from src.core.dependencies import (
|
||||
QdrantDep, OllamaDep, verify_api_key
|
||||
)
|
||||
from src.config import get_settings
|
||||
from src.core.multi_tenancy import DEFAULT_USER
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -220,6 +221,35 @@ class VolatileCleanupResponse(BaseModel):
|
||||
duration_ms: float
|
||||
|
||||
|
||||
class TestDataCleanupResponse(BaseModel):
|
||||
"""Response from test data cleanup operation."""
|
||||
success: bool
|
||||
dry_run: bool
|
||||
wiki_pages_deleted: int
|
||||
graph_nodes_deleted: int
|
||||
vector_chunks_deleted: int
|
||||
pages_found: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
duration_ms: float
|
||||
|
||||
|
||||
# Test data path patterns - restricted to test user namespace only
|
||||
# These are the only paths that can be cleaned up for safety
|
||||
TEST_USER_PATH_PREFIXES = [
|
||||
"users/llm-tester/",
|
||||
"users/llm_tester/",
|
||||
]
|
||||
|
||||
|
||||
def _matches_test_user_path(path: str) -> bool:
|
||||
"""Check if a path is in the test user namespace.
|
||||
|
||||
Only matches paths that START with test user prefixes for safety.
|
||||
This prevents accidental deletion of non-test data.
|
||||
"""
|
||||
path_lower = path.lower()
|
||||
return any(path_lower.startswith(prefix) for prefix in TEST_USER_PATH_PREFIXES)
|
||||
|
||||
|
||||
# ========== Endpoints ==========
|
||||
|
||||
@router.post("/cleanup/vectors", response_model=VectorCleanupResponse)
|
||||
@@ -556,6 +586,95 @@ async def cleanup_volatile(
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/cleanup/test-data", response_model=TestDataCleanupResponse)
|
||||
async def cleanup_test_data(
|
||||
dry_run: bool = Query(default=True, description="Preview only, don't delete"),
|
||||
wiki: WikiJSDep = None,
|
||||
vector_service: VectorServiceDep = None,
|
||||
graph_service: GraphServiceDep = None,
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Purge LLM tester data from wiki, graph, and vectors.
|
||||
|
||||
**Security**: Only deletes pages in the test user namespace:
|
||||
- users/llm-tester/*
|
||||
- users/llm_tester/*
|
||||
|
||||
This endpoint cannot delete data outside these paths.
|
||||
|
||||
**Use dry_run=true (default) to preview what would be deleted.**
|
||||
|
||||
**Scheduler Integration:**
|
||||
```json
|
||||
{
|
||||
"task_name": "test_data_cleanup",
|
||||
"schedule": "0 3 * * 0",
|
||||
"endpoint": "POST /maintenance/cleanup/test-data?dry_run=false",
|
||||
"description": "Weekly cleanup of LLM test data"
|
||||
}
|
||||
```
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# List all wiki pages
|
||||
all_pages = await wiki.list_all_pages(batch_size=500)
|
||||
|
||||
# Filter for test user paths only (security: restricted to test namespace)
|
||||
test_pages = [
|
||||
{"id": p["id"], "path": p["path"], "title": p.get("title", "")}
|
||||
for p in all_pages
|
||||
if _matches_test_user_path(p.get("path", ""))
|
||||
]
|
||||
|
||||
logger.info(f"Found {len(test_pages)} test pages matching patterns: {TEST_USER_PATH_PREFIXES}")
|
||||
|
||||
wiki_deleted = 0
|
||||
graph_deleted = 0
|
||||
vector_deleted = 0
|
||||
|
||||
if not dry_run and test_pages:
|
||||
for page in test_pages:
|
||||
page_id = page["id"]
|
||||
page_path = page["path"]
|
||||
|
||||
try:
|
||||
# Delete vector chunks for this page (using DEFAULT_USER collection)
|
||||
chunks_removed = await vector_service.delete_page_chunks(page_id, DEFAULT_USER)
|
||||
vector_deleted += chunks_removed
|
||||
|
||||
# Delete graph node for this page (returns count, may be 0 if no node)
|
||||
graph_removed = await graph_service.delete_page(page_id, DEFAULT_USER)
|
||||
graph_deleted += graph_removed
|
||||
|
||||
# Delete wiki page (raises exception on failure, returns None on success)
|
||||
await wiki.delete_page(page_id)
|
||||
wiki_deleted += 1
|
||||
|
||||
logger.info(f"Deleted test page: {page_path} (id={page_id})")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete page {page_path}: {e}")
|
||||
continue
|
||||
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
|
||||
return TestDataCleanupResponse(
|
||||
success=True,
|
||||
dry_run=dry_run,
|
||||
wiki_pages_deleted=wiki_deleted,
|
||||
graph_nodes_deleted=graph_deleted,
|
||||
vector_chunks_deleted=vector_deleted,
|
||||
pages_found=test_pages,
|
||||
duration_ms=duration_ms
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Test data cleanup failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/health", response_model=HealthCheckResponse)
|
||||
async def maintenance_health(
|
||||
user: str = Query(..., description="User identifier"),
|
||||
|
||||
Reference in New Issue
Block a user