feat: add test data cleanup endpoint
Build and Push / build (release) Successful in 28s

Add POST /maintenance/cleanup/test-data endpoint to purge LLM test data
from wiki, graph, and vectors. Security-restricted to test user namespace
only (users/llm-tester/*, users/llm_tester/*).

- Supports dry_run=true (default) to preview before deleting
- Cleans vectors, graph nodes, and wiki pages
- Scheduler task configured for weekly cleanup (Sunday 3:00 AM)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-24 21:12:16 +01:00
co-authored by Claude Opus 4.5
parent 37f8e1819e
commit e6e65d6d78
4 changed files with 268 additions and 90 deletions
+119
View File
@@ -22,6 +22,7 @@ from src.core.dependencies import (
QdrantDep, OllamaDep, verify_api_key
)
from src.config import get_settings
from src.core.multi_tenancy import DEFAULT_USER
from datetime import datetime, timezone
logger = logging.getLogger(__name__)
@@ -220,6 +221,35 @@ class VolatileCleanupResponse(BaseModel):
duration_ms: float
class TestDataCleanupResponse(BaseModel):
"""Response from test data cleanup operation."""
success: bool
dry_run: bool
wiki_pages_deleted: int
graph_nodes_deleted: int
vector_chunks_deleted: int
pages_found: List[Dict[str, Any]] = Field(default_factory=list)
duration_ms: float
# Test data path patterns - restricted to test user namespace only
# These are the only paths that can be cleaned up for safety
TEST_USER_PATH_PREFIXES = [
"users/llm-tester/",
"users/llm_tester/",
]
def _matches_test_user_path(path: str) -> bool:
"""Check if a path is in the test user namespace.
Only matches paths that START with test user prefixes for safety.
This prevents accidental deletion of non-test data.
"""
path_lower = path.lower()
return any(path_lower.startswith(prefix) for prefix in TEST_USER_PATH_PREFIXES)
# ========== Endpoints ==========
@router.post("/cleanup/vectors", response_model=VectorCleanupResponse)
@@ -556,6 +586,95 @@ async def cleanup_volatile(
raise HTTPException(status_code=500, detail=str(e))
@router.post("/cleanup/test-data", response_model=TestDataCleanupResponse)
async def cleanup_test_data(
dry_run: bool = Query(default=True, description="Preview only, don't delete"),
wiki: WikiJSDep = None,
vector_service: VectorServiceDep = None,
graph_service: GraphServiceDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Purge LLM tester data from wiki, graph, and vectors.
**Security**: Only deletes pages in the test user namespace:
- users/llm-tester/*
- users/llm_tester/*
This endpoint cannot delete data outside these paths.
**Use dry_run=true (default) to preview what would be deleted.**
**Scheduler Integration:**
```json
{
"task_name": "test_data_cleanup",
"schedule": "0 3 * * 0",
"endpoint": "POST /maintenance/cleanup/test-data?dry_run=false",
"description": "Weekly cleanup of LLM test data"
}
```
"""
start_time = time.time()
try:
# List all wiki pages
all_pages = await wiki.list_all_pages(batch_size=500)
# Filter for test user paths only (security: restricted to test namespace)
test_pages = [
{"id": p["id"], "path": p["path"], "title": p.get("title", "")}
for p in all_pages
if _matches_test_user_path(p.get("path", ""))
]
logger.info(f"Found {len(test_pages)} test pages matching patterns: {TEST_USER_PATH_PREFIXES}")
wiki_deleted = 0
graph_deleted = 0
vector_deleted = 0
if not dry_run and test_pages:
for page in test_pages:
page_id = page["id"]
page_path = page["path"]
try:
# Delete vector chunks for this page (using DEFAULT_USER collection)
chunks_removed = await vector_service.delete_page_chunks(page_id, DEFAULT_USER)
vector_deleted += chunks_removed
# Delete graph node for this page (returns count, may be 0 if no node)
graph_removed = await graph_service.delete_page(page_id, DEFAULT_USER)
graph_deleted += graph_removed
# Delete wiki page (raises exception on failure, returns None on success)
await wiki.delete_page(page_id)
wiki_deleted += 1
logger.info(f"Deleted test page: {page_path} (id={page_id})")
except Exception as e:
logger.error(f"Failed to delete page {page_path}: {e}")
continue
duration_ms = (time.time() - start_time) * 1000
return TestDataCleanupResponse(
success=True,
dry_run=dry_run,
wiki_pages_deleted=wiki_deleted,
graph_nodes_deleted=graph_deleted,
vector_chunks_deleted=vector_deleted,
pages_found=test_pages,
duration_ms=duration_ms
)
except Exception as e:
logger.error(f"Test data cleanup failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
@router.get("/health", response_model=HealthCheckResponse)
async def maintenance_health(
user: str = Query(..., description="User identifier"),