diff --git a/CHANGELOG.md b/CHANGELOG.md index 39fa0cf..d9924de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to Library Desk will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.4] - 2025-12-24 + +### Added + +- **Test Data Cleanup Endpoint** - `POST /maintenance/cleanup/test-data` + - Purges LLM test data from wiki, graph, and vectors + - Security-restricted to test user namespace only (`users/llm-tester/*`, `users/llm_tester/*`) + - Supports `dry_run=true` (default) to preview before deleting + - Scheduler task configured for weekly cleanup (Sunday 3:00 AM) + ## [1.4.3] - 2025-12-24 ### Changed diff --git a/docs/MEMORY_SYSTEM_PLAN.md b/docs/MEMORY_SYSTEM_PLAN.md index dfb7865..ef10ef3 100644 --- a/docs/MEMORY_SYSTEM_PLAN.md +++ b/docs/MEMORY_SYSTEM_PLAN.md @@ -6,15 +6,24 @@ A three-tier memory architecture for Library Desk with intelligent orchestration | Tier | Storage | Purpose | TTL | |------|---------|---------|-----| -| **Volatile** | Redis | Weather, news, financial, ephemeral context | 5min - 2hr | +| **Volatile** | Qdrant (vectors) | Weather, news, financial, ephemeral context | 5min - 2hr | | **Documents** | TBD (research) | Git mirrors, PDFs, video, images | Permanent | | **Knowledge** | Wiki + Neo4j | Personal dossiers, research, summaries | Permanent | -**Implementation Priority**: Cleanup → Volatile → Documents +**Implementation Priority**: Cleanup → Volatile → Documents → Test Data Cleanup + +### Phase Status + +| Phase | Status | Version | +|-------|--------|---------| +| Phase 1: Cleanup System | ✅ Complete | v1.4.0 | +| Phase 2: Volatile Memory | ✅ Complete | v1.4.3 | +| Phase 3: Document Storage | ⏳ Pending | - | +| Phase 4: Test Data Cleanup | ✅ Complete | v1.4.4 | --- -## Phase 1: Cleanup System Completion +## Phase 1: Cleanup System Completion ✅ ### Current State - **COMPLETE** - All Phase 1 tasks implemented @@ -57,9 +66,9 @@ A three-tier memory architecture for Library Desk with intelligent orchestration --- -## Phase 2: Volatile Memory System +## Phase 2: Volatile Memory System ✅ -### Architecture +### Architecture (Final Implementation) ``` ┌─────────────────┐ ┌──────────────┐ ┌─────────────────┐ @@ -71,88 +80,46 @@ A three-tier memory architecture for Library Desk with intelligent orchestration │ ▼ ┌─────────────────┐ -│ Redis │ -│ (DB 4, TTL) │ +│ Qdrant │ +│ (volatile_{user})│ └─────────────────┘ ``` -### Data Model +**Key design decisions:** +- Vector storage in Qdrant (not Redis) for semantic search +- Collection per user: `volatile_{user}` +- TTL via `ttl_expiry` timestamp in payload +- Natural language conversion for embedding structured data +- Integrated into HybridRAG with priority boost -```python -class VolatileRecord(BaseModel): - key: str # e.g., "weather:rotterdam" - namespace: str # e.g., "weather", "news", "financial" - data: dict # Actual content - source: Optional[str] # Origin API/service - created_at: datetime - updated_at: datetime - ttl: int # Seconds until expiration - refresh_schedule: Optional[str] # Cron expression, if repeating - user: str # Multi-tenant isolation -``` - -**Key pattern**: `{user}:volatile:{namespace}:{key_hash}` - -### Implementation Order: Integration-First - -1. **Start with Consolidation Hook** - Understand data flow through existing system -2. **Build Service Layer** - VolatileCacheService with Redis operations -3. **Add API Endpoints** - REST interface for volatile data -4. **Biographer Integration** - Query user preferences for relevance - -### Tasks - -#### 2.1 Integrate with Consolidation (FIRST) -**New file**: `src/services/volatile_service.py` - -```python -class VolatileCacheService: - async def get(user, namespace, key) -> Optional[VolatileRecord] - async def set(user, namespace, key, data, ttl, refresh_schedule=None) - async def delete(user, namespace, key) - async def list_namespace(user, namespace) -> List[str] - async def get_scheduled(user) -> List[VolatileRecord] # For scheduler -``` - -#### 2.2 Create Volatile API Router -**New file**: `src/routers/volatile.py` +### Endpoints (Implemented) | Endpoint | Method | Purpose | |----------|--------|---------| -| `/volatile/{namespace}/{key}` | GET | Retrieve record | -| `/volatile/{namespace}/{key}` | POST | Store/update record | +| `/volatile/search?q=...` | GET | Semantic search across volatile data | +| `/volatile/store?namespace=...&key=...` | POST | Store/update record | +| `/volatile/{namespace}/{key}` | GET | Retrieve specific record | | `/volatile/{namespace}/{key}` | DELETE | Remove record | -| `/volatile/{namespace}` | GET | List keys in namespace | -| `/volatile/scheduled` | GET | List records needing refresh | | `/volatile/stats` | GET | Cache statistics | +| `/volatile/scheduled` | GET | Records needing refresh | +| `/volatile/namespaces` | GET | List available namespaces | +| `/maintenance/cleanup/volatile` | POST | Purge expired records | -#### 2.3 Integrate with Consolidation -**File**: `src/services/consolidation_service.py` +### Namespaces -Add relevance trigger detection: -1. During consolidation, analyze search results for location/interest patterns -2. Query tatlock's Biographer collection for user preferences -3. If match found, create/update volatile refresh schedule - -#### 2.4 Biographer Integration -**File**: `src/core/dependencies.py` - -```python -def get_biographer_qdrant() -> QdrantClientWrapper: - """Direct access to tatlock's Biographer collection.""" - # Configure to connect to tatlock's Qdrant -``` - -#### 2.5 Scheduler-Side Configuration -Document required scheduler tasks: -```json -{ - "task_name": "volatile_refresh", - "schedule": "*/15 * * * *", - "endpoint": "GET /volatile/scheduled", - "follow_up": "For each record, call refresh endpoint with record.refresh_schedule" -} -``` +| Namespace | Default TTL | Use Case | +|-----------|-------------|----------| +| weather | 30 min | Current conditions, forecasts | +| news | 1 hour | Headlines, breaking news | +| financial | 5 min | Stock prices, exchange rates | +| transit | 5 min | Train/bus schedules, delays | +| traffic | 10 min | Commute times, road conditions | +| air_quality | 1 hour | Pollution, pollen counts | +| sports | 1 min | Live scores, matches | +| social | 10 min | Social notifications | +| system | 1 min | Service health status | +| context | 1 hour | Session state | +| custom | 1 hour | User-defined data | --- @@ -219,34 +186,116 @@ Add LLM-powered category descriptor generation: --- -## Files to Modify/Create +## Phase 4: LLM Tester Data Cleanup ✅ -### Phase 1 (Cleanup) -- `src/routers/maintenance.py` - Add timestamp tracking +### Problem + +LLM testing creates accumulated cruft across the system: +- Wiki.js pages under `llm-tester/` and `llm_tester/` paths +- Graph nodes (Document, Entity) linked to test pages +- Vector chunks in Qdrant for test content + +This data accumulates over time and clutters Wiki.js visually (no separate tenant scope for tests). + +### Solution + +Add a maintenance endpoint to purge all LLM tester artifacts across wiki, graph, and vectors. + +### Tasks + +#### 4.1 Identify Test Data Patterns ✅ +**Patterns matched** (security-restricted to test user namespace): +- `users/llm-tester/*` +- `users/llm_tester/*` + +#### 4.2 Add Cleanup Endpoint ✅ +**File**: `src/routers/maintenance.py` + +```python +@router.post("/cleanup/test-data") +async def cleanup_test_data( + dry_run: bool = Query(default=True), + wiki: WikiJSDep = None, + vector_service: VectorServiceDep = None, + graph_service: GraphServiceDep = None, + api_key: str = Depends(verify_api_key) +): + """ + Purge LLM tester data from wiki, graph, and vectors. + + **Security**: Only deletes pages in the test user namespace: + - users/llm-tester/* + - users/llm_tester/* + + Use dry_run=true to preview what would be deleted. + """ +``` + +#### 4.3 Implementation Steps ✅ + +1. **Wiki cleanup**: Delete pages via GraphQL mutation +2. **Graph cleanup**: Delete Document nodes using `delete_page()` method +3. **Vector cleanup**: Delete chunks using `delete_page_chunks()` method + +#### 4.4 Scheduler Integration ✅ +**Recommended schedule**: Weekly (Sunday 3:00 AM) + +```json +{ + "task_name": "test_data_cleanup", + "schedule": "0 3 * * 0", + "endpoint": "POST /maintenance/cleanup/test-data?dry_run=false", + "description": "Weekly cleanup of LLM test data" +} +``` + +### Files to Modify + +- `src/routers/maintenance.py` - Add cleanup endpoint +- `src/services/wiki_service.py` - Add bulk delete by path pattern (if needed) +- `src/services/graph_service.py` - May need pattern-based node deletion +- `src/services/vector_service.py` - Add pattern-based chunk deletion + +--- + +## Files Modified/Created + +### Phase 1 (Cleanup) ✅ +- `src/routers/maintenance.py` - Timestamp tracking, cleanup endpoints - `src/services/graph_service.py` - Bidirectional validation - `src/services/vector_service.py` - Cross-reference checks - `LIBRARIAN_INTEGRATION.md` - Scheduler config docs -### Phase 2 (Volatile) -- `src/services/volatile_service.py` - **NEW** -- `src/routers/volatile.py` - **NEW** -- `src/models/volatile.py` - **NEW** -- `src/core/dependencies.py` - Add Biographer client -- `src/services/consolidation_service.py` - Relevance triggers -- `tests/test_volatile.py` - **NEW** +### Phase 2 (Volatile) ✅ +- `src/services/volatile_service.py` - Qdrant-based volatile cache +- `src/routers/volatile.py` - Simplified endpoints +- `src/models/volatile.py` - Namespaces and models +- `src/models/hybrid_rag.py` - Volatile config options +- `src/services/hybrid_rag_service.py` - Volatile integration +- `src/clients/qdrant_client.py` - Expiry filter methods +- `tests/test_volatile.py` - 37 tests ### Phase 3 (Documents) - `docs/DOCUMENT_STORAGE_RESEARCH.md` - **NEW** - `src/services/document_store_service.py` - **NEW** (post-research) - `src/routers/documents.py` - **NEW** (post-research) +### Phase 4 (Test Data Cleanup) +- `src/routers/maintenance.py` - Add cleanup endpoint +- `src/services/wiki_service.py` - Bulk delete by path pattern +- `src/services/graph_service.py` - Pattern-based node deletion +- `src/services/vector_service.py` - Pattern-based chunk deletion + --- ## Resolved Design Decisions -1. **Biographer Qdrant**: Same Qdrant instance, different collection. Library-Desk queries directly. -2. **Scheduler API**: Has REST API for task registration. Library-Desk can programmatically create refresh schedules. -3. **External API calls**: Library-Desk routes through SearXNG for web search. Consider dedicated API integrations for high-value volatiles (weather, financial) for consistent quality. +1. **Volatile Storage**: Qdrant vectors (not Redis) for semantic search capability +2. **Collection Naming**: `volatile_{user}` for per-user isolation +3. **TTL Mechanism**: `ttl_expiry` timestamp in payload, background cleanup job +4. **HybridRAG Integration**: Volatile as third source with RRF priority boost +5. **Biographer Qdrant**: Same Qdrant instance, different collection +6. **Scheduler API**: Has REST API for task registration --- diff --git a/pyproject.toml b/pyproject.toml index 795e8fb..81e12d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "library-desk" -version = "1.4.3" +version = "1.4.4" description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation" readme = "README.md" requires-python = ">=3.12" diff --git a/src/routers/maintenance.py b/src/routers/maintenance.py index 38aa296..e8855bb 100644 --- a/src/routers/maintenance.py +++ b/src/routers/maintenance.py @@ -22,6 +22,7 @@ from src.core.dependencies import ( QdrantDep, OllamaDep, verify_api_key ) from src.config import get_settings +from src.core.multi_tenancy import DEFAULT_USER from datetime import datetime, timezone logger = logging.getLogger(__name__) @@ -220,6 +221,35 @@ class VolatileCleanupResponse(BaseModel): duration_ms: float +class TestDataCleanupResponse(BaseModel): + """Response from test data cleanup operation.""" + success: bool + dry_run: bool + wiki_pages_deleted: int + graph_nodes_deleted: int + vector_chunks_deleted: int + pages_found: List[Dict[str, Any]] = Field(default_factory=list) + duration_ms: float + + +# Test data path patterns - restricted to test user namespace only +# These are the only paths that can be cleaned up for safety +TEST_USER_PATH_PREFIXES = [ + "users/llm-tester/", + "users/llm_tester/", +] + + +def _matches_test_user_path(path: str) -> bool: + """Check if a path is in the test user namespace. + + Only matches paths that START with test user prefixes for safety. + This prevents accidental deletion of non-test data. + """ + path_lower = path.lower() + return any(path_lower.startswith(prefix) for prefix in TEST_USER_PATH_PREFIXES) + + # ========== Endpoints ========== @router.post("/cleanup/vectors", response_model=VectorCleanupResponse) @@ -556,6 +586,95 @@ async def cleanup_volatile( raise HTTPException(status_code=500, detail=str(e)) +@router.post("/cleanup/test-data", response_model=TestDataCleanupResponse) +async def cleanup_test_data( + dry_run: bool = Query(default=True, description="Preview only, don't delete"), + wiki: WikiJSDep = None, + vector_service: VectorServiceDep = None, + graph_service: GraphServiceDep = None, + api_key: str = Depends(verify_api_key) +): + """ + Purge LLM tester data from wiki, graph, and vectors. + + **Security**: Only deletes pages in the test user namespace: + - users/llm-tester/* + - users/llm_tester/* + + This endpoint cannot delete data outside these paths. + + **Use dry_run=true (default) to preview what would be deleted.** + + **Scheduler Integration:** + ```json + { + "task_name": "test_data_cleanup", + "schedule": "0 3 * * 0", + "endpoint": "POST /maintenance/cleanup/test-data?dry_run=false", + "description": "Weekly cleanup of LLM test data" + } + ``` + """ + start_time = time.time() + + try: + # List all wiki pages + all_pages = await wiki.list_all_pages(batch_size=500) + + # Filter for test user paths only (security: restricted to test namespace) + test_pages = [ + {"id": p["id"], "path": p["path"], "title": p.get("title", "")} + for p in all_pages + if _matches_test_user_path(p.get("path", "")) + ] + + logger.info(f"Found {len(test_pages)} test pages matching patterns: {TEST_USER_PATH_PREFIXES}") + + wiki_deleted = 0 + graph_deleted = 0 + vector_deleted = 0 + + if not dry_run and test_pages: + for page in test_pages: + page_id = page["id"] + page_path = page["path"] + + try: + # Delete vector chunks for this page (using DEFAULT_USER collection) + chunks_removed = await vector_service.delete_page_chunks(page_id, DEFAULT_USER) + vector_deleted += chunks_removed + + # Delete graph node for this page (returns count, may be 0 if no node) + graph_removed = await graph_service.delete_page(page_id, DEFAULT_USER) + graph_deleted += graph_removed + + # Delete wiki page (raises exception on failure, returns None on success) + await wiki.delete_page(page_id) + wiki_deleted += 1 + + logger.info(f"Deleted test page: {page_path} (id={page_id})") + + except Exception as e: + logger.error(f"Failed to delete page {page_path}: {e}") + continue + + duration_ms = (time.time() - start_time) * 1000 + + return TestDataCleanupResponse( + success=True, + dry_run=dry_run, + wiki_pages_deleted=wiki_deleted, + graph_nodes_deleted=graph_deleted, + vector_chunks_deleted=vector_deleted, + pages_found=test_pages, + duration_ms=duration_ms + ) + + except Exception as e: + logger.error(f"Test data cleanup failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/health", response_model=HealthCheckResponse) async def maintenance_health( user: str = Query(..., description="User identifier"),