diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..93caa82 --- /dev/null +++ b/TODO.md @@ -0,0 +1,52 @@ +# TODO + +Outstanding work items for Library Desk. + +## Stub Endpoints to Implement + +The following endpoints in `src/main.py` return stub responses and need real implementations: + +### Ingestion Status Endpoints + +#### `POST /ingest/check-updates` +Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync. + +**Implementation needed:** +1. Query existing documents by path +2. Compare content hashes +3. Return list of updates needed + +#### `GET /ingest/status/{document_id}` +Get processing status for a document. + +**Implementation needed:** +- Status tracking system (Redis or database) +- Track ingestion progress per document + +#### `GET /ingest/repo-status/{repository}` +Get indexing status for an entire repository. + +**Implementation needed:** +- Repository-level statistics +- Track which documents from a repo are indexed + +### Deduplication + +#### `POST /deduplicate/check` +Check for duplicate or highly similar documents using vector similarity and graph analysis. + +**Implementation needed:** +1. Get document embedding from Qdrant +2. Find similar vectors above threshold +3. Check graph relationships +4. Return candidates with similarity scores + +## System Statistics + +#### `GET /stats` +Get system statistics (wiki pages, neo4j nodes, qdrant vectors). + +**Implementation needed:** +- Query Neo4j for node count +- Query Qdrant for vector count +- Query Wiki.js for page count diff --git a/src/main.py b/src/main.py index 65bc223..ae5ece4 100644 --- a/src/main.py +++ b/src/main.py @@ -8,7 +8,7 @@ Following best practices: - OpenAPI documentation """ -from fastapi import FastAPI, HTTPException, Depends +from fastapi import FastAPI, HTTPException, Depends, Query from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from pydantic import BaseModel @@ -17,7 +17,10 @@ import logging from pathlib import Path from src.config import Settings, get_settings, __version__ -from src.core.dependencies import verify_api_key +from src.core.dependencies import ( + verify_api_key, QdrantDep, WikiJSDep, OllamaDep, Neo4jDep +) +from src.core.multi_tenancy import DEFAULT_USER # Configure logging logging.basicConfig( @@ -78,13 +81,6 @@ class HealthResponse(BaseModel): services: Dict[str, Any] -class StatsResponse(BaseModel): - """Statistics response model.""" - wiki_pages: int - neo4j_nodes: int - qdrant_vectors: int - - # Routes @app.get("/", tags=["Root"]) async def root() -> Dict[str, str]: @@ -141,77 +137,6 @@ async def health(settings: Settings = Depends(get_settings)) -> HealthResponse: ) -@app.get("/stats", response_model=StatsResponse, tags=["System"]) -async def stats( - api_key: str = Depends(verify_api_key) -) -> StatsResponse: - """ - Get system statistics. - Protected endpoint - requires API key. - - TODO: Implement actual stats gathering from: - - Neo4j (node count) - - Qdrant (vector count) - - Wiki.js (page count) - """ - return StatsResponse( - wiki_pages=0, - neo4j_nodes=0, - qdrant_vectors=0 - ) - - -# Ingestion endpoints (for Scheduler integration) -@app.post("/ingest/document", tags=["Ingestion"]) -async def ingest_document( - document: Dict[str, Any], - api_key: str = Depends(verify_api_key) -) -> Dict[str, Any]: - """ - Ingest a single document for indexing. - Used by The Scheduler to add mirrored documentation to the knowledge base. - - Expected fields: - - source: str (e.g., "github", "gitea") - - repository: str (e.g., "anthropic-cookbook") - - path: str (file path) - - content: str (document content) - - metadata: dict (commit, author, tags, etc.) - - TODO: Implement document ingestion pipeline: - 1. Chunk content - 2. Generate embeddings (Ollama) - 3. Extract entities (NLP) - 4. Index in Qdrant - 5. Create graph nodes/relationships in Neo4j - """ - return { - "message": "Document ingestion not yet implemented", - "document_id": f"doc_{document.get('path', 'unknown')}", - "status": "stub" - } - - -@app.post("/ingest/batch", tags=["Ingestion"]) -async def batch_ingest( - batch: Dict[str, Any], - api_key: str = Depends(verify_api_key) -) -> Dict[str, Any]: - """ - Ingest multiple documents in a batch. - More efficient than individual ingestion for large syncs. - - TODO: Implement batch processing with task queue - """ - document_count = len(batch.get("documents", [])) - return { - "message": "Batch ingestion not yet implemented", - "batch_id": "batch_stub", - "total_documents": document_count, - "status": "stub" - } - - @app.post("/ingest/check-updates", tags=["Ingestion"]) async def check_updates( documents: Dict[str, Any], @@ -269,41 +194,85 @@ async def get_repo_status( } -# Query endpoints (stubs for future implementation) -# NOTE: /query/hybrid is now implemented in routers/hybrid_rag.py +# Query endpoints +# NOTE: /query/hybrid is implemented in routers/hybrid_rag.py @app.post("/query/semantic", tags=["Query"]) async def semantic_query( - query: Dict[str, Any], + query: str = Query(..., min_length=1, description="Search query text"), + user: str = Query(default=DEFAULT_USER, description="User identifier"), + limit: int = Query(default=10, ge=1, le=100, description="Maximum results"), + score_threshold: float = Query(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score"), + qdrant_client: QdrantDep = None, + wiki_client: WikiJSDep = None, + ollama_client: OllamaDep = None, api_key: str = Depends(verify_api_key) -) -> Dict[str, Any]: +): """ - Semantic search via Qdrant. - Pure vector similarity search. + Semantic search via Qdrant vector similarity. - TODO: Implement semantic search + Searches document chunks using embedding similarity. Returns matching + chunks with relevance scores, page titles, and paths. + + **Example:** + ``` + POST /query/semantic?query=docker%20configuration&user=jpmschweitzer&limit=10 + ``` + + **Returns:** List of matching chunks with similarity scores (0-1) """ - return { - "message": "Semantic search not yet implemented", - "query": query - } + from src.services.vector_service import VectorService + + vector_service = VectorService(qdrant_client, wiki_client, ollama_client) + try: + return await vector_service.search( + query=query, + user=user, + limit=limit, + score_threshold=score_threshold + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Semantic search failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Search failed") @app.post("/query/graph", tags=["Query"]) async def graph_query( - query: Dict[str, Any], + query: str = Query(..., description="Cypher query to execute"), + user: str = Query(default=DEFAULT_USER, description="User for scoping (auto-filters results)"), + neo4j_client: Neo4jDep = None, + wiki_client: WikiJSDep = None, api_key: str = Depends(verify_api_key) -) -> Dict[str, Any]: +): """ - Graph traversal via Neo4j. - Execute Cypher queries. + Execute a Cypher query against the Neo4j knowledge graph. - TODO: Implement graph queries + Queries are automatically scoped to the user's data for security. + Use this for custom graph traversals beyond what /graph/nodes provides. + + **Example:** + ``` + POST /query/graph?query=MATCH%20(d:Document)-[:MENTIONS]->(p:Person)%20RETURN%20d,p&user=jpmschweitzer + ``` + + **Security:** All queries are user-scoped to prevent cross-user data access. """ - return { - "message": "Graph query not yet implemented", - "query": query - } + from src.services.graph_service import GraphService + + graph_service = GraphService(neo4j_client, wiki_client) + try: + return await graph_service.execute_query( + query=query, + parameters={}, + user=user + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Graph query failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Query execution failed") # Deduplication endpoints