library-desk/src/routers/ingestion.py

"""
Document Ingestion API Router

Endpoints for ingesting wiki pages into the knowledge base (vectors + graph).
"""
from fastapi import APIRouter, Depends, HTTPException, Query
from typing import Optional

from src.services.ingestion_service import IngestionService
from src.models.ingestion import (
    IngestionRequest,
    IngestionResult,
    BatchIngestionRequest,
    BatchIngestionResult
)
from src.core.dependencies import get_ingestion_service, verify_api_key

router = APIRouter(prefix="/ingest", tags=["Document Ingestion"])


@router.post("/page", response_model=IngestionResult)
async def ingest_page(
    request: IngestionRequest,
    ingestion: IngestionService = Depends(get_ingestion_service),
    api_key: str = Depends(verify_api_key)
):
    """
    Ingest a single wiki page into the knowledge base.

    This endpoint:
    1. Fetches page content from Wiki.js
    2. Chunks content and generates embeddings (Qdrant)
    3. Extracts entities and updates knowledge graph (Neo4j)

    ## Use Cases

    - **After page creation**: Automatically called by consolidation service
    - **Manual re-indexing**: Force refresh a page after manual edits
    - **Selective ingestion**: Skip vectors or graph if only one is needed

    ## Performance

    - Typical page: 1-3 seconds
    - Large page (>5000 words): 5-10 seconds
    - Vector and graph ingestion run in parallel

    ## Example

    ```bash
    curl -X POST http://192.168.86.149:8089/ingest/page \
      -H "Authorization: Bearer $API_KEY" \
      -H "Content-Type: application/json" \
      -d '{
        "page_id": 19,
        "user": "jpmschweitzer",
        "force_refresh": false
      }'
    ```
    """
    result = await ingestion.ingest_page(
        page_id=request.page_id,
        user=request.user,
        force_refresh=request.force_refresh,
        skip_vectors=request.skip_vectors,
        skip_graph=request.skip_graph
    )

    if not result.success:
        raise HTTPException(
            status_code=500,
            detail=f"Ingestion failed: {result.error}"
        )

    return result


@router.post("/batch", response_model=BatchIngestionResult)
async def ingest_batch(
    request: BatchIngestionRequest,
    ingestion: IngestionService = Depends(get_ingestion_service),
    api_key: str = Depends(verify_api_key)
):
    """
    Ingest multiple wiki pages concurrently.

    ## Concurrency Control

    The `max_concurrent` parameter controls how many pages are processed simultaneously:
    - **Low (1-2)**: Safer for resource-constrained systems
    - **Medium (3-5)**: Good balance of speed and stability
    - **High (6-10)**: Maximum speed, requires good resources

    ## Batch Size Recommendations

    - **Small batches (<10 pages)**: Use max_concurrent=3-5
    - **Medium batches (10-50 pages)**: Use max_concurrent=3
    - **Large batches (>50 pages)**: Use max_concurrent=2, consider splitting

    ## Example

    ```bash
    curl -X POST http://192.168.86.149:8089/ingest/batch \
      -H "Authorization: Bearer $API_KEY" \
      -H "Content-Type: application/json" \
      -d '{
        "page_ids": [19, 20, 21, 22],
        "user": "jpmschweitzer",
        "max_concurrent": 3
      }'
    ```
    """
    result = await ingestion.ingest_batch(
        page_ids=request.page_ids,
        user=request.user,
        force_refresh=request.force_refresh,
        skip_vectors=request.skip_vectors,
        skip_graph=request.skip_graph,
        max_concurrent=request.max_concurrent
    )

    return result


@router.post("/all", response_model=BatchIngestionResult)
async def ingest_all_pages(
    user: str = Query(default="jpmschweitzer", description="User identifier"),
    path_prefix: Optional[str] = Query(None, description="Path prefix filter (e.g., 'users/jpmschweitzer/tech')"),
    force_refresh: bool = Query(False, description="Force re-ingestion of all pages"),
    max_concurrent: int = Query(3, ge=1, le=10, description="Maximum concurrent ingestion tasks"),
    ingestion: IngestionService = Depends(get_ingestion_service),
    api_key: str = Depends(verify_api_key)
):
    """
    Ingest all wiki pages for a user (bulk re-indexing).

    ## Use Cases

    - **Initial setup**: Index all existing pages
    - **Full re-index**: After major schema changes
    - **Path-specific**: Re-index a specific section (e.g., all tech docs)

    ## Performance

    - **Small wiki (<50 pages)**: 2-5 minutes
    - **Medium wiki (50-200 pages)**: 5-20 minutes
    - **Large wiki (>200 pages)**: 20+ minutes

    **Recommendation**: Run as background job for large wikis

    ## Example

    ```bash
    # Ingest all pages for user
    curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer" \
      -H "Authorization: Bearer $API_KEY"

    # Ingest only tech docs
    curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer&path_prefix=users/jpmschweitzer/tech" \
      -H "Authorization: Bearer $API_KEY"
    ```
    """
    result = await ingestion.ingest_all_pages(
        user=user,
        path_prefix=path_prefix,
        force_refresh=force_refresh,
        max_concurrent=max_concurrent
    )

    return result