Initial commit: core-api service extraction from portainer-core

src/controllers/__init__.py

@@ -0,0 +1,5 @@
+"""
+Controllers package for Core-API
+
+Provides controller-based routing architecture for better code organization.
+"""

src/controllers/ai_controller.py

@@ -0,0 +1,219 @@
+"""
+AI Metrics Proxy Controller
+
+Provides proxy endpoints to Core-AI service metrics.
+Allows external access to AI performance stats via core-api.
+"""
+from fastapi import APIRouter, HTTPException
+from typing import Dict, List, Any
+from src.clients.ai_client import get_ai_client
+from src.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+# Create router
+router = APIRouter(
+    prefix="/ai",
+    tags=["AI Metrics"]
+)
+
+
+@router.get(
+    "/health",
+    summary="Check Core-AI service health",
+    description="Verify that the Core-AI service is accessible and responding"
+)
+async def ai_health_check():
+    """
+    Check if Core-AI service is healthy
+
+    Returns:
+        Health status and availability
+    """
+    try:
+        ai_client = get_ai_client()
+        is_healthy = await ai_client.health_check()
+
+        return {
+            "service": "core-ai",
+            "status": "healthy" if is_healthy else "unhealthy",
+            "accessible": is_healthy
+        }
+
+    except Exception as e:
+        logger.error(f"AI health check failed: {e}")
+        return {
+            "service": "core-ai",
+            "status": "error",
+            "accessible": False,
+            "error": str(e)
+        }
+
+
+@router.get(
+    "/metrics",
+    response_model=Dict[str, Any],
+    summary="Get comprehensive AI performance metrics",
+    description="Returns detailed metrics including agent performance, tool execution stats, memory system metrics, and user activity"
+)
+async def get_ai_metrics():
+    """
+    Proxy endpoint for Core-AI metrics
+
+    Returns comprehensive AI performance data:
+    - Agent request statistics (total, by type, response times)
+    - Response time percentiles (p50, p95, p99)
+    - Tool execution metrics (calls, success rates, durations)
+    - Memory system statistics (cache hits, consolidations)
+    - User activity tracking
+    - Concurrency metrics
+
+    Returns:
+        Dict with all collected metrics
+
+    Raises:
+        HTTPException: If Core-AI is unreachable or returns error
+    """
+    try:
+        ai_client = get_ai_client()
+        metrics = await ai_client.get_metrics()
+        return metrics
+
+    except Exception as e:
+        logger.error(f"Failed to fetch AI metrics: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=f"Core-AI service unavailable: {str(e)}"
+        )
+
+
+@router.get(
+    "/metrics/errors",
+    response_model=Dict[str, Any],
+    summary="Get recent request errors",
+    description="Returns recent AI agent request errors with timestamps and details"
+)
+async def get_ai_errors(limit: int = 20):
+    """
+    Get recent AI request errors
+
+    Args:
+        limit: Maximum number of errors to return (default: 20)
+
+    Returns:
+        Dict with error list and total count
+
+    Example response:
+        {
+            "errors": [
+                {
+                    "timestamp": "2025-12-03T19:45:12Z",
+                    "agent_type": "pydantic",
+                    "error": "Connection timeout",
+                    "duration_ms": 5000
+                }
+            ],
+            "total": 1
+        }
+    """
+    try:
+        ai_client = get_ai_client()
+        errors = await ai_client.get_recent_errors(limit=limit)
+
+        return {
+            "errors": errors,
+            "total": len(errors)
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to fetch AI errors: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=f"Core-AI service unavailable: {str(e)}"
+        )
+
+
+@router.get(
+    "/metrics/tool-failures",
+    response_model=Dict[str, Any],
+    summary="Get recent tool execution failures",
+    description="Returns recent tool execution failures with error details"
+)
+async def get_ai_tool_failures(limit: int = 20):
+    """
+    Get recent tool execution failures
+
+    Args:
+        limit: Maximum number of failures to return (default: 20)
+
+    Returns:
+        Dict with failure list and total count
+
+    Example response:
+        {
+            "failures": [
+                {
+                    "timestamp": "2025-12-03T19:50:30Z",
+                    "tool_name": "list_containers",
+                    "error": "Connection refused",
+                    "duration_ms": 150
+                }
+            ],
+            "total": 1
+        }
+    """
+    try:
+        ai_client = get_ai_client()
+        failures = await ai_client.get_tool_failures(limit=limit)
+
+        return {
+            "failures": failures,
+            "total": len(failures)
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to fetch tool failures: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=f"Core-AI service unavailable: {str(e)}"
+        )
+
+
+@router.post(
+    "/metrics/reset",
+    summary="Reset all AI metrics (admin)",
+    description="Clear all collected metrics. This is an administrative operation that resets all counters and history."
+)
+async def reset_ai_metrics():
+    """
+    Reset all AI metrics (admin operation)
+
+    Clears all collected metrics including:
+    - Request history
+    - Tool execution stats
+    - Memory system metrics
+    - Error logs
+
+    Returns:
+        Success confirmation
+
+    Note:
+        This is an administrative operation that should be used carefully.
+        All historical data will be lost.
+    """
+    try:
+        ai_client = get_ai_client()
+        await ai_client.reset_metrics()
+
+        logger.info("AI metrics reset successfully")
+        return {
+            "success": True,
+            "message": "AI metrics reset successfully"
+        }
+
+    except Exception as e:
+        logger.error(f"Failed to reset AI metrics: {e}")
+        raise HTTPException(
+            status_code=503,
+            detail=f"Core-AI service unavailable: {str(e)}"
+        )

src/controllers/base.py

@@ -0,0 +1,50 @@
+"""
+Base controller class for Core-API
+
+Provides common functionality for all controllers.
+"""
+from fastapi import APIRouter
+from abc import ABC, abstractmethod
+
+
+class BaseController(ABC):
+    """
+    Base controller class with common functionality
+
+    All controllers should inherit from this class and implement
+    the create_router() method to define their endpoints.
+    """
+
+    def __init__(self, prefix: str, tags: list[str]):
+        """
+        Initialize base controller
+
+        Args:
+            prefix: URL prefix for this controller's routes
+            tags: OpenAPI tags for documentation grouping
+        """
+        self.prefix = prefix
+        self.tags = tags
+        self._router = None
+
+    @abstractmethod
+    def create_router(self) -> APIRouter:
+        """
+        Create and configure the FastAPI router for this controller
+
+        Returns:
+            Configured APIRouter instance with all endpoints
+        """
+        pass
+
+    @property
+    def router(self) -> APIRouter:
+        """
+        Get the router instance, creating it if needed
+
+        Returns:
+            APIRouter instance
+        """
+        if self._router is None:
+            self._router = self.create_router()
+        return self._router

src/controllers/health_controller.py

@@ -0,0 +1,248 @@
+"""
+Health Controller
+
+Provides service health and information endpoints
+"""
+from fastapi import APIRouter, Response
+from fastapi.responses import JSONResponse
+
+from src.controllers.base import BaseController
+from src.config import get_settings
+from src.logging_config import get_logger
+from src.models.ollama_client import get_ollama_client
+
+# Note: Agent functionality moved to separate core-ai service (Dec 2025)
+# This service (core-api) only provides infrastructure management and tools
+AGENT_AVAILABLE = False
+
+
+logger = get_logger(__name__)
+
+
+class HealthController(BaseController):
+    """
+    Controller for service health and information
+
+    Provides endpoints for:
+    - Service information and status
+    - Health checks
+    """
+
+    def __init__(self):
+        super().__init__(prefix="", tags=["Health"])
+
+    def create_router(self) -> APIRouter:
+        """Create and configure the router"""
+        router = APIRouter(tags=self.tags)
+        settings = get_settings()
+
+        @router.get(
+            "/",
+            summary="Service information",
+            response_class=JSONResponse
+        )
+        async def root():
+            """
+            Get service information and health status
+
+            Returns basic information about the API service and available endpoints.
+            """
+            logger.debug("Root endpoint accessed")
+            return {
+                "service": settings.app_name,
+                "version": settings.app_version,
+                "status": "healthy",
+                "documentation": {
+                    "swagger_ui": "/docs",
+                    "redoc": "/redoc",
+                    "openapi_spec": "/openapi.json"
+                },
+                "endpoints": {
+                    "chat_completions": "/v1/chat/completions",
+                    "models": "/v1/models",
+                    "conversations": "/v1/conversations",
+                    "web_scraper": "/web-scraper/scrape",
+                    "infrastructure": "/infrastructure",
+                    "health": "/health",
+                    "health_full": "/health/full"
+                }
+            }
+
+        @router.get(
+            "/health",
+            summary="Health check",
+            response_class=JSONResponse
+        )
+        async def health_check():
+            """
+            Simple health check endpoint for container orchestration
+
+            Returns a 200 OK status when the service is running properly.
+            Used by Docker, Kubernetes, and load balancers.
+            """
+            ollama_client = get_ollama_client()
+            ollama_healthy = await ollama_client.health_check()
+
+            return {
+                "status": "healthy",
+                "ollama_connected": ollama_healthy
+            }
+
+        @router.get(
+            "/health/full",
+            summary="Fast health check for Docker",
+        )
+        async def full_health_check(response: Response):
+            """
+            Fast health check for container orchestration (Docker/K8s).
+
+            Checks component availability WITHOUT running expensive operations.
+            Returns 200 OK if all components are available, otherwise 503.
+
+            For detailed diagnostics, use /health/diagnostics instead.
+            """
+            import time
+            start_time = time.time()
+
+            # Check 1: Ollama connection + verify agent model is available
+            ollama_client = get_ollama_client()
+            ollama_healthy = False
+            ollama_error = None
+            model_available = False
+
+            try:
+                # Ping Ollama
+                ollama_healthy = await ollama_client.health_check()
+
+                # Verify the agent model is pulled and check what's currently loaded
+                models_info = {}
+                if ollama_healthy:
+                    try:
+                        models_response = await ollama_client.list_models()
+                        available_models = [m.get('name', '') for m in models_response.get('models', [])]
+                        model_available = settings.agent_model in available_models
+
+                        # Get info about currently loaded models (those with size in memory)
+                        loaded_models = [
+                            m.get('name', '') for m in models_response.get('models', [])
+                            if m.get('size', 0) > 0
+                        ]
+
+                        models_info = {
+                            "configured": settings.agent_model,
+                            "available": model_available,
+                            "total_in_ollama": len(available_models),
+                            "currently_loaded": loaded_models if loaded_models else ["none"]
+                        }
+
+                        if not model_available:
+                            ollama_error = f"Model '{settings.agent_model}' not found in Ollama. Available: {', '.join(available_models[:3])}"
+                            ollama_healthy = False
+                    except Exception as e:
+                        ollama_error = f"Could not list Ollama models: {str(e)}"
+                        ollama_healthy = False
+
+            except Exception as e:
+                ollama_error = str(e)
+                logger.warning(f"Ollama health check failed: {ollama_error}")
+
+            # Note: Agent functionality moved to separate core-ai service
+            # This service only needs Ollama for embeddings (infrastructure tools)
+            # Agent health is checked separately in core-ai service
+
+            # Determine overall status (only Ollama required for core-api)
+            is_healthy = ollama_healthy
+
+            elapsed_ms = int((time.time() - start_time) * 1000)
+            status_code = 200 if is_healthy else 503
+            response.status_code = status_code
+
+            return {
+                "status": "healthy" if is_healthy else "unhealthy",
+                "status_code": status_code,
+                "response_time_ms": elapsed_ms,
+                "components": {
+                    "ollama": {
+                        "status": "✅ healthy" if ollama_healthy else "❌ unhealthy",
+                        "models": models_info if models_info else {
+                            "configured": settings.agent_model,
+                            "available": False
+                        },
+                        "error": ollama_error
+                    },
+                    "note": "AI agent functionality available in separate core-ai service (port 8086)"
+                }
+            }
+
+        @router.get(
+            "/health/diagnostics",
+            summary="Detailed system diagnostics",
+        )
+        async def diagnostics(deep_test: bool = False):
+            """
+            Comprehensive system diagnostics with detailed component information.
+
+            Query Parameters:
+            - deep_test: Set to true to actually test agent generation (slow, ~5-10s)
+
+            Returns detailed information about all system components.
+            """
+            import time
+
+            start_time = time.time()
+            diagnostics = {
+                "timestamp": time.time(),
+                "service": {
+                    "name": settings.app_name,
+                    "version": settings.app_version,
+                    "purpose": "Infrastructure management and tools API"
+                },
+                "components": {}
+            }
+
+            # 1. Ollama Connection
+            ollama_client = get_ollama_client()
+            try:
+                ollama_healthy = await ollama_client.health_check()
+                diagnostics["components"]["ollama"] = {
+                    "status": "✅ connected",
+                    "url": settings.ollama_base_url,
+                    "timeout": settings.ollama_timeout,
+                    "default_model": settings.default_model
+                }
+            except Exception as e:
+                diagnostics["components"]["ollama"] = {
+                    "status": "❌ error",
+                    "error": str(e)
+                }
+
+            # 2. Agent Stack - Moved to separate core-ai service
+            diagnostics["components"]["agent"] = {
+                "status": "N/A",
+                "note": "AI agent functionality moved to separate core-ai service (port 8086)",
+                "check_url": "http://core-ai:8086/health"
+            }
+
+            # 3. Memory System (Qdrant) - Moved to core-ai service
+            diagnostics["components"]["qdrant"] = {
+                "status": "N/A",
+                "note": "Memory system managed by core-ai service (port 8086)"
+            }
+
+            # 4. Configuration
+            diagnostics["configuration"] = {
+                "agent_fallback_enabled": settings.agent_fallback_enabled,
+                "memory_tier1_max_turns": settings.memory_tier1_max_turns,
+                "cors_origins": settings.cors_origins[:2] if len(settings.cors_origins) > 2 else settings.cors_origins
+            }
+
+            elapsed_ms = int((time.time() - start_time) * 1000)
+            diagnostics["response_time_ms"] = elapsed_ms
+
+            return diagnostics
+
+        return router
+
+
+# Create controller instance
+health_controller = HealthController()

src/controllers/static_controller.py

@@ -0,0 +1,116 @@
+"""
+Static Files Controller
+
+Serves static files for widgets and other frontend assets.
+"""
+from fastapi import APIRouter
+from fastapi.responses import FileResponse, HTMLResponse
+from pathlib import Path
+import os
+
+from src.controllers.base import BaseController
+from src.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class StaticController(BaseController):
+    """
+    Controller for serving static files
+
+    Provides endpoints for:
+    - Organizr widgets
+    - Other static assets
+    """
+
+    def __init__(self):
+        super().__init__(prefix="/static", tags=["Static"])
+        self.static_dir = Path(__file__).parent.parent.parent / "static"
+
+    def create_router(self) -> APIRouter:
+        """Create and configure the router"""
+        router = APIRouter(prefix=self.prefix, tags=self.tags)
+
+        @router.get(
+            "/widgets/{filename}",
+            response_class=HTMLResponse,
+            summary="Get widget file"
+        )
+        async def get_widget(filename: str):
+            """
+            Serve widget HTML files
+
+            Args:
+                filename: Widget filename (e.g., service-control.html)
+
+            Returns:
+                HTML file content
+            """
+            widget_path = self.static_dir / "widgets" / filename
+
+            if not widget_path.exists():
+                return HTMLResponse(
+                    content=f"<h1>404 - Widget not found</h1><p>{filename}</p>",
+                    status_code=404
+                )
+
+            if not widget_path.is_file():
+                return HTMLResponse(
+                    content=f"<h1>400 - Not a file</h1>",
+                    status_code=400
+                )
+
+            # Security: Ensure the path is within the static directory
+            try:
+                widget_path.resolve().relative_to(self.static_dir.resolve())
+            except ValueError:
+                return HTMLResponse(
+                    content=f"<h1>403 - Forbidden</h1>",
+                    status_code=403
+                )
+
+            logger.info(f"Serving widget: {filename}")
+            return FileResponse(
+                widget_path,
+                media_type="text/html",
+                headers={
+                    "Cache-Control": "no-cache, no-store, must-revalidate",
+                    "Pragma": "no-cache",
+                    "Expires": "0"
+                }
+            )
+
+        @router.get(
+            "/widgets",
+            summary="List available widgets"
+        )
+        async def list_widgets():
+            """
+            List all available widget files
+
+            Returns:
+                List of widget filenames
+            """
+            widgets_dir = self.static_dir / "widgets"
+
+            if not widgets_dir.exists():
+                return {"widgets": [], "message": "Widgets directory not found"}
+
+            widgets = []
+            for file in widgets_dir.glob("*.html"):
+                widgets.append({
+                    "name": file.name,
+                    "url": f"/static/widgets/{file.name}",
+                    "size": file.stat().st_size
+                })
+
+            return {
+                "widgets": widgets,
+                "count": len(widgets)
+            }
+
+        return router
+
+
+# Create controller instance
+static_controller = StaticController()

src/controllers/tools_controller.py

@@ -0,0 +1,168 @@
+"""
+Tools Controller
+
+Provides utility tool endpoints including:
+- Web scraping and content extraction
+- DNS lookups
+"""
+from fastapi import APIRouter, HTTPException, status
+
+from src.controllers.base import BaseController
+from src.logging_config import get_logger
+from src.web_scraper.schemas import WebScraperRequest, WebScraperResponse
+from src.web_scraper.service import WebScraperService
+from src.web_scraper.exceptions import FetchError, ScrapingError
+from src.dns.schemas import DNSLookupRequest, DNSLookupResponse
+from src.dns.service import DNSService
+from src.dns.exceptions import DNSQueryError
+
+logger = get_logger(__name__)
+
+
+class ToolsController(BaseController):
+    """
+    Controller for utility tools
+
+    Provides endpoints for:
+    - Web scraping and content extraction
+    - DNS lookups
+    """
+
+    def __init__(self):
+        super().__init__(prefix="/tools", tags=["Tools"])
+        # Initialize services (could be dependency injected for testing)
+        self.scraper_service = WebScraperService()
+        self.dns_service = DNSService()
+
+    def create_router(self) -> APIRouter:
+        """Create and configure the router"""
+        router = APIRouter(prefix=self.prefix, tags=self.tags)
+
+        @router.post(
+            "/scrape",
+            response_model=WebScraperResponse,
+            status_code=status.HTTP_200_OK,
+            summary="Scrape website content",
+            description="""
+            Scrape and extract main content from a website.
+
+            Uses trafilatura for intelligent content extraction (articles, blog posts, documentation),
+            with BeautifulSoup as fallback. Perfect for feeding webpage content to LLMs.
+
+            **Features:**
+            - Intelligent main content extraction
+            - Removes navigation, ads, footers
+            - Optional link extraction
+            - Configurable content length limits
+
+            **Rate Limiting:** None (internal network use only)
+            """
+        )
+        async def scrape_website(request: WebScraperRequest) -> WebScraperResponse:
+            """
+            Scrape a website and extract its main content
+
+            Args:
+                request: Scraping request with URL and options
+
+            Returns:
+                Extracted content with metadata
+
+            Raises:
+                HTTPException: 400 for fetch errors, 500 for processing errors
+            """
+            try:
+                logger.info(f"Received scrape request for: {request.url}")
+                result = await self.scraper_service.scrape_url(request)
+                return result
+
+            except FetchError as e:
+                logger.warning(f"Fetch failed: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=f"Failed to fetch URL: {str(e)}"
+                )
+
+            except ScrapingError as e:
+                logger.error(f"Scraping failed: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=f"Failed to extract content: {str(e)}"
+                )
+
+            except Exception as e:
+                logger.error(f"Unexpected error: {str(e)}", exc_info=True)
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail="An unexpected error occurred"
+                )
+
+        @router.post(
+            "/dns/lookup",
+            response_model=DNSLookupResponse,
+            status_code=status.HTTP_200_OK,
+            summary="Perform DNS lookup",
+            description="""
+            Perform DNS lookups for various record types.
+
+            Uses dnspython for reliable DNS queries with support for multiple record types
+            and custom nameservers. Perfect for troubleshooting DNS issues and checking
+            domain configurations.
+
+            **Supported Record Types:**
+            - A: IPv4 address records
+            - AAAA: IPv6 address records
+            - MX: Mail exchange records
+            - TXT: Text records (SPF, DKIM, etc.)
+            - CNAME: Canonical name records
+            - NS: Nameserver records
+            - SOA: Start of authority records
+            - PTR: Pointer records (reverse DNS)
+            - CAA: Certification authority authorization
+            - SRV: Service records
+
+            **Features:**
+            - Custom nameserver support (e.g., 8.8.8.8, 1.1.1.1)
+            - Query time measurement
+            - Detailed error messages
+
+            **Rate Limiting:** None (internal network use only)
+            """
+        )
+        async def dns_lookup(request: DNSLookupRequest) -> DNSLookupResponse:
+            """
+            Perform DNS lookup for a domain
+
+            Args:
+                request: DNS lookup request with domain, record type, and optional nameserver
+
+            Returns:
+                DNS lookup results with records and metadata
+
+            Raises:
+                HTTPException: 400 for invalid queries, 500 for processing errors
+            """
+            try:
+                logger.info(f"Received DNS lookup request for: {request.domain} ({request.record_type})")
+                result = await self.dns_service.lookup(request)
+                return result
+
+            except DNSQueryError as e:
+                logger.warning(f"DNS query error: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=f"DNS query failed: {str(e)}"
+                )
+
+            except Exception as e:
+                logger.error(f"Unexpected error during DNS lookup: {str(e)}", exc_info=True)
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail="An unexpected error occurred during DNS lookup"
+                )
+
+        return router
+
+
+# Create controller instance
+tools_controller = ToolsController()